{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 0 - Importing packages","metadata":{}},{"cell_type":"code","source":"# imports\nimport os\nimport random\nimport datetime\nimport csv\n\n# Data analysis and manipulation\nimport numpy as np\nimport pandas as pd\n\n# Data visualization\nfrom matplotlib import animation\nimport matplotlib.pyplot as plt\nfrom IPython import display\nimport seaborn as sns\nimport plotly\n\n# ML, DL & Modelling\n# from sklearn import \nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, callbacks\nfrom tensorflow.keras.models import Sequential\n\n# Garbage collect for generator\nimport gc\n\n# https://www.kaggle.com/datasets/bencetar/prep-valid-masks\n# https://www.kaggle.com/datasets/bencetar/prep-train-masks\n# https://www.kaggle.com/datasets/bencetar/contrail-valid-imgs\n# https://www.kaggle.com/datasets/bencetar/contrail-train-imgs","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-18T12:12:36.944151Z","iopub.execute_input":"2023-09-18T12:12:36.944532Z","iopub.status.idle":"2023-09-18T12:12:36.952475Z","shell.execute_reply.started":"2023-09-18T12:12:36.944502Z","shell.execute_reply":"2023-09-18T12:12:36.951333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nRUN_CELL = True\nif RUN_CELL:\n\n BASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation'\n band_choice = ['band_11.npy','band_14.npy','band_15.npy']\n target_suffix = 'human_pixel_masks.npy'\n N_TIMES_BEFORE = 4\n\n # Building list of record_ids\n record_ids = os.listdir(BASE_DIR)\n len_dataset = len(record_ids)\n print(f'The complete dataset contains {len_dataset} observations.')\n\n\n # Keeping only observations that contains contrails\n contrail_record_ids = []\n len(contrail_record_ids)\n for record_id in record_ids:\n # Building target paths\n target_path = os.path.join(BASE_DIR, record_id, target_suffix)\n target = np.load(open(target_path, 'rb'))\n\n # Jumping over observations with no contrails\n if target.sum()==0:\n continue\n else:\n contrail_record_ids.append(record_id)\n\nprint(f'The dataset contains {len(contrail_record_ids)} observations with contrails in them.')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_number = 20\nsample_contrails_id = contrail_record_ids[0:sample_number]\nsample_contrails_id","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_list_paths = []\nfor contrail_id in sample_contrails_id:\n record_path = os.path.join(BASE_DIR, contrail_id)\n sample_list_paths.append(record_path)\nsample_list_paths","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil \nfor path in sample_list_paths:\n target_directory = f\"/kaggle/working/{path.split('/')[-1]}\"\n shutil.copytree(path, target_directory)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/model_histories","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:12:37.988126Z","iopub.execute_input":"2023-09-18T12:12:37.988523Z","iopub.status.idle":"2023-09-18T12:12:38.962358Z","shell.execute_reply.started":"2023-09-18T12:12:37.988492Z","shell.execute_reply":"2023-09-18T12:12:38.961049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# I - Retrieving and prepping the datasets","metadata":{}},{"cell_type":"code","source":"%%time\n# Checking the length of total dataset and defining length of each chunck\n\nRUN_CELL = True\nif RUN_CELL:\n\n BASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\n band_choice = ['band_11.npy','band_14.npy','band_15.npy']\n target_suffix = 'human_pixel_masks.npy'\n N_TIMES_BEFORE = 4\n\n # Building list of record_ids\n record_ids = os.listdir(BASE_DIR)\n len_dataset = len(record_ids)\n print(f'The complete dataset contains {len_dataset} observations.')\n\n\n # Keeping only observations that contains contrails\n contrail_record_ids = []\n len(contrail_record_ids)\n for record_id in record_ids:\n # Building target paths\n target_path = os.path.join(BASE_DIR, record_id, target_suffix)\n target = np.load(open(target_path, 'rb'))\n\n # Jumping over observations with no contrails\n if target.sum()==0:\n continue\n else:\n contrail_record_ids.append(record_id)\n\nprint(f'The dataset contains {len(contrail_record_ids)} observations with contrails in them.')","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:12:39.297416Z","iopub.execute_input":"2023-09-18T12:12:39.300768Z","iopub.status.idle":"2023-09-18T12:15:48.26939Z","shell.execute_reply.started":"2023-09-18T12:12:39.30072Z","shell.execute_reply":"2023-09-18T12:15:48.268405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"contrail_record_ids","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:20:22.538057Z","iopub.execute_input":"2023-09-18T12:20:22.538468Z","iopub.status.idle":"2023-09-18T12:20:22.565096Z","shell.execute_reply.started":"2023-09-18T12:20:22.538436Z","shell.execute_reply":"2023-09-18T12:20:22.564068Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining the number of chuncks\nchunck_size = 500\nnb_chuncks = int(len(contrail_record_ids)/chunck_size) + 1","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:48.271428Z","iopub.execute_input":"2023-09-18T12:15:48.271763Z","iopub.status.idle":"2023-09-18T12:15:48.276576Z","shell.execute_reply.started":"2023-09-18T12:15:48.27173Z","shell.execute_reply":"2023-09-18T12:15:48.275515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining X normalization function\n\ndef normalize_range(data, bounds):\n return (data - bounds[0]) / (bounds[1] - bounds[0])","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:48.278211Z","iopub.execute_input":"2023-09-18T12:15:48.278833Z","iopub.status.idle":"2023-09-18T12:15:48.293144Z","shell.execute_reply.started":"2023-09-18T12:15:48.2788Z","shell.execute_reply":"2023-09-18T12:15:48.292183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Building a dictionnary containing each chunck\n\nRUN_CELL = True\nif RUN_CELL:\n\n list_chuncks = []\n chunck_names = []\n for i in range(0, nb_chuncks):\n chunck_name = f'chunck_{i}'\n chunk_i_records = contrail_record_ids[(i*chunck_size):((i+1)*chunck_size)]\n chunck_names.append(chunck_name)\n list_chuncks.append(chunk_i_records)\n\n chuncks_record_dict = dict(zip(chunck_names, list_chuncks))","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:48.297089Z","iopub.execute_input":"2023-09-18T12:15:48.297446Z","iopub.status.idle":"2023-09-18T12:15:48.305952Z","shell.execute_reply.started":"2023-09-18T12:15:48.297414Z","shell.execute_reply":"2023-09-18T12:15:48.304844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Function to load a X chunck and normalizing its data\n\ndef load_normalize_X_chunck(chunck):\n \n X_chunck = []\n for record_id in chuncks_record_dict[chunck]:\n # Building band paths\n record_first_band_path = os.path.join(BASE_DIR, record_id, band_choice[0])\n record_second_band_path = os.path.join(BASE_DIR, record_id, band_choice[1])\n record_third_band_path = os.path.join(BASE_DIR, record_id, band_choice[2])\n \n # Loading each band\n first_band = np.load(open(record_first_band_path, 'rb'))[:,:,N_TIMES_BEFORE]\n second_band = np.load(open(record_second_band_path, 'rb'))[:,:,N_TIMES_BEFORE]\n third_band = np.load(open(record_third_band_path, 'rb'))[:,:,N_TIMES_BEFORE]\n \n # Normalizing each band with its relevant bounds\n # a - Defining bounds for each band\n _T11_BOUNDS = (243, 303)\n _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n _TDIFF_BOUNDS = (-4, 2)\n \n # b - Applying normalization functions\n normalized_r = normalize_range(third_band - second_band, _TDIFF_BOUNDS)\n normalized_g = normalize_range(second_band - first_band, _CLOUD_TOP_TDIFF_BOUNDS)\n normalized_b = normalize_range(second_band, _T11_BOUNDS)\n \n # Building a single record from all bands\n record = np.clip(np.stack([normalized_r, normalized_g, normalized_b], axis=2), 0, 1)\n \n # Appending chunck list\n X_chunck.append(record)\n \n # Building the chunck array\n X_chunck_array = np.stack(X_chunck, axis = 0)\n \n return X_chunck_array","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:48.307481Z","iopub.execute_input":"2023-09-18T12:15:48.307814Z","iopub.status.idle":"2023-09-18T12:15:48.322705Z","shell.execute_reply.started":"2023-09-18T12:15:48.307784Z","shell.execute_reply":"2023-09-18T12:15:48.321633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Function to load a y chunck\n\ndef load_y_chunck(chunck):\n \n y_chunck = []\n for record_id in chuncks_record_dict[chunck]:\n \n # Building target paths and loading data\n target_path = os.path.join(BASE_DIR, record_id, target_suffix)\n target = np.load(open(target_path, 'rb'))\n \n # Appending chunck list\n y_chunck.append(target)\n \n # Building the chunck array\n y_chunck_array = np.stack(y_chunck, axis = 0).astype(float)\n \n return y_chunck_array","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:48.324262Z","iopub.execute_input":"2023-09-18T12:15:48.324915Z","iopub.status.idle":"2023-09-18T12:15:48.340052Z","shell.execute_reply.started":"2023-09-18T12:15:48.324883Z","shell.execute_reply":"2023-09-18T12:15:48.339099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# II - Building the evaluation metrics","metadata":{}},{"cell_type":"code","source":"# Defining the dice metric to be used in model compilation\n\ndef proba_to_pixel(y):\n return tf.where(y > 0.5, tf.ones_like(y),tf.zeros_like(y))\ndef dice_metric(y_true, y_pred):\n y_pred = proba_to_pixel(y_pred)\n y_true = proba_to_pixel(y_true)\n smooth = 1e-5 \n y_true_sum = tf.reduce_sum(y_true)\n y_pred_sum = tf.reduce_sum(y_pred)\n intersection = tf.reduce_sum(y_true * y_pred)\n union = y_true_sum + y_pred_sum\n dice = (2. * intersection + smooth) / (union + smooth)\n return dice\ndef dice_loss(y_true, y_pred):\n smooth = 1e-5 \n y_true_sum = tf.reduce_sum(y_true)\n y_pred_sum = tf.reduce_sum(y_pred)\n intersection = tf.reduce_sum(y_true * y_pred)\n union = y_true_sum + y_pred_sum\n dice = (2. * intersection + smooth) / (union + smooth)\n return 1 - dice \ndef binary_crossentropy(y_true, y_pred) :\n return (-1)*tf.math.reduce_sum(y_true * tf.math.log(y_pred + 1e-7) + (1-y_true)*tf.math.log(1-y_pred + 1e-7))\ndef weighted_binary_crossentropy_func(weight_of_1) :\n def weighted_binary_crossentropy(y_true, y_pred):\n return (-1)*tf.math.reduce_sum((1-weight_of_1)*y_true * tf.math.log(y_pred + 1e-7) + weight_of_1*(1-y_true)*tf.math.log(1-y_pred + 1e-7))\n return weighted_binary_crossentropy","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:48.341782Z","iopub.execute_input":"2023-09-18T12:15:48.342545Z","iopub.status.idle":"2023-09-18T12:15:48.358856Z","shell.execute_reply.started":"2023-09-18T12:15:48.342504Z","shell.execute_reply":"2023-09-18T12:15:48.357913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining a function to visualize the history\ndef plot_history(history, title='', axs=None, exp_name=\"\"):\n if axs is not None:\n ax1, ax2 = axs\n else:\n f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))\n if len(exp_name) > 0 and exp_name[0] != '_':\n exp_name = '_' + exp_name\n ax1.plot(history.history['loss'], label = 'train' + exp_name)\n ax1.plot(history.history['val_loss'], label = 'val' + exp_name)\n ax1.set_title('loss')\n ax1.legend()\n ax2.plot(history.history['dice_metric'], label='train dice metric' + exp_name)\n ax2.plot(history.history['val_dice_metric'], label='val dice metric' + exp_name)\n ax2.set_title('Dice metric')\n ax2.legend()\n return (ax1, ax2)","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:48.362265Z","iopub.execute_input":"2023-09-18T12:15:48.362549Z","iopub.status.idle":"2023-09-18T12:15:48.372274Z","shell.execute_reply.started":"2023-09-18T12:15:48.362525Z","shell.execute_reply":"2023-09-18T12:15:48.371321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# III - Building the advanced U-Net model","metadata":{}},{"cell_type":"code","source":"# 1. Building the Unet model \ndef build_model(input_layer, start_neurons, drop_out_factor):\n \n # Downsampling path / Decoder = convolutional layers extracting features from the image while \n # reducing it size\n conv1 = layers.Conv2D(start_neurons * 1, (3, 3), activation=\"relu\", padding=\"same\")(input_layer)\n conv1 = layers.Conv2D(start_neurons * 1, (3, 3), activation=\"relu\", padding=\"same\")(conv1)\n pool1 = layers.MaxPooling2D((2, 2))(conv1)\n pool1 = layers.Dropout(0.25 * drop_out_factor)(pool1)\n\n conv2 = layers.Conv2D(start_neurons * 2, (3, 3), activation=\"relu\", padding=\"same\")(pool1)\n conv2 = layers.Conv2D(start_neurons * 2, (3, 3), activation=\"relu\", padding=\"same\")(conv2)\n pool2 = layers.MaxPooling2D((2, 2))(conv2)\n pool2 = layers.Dropout(0.5 * drop_out_factor)(pool2)\n\n conv3 = layers.Conv2D(start_neurons * 4, (3, 3), activation=\"relu\", padding=\"same\")(pool2)\n conv3 = layers.Conv2D(start_neurons * 4, (3, 3), activation=\"relu\", padding=\"same\")(conv3)\n pool3 = layers.MaxPooling2D((2, 2))(conv3)\n pool3 = layers.Dropout(0.5 * drop_out_factor)(pool3)\n\n conv4 = layers.Conv2D(start_neurons * 8, (3, 3), activation=\"relu\", padding=\"same\")(pool3)\n conv4 = layers.Conv2D(start_neurons * 8, (3, 3), activation=\"relu\", padding=\"same\")(conv4)\n pool4 = layers.MaxPooling2D((2, 2))(conv4)\n pool4 = layers.Dropout(0.5 * drop_out_factor)(pool4)\n\n # Middle path / Bottleneck = CNN with large number of layers to extract the image's most important /\n # complex features\n convm = layers.Conv2D(start_neurons * 16, (3, 3), activation=\"relu\", padding=\"same\")(pool4)\n convm = layers.Conv2D(start_neurons * 16, (3, 3), activation=\"relu\", padding=\"same\")(convm)\n \n # Upsampling / Decoder using Transpose convolution = expanding the size of the image to reach an output \n # (the mask) with same size as input image \n \n # Skip connections: it helps the model learn both detailed information from the \n # downsampling / decoder and high-level info from the upsampling / decoder \n \n # In practice, after the transposed convolution, the image is upsized from 28x28x1024 β†’ 56x56x512\n # this image is then concatenated with the corresponding image from the downsampling path \n # and together makes an image of size 56x56x1024. \n \n # upsamppling \n deconv4 = layers.Conv2DTranspose(start_neurons * 8, (3, 3), strides=(2, 2), padding=\"same\")(convm)\n # skip-connection\n uconv4 = layers.concatenate([deconv4, conv4])\n uconv4 = layers.Dropout(0.5 * drop_out_factor)(uconv4)\n uconv4 = layers.Conv2D(start_neurons * 8, (3, 3), activation=\"relu\", padding=\"same\")(uconv4)\n uconv4 = layers.Conv2D(start_neurons * 8, (3, 3), activation=\"relu\", padding=\"same\")(uconv4)\n\n deconv3 = layers.Conv2DTranspose(start_neurons * 4, (3, 3), strides=(2, 2), padding=\"same\")(uconv4)\n uconv3 = layers.concatenate([deconv3, conv3])\n uconv3 = layers.Dropout(0.5 * drop_out_factor)(uconv3)\n uconv3 = layers.Conv2D(start_neurons * 4, (3, 3), activation=\"relu\", padding=\"same\")(uconv3)\n uconv3 = layers.Conv2D(start_neurons * 4, (3, 3), activation=\"relu\", padding=\"same\")(uconv3)\n\n deconv2 = layers.Conv2DTranspose(start_neurons * 2, (3, 3), strides=(2, 2), padding=\"same\")(uconv3)\n uconv2 = layers.concatenate([deconv2, conv2])\n uconv2 = layers.Dropout(0.5 * drop_out_factor)(uconv2)\n uconv2 = layers.Conv2D(start_neurons * 2, (3, 3), activation=\"relu\", padding=\"same\")(uconv2)\n uconv2 = layers.Conv2D(start_neurons * 2, (3, 3), activation=\"relu\", padding=\"same\")(uconv2)\n\n deconv1 = layers.Conv2DTranspose(start_neurons * 1, (3, 3), strides=(2, 2), padding=\"same\")(uconv2)\n uconv1 = layers.concatenate([deconv1, conv1])\n uconv1 = layers.Dropout(0.5 * drop_out_factor)(uconv1)\n uconv1 = layers.Conv2D(start_neurons * 1, (3, 3), activation=\"relu\", padding=\"same\")(uconv1)\n uconv1 = layers.Conv2D(start_neurons * 1, (3, 3), activation=\"relu\", padding=\"same\")(uconv1)\n \n output_layer = layers.Conv2D(1, (1,1), padding=\"same\", activation=\"sigmoid\")(uconv1)\n \n return output_layer","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:48.375508Z","iopub.execute_input":"2023-09-18T12:15:48.375806Z","iopub.status.idle":"2023-09-18T12:15:48.398539Z","shell.execute_reply.started":"2023-09-18T12:15:48.375768Z","shell.execute_reply":"2023-09-18T12:15:48.397638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" # Instantiating the model\n img_size_target = 256\n number_channels_target = 256\n start_neurons = 64\n \n input_layer = layers.Input((img_size_target, img_size_target, 3))\n output_layer = build_model(input_layer, start_neurons, drop_out_factor=0.5)\n \n # U-Net model with Functional API from Keras\n unet_model = tf.keras.Model(input_layer, output_layer, name=\"U-Net\")\n unet_model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:48.402373Z","iopub.execute_input":"2023-09-18T12:15:48.402626Z","iopub.status.idle":"2023-09-18T12:15:53.814573Z","shell.execute_reply.started":"2023-09-18T12:15:48.402604Z","shell.execute_reply":"2023-09-18T12:15:53.813769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#unet_model.load_weights('/kaggle/working/tf_checkpoint')","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:53.815678Z","iopub.execute_input":"2023-09-18T12:15:53.816065Z","iopub.status.idle":"2023-09-18T12:15:53.971438Z","shell.execute_reply.started":"2023-09-18T12:15:53.816032Z","shell.execute_reply":"2023-09-18T12:15:53.969674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndef train_on_chunck(chunck, start_neurons, drop_out_factor):\n \n # Loading and normalizing X_train\n X_train = load_normalize_X_chunck(chunck)\n \n # Loading y_train\n y_train = load_y_chunck(chunck)\n \n # Instantiating the model\n img_size_target = X_train.shape[1]\n number_channels_target = X_train.shape[-1]\n #start_neurons = 64\n \n input_layer = layers.Input((img_size_target, img_size_target, number_channels_target))\n output_layer = build_model(input_layer, start_neurons, drop_out_factor)\n \n # U-Net model with Functional API from Keras\n unet_model = tf.keras.Model(input_layer, output_layer, name=\"U-Net\")\n \n \n # Loading weights from previous training chunk \n #Mettre une condition pour que Γ§a passe au premier\n \n if chunck != 'chunck_0':\n load_model_weights = True\n if load_model_weights:\n unet_model.load_weights('/kaggle/working/tf_checkpoint')\n \n \n # Compiling the model\n optimizer = tf.keras.optimizers.legacy.Adam(learning_rate=5e-4)\n weight_of_1 = 0.01183131632324377 #on training dataset excluding images without contrails\n \n unet_model.compile(optimizer=optimizer,\n loss=dice_loss,\n metrics=[dice_metric, dice_loss, weighted_binary_crossentropy_func(weight_of_1)])\n \n \n # Building callbacks\n es = tf.keras.callbacks.EarlyStopping(patience=20, restore_best_weights=True)\n lrp = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=5,min_lr=0.00001)\n mcc = tf.keras.callbacks.ModelCheckpoint(filepath='/kaggle/working/tf_checkpoint',save_weights_only=True,save_best_only=True)\n \n \n # Fitting the model\n history_unet_model = unet_model.fit(X_train,\n y_train,\n batch_size=16,\n epochs=15,\n validation_split=0.3,\n callbacks=[es, lrp, mcc],\n verbose=1)\n \n # Delete chunck from memory\n del X_train\n del y_train\n gc.collect \n \n return history_unet_model","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:30:41.356809Z","iopub.execute_input":"2023-09-18T12:30:41.357208Z","iopub.status.idle":"2023-09-18T12:30:41.370099Z","shell.execute_reply.started":"2023-09-18T12:30:41.357176Z","shell.execute_reply":"2023-09-18T12:30:41.369087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# IV - Training the model on chuncked data","metadata":{}},{"cell_type":"code","source":"# Initializing the training history dictionnary\ntraining_history = {f'{chunck}' : \"\" for chunck in chuncks_record_dict.keys()} # Dictionnaire avec batch number + historique","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:30:43.488092Z","iopub.execute_input":"2023-09-18T12:30:43.488832Z","iopub.status.idle":"2023-09-18T12:30:43.496083Z","shell.execute_reply.started":"2023-09-18T12:30:43.488797Z","shell.execute_reply":"2023-09-18T12:30:43.495001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Starting the training loop on all chuncks\n\n# # Creating a sub-folder for the current training session\n# !mkdir /kaggle/working/model_histories/training_$(date +%Y%m%d_%H%M%S)\n# base_dir = '/kaggle/working/model_histories/training_' +datetime.datetime.now().strftime('%Y%m%d_%H%M%S')\n\n# total_training_chunks = len(list(chuncks_record_dict.keys()))\n# for chunck in list(chuncks_record_dict.keys()):\n \n# print(f'Launching training for {chunck} out of {total_training_chunks}πŸš€πŸ‘©β€πŸš€πŸ‹πŸ»πŸ’¦')\n \n# # Calling the training function\n# history_unet_model = train_on_chunck(chunck, start_neurons=64, drop_out_factor=0.5)\n \n# # Building a temporary dictionnary for current chunck\n# dict_history_from_current_chunck = history_unet_model.history\n \n# # Storing training history for this chunck in the global dictionnary\n# training_history[chunck] = dict_history_from_current_chunck\n \n# # Storing fields for the CSV file\n# csv_fields = list(dict_history_from_current_chunck.keys())\n \n# # Creating the folder and openning the CSV file with write permission\n \n# with open(f\"{base_dir}/training_history_{chunck}.csv\", \"w\", newline=\"\",) as csvfile:\n \n# # Create a CSV writer using the field/column names\n# writer = csv.DictWriter(csvfile, fieldnames=csv_fields)\n \n# # Write the header row (column names)\n# writer.writeheader()\n \n# # Write the data\n# for line in range(len(dict_history_from_current_chunck[\"loss\"])):\n# dict_row = {index: dict_history_from_current_chunck[index][line] for index in csv_fields}\n# writer.writerow(dict_row)\n \n# print(f'Training finished for {chunck}! Time for a break ! πŸΊπŸ»β˜•πŸ‘¬')","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:30:43.848982Z","iopub.execute_input":"2023-09-18T12:30:43.850397Z","iopub.status.idle":"2023-09-18T12:30:45.024045Z","shell.execute_reply.started":"2023-09-18T12:30:43.850352Z","shell.execute_reply":"2023-09-18T12:30:45.022532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# V. Visualizing predicted image","metadata":{}},{"cell_type":"code","source":"%%time \n\nRUN_CELL = True\nif RUN_CELL:\n\n BASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation'\n band_choice = ['band_11.npy','band_14.npy','band_15.npy']\n target_suffix = 'human_pixel_masks.npy'\n N_TIMES_BEFORE = 4\n\n # Building list of record_ids\n record_ids = os.listdir(BASE_DIR)\n len_dataset = len(record_ids)\n print(f'The complete dataset contains {len_dataset} observations.')\n\n\n # Keeping only observations that contains contrails\n contrail_record_ids = []\n len(contrail_record_ids)\n for record_id in record_ids:\n # Building target paths\n target_path = os.path.join(BASE_DIR, record_id, target_suffix)\n target = np.load(open(target_path, 'rb'))\n\n # Jumping over observations with no contrails\n if target.sum()==0:\n continue\n else:\n contrail_record_ids.append(record_id)\n\nprint(f'The dataset contains {len(contrail_record_ids)} observations with contrails in them.')","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:27:58.666252Z","iopub.execute_input":"2023-09-18T12:27:58.666616Z","iopub.status.idle":"2023-09-18T12:28:19.401524Z","shell.execute_reply.started":"2023-09-18T12:27:58.666578Z","shell.execute_reply":"2023-09-18T12:28:19.400555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_number = 20\nsample_contrails_id = contrail_record_ids[0:sample_number]\nsample_contrails_id","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:39:47.928026Z","iopub.execute_input":"2023-09-18T12:39:47.928787Z","iopub.status.idle":"2023-09-18T12:39:47.936392Z","shell.execute_reply.started":"2023-09-18T12:39:47.928738Z","shell.execute_reply":"2023-09-18T12:39:47.935308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_list_paths = []\nfor contrail_id in sample_contrails_id:\n record_path = os.path.join(BASE_DIR, contrail_id)\n sample_list_paths.append(record_path)\nsample_list_paths","metadata":{"execution":{"iopub.status.busy":"2023-09-18T13:07:29.281899Z","iopub.execute_input":"2023-09-18T13:07:29.282289Z","iopub.status.idle":"2023-09-18T13:07:29.292096Z","shell.execute_reply.started":"2023-09-18T13:07:29.282257Z","shell.execute_reply":"2023-09-18T13:07:29.291179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil \nfor path in sample_list_paths:\n target_directory = f\"/kaggle/working/{path.split('/')[-1]}\"\n shutil.copytree(path, target_directory)","metadata":{"execution":{"iopub.status.busy":"2023-09-18T13:07:33.386222Z","iopub.execute_input":"2023-09-18T13:07:33.386624Z","iopub.status.idle":"2023-09-18T13:07:41.349643Z","shell.execute_reply.started":"2023-09-18T13:07:33.386592Z","shell.execute_reply":"2023-09-18T13:07:41.348613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"band_choice = ['band_11.npy','band_14.npy','band_15.npy', 'human_pixel_masks']\nX_chunck = []\nfor record_id in chuncks_record_dict[chunck]:\n # Building band paths\n record_first_band_path = os.path.join(BASE_DIR, record_id, band_choice[0])\n record_second_band_path = os.path.join(BASE_DIR, record_id, band_choice[1])\n record_third_band_path = os.path.join(BASE_DIR, record_id, band_choice[2])\n record_target = os.path.join(BASE_DIR, record_id, band_choice[3])\n\n # Loading each band\n first_band = np.load(open(record_first_band_path, 'rb'))\n second_band = np.load(open(record_second_band_path, 'rb'))\n third_band = np.load(open(record_third_band_path, 'rb'))\n target = np.load(open(record_target, 'rb'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Choosing a random image from validation set and defining the corresponding true mask and predicted mask \nimport random \n\nBASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation'\n\n# Choosing a random image from validation set \nvalid_imgs_record_list = os.listdir('/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation')\nrecord_id = '1393637649251558618'\n#record_id = random.sample(valid_imgs_id_list, 1)[0]\n\n# loading 3 bands required for normalization and the mask\nwith open(os.path.join(BASE_DIR, record_id, 'band_11.npy'), 'rb') as f:\n band11 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'band_14.npy'), 'rb') as f:\n band14 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'band_15.npy'), 'rb') as f:\n band15 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'human_pixel_masks.npy'), 'rb') as f:\n output_mask = np.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:31:37.985505Z","iopub.execute_input":"2023-09-18T12:31:37.985901Z","iopub.status.idle":"2023-09-18T12:31:38.106822Z","shell.execute_reply.started":"2023-09-18T12:31:37.985868Z","shell.execute_reply":"2023-09-18T12:31:38.105825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# normalizing the selected image to plot it in RGB ash\n\n_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\ndef normalize_range(data, bounds):\n \"\"\"Maps data to the range [0, 1].\"\"\"\n return (data - bounds[0]) / (bounds[1] - bounds[0])\n\nr = normalize_range(band15 - band14, _TDIFF_BOUNDS)\ng = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\nb = normalize_range(band14, _T11_BOUNDS)\nimage_full_sequences = np.clip(np.stack([r, g, b], axis=2), 0, 1)\nimage_full_sequences.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:31:38.514361Z","iopub.execute_input":"2023-09-18T12:31:38.514704Z","iopub.status.idle":"2023-09-18T12:31:38.540075Z","shell.execute_reply.started":"2023-09-18T12:31:38.514676Z","shell.execute_reply":"2023-09-18T12:31:38.538785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# choosing only the 5th sequence to plot the image \nN_TIMES_BEFORE = 4\ninput_image = image_full_sequences[..., N_TIMES_BEFORE]\n\n# artificially adding 1 dimension to feed to the model, expecting batches\ninput_image_model = tf.expand_dims(input_image, 0)\ninput_image_model.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:31:38.875994Z","iopub.execute_input":"2023-09-18T12:31:38.876708Z","iopub.status.idle":"2023-09-18T12:31:38.886551Z","shell.execute_reply.started":"2023-09-18T12:31:38.876672Z","shell.execute_reply":"2023-09-18T12:31:38.885517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use the model to predict a mask on the input image \npredicted_mask = unet_model.predict(input_image_model)\n\n# Removing the additional dimension to plot the image \npredicted_mask_image = predicted_mask[0,:,:,:]\npredicted_mask_image","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:32:14.239274Z","iopub.execute_input":"2023-09-18T12:32:14.239634Z","iopub.status.idle":"2023-09-18T12:32:14.343152Z","shell.execute_reply.started":"2023-09-18T12:32:14.239604Z","shell.execute_reply":"2023-09-18T12:32:14.342161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plotting the results on a graph \n\nplt.figure(figsize=(18, 6))\nax = plt.subplot(2, 3, 1)\nax.imshow(input_image)\nax.set_title('Input image')\n\nax = plt.subplot(2, 3, 2)\nax.imshow(output_mask, interpolation='none')\nax.set_title('Contrail mask from validation set')\n\nax = plt.subplot(2, 3, 3)\nax.imshow(input_image)\nax.imshow(output_mask, cmap='Reds', alpha=.4, interpolation='none')\nax.set_title('Contrail mask on input image')\n\nplt.figure(figsize=(18, 6))\nax = plt.subplot(2, 3, 4)\nax.imshow(input_image)\nax.set_title('Input image')\n\nax = plt.subplot(2, 3, 5)\nax.imshow(predicted_mask_image, interpolation='none')\nax.set_title('Predicted contrail mask using model')\n\nax = plt.subplot(2, 3, 6)\nax.imshow(input_image)\nax.imshow(predicted_mask_image, cmap='Reds', alpha=.4, interpolation='none')\nax.set_title('Predicted contrail mask on input image');","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:31:39.900122Z","iopub.execute_input":"2023-09-18T12:31:39.900539Z","iopub.status.idle":"2023-09-18T12:31:41.378877Z","shell.execute_reply.started":"2023-09-18T12:31:39.900507Z","shell.execute_reply":"2023-09-18T12:31:41.378028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_prep_imgs_dir = '/kaggle/input/contrail-valid-imgs/contrail_valid'\nval_prep_masks_dir = '/kaggle/input/prep-valid-masks'\n\nvalid_prep_imgs_record_list = os.listdir('/kaggle/input/contrail-valid-imgs/contrail_valid')\nrecord_id = random.sample(valid_prep_imgs_record_list, 1)[0]\ninput_image_path = os.path.join(val_prep_dir, record_id)\ninput_image = np.load(open(input_image_path, 'rb'))\n\nunique_id = record_id.split(\"_\")[2]\noutput_mask_id = f\"prep_mask_{unique_id}\"\noutput_mask_path = os.path.join(val_prep_masks_dir, output_mask_id)\noutput_mask = np.load(open(output_mask_path, 'rb'))\n\n#predicted_mask = unet_model.predict(input_image)","metadata":{"execution":{"iopub.status.busy":"2023-09-18T12:15:53.987964Z","iopub.status.idle":"2023-09-18T12:15:53.988803Z","shell.execute_reply.started":"2023-09-18T12:15:53.988562Z","shell.execute_reply":"2023-09-18T12:15:53.988585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}