{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":51753,"databundleVersionId":5692552,"sourceType":"competition"}],"dockerImageVersionId":30497,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Load Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\n\nimport os\nos.getcwd()\n\n\nimport matplotlib.pyplot as plt\nfrom matplotlib import animation\nimport seaborn as sns\nfrom pathlib import Path\nfrom IPython import display\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import TensorDataset\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Command to print multiple outputs in a single cell\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = \"all\"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# use Cuda\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Input data","metadata":{}},{"cell_type":"code","source":"# Custom dataset class\nbase_dir = '/kaggle/input/google-research-identify-contrails-reduce-global-warming'\n\nclass CustomContrailDataset(Dataset):\n    def __init__(self, base_dir, dataset_type, transform=None, target_transform=None):\n        assert dataset_type in ['train','test','validation']\n        self.base_dir = base_dir\n        self.dataset_type = dataset_type\n        self.tramsform = transform\n        self.target_transform = target_transform\n        self.records = os.listdir(self.base_dir + '/' + self.dataset_type)\n\n    def __len__(self):\n        return len(self.records)\n\n    def __getitem__(self, idx):\n        record_id = self.records[idx]\n        record_dir = os.path.join(self.base_dir,self.dataset_type,record_id)\n        \n        # Load npy data\n        bands_data = []\n        \n        for i in range(8,17):\n            band_dir = os.path.join(record_dir,f'band_{i:02}.npy')\n            band_data = np.load(band_dir)\n            bands_data.append(band_data[:,:,4])\n        \n        bands_data = np.stack(bands_data,axis=-1)\n\n        \n        # load the masks\n        if self.dataset_type in [\"train\",\"validation\"]:\n            target_dir = os.path.join(record_dir,'human_pixel_masks.npy')\n            target = np.load(target_dir)\n            target = target[:,:,0]\n        \n        else:\n            target = 1\n\n        bands_data = torch.tensor(bands_data)\n        target = torch.tensor(target)\n        target = target.to(torch.float32)\n        \n        bands_data = bands_data.permute(2, 0, 1)\n        \n        target = target.view(-1)\n        \n        \n        return(bands_data,target)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create train, validation and test dataset\ntrain_dataset = CustomContrailDataset(base_dir,'train')\ntest_dataset = CustomContrailDataset(base_dir,'test')\nvalidation_dataset = CustomContrailDataset(base_dir,'validation')\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train, test and validation loader\ntorch.manual_seed(42)\nbatch_size = 32\n\ntrain_dl = DataLoader(train_dataset, batch_size, shuffle = True, drop_last=False)\nvalid_dl = DataLoader(validation_dataset, batch_size, shuffle = False, drop_last=False)\ntest_dl = DataLoader(test_dataset, batch_size, shuffle = False, drop_last=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Network Architecture","metadata":{}},{"cell_type":"code","source":"def double_conv(in_c, out_c):\n    conv = nn.Sequential(\n        nn.Conv2d(in_c, out_c, kernel_size=3,padding=1),\n        nn.ReLU(inplace=True),\n        nn.Conv2d(out_c, out_c, kernel_size=3,padding=1),\n        nn.ReLU(inplace=True)\n    )\n    \n    return conv\n\nclass UNet(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.max_pool_2x2 = nn.MaxPool2d(kernel_size=2,stride=2)\n        self.down_conv_1 = double_conv(9,64)\n        self.down_conv_2 = double_conv(64,128)\n        self.down_conv_3 = double_conv(128,256)\n        self.down_conv_4 = double_conv(256,512)\n        self.down_conv_5 = double_conv(512,1024)\n        \n        self.up_trans_1 = nn.ConvTranspose2d(in_channels = 1024, out_channels =512,\n                                             kernel_size =2, stride =2)\n        self.up_conv_1 = double_conv(1024,512)\n        \n        self.up_trans_2 = nn.ConvTranspose2d(in_channels = 512, out_channels = 256,\n                                             kernel_size =2, stride =2)\n        self.up_conv_2 = double_conv(512,256)\n        \n        self.up_trans_3 = nn.ConvTranspose2d(in_channels = 256, out_channels = 128,\n                                             kernel_size =2, stride =2)\n        self.up_conv_3 = double_conv(256,128)\n        \n        self.up_trans_4 = nn.ConvTranspose2d(in_channels = 128, out_channels = 64,\n                                             kernel_size =2, stride =2)\n        self.up_conv_4 = double_conv(128,64)\n        \n        self.out = nn.Conv2d(in_channels=64, out_channels=1, kernel_size =1)\n        self.flat = nn.Flatten()\n        self.sigmoid = nn.Sigmoid()\n        \n        \n        \n        \n    def forward(self,image):\n        x1 = self.down_conv_1(image) #\n        #print(x1.size())\n        x2 = self.max_pool_2x2(x1) \n        x3 = self.down_conv_2(x2)  #\n        #print(x3.size())\n        x4 = self.max_pool_2x2(x3)\n        x5 = self.down_conv_3(x4)  #\n        #print(x5.size())\n        x6 = self.max_pool_2x2(x5)\n        x7 = self.down_conv_4(x6)  #\n        #print(x7.size())\n        x8 = self.max_pool_2x2(x7)\n        x9 = self.down_conv_5(x8)  \n        #print(x9.size())\n        \n        #Upconv 1\n        x = self.up_trans_1(x9)\n        x = self.up_conv_1(torch.cat([x,x7],axis=1))\n        #print(x.size())\n        \n        #Upconv 2\n        x = self.up_trans_2(x)\n        x = self.up_conv_2(torch.cat([x,x5],axis=1))\n        #print(x.size())\n        \n        #Upconv 3\n        x = self.up_trans_3(x)\n        x = self.up_conv_3(torch.cat([x,x3],axis=1))\n        #print(x.size())\n        \n        #Upconv 4\n        x = self.up_trans_4(x)\n        x = self.up_conv_4(torch.cat([x,x1],axis=1))\n        #print(x.size())\n        \n        #Final Out\n        x = self.out(x)\n        #print(x.size())\n        \n        x = self.flat(x)\n        x = self.sigmoid(x)\n        \n        return x\n        \n        \n    \n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = UNet().to(device)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#loss function - Optimizer\nloss_fn = nn.BCELoss()\noptimizer = torch.optim.Adam(model.parameters(),lr=0.01)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for t,(i,j) in enumerate(train_dl):\n    #i = torch.permute(i,(0,3,1,2))\n    #print(i.shape)\n    i = i.to(device)\n    j = j.to(device)\n    print(i.shape)\n    print(j.shape)\n\n    pred = model.forward(i)\n    print(pred.size())\n    print(j)\n    print(pred)\n    \n    if t>=0:\n        break","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def dice_coef(y_true, y_pred, thr=0.5, epsilon=0.001):\n    #y_true = y_true.flatten()\n    y_pred = (y_pred>thr)\n    \n    inter = (y_true*y_pred).sum()\n    den = y_true.sum() + y_pred.sum()\n    dice = ((2*inter+epsilon)/(den+epsilon))\n    return dice","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_epochs = 10\n\ndef train_model(model,num_epochs,train_dl,valid_dl):\n\n    torch.manual_seed(42)\n\n    loss_hist_train = [0.]*num_epochs   \n    dice_hist_train = [0.]*num_epochs\n\n    loss_hist_valid = [0.]*num_epochs\n    dice_hist_valid = [0.]*num_epochs\n\n    for epoch in range(num_epochs):\n        \n        #model.train()  # Turn on Dropout Layer\n\n        for x_batch, y_batch in train_dl:\n            \n            x_batch, y_batch = x_batch.to(device), y_batch.to(device)\n\n            pred = model(x_batch)            #1. Make prediction\n\n            loss = loss_fn(pred,y_batch)     #2. calculate loss\n\n            loss.backward()                  #3. Compute gradients\n\n            optimizer.step()                 #4. Update parameters using gradients\n\n            optimizer.zero_grad()            #5. Reset gradients\n\n            loss_hist_train[epoch] += loss.item()*batch_size\n            dice_batch = dice_coef(y_batch,pred)*batch_size\n            dice_hist_train[epoch] += dice_batch\n            \n            print(f'interim-loss_hist_train: {loss_hist_train[epoch]}  interim-dice score: {dice_batch} interim-dice total: {dice_hist_train[epoch]} ')\n            #is_correct = (torch.argmax(pred,dim=1)==y_batch).float()\n            #accuracy_hist_train[epoch] += is_correct.sum()\n            \n        loss_hist_train[epoch] /= len(train_dataset)\n        dice_hist_train[epoch] /= len(train_dataset)\n        \n        #model.eval()   # Turn off Dropout Layer\n        \n        with torch.no_grad():\n            for x_batch, y_batch in valid_dl:\n                \n                x_batch, y_batch = x_batch.to(device), y_batch.to(device)\n                pred = model(x_batch)\n                loss = loss_fn(pred,y_batch)\n                \n                loss_hist_valid[epoch] += loss.item()*batch_size\n                dice_batch = dice_coef(y_batch,pred)*batch_size\n                dice_hist_valid[epoch] += dice_batch\n                #is_correct = (torch.argmax(pred,dim=1)==y_batch).float()\n                #accuracy_hist_valid[epoch] += is_correct.sum()\n            \n            loss_hist_valid[epoch] /= len(validation_dataset)\n            dice_hist_valid[epoch] /= len(validation_dataset)       \n        \n        \n        print(f'Epoch:{epoch+1}  Train Loss:{loss_hist_train[epoch]:.4f} Train DICE:{dice_hist_train[epoch]:0.4f}')\n        print(f'Epoch:{epoch+1}  Val Loss:{loss_hist_valid[epoch]:.4f} Val DICE:{dice_hist_valid[epoch]:0.4f}\\n')\n\n\n    return(loss_hist_train,accuracy_hist_train,loss_hist_valid,accuracy_hist_valid)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loss, train_dice, val_loss, val_dice = train_model(model,num_epochs,train_dl,valid_dl)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"contrail_test.records","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x = iter(contrail_test)\ny = next(x)\nz = next(x)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"z","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_path = Path('/kaggle/input/google-research-identify-contrails-reduce-global-warming/test')\npath_new = os.path.join(img_path,\"1002653297254493116\")\npath_new","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for root,dirs,files in os.walk(img_path):\n    for dirs_1 in dirs:\n        path_new = os.path.join(img_path,dirs_1)\n        path_new = Path(path_new)\n        print(type(path_new))\n        file_list = [path for path in path_new.glob('*.npy')]\n    break","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_dir = Path('/kaggle/input/google-research-identify-contrails-reduce-global-warming')\n\nrecord = os.listdir(str(base_dir)+'/'+'test')\ntype(record)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"i=3\nf'band_{i:02}'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_10002164 = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000603527582775543/band_14.npy')\nimg_10002164_mask = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000603527582775543/human_pixel_masks.npy')\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_10002164_08 = img_10002164[:,:,4]\nplt.imshow(img_10002164_08)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_10002164_mask1 = img_10002164_mask[:,:,0]\nplt.imshow(img_10002164_mask1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.get_cmap()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_10002164_mask.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.unique(img_10002164_mask1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom matplotlib import animation\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tensor = torch.randn(3, 2, 2)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tensor","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"flattened_tensor = tensor.view(3, -1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"flattened_tensor ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dir = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_metadata = pd.read_json('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train_metadata.json')\nvalidation_metadata = pd.read_json('/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation_metadata.json')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_metadata.shape\nvalidation_metadata.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_metadata.sort_values('record_id',ascending=True).head()","metadata":{"_kg_hide-output":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sample_1000216489776414077_band_08 = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/band_08.npy')\ntrain_sample_1000216489776414077_band_12 = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/band_12.npy')\ntrain_sample_human_mask = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000603527582775543/human_individual_masks.npy')\ntrain_sample_human_pixel_mask = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000603527582775543/human_pixel_masks.npy')\n","metadata":{"_kg_hide-output":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.min(train_sample_1000216489776414077_band_08)\nnp.min(train_sample_1000216489776414077_band_12)\nnp.min(train_sample_human_mask)\nnp.min(train_sample_human_pixel_mask)\n\nnp.max(train_sample_1000216489776414077_band_08)\nnp.max(train_sample_1000216489776414077_band_12)\nnp.max(train_sample_human_mask)\nnp.max(train_sample_human_pixel_mask)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sample_1000216489776414077_band_08.shape\ntrain_sample_1000216489776414077_band_12.shape\ntrain_sample_human_mask.shape\ntrain_sample_human_pixel_mask.shape","metadata":{"_kg_hide-input":false,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sample_human_pixel_mask","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sample_human_mask","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}