{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_validate = False","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:29.225583Z","iopub.execute_input":"2023-08-09T22:08:29.226074Z","iopub.status.idle":"2023-08-09T22:08:29.234626Z","shell.execute_reply.started":"2023-08-09T22:08:29.226038Z","shell.execute_reply":"2023-08-09T22:08:29.233569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport copy\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.nn as nn\nfrom timm.scheduler import CosineLRScheduler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-08-09T22:08:29.265842Z","iopub.execute_input":"2023-08-09T22:08:29.266171Z","iopub.status.idle":"2023-08-09T22:08:29.275379Z","shell.execute_reply.started":"2023-08-09T22:08:29.266142Z","shell.execute_reply":"2023-08-09T22:08:29.274431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_device():\n device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n print(f\"Using {device} device\")\n return device\n\ndevice = get_device()","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:29.313074Z","iopub.execute_input":"2023-08-09T22:08:29.315978Z","iopub.status.idle":"2023-08-09T22:08:29.326014Z","shell.execute_reply.started":"2023-08-09T22:08:29.315937Z","shell.execute_reply":"2023-08-09T22:08:29.325149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n # Data\n base_dir = \"../input/google-research-identify-contrails-reduce-global-warming\"\n train_path = os.path.join(base_dir,\"train\")\n val_path = os.path.join(base_dir,\"validation\")\n \n # Train\n num_epochs = 10\n batch_size = 48\n num_workers = 2\n \n # Optimizer & Scheduler\n lr_max = 3e-4\n epochs_warmup = 5\n warmup_lr_init = 5e-4\n lr_min = 1e-6\n scheduler_name = \"CosineAnnealingLR\"\n \n # threshold\n threshold = 0.4","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:29.354033Z","iopub.execute_input":"2023-08-09T22:08:29.354417Z","iopub.status.idle":"2023-08-09T22:08:29.365317Z","shell.execute_reply.started":"2023-08-09T22:08:29.354383Z","shell.execute_reply":"2023-08-09T22:08:29.364271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\ndef normalize_range(data, bounds):\n \"\"\"Maps data to the range [0, 1].\"\"\"\n return (data - bounds[0]) / (bounds[1] - bounds[0])\n\ndef normalize_std(spec):\n return (spec- np.mean(spec))/np.std(spec)\n\nclass Dataset(torch.utils.data.Dataset):\n def __init__(self, data_path, mode='train'):\n self.data_path = data_path\n self.file_name = os.listdir(data_path)\n self.mode = mode\n \n\n def __len__(self):\n return len(self.file_name)\n\n def __getitem__(self, i):\n \n band11 = np.load(os.path.join(self.data_path, self.file_name[i], 'band_11.npy'))\n band14 = np.load(os.path.join(self.data_path, self.file_name[i], 'band_14.npy'))\n band15 = np.load(os.path.join(self.data_path, self.file_name[i], 'band_15.npy'))\n \n r = normalize_range(band15 - band14, _TDIFF_BOUNDS)\n g = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\n b = normalize_range(band14, _T11_BOUNDS)\n x = np.transpose(np.clip(np.stack([r, g, b], axis=2), 0, 1)[:,:,:,4],(2,0,1))\n x = normalize_std(x)\n \n if self.mode == 'train':\n y = np.load(os.path.join(self.data_path, self.file_name[i], 'human_pixel_masks.npy')).astype(np.float32).transpose(2,0,1)\n elif self.mode == 'test':\n y = self.file_name[i]\n \n return x, y","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:29.398133Z","iopub.execute_input":"2023-08-09T22:08:29.39848Z","iopub.status.idle":"2023-08-09T22:08:29.428692Z","shell.execute_reply.started":"2023-08-09T22:08:29.398449Z","shell.execute_reply":"2023-08-09T22:08:29.427729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = Dataset(CFG.train_path)\na, b = dataset[3]\n\nplt.figure(figsize=(12, 6))\nax = plt.subplot(1, 2, 1)\nax.imshow(np.transpose(a,(1,2,0)))\nax = plt.subplot(1, 2, 2)\nax.imshow(np.transpose(b,(1,2,0)), interpolation='none') \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:29.443439Z","iopub.execute_input":"2023-08-09T22:08:29.443741Z","iopub.status.idle":"2023-08-09T22:08:30.585373Z","shell.execute_reply.started":"2023-08-09T22:08:29.443712Z","shell.execute_reply":"2023-08-09T22:08:30.584213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/pretrained-models-pytorch\")\nsys.path.append(\"../input/efficientnet-pytorch\")\nsys.path.append(\"/kaggle/input/smp-github/segmentation_models.pytorch-master\")\nsys.path.append(\"/kaggle/input/timm-pretrained-resnest/resnest/\")\nimport segmentation_models_pytorch as smp","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:30.587848Z","iopub.execute_input":"2023-08-09T22:08:30.588649Z","iopub.status.idle":"2023-08-09T22:08:30.600046Z","shell.execute_reply.started":"2023-08-09T22:08:30.588609Z","shell.execute_reply":"2023-08-09T22:08:30.599022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.cache/torch/hub/checkpoints/\n!cp /kaggle/input/timm-pretrained-resnest/resnest/gluon_resnest26-50eb607c.pth /root/.cache/torch/hub/checkpoints/gluon_resnest26-50eb607c.pth","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:30.601801Z","iopub.execute_input":"2023-08-09T22:08:30.602526Z","iopub.status.idle":"2023-08-09T22:08:33.510816Z","shell.execute_reply.started":"2023-08-09T22:08:30.602489Z","shell.execute_reply":"2023-08-09T22:08:33.509346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = torch.load('/kaggle/input/ic2rgw-pytorch-baseline-train-inference/unet-smp-sample.pth').to(device);","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:33.517879Z","iopub.execute_input":"2023-08-09T22:08:33.520223Z","iopub.status.idle":"2023-08-09T22:08:33.78454Z","shell.execute_reply.started":"2023-08-09T22:08:33.52018Z","shell.execute_reply":"2023-08-09T22:08:33.783457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sigmoid = nn.Sigmoid()","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:33.79024Z","iopub.execute_input":"2023-08-09T22:08:33.792595Z","iopub.status.idle":"2023-08-09T22:08:33.798891Z","shell.execute_reply.started":"2023-08-09T22:08:33.792557Z","shell.execute_reply":"2023-08-09T22:08:33.797925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dice(nn.Module):\n def __init__(self, weight=None, size_average=True):\n super(Dice, self).__init__()\n self.sigmoid = nn.Sigmoid()\n\n def forward(self, inputs, targets, smooth=1):\n \n inputs = self.sigmoid(inputs) \n \n inputs = inputs.view(-1)\n targets = targets.view(-1)\n \n intersection = (inputs * targets).sum() \n dice = (2.*intersection + smooth)/(inputs.sum() + targets.sum() + smooth) \n \n return dice\n \ndice = Dice()","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:33.803121Z","iopub.execute_input":"2023-08-09T22:08:33.805729Z","iopub.status.idle":"2023-08-09T22:08:33.816121Z","shell.execute_reply.started":"2023-08-09T22:08:33.805693Z","shell.execute_reply":"2023-08-09T22:08:33.815039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode(x, fg_val=1):\n \"\"\"\n Args:\n x: numpy array of shape (height, width), 1 - mask, 0 - background\n Returns: run length encoding as list\n \"\"\"\n\n dots = np.where(\n x.T.flatten() == fg_val)[0] # .T sets Fortran order down-then-right\n run_lengths = []\n prev = -2\n for b in dots:\n if b > prev + 1:\n run_lengths.extend((b + 1, 0))\n run_lengths[-1] += 1\n prev = b\n return run_lengths\n\n\ndef list_to_string(x):\n \"\"\"\n Converts list to a string representation\n Empty list returns '-'\n \"\"\"\n if x: # non-empty list\n s = str(x).replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\")\n else:\n s = '-'\n return s\n\n\ndef rle_decode(mask_rle, shape=(256, 256)):\n '''\n mask_rle: run-length as string formatted (start length)\n empty predictions need to be encoded with '-'\n shape: (height, width) of array to return \n Returns numpy array, 1 - mask, 0 - background\n '''\n\n img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n if mask_rle != '-': \n s = mask_rle.split()\n starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n starts -= 1\n ends = starts + lengths\n for lo, hi in zip(starts, ends):\n img[lo:hi] = 1\n return img.reshape(shape, order='F') # Needed to align to RLE direction","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:33.820355Z","iopub.execute_input":"2023-08-09T22:08:33.823362Z","iopub.status.idle":"2023-08-09T22:08:33.838448Z","shell.execute_reply.started":"2023-08-09T22:08:33.823303Z","shell.execute_reply":"2023-08-09T22:08:33.837218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_validate:\n test_recs = os.listdir(os.path.join(CFG.base_dir,\"validation\"))\nelse:\n test_recs = os.listdir(os.path.join(CFG.base_dir,\"test\"))\nprint(test_recs[:5])","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:33.84287Z","iopub.execute_input":"2023-08-09T22:08:33.845308Z","iopub.status.idle":"2023-08-09T22:08:33.925203Z","shell.execute_reply.started":"2023-08-09T22:08:33.84525Z","shell.execute_reply":"2023-08-09T22:08:33.924352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32\nnum_workers = 2\n\nif is_validate:\n test_dataset = Dataset('/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation', mode='test')\nelse:\n test_dataset = Dataset('/kaggle/input/google-research-identify-contrails-reduce-global-warming/test', mode='test')\n\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:33.929249Z","iopub.execute_input":"2023-08-09T22:08:33.93148Z","iopub.status.idle":"2023-08-09T22:08:33.941386Z","shell.execute_reply.started":"2023-08-09T22:08:33.931445Z","shell.execute_reply":"2023-08-09T22:08:33.940351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/google-research-identify-contrails-reduce-global-warming/sample_submission.csv', index_col='record_id')\nbest_model.eval()\nwith torch.no_grad():\n for X, rec in tqdm(test_loader):\n X = X.to(device)\n pred = sigmoid(best_model(X)).cpu().detach().numpy().copy()[:,0,:,:] \n mask = np.zeros((len(rec), 256, 256))\n mask[predCFG.threshold] = 1\n \n for file_id, file_name in enumerate(rec):\n submission.loc[int(file_name), 'encoded_pixels'] = list_to_string(rle_encode(mask[file_id,:,:]))\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:08:33.949694Z","iopub.execute_input":"2023-08-09T22:08:33.952124Z","iopub.status.idle":"2023-08-09T22:10:04.182539Z","shell.execute_reply.started":"2023-08-09T22:08:33.95209Z","shell.execute_reply":"2023-08-09T22:10:04.181344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:04.187515Z","iopub.execute_input":"2023-08-09T22:10:04.191218Z","iopub.status.idle":"2023-08-09T22:10:04.237244Z","shell.execute_reply.started":"2023-08-09T22:10:04.191176Z","shell.execute_reply":"2023-08-09T22:10:04.236355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## dikkat burada ismi sample submission degil : submission !!!!","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nfrom PIL import Image\n\n\ndef dice_coefficient(predicted_masks, gt_masks):\n predicted_masks = predicted_masks.astype(np.uint8)\n gt_masks = gt_masks.astype(np.uint8)\n\n # Calculate intersection and the sum of pixels for predicted and ground truth masks\n intersection = np.sum(predicted_masks * gt_masks)\n total = np.sum(predicted_masks) + np.sum(gt_masks)\n\n # Compute Dice coefficient\n dice = (2. * intersection) / total\n return dice\n\n\nif is_validate:\n \n ## get the masks\n gt_masks = np.array([np.array(Image.open(f'/kaggle/input/another-trial/mask/{x}.png')) for x in submission.index.values])\n ## get the decoded masks\n predicted_masks = []\n for line in tqdm(submission.itertuples()):\n maske = rle_decode(line.encoded_pixels)\n predicted_masks.append(maske)\n# break\n predicted_masks = np.array(predicted_masks)\n \n # Example usage:\n dsc = dice_coefficient(predicted_masks, gt_masks)\n print(f\"Global Dice Coefficient: {dsc:.4f}\")\n \n## HER 0405 agu modelleri: a ve b tip birlikte MODEL .6144 VERDI TUM DATA DA....\n## sadece tek resimli baseline model tip a: 6116 verdi...\n## tek basina daha iyi tekli olan 6148!!!!","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:04.241508Z","iopub.execute_input":"2023-08-09T22:10:04.243822Z","iopub.status.idle":"2023-08-09T22:10:25.74385Z","shell.execute_reply.started":"2023-08-09T22:10:04.243787Z","shell.execute_reply":"2023-08-09T22:10:25.742863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nfor f in glob.glob('*'):\n if not f.startswith('subm'):\n !rm -rf {f}","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:25.748229Z","iopub.execute_input":"2023-08-09T22:10:25.750514Z","iopub.status.idle":"2023-08-09T22:10:25.758426Z","shell.execute_reply.started":"2023-08-09T22:10:25.750478Z","shell.execute_reply":"2023-08-09T22:10:25.757468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%reset -f","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:25.76257Z","iopub.execute_input":"2023-08-09T22:10:25.764977Z","iopub.status.idle":"2023-08-09T22:10:26.836594Z","shell.execute_reply.started":"2023-08-09T22:10:25.764943Z","shell.execute_reply":"2023-08-09T22:10:26.83551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_validate=False","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:26.842125Z","iopub.execute_input":"2023-08-09T22:10:26.844509Z","iopub.status.idle":"2023-08-09T22:10:26.850652Z","shell.execute_reply.started":"2023-08-09T22:10:26.844472Z","shell.execute_reply":"2023-08-09T22:10:26.849554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/pretrained-models-pytorch\")\nsys.path.append(\"../input/efficientnet-pytorch\")\nsys.path.append(\"/kaggle/input/smp-github/segmentation_models.pytorch-master\")\nsys.path.append(\"/kaggle/input/timm-pretrained-resnest/resnest/\")\nimport segmentation_models_pytorch as smp","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:26.854758Z","iopub.execute_input":"2023-08-09T22:10:26.857317Z","iopub.status.idle":"2023-08-09T22:10:26.867065Z","shell.execute_reply.started":"2023-08-09T22:10:26.857119Z","shell.execute_reply":"2023-08-09T22:10:26.865961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /root/.cache/torch/hub/checkpoints/\n!cp /kaggle/input/timm-pretrained-resnest/resnest/gluon_resnest26-50eb607c.pth /root/.cache/torch/hub/checkpoints/gluon_resnest26-50eb607c.pth","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:26.871479Z","iopub.execute_input":"2023-08-09T22:10:26.873914Z","iopub.status.idle":"2023-08-09T22:10:30.146879Z","shell.execute_reply.started":"2023-08-09T22:10:26.873879Z","shell.execute_reply":"2023-08-09T22:10:30.145427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile config.yaml\n\ndata_path: \"/kaggle/input/contrails-images-ash-color\"\noutput_dir: \"models\"\n\nseed: 420\n\ntrain_bs: 48\nvalid_bs: 128\nworkers: 2\n\nprogress_bar_refresh_rate: 1\n\nearly_stop:\n monitor: \"val_loss\"\n mode: \"min\"\n patience: 999\n verbose: 1\n\ntrainer:\n max_epochs: 29\n min_epochs: 27\n enable_progress_bar: True\n precision: \"16-mixed\"\n devices: 2\n\nmodel:\n seg_model: \"Unet\"\n encoder_name: \"timm-resnest26d\"\n loss_smooth: 1.0\n image_size: 384\n optimizer_params:\n lr: 0.0005\n weight_decay: 0.01 #0.0\n scheduler:\n name: \"CosineAnnealingLR\"\n params:\n CosineAnnealingLR:\n T_max: 2\n eta_min: 1.0e-6\n last_epoch: -1\n ReduceLROnPlateau:\n mode: \"min\"\n factor: 0.31622776602\n patience: 4\n verbose: True","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:30.152908Z","iopub.execute_input":"2023-08-09T22:10:30.155245Z","iopub.status.idle":"2023-08-09T22:10:30.169827Z","shell.execute_reply.started":"2023-08-09T22:10:30.155204Z","shell.execute_reply":"2023-08-09T22:10:30.168952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport torch\nimport yaml\nimport pandas as pd\nimport pytorch_lightning as pl\nfrom pprint import pprint\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, TQDMProgressBar\nfrom torch.utils.data import DataLoader\n","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:30.173093Z","iopub.execute_input":"2023-08-09T22:10:30.177678Z","iopub.status.idle":"2023-08-09T22:10:30.354537Z","shell.execute_reply.started":"2023-08-09T22:10:30.177642Z","shell.execute_reply":"2023-08-09T22:10:30.353371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"config.yaml\", \"r\") as file_obj:\n config = yaml.safe_load(file_obj)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:30.357393Z","iopub.execute_input":"2023-08-09T22:10:30.358137Z","iopub.status.idle":"2023-08-09T22:10:30.379981Z","shell.execute_reply.started":"2023-08-09T22:10:30.358101Z","shell.execute_reply":"2023-08-09T22:10:30.379073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 128\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndata = '/kaggle/input/google-research-identify-contrails-reduce-global-warming'\n\nif is_validate:\n data_root = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation/'\nelse:\n data_root = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/test/'","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:30.384174Z","iopub.execute_input":"2023-08-09T22:10:30.386443Z","iopub.status.idle":"2023-08-09T22:10:30.393866Z","shell.execute_reply.started":"2023-08-09T22:10:30.386408Z","shell.execute_reply":"2023-08-09T22:10:30.3929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filenames = os.listdir(data_root)\ntest_df = pd.DataFrame(filenames, columns=['record_id'])\ntest_df['path'] = data_root + test_df['record_id'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:30.39825Z","iopub.execute_input":"2023-08-09T22:10:30.400988Z","iopub.status.idle":"2023-08-09T22:10:30.421127Z","shell.execute_reply.started":"2023-08-09T22:10:30.400948Z","shell.execute_reply":"2023-08-09T22:10:30.420269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ContrailsDataset(torch.utils.data.Dataset): ## test dedigimiz sey: 'validation' or 'test'\n def __init__(self, df, test, image_size=256, train=True):\n \n self.df = df\n self.trn = train\n self.df_idx: pd.DataFrame = pd.DataFrame({'idx': os.listdir(f'/kaggle/input/google-research-identify-contrails-reduce-global-warming/{test}')})\n self.normalize_image = T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n self.image_size = image_size\n if image_size != 256:\n self.resize_image = T.transforms.Resize(image_size)\n \n def read_record(self, directory):\n record_data = {}\n for x in [\n \"band_11\", \n \"band_14\", \n \"band_15\"\n ]:\n\n record_data[x] = np.load(os.path.join(directory, x + \".npy\"))\n\n return record_data\n\n def normalize_range(self, data, bounds):\n \"\"\"Maps data to the range [0, 1].\"\"\"\n return (data - bounds[0]) / (bounds[1] - bounds[0])\n \n def get_false_color(self, record_data):\n _T11_BOUNDS = (243, 303)\n _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n _TDIFF_BOUNDS = (-4, 2)\n \n N_TIMES_BEFORE = 4\n\n r = self.normalize_range(record_data[\"band_15\"] - record_data[\"band_14\"], _TDIFF_BOUNDS)\n g = self.normalize_range(record_data[\"band_14\"] - record_data[\"band_11\"], _CLOUD_TOP_TDIFF_BOUNDS)\n b = self.normalize_range(record_data[\"band_14\"], _T11_BOUNDS)\n false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n img = false_color[..., N_TIMES_BEFORE]\n\n return img\n \n def __getitem__(self, index):\n row = self.df.iloc[index]\n con_path = row.path\n data = self.read_record(con_path) \n \n img = self.get_false_color(data)\n \n img = torch.tensor(np.reshape(img, (256, 256, 3))).to(torch.float32).permute(2, 0, 1)\n \n if self.image_size != 256:\n img = self.resize_image(img)\n \n img = self.normalize_image(img)\n \n image_id = int(self.df_idx.iloc[index]['idx'])\n \n return img.float(), torch.tensor(image_id)\n \n def __len__(self):\n return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:30.425264Z","iopub.execute_input":"2023-08-09T22:10:30.427778Z","iopub.status.idle":"2023-08-09T22:10:30.448814Z","shell.execute_reply.started":"2023-08-09T22:10:30.427744Z","shell.execute_reply":"2023-08-09T22:10:30.447732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset\n\nimport torch\nimport numpy as np\nimport torchvision.transforms as T\n\nif is_validate:\n tester = 'validation'\nelse:\n tester = 'test'\n\ntest_ds = ContrailsDataset(\n test_df,\n tester,\n config[\"model\"][\"image_size\"],\n train = False,\n )\n \ntest_dl = DataLoader(test_ds, batch_size=batch_size, num_workers = 1)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:30.453446Z","iopub.execute_input":"2023-08-09T22:10:30.455944Z","iopub.status.idle":"2023-08-09T22:10:30.468772Z","shell.execute_reply.started":"2023-08-09T22:10:30.455889Z","shell.execute_reply":"2023-08-09T22:10:30.467827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LightningModule(pl.LightningModule):\n\n def __init__(self):\n super().__init__()\n self.model = smp.Unet(encoder_name=\"timm-resnest26d\",\n encoder_weights=None,\n in_channels=3,\n classes=1,\n activation=None,\n )\n\n def forward(self, batch):\n return self.model(batch)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:30.47339Z","iopub.execute_input":"2023-08-09T22:10:30.475782Z","iopub.status.idle":"2023-08-09T22:10:30.483662Z","shell.execute_reply.started":"2023-08-09T22:10:30.475748Z","shell.execute_reply":"2023-08-09T22:10:30.482581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LightningModule().load_from_checkpoint(\"/kaggle/input/lb-0-653-pl-pipeline-improved-gr-icrgw/models/model.ckpt\")\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\nmodel.eval()\nmodel.zero_grad()","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:30.488551Z","iopub.execute_input":"2023-08-09T22:10:30.491082Z","iopub.status.idle":"2023-08-09T22:10:33.123645Z","shell.execute_reply.started":"2023-08-09T22:10:30.491047Z","shell.execute_reply":"2023-08-09T22:10:33.122587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode(x, fg_val=1):\n \"\"\"\n Args:\n x: numpy array of shape (height, width), 1 - mask, 0 - background\n Returns: run length encoding as list\n \"\"\"\n\n dots = np.where(\n x.T.flatten() == fg_val)[0] # .T sets Fortran order down-then-right\n run_lengths = []\n prev = -2\n for b in dots:\n if b > prev + 1:\n run_lengths.extend((b + 1, 0))\n run_lengths[-1] += 1\n prev = b\n return run_lengths\n\ndef list_to_string(x):\n \"\"\"\n Converts list to a string representation\n Empty list returns '-'\n \"\"\"\n if x: # non-empty list\n s = str(x).replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\")\n else:\n s = '-'\n return s\n\ndef rle_decode(mask_rle, shape=(256, 256)):\n '''\n mask_rle: run-length as string formatted (start length)\n empty predictions need to be encoded with '-'\n shape: (height, width) of array to return \n Returns numpy array, 1 - mask, 0 - background\n '''\n\n img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n if mask_rle != '-': \n s = mask_rle.split()\n starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n starts -= 1\n ends = starts + lengths\n for lo, hi in zip(starts, ends):\n img[lo:hi] = 1\n return img.reshape(shape, order='F') # Needed to align to RLE direction","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:33.128618Z","iopub.execute_input":"2023-08-09T22:10:33.130959Z","iopub.status.idle":"2023-08-09T22:10:33.147456Z","shell.execute_reply.started":"2023-08-09T22:10:33.130923Z","shell.execute_reply":"2023-08-09T22:10:33.146121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/google-research-identify-contrails-reduce-global-warming/sample_submission.csv', index_col='record_id')","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:33.165164Z","iopub.execute_input":"2023-08-09T22:10:33.167756Z","iopub.status.idle":"2023-08-09T22:10:33.180277Z","shell.execute_reply.started":"2023-08-09T22:10:33.167717Z","shell.execute_reply":"2023-08-09T22:10:33.179129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import tqdm\nfor i, data in tqdm(enumerate(test_dl)):\n images, image_id = data\n \n images = images.to(device)\n with torch.no_grad():\n predicted_mask = model.forward(images[:, :, :, :])\n if config[\"model\"][\"image_size\"] != 256:\n predicted_mask = torch.nn.functional.interpolate(predicted_mask, size=256, mode='bilinear')\n predicted_mask = torch.sigmoid(predicted_mask).cpu().detach().numpy()\n \n predicted_mask_with_threshold = np.zeros((images.shape[0], 256, 256))\n predicted_mask_with_threshold[predicted_mask[:, 0, :, :] < 0.5] = 0\n predicted_mask_with_threshold[predicted_mask[:, 0, :, :] > 0.5] = 1\n \n for img_num in range(0, images.shape[0]):\n current_mask = predicted_mask_with_threshold[img_num, :, :]\n current_image_id = image_id[img_num].item()\n \n submission.loc[int(current_image_id), 'encoded_pixels'] = list_to_string(rle_encode(current_mask))","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:10:33.185224Z","iopub.execute_input":"2023-08-09T22:10:33.187803Z","iopub.status.idle":"2023-08-09T22:13:39.809927Z","shell.execute_reply.started":"2023-08-09T22:10:33.187766Z","shell.execute_reply":"2023-08-09T22:13:39.808607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission_653.csv')","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:39.811914Z","iopub.execute_input":"2023-08-09T22:13:39.813581Z","iopub.status.idle":"2023-08-09T22:13:39.843331Z","shell.execute_reply.started":"2023-08-09T22:13:39.813542Z","shell.execute_reply":"2023-08-09T22:13:39.842339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nfrom PIL import Image\n\n\ndef dice_coefficient(predicted_masks, gt_masks):\n predicted_masks = predicted_masks.astype(np.uint8)\n gt_masks = gt_masks.astype(np.uint8)\n\n # Calculate intersection and the sum of pixels for predicted and ground truth masks\n intersection = np.sum(predicted_masks * gt_masks)\n total = np.sum(predicted_masks) + np.sum(gt_masks)\n\n # Compute Dice coefficient\n dice = (2. * intersection) / total\n return dice\n\n\nif is_validate:\n \n ## get the masks\n gt_masks = np.array([np.array(Image.open(f'/kaggle/input/another-trial/mask/{x}.png')) for x in submission.index.values])\n ## get the decoded masks\n predicted_masks = []\n for line in tqdm(submission.itertuples()):\n maske = rle_decode(line.encoded_pixels)\n predicted_masks.append(maske)\n# break\n predicted_masks = np.array(predicted_masks)\n \n # Example usage:\n dsc = dice_coefficient(predicted_masks, gt_masks)\n print(f\"Global Dice Coefficient: {dsc:.4f}\")\n \n## HER 0405 agu modelleri: a ve b tip birlikte MODEL .6144 VERDI TUM DATA DA....\n## sadece tek resimli baseline model tip a: 6116 verdi...\n## tek basina daha iyi tekli olan 6148!!!!","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:39.844983Z","iopub.execute_input":"2023-08-09T22:13:39.845382Z","iopub.status.idle":"2023-08-09T22:13:45.858097Z","shell.execute_reply.started":"2023-08-09T22:13:39.845345Z","shell.execute_reply":"2023-08-09T22:13:45.857035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nfor f in glob.glob('*'):\n if not f.startswith('subm'):\n !rm -rf {f}","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:45.859482Z","iopub.execute_input":"2023-08-09T22:13:45.860141Z","iopub.status.idle":"2023-08-09T22:13:46.882764Z","shell.execute_reply.started":"2023-08-09T22:13:45.860102Z","shell.execute_reply":"2023-08-09T22:13:46.880928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%reset -f","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:46.884718Z","iopub.execute_input":"2023-08-09T22:13:46.885549Z","iopub.status.idle":"2023-08-09T22:13:47.833423Z","shell.execute_reply.started":"2023-08-09T22:13:46.885505Z","shell.execute_reply":"2023-08-09T22:13:47.832373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_validate = False","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:47.837836Z","iopub.execute_input":"2023-08-09T22:13:47.839539Z","iopub.status.idle":"2023-08-09T22:13:47.847421Z","shell.execute_reply.started":"2023-08-09T22:13:47.839501Z","shell.execute_reply":"2023-08-09T22:13:47.846189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport os\nimport random\nimport math\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2\n\nimport torch\nfrom torch import nn\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nimport torch.nn.functional as F\n\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom transformers import get_cosine_schedule_with_warmup\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:47.849267Z","iopub.execute_input":"2023-08-09T22:13:47.850813Z","iopub.status.idle":"2023-08-09T22:13:47.864629Z","shell.execute_reply.started":"2023-08-09T22:13:47.850775Z","shell.execute_reply":"2023-08-09T22:13:47.863662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/pretrained-models-pytorch\")\nsys.path.append(\"../input/efficientnet-pytorch\")\nsys.path.append(\"/kaggle/input/smp-github/segmentation_models.pytorch-master\")\nimport segmentation_models_pytorch as smp\n\nprint(f\"Segmentation Models version: {smp.__version__}\")","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:47.867595Z","iopub.execute_input":"2023-08-09T22:13:47.869075Z","iopub.status.idle":"2023-08-09T22:13:47.880832Z","shell.execute_reply.started":"2023-08-09T22:13:47.869039Z","shell.execute_reply":"2023-08-09T22:13:47.879829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n batch_size = 32\n seed = 42\n thr = 0.01\n \n encoder = 'efficientnet-b3'\n pretrained = False\n weights = None\n classes = ['contrail']\n activation = None\n in_chans = 3\n \n device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n \n image_size = 256\n \n model_ckpt = '/kaggle/input/unet-model/epoch-29.pth'\n \nclass Paths:\n data = '/kaggle/input/google-research-identify-contrails-reduce-global-warming'\n if is_validate == True:\n data_root = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation/'\n else:\n data_root = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/test/'","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:47.884808Z","iopub.execute_input":"2023-08-09T22:13:47.886629Z","iopub.status.idle":"2023-08-09T22:13:47.899133Z","shell.execute_reply.started":"2023-08-09T22:13:47.886585Z","shell.execute_reply":"2023-08-09T22:13:47.897657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=1234):\n random.seed(seed)\n np.random.seed(seed)\n os.environ[\"PYTHONHASHSEED\"] = str(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = False\n torch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:47.900339Z","iopub.execute_input":"2023-08-09T22:13:47.900693Z","iopub.status.idle":"2023-08-09T22:13:47.922391Z","shell.execute_reply.started":"2023-08-09T22:13:47.90066Z","shell.execute_reply":"2023-08-09T22:13:47.918565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set_seed(Config.seed)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:47.924583Z","iopub.execute_input":"2023-08-09T22:13:47.925727Z","iopub.status.idle":"2023-08-09T22:13:47.934696Z","shell.execute_reply.started":"2023-08-09T22:13:47.925692Z","shell.execute_reply":"2023-08-09T22:13:47.932881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filenames = os.listdir(Paths.data_root)\ntest_df = pd.DataFrame(filenames, columns=['record_id'])\n\ntest_df['path'] = Paths.data_root + test_df['record_id'].astype(str)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:47.936795Z","iopub.execute_input":"2023-08-09T22:13:47.937896Z","iopub.status.idle":"2023-08-09T22:13:48.039432Z","shell.execute_reply.started":"2023-08-09T22:13:47.937859Z","shell.execute_reply":"2023-08-09T22:13:48.038545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(test_df), test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:48.043396Z","iopub.execute_input":"2023-08-09T22:13:48.045973Z","iopub.status.idle":"2023-08-09T22:13:48.060125Z","shell.execute_reply.started":"2023-08-09T22:13:48.045937Z","shell.execute_reply":"2023-08-09T22:13:48.059045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_size = A.Compose([\n A.Resize(Config.image_size, Config.image_size, interpolation=cv2.INTER_LANCZOS4, always_apply=True)\n])","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:48.064878Z","iopub.execute_input":"2023-08-09T22:13:48.067269Z","iopub.status.idle":"2023-08-09T22:13:48.074936Z","shell.execute_reply.started":"2023-08-09T22:13:48.067233Z","shell.execute_reply":"2023-08-09T22:13:48.07387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ContrailsDataset(torch.utils.data.Dataset):\n def __init__(self, df, train=True):\n \n self.df = df\n self.trn = train\n \n def read_record(self, directory):\n record_data = {}\n for x in [\n \"band_11\", \n \"band_14\", \n \"band_15\"\n ]:\n\n record_data[x] = np.load(os.path.join(directory, x + \".npy\"))\n\n return record_data\n\n def normalize_range(self, data, bounds):\n \"\"\"Maps data to the range [0, 1].\"\"\"\n return (data - bounds[0]) / (bounds[1] - bounds[0])\n \n def get_false_color(self, record_data):\n _T11_BOUNDS = (243, 303)\n _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n _TDIFF_BOUNDS = (-4, 2)\n \n N_TIMES_BEFORE = 4\n\n r = self.normalize_range(record_data[\"band_15\"] - record_data[\"band_14\"], _TDIFF_BOUNDS)\n g = self.normalize_range(record_data[\"band_14\"] - record_data[\"band_11\"], _CLOUD_TOP_TDIFF_BOUNDS)\n b = self.normalize_range(record_data[\"band_14\"], _T11_BOUNDS)\n false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n img = false_color[..., N_TIMES_BEFORE]\n\n return img\n \n def __getitem__(self, index):\n row = self.df.iloc[index]\n con_path = row.path\n data = self.read_record(con_path) \n \n img = self.get_false_color(data)\n \n if Config.image_size != 256:\n img = transform_size(image=img)[\"image\"]\n \n img = torch.tensor(img)\n img = img.permute(2, 0, 1)\n \n return img.float()\n \n def __len__(self):\n return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:48.080744Z","iopub.execute_input":"2023-08-09T22:13:48.083521Z","iopub.status.idle":"2023-08-09T22:13:48.102822Z","shell.execute_reply.started":"2023-08-09T22:13:48.083483Z","shell.execute_reply":"2023-08-09T22:13:48.101715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ds = ContrailsDataset(\n test_df,\n train = False\n )\n \ntest_dl = DataLoader(test_ds, batch_size=Config.batch_size, num_workers = 2)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:48.107819Z","iopub.execute_input":"2023-08-09T22:13:48.110964Z","iopub.status.idle":"2023-08-09T22:13:48.119115Z","shell.execute_reply.started":"2023-08-09T22:13:48.110921Z","shell.execute_reply":"2023-08-09T22:13:48.118054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UNet(nn.Module):\n def __init__(self, cfg):\n super(UNet, self).__init__()\n \n self.cfg = cfg\n self.training = True\n \n self.model = smp.Unet(\n encoder_name=cfg.encoder, \n encoder_weights=cfg.weights, \n decoder_use_batchnorm=True,\n classes=len(cfg.classes), \n activation=cfg.activation,\n )\n \n self.loss_fn = smp.losses.DiceLoss(mode='binary')\n \n def forward(self, imgs):\n \n x = imgs\n logits = self.model(x)\n if Config.image_size != 256:\n logits = F.interpolate(logits, size=(256, 256), mode='nearest-exact')\n \n return {\"logits\": logits.sigmoid()}","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:48.12455Z","iopub.execute_input":"2023-08-09T22:13:48.127399Z","iopub.status.idle":"2023-08-09T22:13:48.138584Z","shell.execute_reply.started":"2023-08-09T22:13:48.127361Z","shell.execute_reply":"2023-08-09T22:13:48.137448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = UNet(Config).to(Config.device)\nmodel.load_state_dict(torch.load(Config.model_ckpt, map_location=torch.device('cuda')))","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:48.143654Z","iopub.execute_input":"2023-08-09T22:13:48.146802Z","iopub.status.idle":"2023-08-09T22:13:49.221462Z","shell.execute_reply.started":"2023-08-09T22:13:48.146764Z","shell.execute_reply":"2023-08-09T22:13:49.220427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\ntorch.set_grad_enabled(False)\n\nval_data = defaultdict(list)\npbar = tqdm(enumerate(test_dl), total=len(test_dl), desc='Test')\nfor step, X in pbar: \n X = X.to(Config.device)\n\n output = model(X)\n for key, val in output.items():\n val_data[key] += [output[key]]\n\nfor key, val in output.items():\n value = val_data[key]\n if len(value[0].shape) == 0:\n val_data[key] = torch.stack(value)\n else:\n val_data[key] = torch.cat(value, dim=0).cpu().detach().numpy()","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:49.222878Z","iopub.execute_input":"2023-08-09T22:13:49.223263Z","iopub.status.idle":"2023-08-09T22:13:58.017885Z","shell.execute_reply.started":"2023-08-09T22:13:49.223229Z","shell.execute_reply":"2023-08-09T22:13:58.015746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rle_encode(x, fg_val=1):\n \"\"\"\n Args:\n x: numpy array of shape (height, width), 1 - mask, 0 - background\n Returns: run length encoding as list\n \"\"\"\n\n dots = np.where(\n x.T.flatten() == fg_val)[0] # .T sets Fortran order down-then-right\n run_lengths = []\n prev = -2\n for b in dots:\n if b > prev + 1:\n run_lengths.extend((b + 1, 0))\n run_lengths[-1] += 1\n prev = b\n return run_lengths\n\ndef list_to_string(x):\n \"\"\"\n Converts list to a string representation\n Empty list returns '-'\n \"\"\"\n if x: # non-empty list\n s = str(x).replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\")\n else:\n s = '-'\n return s\n\ndef rle_decode(mask_rle, shape=(256, 256)):\n '''\n mask_rle: run-length as string formatted (start length)\n empty predictions need to be encoded with '-'\n shape: (height, width) of array to return \n Returns numpy array, 1 - mask, 0 - background\n '''\n\n img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n if mask_rle != '-': \n s = mask_rle.split()\n starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n starts -= 1\n ends = starts + lengths\n for lo, hi in zip(starts, ends):\n img[lo:hi] = 1\n return img.reshape(shape, order='F') # Needed to align to RLE direction","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.019335Z","iopub.status.idle":"2023-08-09T22:13:58.020071Z","shell.execute_reply.started":"2023-08-09T22:13:58.01981Z","shell.execute_reply":"2023-08-09T22:13:58.019835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(Paths.data + '/sample_submission.csv', index_col='record_id')\n\nfor i, pred in enumerate(val_data['logits']):\n rec = test_df['record_id'][i]\n mask = (pred[0]>Config.thr).astype(np.float32)\n submission.loc[int(rec), 'encoded_pixels'] = list_to_string(rle_encode(mask))\n\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.022045Z","iopub.status.idle":"2023-08-09T22:13:58.022585Z","shell.execute_reply.started":"2023-08-09T22:13:58.022323Z","shell.execute_reply":"2023-08-09T22:13:58.022349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission_rahvan.csv')","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.024391Z","iopub.status.idle":"2023-08-09T22:13:58.024864Z","shell.execute_reply.started":"2023-08-09T22:13:58.024622Z","shell.execute_reply":"2023-08-09T22:13:58.024646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nfrom PIL import Image\n\n\ndef dice_coefficient(predicted_masks, gt_masks):\n predicted_masks = predicted_masks.astype(np.uint8)\n gt_masks = gt_masks.astype(np.uint8)\n\n # Calculate intersection and the sum of pixels for predicted and ground truth masks\n intersection = np.sum(predicted_masks * gt_masks)\n total = np.sum(predicted_masks) + np.sum(gt_masks)\n\n # Compute Dice coefficient\n dice = (2. * intersection) / total\n return dice\n\n\nif is_validate:\n \n ## get the masks\n gt_masks = np.array([np.array(Image.open(f'/kaggle/input/another-trial/mask/{x}.png')) for x in submission.index.values])\n ## get the decoded masks\n predicted_masks = []\n for line in tqdm(submission.itertuples()):\n maske = rle_decode(line.encoded_pixels)\n predicted_masks.append(maske)\n# break\n predicted_masks = np.array(predicted_masks)\n \n # Example usage:\n dsc = dice_coefficient(predicted_masks, gt_masks)\n print(f\"Global Dice Coefficient: {dsc:.4f}\")\n \n## HER 0405 agu modelleri: a ve b tip birlikte MODEL .6144 VERDI TUM DATA DA....\n## sadece tek resimli baseline model tip a: 6116 verdi...\n## tek basina daha iyi tekli olan 6148!!!!","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.026502Z","iopub.status.idle":"2023-08-09T22:13:58.026972Z","shell.execute_reply.started":"2023-08-09T22:13:58.026729Z","shell.execute_reply":"2023-08-09T22:13:58.026752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nfor f in glob.glob('*'):\n if not f.startswith('subm'):\n !rm -rf {f}","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.028612Z","iopub.status.idle":"2023-08-09T22:13:58.029074Z","shell.execute_reply.started":"2023-08-09T22:13:58.028841Z","shell.execute_reply":"2023-08-09T22:13:58.028864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%reset -f","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.037157Z","iopub.status.idle":"2023-08-09T22:13:58.037697Z","shell.execute_reply.started":"2023-08-09T22:13:58.03741Z","shell.execute_reply":"2023-08-09T22:13:58.037474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CHECK LEARNER CPU OR GPU BASED ON CHOICE !!!!!!\n\nis_validate = False\n# make_short = True \nonly_contrail = False # bu sadece is_validate durumunda calisiyor sorun yok","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.039469Z","iopub.status.idle":"2023-08-09T22:13:58.039938Z","shell.execute_reply.started":"2023-08-09T22:13:58.039696Z","shell.execute_reply":"2023-08-09T22:13:58.039719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.vision.all import *\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.04161Z","iopub.status.idle":"2023-08-09T22:13:58.042068Z","shell.execute_reply.started":"2023-08-09T22:13:58.041834Z","shell.execute_reply":"2023-08-09T22:13:58.041858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import segmentation_models_pytorch as smp","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.043557Z","iopub.status.idle":"2023-08-09T22:13:58.044564Z","shell.execute_reply.started":"2023-08-09T22:13:58.044315Z","shell.execute_reply":"2023-08-09T22:13:58.044343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## the required functiona\ndef rle_encode(x, fg_val=1):\n \"\"\"\n Args:\n x: numpy array of shape (height, width), 1 - mask, 0 - background\n Returns: run length encoding as list\n \"\"\"\n\n dots = np.where(\n x.T.flatten() == fg_val)[0] # .T sets Fortran order down-then-right\n run_lengths = []\n prev = -2\n for b in dots:\n if b > prev + 1:\n run_lengths.extend((b + 1, 0))\n run_lengths[-1] += 1\n prev = b\n return run_lengths\n\n\ndef list_to_string(x):\n \"\"\"\n Converts list to a string representation\n Empty list returns '-'\n \"\"\"\n if x: # non-empty list\n s = str(x).replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\")\n else:\n s = '-'\n return s\n\n\ndef rle_decode(mask_rle, shape=(256, 256)):\n '''\n mask_rle: run-length as string formatted (start length)\n empty predictions need to be encoded with '-'\n shape: (height, width) of array to return \n Returns numpy array, 1 - mask, 0 - background\n '''\n\n img = np.zeros(shape[0]*shape[1], dtype=np.uint8)\n if mask_rle != '-': \n s = mask_rle.split()\n starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n starts -= 1\n ends = starts + lengths\n for lo, hi in zip(starts, ends):\n img[lo:hi] = 1\n return img.reshape(shape, order='F') # Needed to align to RLE direction","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:14:37.136998Z","iopub.execute_input":"2023-08-09T22:14:37.137415Z","iopub.status.idle":"2023-08-09T22:14:37.149822Z","shell.execute_reply.started":"2023-08-09T22:14:37.137382Z","shell.execute_reply":"2023-08-09T22:14:37.148788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_5 = smp.Unet(encoder_name = 'efficientnet-b5',\n encoder_weights = None,\n in_channels = 3,\n classes = 2)\n\nmodel_4 = smp.Unet(encoder_name = 'efficientnet-b4',\n encoder_weights = None,\n in_channels = 3,\n classes = 2)\n\nmodel_3 = smp.Unet(encoder_name = 'efficientnet-b3',\n encoder_weights = None,\n in_channels = 3,\n classes = 2)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.048025Z","iopub.status.idle":"2023-08-09T22:13:58.048926Z","shell.execute_reply.started":"2023-08-09T22:13:58.048675Z","shell.execute_reply":"2023-08-09T22:13:58.048702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = Path('/kaggle/input/google-research-identify-contrails-reduce-global-warming')","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.050465Z","iopub.status.idle":"2023-08-09T22:13:58.051548Z","shell.execute_reply.started":"2023-08-09T22:13:58.051279Z","shell.execute_reply":"2023-08-09T22:13:58.051325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_validate:\n \n # Specify the main folder path\n# main_folder_path = 'your_folder_path_here' # Replace with your folder path\n main_folder = data_path / 'validation'\n\n # Grab the first child subfolder names\n subfolder_names = [subfolder.name for subfolder in main_folder.iterdir() if subfolder.is_dir()]\n\n # Create a pandas DataFrame\n sample_submission = pd.DataFrame(index=subfolder_names)\n sample_submission['encoded_pixels'] = '-' \n sample_submission.index.name = 'record_id'\n sample_submission.index =sample_submission.index.astype(int) \n\n \n ## burada istersen validation icinden refine edebilirsin....\n ## mesela sadece contrail + ise ... isimleri ona gore planlarsin...\n \n ##............process simple submission to get desired records....\n train_dff = pd.read_csv('/kaggle/input/05agu-train-df-true-false/train_dff.csv')\n \n if only_contrail == True: ## SAMPLE SUBMISSION FILTRESI BU...\n train_dff = train_dff.query(\"label == True and is_valid == True\").record_id.values\n sample_submission = sample_submission[sample_submission.index.isin(train_dff)]\n\n# if make_short: ## daha az ornek istyorsa...\n# sample_submission = sample_submission.sample(frac=.1).reset_index(drop=True)\n \n ## bir de masklari da kaydedecegim.. bunun folde rolusgturayim\n path_mask = Path('mask')\n path_mask.mkdir(exist_ok=True, parents=True)\n \nelse:\n sample_submission = pd.read_csv(data_path / 'sample_submission.csv', index_col='record_id')\nsample_submission.head(),len(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.05308Z","iopub.status.idle":"2023-08-09T22:13:58.053577Z","shell.execute_reply.started":"2023-08-09T22:13:58.053339Z","shell.execute_reply":"2023-08-09T22:13:58.053364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## directory test olacak!!!\ndef read_record(record_id, directory):\n record_data = {}\n if is_validate:\n basliklar = [\n \"band_11\", \n \"band_14\", \n \"band_15\", \n 'human_pixel_masks'\n ]\n else:\n basliklar = [\n \"band_11\", \n \"band_14\", \n \"band_15\", \n ]\n \n \n for x in basliklar:\n record_data[x] = np.load(os.path.join(directory, record_id, x + \".npy\"))\n \n return record_data","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.05517Z","iopub.status.idle":"2023-08-09T22:13:58.055662Z","shell.execute_reply.started":"2023-08-09T22:13:58.055423Z","shell.execute_reply":"2023-08-09T22:13:58.055446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\ndef normalize_range(data, bounds):\n \"\"\"Maps data to the range [0, 1].\"\"\"\n return (data - bounds[0]) / (bounds[1] - bounds[0])\n\nN_TIMES_BEFORE = 4","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.057261Z","iopub.status.idle":"2023-08-09T22:13:58.057756Z","shell.execute_reply.started":"2023-08-09T22:13:58.057519Z","shell.execute_reply":"2023-08-09T22:13:58.057543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_false_color(record_data,framim): ## added framim...\n _T11_BOUNDS = (243, 303)\n _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n _TDIFF_BOUNDS = (-4, 2)\n\n r = normalize_range(record_data[\"band_15\"] - record_data[\"band_14\"], _TDIFF_BOUNDS)\n g = normalize_range(record_data[\"band_14\"] - record_data[\"band_11\"], _CLOUD_TOP_TDIFF_BOUNDS)\n b = normalize_range(record_data[\"band_14\"], _T11_BOUNDS)\n false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n img = false_color[..., framim]\n \n return img","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.059409Z","iopub.status.idle":"2023-08-09T22:13:58.05987Z","shell.execute_reply.started":"2023-08-09T22:13:58.059632Z","shell.execute_reply":"2023-08-09T22:13:58.059655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = Path('frame_4')\npath.mkdir(exist_ok=True, parents=True)\npath = Path('all_frames')\npath.mkdir(exist_ok=True, parents=True)\n# path = Path('frame_3')\n# path.mkdir(exist_ok=True, parents=True)\n# path = Path('frame_5')\n# path.mkdir(exist_ok=True, parents=True)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.061838Z","iopub.status.idle":"2023-08-09T22:13:58.062377Z","shell.execute_reply.started":"2023-08-09T22:13:58.062107Z","shell.execute_reply":"2023-08-09T22:13:58.06213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # ..... DIKKAT RERUN DA BUNA GEREK YOK..................\n \nfor record_id in tqdm(sample_submission.index.values):\n if is_validate==True:\n folder_name = 'validation'\n else:\n folder_name = 'test'\n data = read_record(str(record_id), data_path / folder_name)\n# break\n imgs = [get_false_color(data,hangi_frame) for hangi_frame in [2,3,4,5]] ## this is the 5th,.. 2 3 4, so the labelframe\n ## 4ncuyu ayri olarak save et...\n ## digerlerini de komple olarak save et\n matplotlib.image.imsave(fname =f'frame_4/{record_id}.png',arr= imgs[2])\n \n # simdi burada imgs islenmeli....\n ## stack edilmeli nasil olacak...\n imgs = np.vstack((np.hstack((np.array(imgs[0]), np.array(imgs[1]))),\n np.hstack((np.array(imgs[2]), np.array(imgs[3])))))\n matplotlib.image.imsave(fname =f'all_frames/{record_id}.png',arr= imgs)\n \n \n if is_validate: ## o zaman masklar da lazim...\n pathm = path_mask/f\"{record_id}.png\" \n Image.fromarray((data['human_pixel_masks'].squeeze()).astype(np.uint8)).save(pathm)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.064081Z","iopub.status.idle":"2023-08-09T22:13:58.064572Z","shell.execute_reply.started":"2023-08-09T22:13:58.064333Z","shell.execute_reply":"2023-08-09T22:13:58.064356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image.fromarray(np.zeros((256,256)).astype(np.uint8)).save('example_mask.png')","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.066015Z","iopub.status.idle":"2023-08-09T22:13:58.066897Z","shell.execute_reply.started":"2023-08-09T22:13:58.066645Z","shell.execute_reply":"2023-08-09T22:13:58.066675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## now I can build the dls for training from frame_4 folder\n## test icindekileri train ve valid yap..\n## dls olarak tanimla\n## sonra da dogrudan test_dl olarak tanimlarsin.\n\ndef input_func(x):\n return Path(f'frame_4/{x.name}.png') ## bu indexi aliyor...\n\n\ndef label_func(x): ## a dummy mask!!\n# print(path / Path(f'mask/{x.record_id}.png'))\n return Path('example_mask.png')\n\n## classifcation modellerinde dls olmayacak... DOGRUDAN LOAD LEARNER....!!\n## O YUZDEN SADECE DUMMY FONK BURAYA ALIYROUM..\ndef get_composite_input(x): ## bunu dummy olarak koydum\n return x ## bu indexi aliyor...","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.068509Z","iopub.status.idle":"2023-08-09T22:13:58.068978Z","shell.execute_reply.started":"2023-08-09T22:13:58.068738Z","shell.execute_reply":"2023-08-09T22:13:58.06876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RandomResize(Transform):\n def __init__(self,min_size,max_size):\n self.min_size = min_size\n self.max_size = max_size\n def encoded(self, b:Tuple[TensorImage,TensorMask]):\n size = random.randint(self.min_size, self.max_size)\n resize_tfms = Resize(size) # once instantiate sonra call!!\n \n return resize_tfms(b)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.070654Z","iopub.status.idle":"2023-08-09T22:13:58.071115Z","shell.execute_reply.started":"2023-08-09T22:13:58.07088Z","shell.execute_reply":"2023-08-09T22:13:58.070903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"contrail_random384 = DataBlock(blocks=(ImageBlock, MaskBlock(codes = ['background','contrail'])),\n get_x = input_func,\n get_y = label_func, \n splitter=RandomSplitter(),\n item_tfms = Resize(384),\n batch_tfms=[RandomResize(256,512),\n Normalize.from_stats(*imagenet_stats)],\n# batch_tfms=[RandomResizedCrop(160),aug_transforms()],\n )\n\ndls_random384 = contrail_random384.dataloaders(sample_submission,bs = 64)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.072437Z","iopub.status.idle":"2023-08-09T22:13:58.073787Z","shell.execute_reply.started":"2023-08-09T22:13:58.073516Z","shell.execute_reply":"2023-08-09T22:13:58.073541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"contrail_random640 = DataBlock(blocks=(ImageBlock, MaskBlock(codes = ['background','contrail'])),\n get_x = input_func,\n get_y = label_func, \n splitter=RandomSplitter(),\n item_tfms = Resize(640),\n batch_tfms=[RandomResize(512,768),\n Normalize.from_stats(*imagenet_stats)],\n# batch_tfms=[RandomResizedCrop(160),aug_transforms()],\n )\n\ndls_random640 = contrail_random640.dataloaders(sample_submission,bs = 64)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.076321Z","iopub.status.idle":"2023-08-09T22:13:58.076786Z","shell.execute_reply.started":"2023-08-09T22:13:58.076549Z","shell.execute_reply":"2023-08-09T22:13:58.076571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"contrail_rrc512 = DataBlock(blocks=(ImageBlock, MaskBlock(codes = ['background','contrail'])),\n get_x = input_func,\n get_y = label_func, \n splitter=RandomSplitter(),\n item_tfms = Resize(512),\n batch_tfms=[RandomResize(384,640),\n Normalize.from_stats(*imagenet_stats)],\n# batch_tfms=[RandomResizedCrop(160),aug_transforms()],\n )\n\ndls_rrc512 = contrail_rrc512.dataloaders(sample_submission,bs = 64)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.078466Z","iopub.status.idle":"2023-08-09T22:13:58.078931Z","shell.execute_reply.started":"2023-08-09T22:13:58.07869Z","shell.execute_reply":"2023-08-09T22:13:58.078713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"contrail = DataBlock(blocks=(ImageBlock, MaskBlock(codes = ['background','contrail'])),\n get_x = input_func,\n get_y = label_func, \n splitter=RandomSplitter(),\n batch_tfms=[Normalize.from_stats(*imagenet_stats)],\n# batch_tfms=[RandomResizedCrop(160),aug_transforms()],\n )\n\ndls = contrail.dataloaders(sample_submission,bs = 64)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.080658Z","iopub.status.idle":"2023-08-09T22:13:58.081126Z","shell.execute_reply.started":"2023-08-09T22:13:58.080885Z","shell.execute_reply":"2023-08-09T22:13:58.080908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn_3_384 = Learner(dls_random384, model_3 ,metrics = Dice(),#loss_func=new_loss_func,\n ).to_fp16()\n\nlearn_3_640 = Learner(dls_random640, model_3 ,metrics = Dice(),#loss_func=new_loss_func,\n ).to_fp16()\n\nlearn_5_eski= Learner(dls, model_5 ,metrics = Dice(),#loss_func=new_loss_func,\n ).to_fp16()\n\nlearn_3_rrc = Learner(dls_rrc512, model_3 ,metrics = Dice(),#loss_func=new_loss_func,\n ).to_fp16()\n\nlearn_4_rrc = Learner(dls_rrc512, model_4 ,metrics = Dice(),#loss_func=new_loss_func,\n ).to_fp16()","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.082784Z","iopub.status.idle":"2023-08-09T22:13:58.083245Z","shell.execute_reply.started":"2023-08-09T22:13:58.083009Z","shell.execute_reply":"2023-08-09T22:13:58.083033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn_5_eski.load('/kaggle/input/07agu-segm-models/model_17_colab_continued_training')\nlearn_4_rrc.load('/kaggle/input/09agu-submissions/model_10_local_6145_klasikRRC512_b4')\nlearn_3_rrc.load('/kaggle/input/09agu-submissions/model_10_local_6167_klasikRRC512_b3')\nlearn_3_640.load('/kaggle/input/09agu-submissions/effnetb3_CUSTOM640RRC')\nlearn_3_384.load('/kaggle/input/09agu-submissions/effnetb3_CUSTOM384RRC')\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.08514Z","iopub.status.idle":"2023-08-09T22:13:58.085636Z","shell.execute_reply.started":"2023-08-09T22:13:58.085393Z","shell.execute_reply":"2023-08-09T22:13:58.085417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# seg_models = [learn_5_eski,learn_4_rrc,learn_3_rrc,learn_3_640,learn_3_384]\n\nseg_models = [learn_5_eski]","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.087481Z","iopub.status.idle":"2023-08-09T22:13:58.087951Z","shell.execute_reply.started":"2023-08-09T22:13:58.087711Z","shell.execute_reply":"2023-08-09T22:13:58.087734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clsf_paths = [\n '/kaggle/input/07agustos-clsf-models-convnextbase-local/07agu_baseline_stage1model_convnextv2_base.pkl',\n# '/kaggle/input/07agustos-clsf-models-convnextbase-local/07agu_baseline_stage1model_convnextv2_base_ftune.pkl',\n ]\n\nclsf_models = [load_learner(x,cpu =False) for x in clsf_paths]","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.089381Z","iopub.status.idle":"2023-08-09T22:13:58.090264Z","shell.execute_reply.started":"2023-08-09T22:13:58.090009Z","shell.execute_reply":"2023-08-09T22:13:58.090039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# burada path to images vermen yeterli oluyor\ntest_items = [Path(f'frame_4/{zz}.png') for zz in sample_submission.index.values]\ntest_items[:5],len(test_items)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.091894Z","iopub.status.idle":"2023-08-09T22:13:58.092387Z","shell.execute_reply.started":"2023-08-09T22:13:58.092123Z","shell.execute_reply":"2023-08-09T22:13:58.092146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## clsf ensemble functions\ndef simple_voting(preds):\n # Thresholding each model's output\n binary_preds = np.argmax(preds, axis=2)\n # Voting\n summed_votes = np.sum(binary_preds, axis=0)\n final_preds = np.where(summed_votes > (len(preds) / 2), 1, 0)\n return final_preds\n\ndef weighted_average(preds, weights, threshold=0.5):\n # Calculate the weighted average for class 1\n weighted_probs = np.sum(preds[:,:,1] * weights[:, np.newaxis], axis=0) / np.sum(weights)\n # Thresholding\n final_preds = np.where(weighted_probs > threshold, 1, 0)\n return final_preds\n\ndef unweighted_average(preds, threshold=0.5):\n # Calculate the unweighted average for class 1\n avg_probs = np.mean(preds[:,:,1], axis=0)\n # Thresholding\n final_preds = np.where(avg_probs > threshold, 1, 0)\n return final_preds\n\ndef or_logic_ensemble(preds, threshold=0.37):\n # Check if any model predicts class 1 with probability greater than threshold\n binary_preds = (preds[:, :, 1] > threshold).astype(int)\n final_preds = np.any(binary_preds, axis=0).astype(int)\n return final_preds\n\n\n# Sample segmentation predictions (using the previously generated predictions)\n# seg_predictions = np.random.rand(3, 32, 2, 256, 256)\n\ndef simple_voting_segmentation_threshold(preds, threshold=0.5):\n # Thresholding each model's output\n binary_preds = (preds[:, :, 1, :, :] > threshold).astype(int)\n # Voting\n summed_votes = np.sum(binary_preds, axis=0)\n final_preds = np.where(summed_votes > (len(preds) / 2), 1, 0)\n return final_preds\n\ndef unweighted_average_segmentation_threshold(preds, threshold=0.5):\n # Calculate the unweighted average for class 1 (object)\n avg_probs = np.mean(preds[:, :, 1, :, :], axis=0)\n # Thresholding\n final_preds = np.where(avg_probs > threshold, 1, 0)\n return final_preds\n\ndef weighted_thresholded_ensemble_segmentation(predictions, weights, threshold=0.5):\n \"\"\"\n Ensembles the predictions using weighted average and applies a threshold.\n \n Args:\n - predictions (np.array): The predictions array of shape (models, batch_size, classes, height, width)\n - weights (list): Weights for each model\n - threshold (float): Threshold value to classify pixel as object or background\n \n Returns:\n - np.array: The ensembled predictions of shape (batch_size, height, width)\n \"\"\"\n # Weighted average of predictions\n weighted_predictions = np.average(predictions, axis=0, weights=weights)\n \n # Extracting the object probabilities\n object_probabilities = weighted_predictions[:, 1, :, :]\n \n # Applying threshold\n ensembled_output = (object_probabilities > threshold).astype(int)\n \n return ensembled_output","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.09579Z","iopub.status.idle":"2023-08-09T22:13:58.096375Z","shell.execute_reply.started":"2023-08-09T22:13:58.096095Z","shell.execute_reply":"2023-08-09T22:13:58.096133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32\n# all_preds = [] ## buna gerek yok....\n\n# Iterate over batches of test_items and location\nfor i in tqdm(range(0, len(test_items), batch_size)):\n batch_test_items = test_items[i:i+batch_size]\n batch_test_items_clsf = [Path('all_frames') / p.name for p in batch_test_items]\n \n batch_location = sample_submission.index.values[i:i+batch_size]\n \n # Create test DataLoader for the batch\n test_dl = dls.test_dl(batch_test_items)\n test_dl_clsf = dls.test_dl(batch_test_items_clsf)\n# break\n \n # Get predictions for the batch,,, batchin kendisi dataloader'a sokuluyor\n ## yani dataloaderdan alip batch yapip degil, batchi al dataloader sonra get_pred\n ## ..............ONCE CLSF PREDICTIONS..........................\n preds_clsf = [x.get_preds(dl=test_dl_clsf)[0] for x in clsf_models]\n preds_clsf = np.array([x.detach().cpu().numpy() for x in preds_clsf])\n \n # Get predictions for each ensemble method\n# preds_clsf = simple_voting(preds_clsf)\n# preds_clsf = weighted_average(preds_clsf, weights=np.array([1, 2, 4]))\n# preds_clsf = unweighted_average(preds_clsf)\n# preds_clsf = weighted_average(preds_clsf, weights=np.array([1, 2, 4]), threshold=0.25)\n# preds_clsf = unweighted_average(preds_clsf, threshold=0.25)\n preds_clsf = unweighted_average(preds_clsf, threshold=0.35) ## bunun 50si zaten klasik..\n# preds_clsf = or_logic_ensemble(preds_clsf)\n \n \n ##..............simdi segmentators....................\n preds = [x.tta(dl=test_dl)[0] for x in seg_models]\n preds = np.array([x.detach().cpu().numpy() for x in preds])\n \n # Get predictions for each ensemble method\n# preds = simple_voting_segmentation_threshold(preds)\n# preds = simple_voting_segmentation_threshold(preds, threshold=0.44)\n preds = unweighted_average_segmentation_threshold(preds)\n# preds = unweighted_average_segmentation_threshold(preds, threshold=0.44)\n# preds = unweighted_average_segmentation_threshold(preds, threshold=0.55)\n# preds = weighted_thresholded_ensemble_segmentation(preds, np.array([4, 3 ,1.5 ]), .5)\n\n \n for location,mask,yes_or_no in zip(batch_location,preds,preds_clsf):\n if yes_or_no==1:\n sample_submission.loc[int(location), 'encoded_pixels'] = list_to_string(rle_encode(mask))\n else: ## bos mask gonderiyorum....\n# print('bos mask')\n sample_submission.loc[int(location), 'encoded_pixels'] = list_to_string(rle_encode(np.zeros((256,256)).astype(np.uint8)))\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.098209Z","iopub.status.idle":"2023-08-09T22:13:58.098744Z","shell.execute_reply.started":"2023-08-09T22:13:58.098463Z","shell.execute_reply":"2023-08-09T22:13:58.098486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not is_validate: ## validate ise o filelar bana lazim\n# print('submissin disidakileri siliyorum')\n for f in glob.glob('*'):\n if not f.startswith('subm'):\n !rm -rf {f}","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.100234Z","iopub.status.idle":"2023-08-09T22:13:58.101143Z","shell.execute_reply.started":"2023-08-09T22:13:58.100889Z","shell.execute_reply":"2023-08-09T22:13:58.100916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dice_coefficient(predicted_masks, gt_masks):\n predicted_masks = predicted_masks.astype(np.uint8)\n gt_masks = gt_masks.astype(np.uint8)\n\n # Calculate intersection and the sum of pixels for predicted and ground truth masks\n intersection = np.sum(predicted_masks * gt_masks)\n total = np.sum(predicted_masks) + np.sum(gt_masks)\n\n # Compute Dice coefficient\n dice = (2. * intersection) / total\n return dice\n\n\nif is_validate:\n ## get the masks\n gt_masks = np.array([np.array(Image.open(f'mask/{x}.png')) for x in sample_submission.index.values])\n ## get the decoded masks\n predicted_masks = []\n for line in sample_submission.itertuples():\n maske = rle_decode(line.encoded_pixels)\n predicted_masks.append(maske)\n # break\n predicted_masks = np.array(predicted_masks)\n \n # Example usage:\n dsc = dice_coefficient(predicted_masks, gt_masks)\n print(f\"Global Dice Coefficient: {dsc:.4f}\")\n \n## HER 0405 agu modelleri: a ve b tip birlikte MODEL .6144 VERDI TUM DATA DA....\n## sadece tek resimli baseline model tip a: 6116 verdi...\n## tek basina daha iyi tekli olan 6148!!!!","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.102742Z","iopub.status.idle":"2023-08-09T22:13:58.10321Z","shell.execute_reply.started":"2023-08-09T22:13:58.102969Z","shell.execute_reply":"2023-08-09T22:13:58.102993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv('submission_x.csv')\n# pd.read_csv('submission_x.csv')","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.104879Z","iopub.status.idle":"2023-08-09T22:13:58.105371Z","shell.execute_reply.started":"2023-08-09T22:13:58.105109Z","shell.execute_reply":"2023-08-09T22:13:58.105132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_01 = pd.read_csv('/kaggle/working/submission.csv')\nsubmission_02 = pd.read_csv('/kaggle/working/submission_653.csv')\nsubmission_03 = pd.read_csv('/kaggle/working/submission_rahvan.csv')\nsubmission_x = pd.read_csv('/kaggle/working/submission_x.csv')","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.106783Z","iopub.status.idle":"2023-08-09T22:13:58.107608Z","shell.execute_reply.started":"2023-08-09T22:13:58.107361Z","shell.execute_reply":"2023-08-09T22:13:58.107384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for line in submission_x.itertuples():\n# mask_1 = rle_decode(submission_01[submission_01.record_id==line.record_id].encoded_pixels.values[0])\n# mask_2 = rle_decode(submission_02[submission_02.record_id==line.record_id].encoded_pixels.values[0])\n# mask_3 = rle_decode(submission_03[submission_03.record_id==line.record_id].encoded_pixels.values[0])\n# mask_x = rle_decode(line.encoded_pixels)\n\n# ## en kotu, en iyi, orta, ortadan iyi benimki\n# weights = np.array([0.1, 0.4, 0.2, 0.3])\n# weighted_avg = np.average(np.array([mask_1,mask_2,mask_3,mask_x]), axis=0, weights=weights)\n# weighted_avg = (weighted_avg>.5).astype(np.uint8)\n# sample_submission.loc[int(line.record_id), 'encoded_pixels'] = list_to_string(rle_encode(weighted_avg))","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.109251Z","iopub.status.idle":"2023-08-09T22:13:58.109739Z","shell.execute_reply.started":"2023-08-09T22:13:58.109501Z","shell.execute_reply":"2023-08-09T22:13:58.109524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for line in submission_x.itertuples():\n mask_1 = rle_decode(submission_01[submission_01.record_id==line.record_id].encoded_pixels.values[0])\n mask_2 = rle_decode(submission_02[submission_02.record_id==line.record_id].encoded_pixels.values[0])\n mask_3 = rle_decode(submission_03[submission_03.record_id==line.record_id].encoded_pixels.values[0])\n mask_x = rle_decode(line.encoded_pixels)\n\n ## en kotu, en iyi, orta, ortadan iyi benimki\n weights = np.array([0.1, 0.4, 0.2, 0.3])\n weighted_avg = np.average(np.array([mask_1,mask_2,mask_3,mask_x]), axis=0, weights=weights)\n weighted_avg = (weighted_avg>.5).astype(np.uint8)\n if np.sum(weighted_avg)<20:\n sample_submission.loc[int(line.record_id), 'encoded_pixels'] = list_to_string(rle_encode(np.zeros((256,256)).astype(np.uint8)))\n else:\n sample_submission.loc[int(line.record_id), 'encoded_pixels'] = list_to_string(rle_encode(weighted_avg))","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.109251Z","iopub.status.idle":"2023-08-09T22:13:58.109739Z","shell.execute_reply.started":"2023-08-09T22:13:58.109501Z","shell.execute_reply":"2023-08-09T22:13:58.109524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:13:58.111159Z","iopub.status.idle":"2023-08-09T22:13:58.111978Z","shell.execute_reply.started":"2023-08-09T22:13:58.111727Z","shell.execute_reply":"2023-08-09T22:13:58.111752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not is_validate: ## validate ise o filelar bana lazim\n# print('submissin disidakileri siliyorum')\n for f in glob.glob('*'):\n \n if not f.endswith('mission.csv'):\n !rm -rf {f}","metadata":{"execution":{"iopub.status.busy":"2023-08-09T22:22:15.792594Z","iopub.execute_input":"2023-08-09T22:22:15.793041Z","iopub.status.idle":"2023-08-09T22:22:15.800348Z","shell.execute_reply.started":"2023-08-09T22:22:15.793003Z","shell.execute_reply":"2023-08-09T22:22:15.799393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}