{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# HELP IS NEEDED INFERENCE\n\n### So far I have tried everything to avoid running out of memory;\n\n> Smallest batch size = 1, max = 4\n\n> Number of workers, min = 1, max = 2\n\n> Patch size, 512\n\nI hope you can help me fix this error\n\nAlso, I have another inference notebook that gets submission-scoring error; \n\nHere is the link; [inference-submission-scoring-error](https://www.kaggle.com/samu2505/help-inference-submission-scoring-error/edit)","metadata":{}},{"cell_type":"code","source":"!yes | sudo dpkg -i /kaggle/input/libvips-pyvips-installation-and-getting-started/libvips/*.deb\n!pip install /kaggle/input/libvips-pyvips-installation-and-getting-started/pyvips/pyvips-2.2.1-py2.py3-none-any.whl --no-index --find-links /kaggle/input/libvips-pyvips-installation-and-getting-started/pyvips","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:04:22.362985Z","iopub.execute_input":"2023-11-14T09:04:22.363371Z","iopub.status.idle":"2023-11-14T09:04:59.788611Z","shell.execute_reply.started":"2023-11-14T09:04:22.36334Z","shell.execute_reply":"2023-11-14T09:04:59.787357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, gc, time, copy\nimport h5py\nos.environ[\"OPENCV_IO_MAX_IMAGE_PIXELS\"] = pow(2,40).__str__()\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\ntqdm.pandas()\nfrom collections import defaultdict\n\nimport math\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn import model_selection\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\nimport tensorflow as tf\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\n# from torchvision.transforms import v2\n\nimport timm\nfrom timm.data import resolve_data_config\nfrom timm.data.transforms_factory import create_transform\n\nimport IPython.display as display\n\nfrom PIL import Image\nimport cv2 as cv\nimport pyvips\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:04:59.790843Z","iopub.execute_input":"2023-11-14T09:04:59.791174Z","iopub.status.idle":"2023-11-14T09:05:14.669729Z","shell.execute_reply.started":"2023-11-14T09:04:59.791144Z","shell.execute_reply":"2023-11-14T09:05:14.668682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = dict(\n    seed = 42,\n    folds = 5,\n    img_size = [384, 384],\n    learning_rate = 23e-5, # 2e-5, 3e-4\n    eta_min = 23e-6,\n    epochs = 15,\n    batch_size=1,\n    num_workers=1,\n)\n\ndef seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(SEED)\n        torch.cuda.manual_seed_all(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n#     os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)\n#     tf.random.set_seed(SEED)\n#     keras.utils.set_random_seed(seed=SEED)\n    print('seeding done!!!')\n    \ndef flush():\n    gc.collect()\n#     if torch.cuda.is_available():\n    torch.cuda.empty_cache()\n    torch.cuda.reset_peak_memory_stats()\n    \nseeding(config['seed'])\n# seeding(config['seed'])","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:14.671279Z","iopub.execute_input":"2023-11-14T09:05:14.672314Z","iopub.status.idle":"2023-11-14T09:05:14.710466Z","shell.execute_reply.started":"2023-11-14T09:05:14.672276Z","shell.execute_reply":"2023-11-14T09:05:14.709569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = Path(\"../input/UBC-OCEAN/\")\nos.listdir(DATA_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:14.713044Z","iopub.execute_input":"2023-11-14T09:05:14.713703Z","iopub.status.idle":"2023-11-14T09:05:14.725235Z","shell.execute_reply.started":"2023-11-14T09:05:14.713674Z","shell.execute_reply":"2023-11-14T09:05:14.724237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'/kaggle/input/UBC-OCEAN/test_images'","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:14.726428Z","iopub.execute_input":"2023-11-14T09:05:14.726762Z","iopub.status.idle":"2023-11-14T09:05:14.733704Z","shell.execute_reply.started":"2023-11-14T09:05:14.726726Z","shell.execute_reply":"2023-11-14T09:05:14.732697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(DATA_PATH/'train.csv')\ntest_df = pd.read_csv(DATA_PATH/'test.csv')\nsample_df = pd.read_csv(DATA_PATH/'sample_submission.csv')\n\nget_train_images = lambda x: \"/kaggle/input/UBC-OCEAN/train_thumbnails/\" + str(x) + \"_thumbnail\" + \".png\"\nget_test_images = lambda x: \"/kaggle/input/UBC-OCEAN/test_images/\" + str(x) + \".png\"\n\ncheck_path = lambda path: tf.io.gfile.exists(path)\n\ntrain_df['image_path'] = train_df.loc[:, 'image_id'].progress_apply(get_train_images)\ntrain_df['exists'] = train_df.loc[:, 'image_path'].map(check_path)\n\nprint(\"Checking training data ...\")\ndisplay.display(train_df['exists'].value_counts())\ntrain_df = train_df[train_df['exists'] == True]\ntrain_df.reset_index(drop=True, inplace=True)\n\ntest_df['image_path'] = test_df.loc[:, 'image_id'].progress_apply(get_test_images)\ntest_df['exists'] = test_df.loc[:, 'image_path'].map(check_path)\n\nprint(\"Checking test data ...\")\ndisplay.display(test_df['exists'].value_counts())\ntest_df = test_df[test_df['exists'] == True]\ntest_df.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:14.734959Z","iopub.execute_input":"2023-11-14T09:05:14.735325Z","iopub.status.idle":"2023-11-14T09:05:15.732988Z","shell.execute_reply.started":"2023-11-14T09:05:14.735291Z","shell.execute_reply":"2023-11-14T09:05:15.732151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_thumbnails = os.listdir(DATA_PATH/'test_thumbnails')\n# get_thumbnail = lambda thumbnail: int(thumbnail.split(\"_\")[0])\n# test_thumbnails = [get_thumbnail(t) for t in test_thumbnails]\n# test_df['is_tma'] = True\n# test_df.loc[test_df['image_id'].isin(test_thumbnails), 'is_tma'] = False\n# # test_df","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:15.73433Z","iopub.execute_input":"2023-11-14T09:05:15.73513Z","iopub.status.idle":"2023-11-14T09:05:15.73922Z","shell.execute_reply.started":"2023-11-14T09:05:15.735101Z","shell.execute_reply":"2023-11-14T09:05:15.738157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = train_df['label'].unique().tolist()\nid2label = {l:i for i, l in enumerate(labels)}\nlabel2id = {i:l for i, l in enumerate(labels)}\n\nprint(f\"id2label: {id2label}\")\nprint(f\"label2id: {label2id}\")\n\ntrain_df['target'] = train_df['label'].map(id2label)\n# test_df['target'] = 0\n# train_df['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:15.740541Z","iopub.execute_input":"2023-11-14T09:05:15.740901Z","iopub.status.idle":"2023-11-14T09:05:15.759749Z","shell.execute_reply.started":"2023-11-14T09:05:15.740837Z","shell.execute_reply":"2023-11-14T09:05:15.758661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_images(image_path, scale_factor=2):\n    image = pyvips.Image.new_from_file(image_path, access='sequential')\n    return image.resize(1.0/scale_factor).numpy()\n\ndef vips_read_image(image_path, longest_edge):\n    \n    \"\"\"\n    Read image using libvips\n\n    Parameters\n    ----------\n    image_path: str\n        Path of the image\n\n    Returns\n    -------\n    image: numpy.ndarray of shape (height, width, 3)\n        Image array\n    \"\"\"\n    \n    image_thumbnail = pyvips.Image.thumbnail(image_path, longest_edge)\n\n    return np.ndarray(\n        buffer=image_thumbnail.write_to_memory(),\n        dtype=np.uint8,\n        shape=[image_thumbnail.height, image_thumbnail.width, image_thumbnail.bands]\n    )","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:15.761153Z","iopub.execute_input":"2023-11-14T09:05:15.761468Z","iopub.status.idle":"2023-11-14T09:05:15.770616Z","shell.execute_reply.started":"2023-11-14T09:05:15.761443Z","shell.execute_reply":"2023-11-14T09:05:15.769735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create patches","metadata":{}},{"cell_type":"code","source":"def create_patches(data, patch_size=512, scale_factor=6):\n    flush()\n\n    k=0\n    IMAGES = list()\n    LABELS = list()\n\n    # define the lower and upper ranges for the black and white colors in HSV\n    lower_black = np.array([0, 0, 0])\n    upper_black = np.array([30, 30, 30])     # [179, 50, 50]\n    lower_white = np.array([179, 30, 30])   # [0, 0, 200]\n    upper_white = np.array([179, 240, 50])   # [179, 50, 255]\n\n    for i in tqdm(range(data.shape[0])):\n#         img = cv.imread(data.loc[i, 'image_path'])\n#         img = read_images(data.loc[i, 'image_path'], scale_factor=scale_factor)\n        img = vips_read_image(data.loc[i, 'image_path'], longest_edge=5000)\n        img = cv.resize(img, (4096, 2048))\n        h, w = img.shape[:2]\n        rows = np.split(img, h // patch_size, axis=0)\n        patches = np.array([np.split(row, w // patch_size, axis=1) for row in rows])\n        img_mean = np.mean(img)\n\n        black = np.array([0, 0, 0])\n        not_black = np.any(patches != black, axis=(2, 3, 4))\n        non_black_patches = patches[not_black]\n\n        for j in range(non_black_patches.shape[0]):\n            patch = non_black_patches[j]\n            p_mean = np.mean(patch)\n            img_id = data.loc[i, 'image_id']\n#             label = data.loc[i, 'target']\n\n            # convert the patch to HSV\n            patch_hsv = cv.cvtColor(patch, cv.COLOR_BGR2HSV)\n\n            # create masks for the black and white colors\n            black_mask = cv.inRange(patch_hsv, lower_black, upper_black)\n            white_mask = cv.inRange(patch_hsv, lower_white, upper_white)\n\n            # check if any of the masks contain any non-zero values\n            if not np.any(black_mask) and not np.any(white_mask):\n                IMAGES.append(patch)\n                LABELS.append([img_id])\n\n    flush()\n    \n    return {\"images\": np.array(IMAGES), \"target\": np.array(LABELS)}\n\nPATCHES = create_patches(test_df)\nflush()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:15.773576Z","iopub.execute_input":"2023-11-14T09:05:15.773864Z","iopub.status.idle":"2023-11-14T09:05:41.447075Z","shell.execute_reply.started":"2023-11-14T09:05:15.773829Z","shell.execute_reply":"2023-11-14T09:05:41.44599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UBCPatchDataset(Dataset):\n    \n    def __init__(self, image_data, labels, transform):\n        super(UBCPatchDataset, self).__init__()\n        self.data = image_data\n        self.labels = labels\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        image = self.data[idx]\n        image = image.astype(np.float32) / 255.0\n    \n        label = self.labels[idx]\n        image = self.transform(image=image)['image']\n        return {'image': image, 'target': label}","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:41.448316Z","iopub.execute_input":"2023-11-14T09:05:41.448638Z","iopub.status.idle":"2023-11-14T09:05:41.455833Z","shell.execute_reply.started":"2023-11-14T09:05:41.44861Z","shell.execute_reply":"2023-11-14T09:05:41.454902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(img_size):\n    test_tsfm = A.Compose(\n        [\n            A.Resize(height=img_size[0], width=img_size[1]),\n            ToTensorV2(),\n        ]\n    )\n    \n    return {\"test\": test_tsfm}\n\n\ndef get_dataloaders(patches, img_size, batch_size, num_workers, split='test'):\n    tsfm = get_transforms(img_size=img_size)\n    if split.lower() == 'test':\n#         ds = MemCornDataset(data, transform=tsfm[split], labels=labels)\n        ds = UBCPatchDataset(image_data=patches['images'], labels=patches['target'], transform=tsfm[split])\n        dls = DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=False)\n        \n    else:\n        raise ValueError('Invalid split, test only used for inference')\n    return dls","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:41.457073Z","iopub.execute_input":"2023-11-14T09:05:41.457395Z","iopub.status.idle":"2023-11-14T09:05:41.470889Z","shell.execute_reply.started":"2023-11-14T09:05:41.45735Z","shell.execute_reply":"2023-11-14T09:05:41.469942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config['img_size'][0] > 224:\n    model_id_timm = \"timm/tf_efficientnetv2_b3.in21k_ft_in1k\"\nelif config['img_size'][0] == 224:\n#     model_id_timm = \"timm/vit_base_patch32_clip_224.openai_ft_in1k\"\n    model_id_timm = \"timm/vit_base_patch8_224.dino\"\n# timm.list_pretrained(\"*swin*\")\nmodel = timm.create_model(model_id_timm, pretrained=False, num_classes=512)\n\nclass Regr(torch.nn.Module):\n    \n    def __init__(self, base_model, rate=0.0):\n        super(Regr, self).__init__()\n        self.base_model = base_model\n        self.base_model.classifier = nn.Identity()\n        self.base_model.global_pool = nn.Identity()\n        self.pooling = GeM()\n        \n        self.linear_model = torch.nn.Sequential(\n            nn.LazyBatchNorm1d(),\n            nn.LazyLinear(512),\n            nn.ReLU(),\n            nn.Dropout(rate),\n            nn.LazyBatchNorm1d(),\n            nn.LazyLinear(128),\n            nn.ReLU(),\n            nn.Dropout(rate),\n            nn.LazyLinear(5),\n            nn.Softmax(dim=1),\n        )\n        \n        \n    def forward(self, x):\n        x = self.base_model(x)\n        x = self.pooling(x)\n        x = x.view(x.size(0), -1)\n        x = self.linear_model(x)\n        return x\n\n\nclass LSTMClassifier(torch.nn.Module):\n\n    def __init__(self, base_model, rate=0.0):\n        super(LSTMClassifier, self).__init__()\n        self.base_model = base_model\n\n        self.lstm_model = nn.LSTM(input_size=512, hidden_size=256, num_layers=2, dropout=0.0, bidirectional=True, batch_first=True)\n\n        # Maintain the same output layer\n        self.output_layer = nn.Sequential(\n            nn.LazyLinear(128),\n            nn.LazyBatchNorm1d(),\n            nn.Dropout(rate),\n#             nn.LeakyReLU(0.1),\n            nn.ReLU(),\n            nn.LazyLinear(5),\n            nn.Softmax(dim=1),\n        )\n\n    def forward(self, x):\n        x = self.base_model(x)\n        x, _ = self.lstm_model(x)\n        x = x.view(x.size(0), -1)\n\n        x = self.output_layer(x)\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:41.472197Z","iopub.execute_input":"2023-11-14T09:05:41.473074Z","iopub.status.idle":"2023-11-14T09:05:41.782291Z","shell.execute_reply.started":"2023-11-14T09:05:41.47304Z","shell.execute_reply":"2023-11-14T09:05:41.781292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MetricMonitor:\n    def __init__(self, float_precision=4):\n        self.float_precision = float_precision\n        self.reset()\n\n    def reset(self):\n        self.metrics = defaultdict(lambda: {\"val\": 0, \"count\": 0, \"avg\": 0})\n\n    def update(self, metric_name, val):\n        metric = self.metrics[metric_name]\n\n        metric[\"val\"] += val\n        metric[\"count\"] += 1\n        metric[\"avg\"] = metric[\"val\"] / metric[\"count\"]\n\n    def __str__(self):\n        return \" | \".join(\n            [\n                \"{metric_name}: {avg:.{float_precision}f}\".format(\n                    metric_name=metric_name, avg=metric[\"avg\"], float_precision=self.float_precision\n                )\n                for (metric_name, metric) in self.metrics.items()\n            ]\n        )\n\n\ndef ACC(y_true, y_preds):\n    y_true = y_true.detach().cpu().numpy()\n    y_preds = torch.argmax(y_preds, dim=1)\n    y_preds = y_preds.detach().cpu().numpy()\n    return metrics.balanced_accuracy_score(y_true, y_preds)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:41.783683Z","iopub.execute_input":"2023-11-14T09:05:41.784426Z","iopub.status.idle":"2023-11-14T09:05:41.793509Z","shell.execute_reply.started":"2023-11-14T09:05:41.784389Z","shell.execute_reply":"2023-11-14T09:05:41.792509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference(model, data_loader, device='cpu', epoch=1):\n    model.to(device)\n    model.eval()\n\n    embeds = list()\n    for i, batch in enumerate(tqdm(data_loader, total=len(data_loader))):\n        xb= batch['image']\n        xb = xb.to(device, non_blocking=True)\n        \n        with torch.autocast(device_type=device, dtype=torch.float16):\n            with torch.inference_mode():\n                outputs = model(xb)\n                embeds.append(outputs.detach().cpu().numpy())\n                \n    flush()\n    return embeds","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:41.794955Z","iopub.execute_input":"2023-11-14T09:05:41.795353Z","iopub.status.idle":"2023-11-14T09:05:41.806546Z","shell.execute_reply.started":"2023-11-14T09:05:41.79532Z","shell.execute_reply":"2023-11-14T09:05:41.805676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nname = model_id_timm.split('/')[-1]\nfold = 4\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f\"Using {device} ...\")\n\ntest_loader = get_dataloaders(PATCHES, config['img_size'], config['num_workers'], config['batch_size'])\n# train_loader = get_dataloaders(train_df, img_size=config['img_size'], batch_size=config['batch_size'])\n\ndef run_inference(fold, dataloader):\n#     run = wandb.init()\n    print(f\"Fold {fold} weights being downloaded ...\")\n    fold_path = Path(f\"../input/ubc-training-with-patches/{name}_fold_{fold}.pth\")\n#     artifact = run.use_artifact(f'samu2505/cgair-pytorch-baseline/fold_{i}_weights:v0', type='model')\n#     artifact_dir = artifact.download()\n    WEIGHTS = torch.load(fold_path, map_location=torch.device('cpu'))\n#     regr_model = Regr(base_model=model, rate=0.0)\n    regr_model = LSTMClassifier(base_model=model, rate=0.0)\n    regr_model.load_state_dict(WEIGHTS)\n\n    TEST_PREDS = inference(regr_model, dataloader, device=device)\n    # TRAIN_PREDS = inference(regr_model, criterion, train_loader, device=device)\n\n    TEST_PREDS = np.concatenate(TEST_PREDS)\n    np.save(f\"test_preds_fold_{fold}_{config['img_size'][0]}.npy\", np.argmax(TEST_PREDS, axis=1))\n    del dataloader, fold_path, WEIGHTS, regr_model, TEST_PREDS\n    flush()\n    \nrun_inference(fold=fold, dataloader=test_loader)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:41.807759Z","iopub.execute_input":"2023-11-14T09:05:41.808092Z","iopub.status.idle":"2023-11-14T09:05:58.578188Z","shell.execute_reply.started":"2023-11-14T09:05:41.808064Z","shell.execute_reply":"2023-11-14T09:05:58.576954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.load(f'/kaggle/working/test_preds_fold_{fold}_384.npy')\ndf = pd.DataFrame(PATCHES['target'], columns=['image_id'])\ndf['preds'] = preds\nfinal_preds = np.round(df.groupby('image_id')['preds'].mean().values).astype(int)\nflush()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:58.57973Z","iopub.execute_input":"2023-11-14T09:05:58.580109Z","iopub.status.idle":"2023-11-14T09:05:58.860878Z","shell.execute_reply.started":"2023-11-14T09:05:58.580074Z","shell.execute_reply":"2023-11-14T09:05:58.859843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df['preds'] = final_preds\nsample_df['label'] = sample_df['preds'].map(label2id)\nsample_df.drop('preds', axis=1, inplace=True)\nsample_df.to_csv('submission.csv', index=False)\npd.read_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:05:58.862469Z","iopub.execute_input":"2023-11-14T09:05:58.862755Z","iopub.status.idle":"2023-11-14T09:05:58.886191Z","shell.execute_reply.started":"2023-11-14T09:05:58.862729Z","shell.execute_reply":"2023-11-14T09:05:58.884974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}