{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Inference\n\nPreviously, I had used this code to make submissions and it worked by right now, it's giving me submission error. The last version of this notebook was adjusted to use Patches and it runs out of memory after 4 to 6 hours of scoring. Here is the link. \n\n[patch-inference-out-of-memory](https://www.kaggle.com/samu2505/help-patch-inference-out-of-memory/edit)\n\nHelp is needed for both","metadata":{}},{"cell_type":"code","source":"!yes | sudo dpkg -i /kaggle/input/libvips-pyvips-installation-and-getting-started/libvips/*.deb\n!pip install /kaggle/input/libvips-pyvips-installation-and-getting-started/pyvips/pyvips-2.2.1-py2.py3-none-any.whl --no-index --find-links /kaggle/input/libvips-pyvips-installation-and-getting-started/pyvips","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:20:42.032702Z","iopub.execute_input":"2023-11-14T09:20:42.03361Z","iopub.status.idle":"2023-11-14T09:21:20.463223Z","shell.execute_reply.started":"2023-11-14T09:20:42.033562Z","shell.execute_reply":"2023-11-14T09:21:20.461073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, gc, time, copy\nimport h5py\nos.environ[\"OPENCV_IO_MAX_IMAGE_PIXELS\"] = pow(2,40).__str__()\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\ntqdm.pandas()\nfrom collections import defaultdict\n\nimport math\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn import model_selection\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\nimport tensorflow as tf\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\n# from torchvision.transforms import v2\n\nimport timm\nfrom timm.data import resolve_data_config\nfrom timm.data.transforms_factory import create_transform\n\nimport IPython.display as display\n\nfrom PIL import Image\nimport cv2 as cv\nimport pyvips\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:20.467341Z","iopub.execute_input":"2023-11-14T09:21:20.467933Z","iopub.status.idle":"2023-11-14T09:21:38.577296Z","shell.execute_reply.started":"2023-11-14T09:21:20.467874Z","shell.execute_reply":"2023-11-14T09:21:38.576434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = dict(\n    seed = 42,\n    folds = 5,\n    img_size = [512, 512],\n    learning_rate = 23e-5, # 2e-5, 3e-4\n    eta_min = 23e-6,\n    epochs = 15,\n    batch_size=1,\n    num_workers=1,\n)\n\ndef seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(SEED)\n        torch.cuda.manual_seed_all(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n#     os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)\n#     tf.random.set_seed(SEED)\n#     keras.utils.set_random_seed(seed=SEED)\n    print('seeding done!!!')\n    \ndef flush():\n    gc.collect()\n#     if torch.cuda.is_available():\n    torch.cuda.empty_cache()\n    torch.cuda.reset_peak_memory_stats()\n    \nseeding(config['seed'])\n# seeding(config['seed'])","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:38.578695Z","iopub.execute_input":"2023-11-14T09:21:38.579317Z","iopub.status.idle":"2023-11-14T09:21:38.619142Z","shell.execute_reply.started":"2023-11-14T09:21:38.579287Z","shell.execute_reply":"2023-11-14T09:21:38.618136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = Path(\"../input/UBC-OCEAN/\")\nos.listdir(DATA_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:38.621735Z","iopub.execute_input":"2023-11-14T09:21:38.62204Z","iopub.status.idle":"2023-11-14T09:21:38.63342Z","shell.execute_reply.started":"2023-11-14T09:21:38.622012Z","shell.execute_reply":"2023-11-14T09:21:38.63256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'/kaggle/input/UBC-OCEAN/train_images'","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:38.634616Z","iopub.execute_input":"2023-11-14T09:21:38.634926Z","iopub.status.idle":"2023-11-14T09:21:38.643005Z","shell.execute_reply.started":"2023-11-14T09:21:38.634899Z","shell.execute_reply":"2023-11-14T09:21:38.642154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(DATA_PATH/'train.csv')\ntest_df = pd.read_csv(DATA_PATH/'test.csv')\nsample_df = pd.read_csv(DATA_PATH/'sample_submission.csv')\n\nget_train_images = lambda x: \"/kaggle/input/UBC-OCEAN/train_thumbnails/\" + str(x) + \"_thumbnail\" + \".png\"\nget_test_images = lambda x: \"/kaggle/input/UBC-OCEAN/test_images/\" + str(x) + \".png\"\n\ncheck_path = lambda path: tf.io.gfile.exists(path)\n\ntrain_df['image_path'] = train_df.loc[:, 'image_id'].progress_apply(get_train_images)\ntrain_df['exists'] = train_df.loc[:, 'image_path'].map(check_path)\n\nprint(\"Checking training data ...\")\ndisplay.display(train_df['exists'].value_counts())\ntrain_df = train_df[train_df['exists'] == True]\ntrain_df.reset_index(drop=True, inplace=True)\n\ntest_df['image_path'] = test_df.loc[:, 'image_id'].progress_apply(get_test_images)\ntest_df['exists'] = test_df.loc[:, 'image_path'].map(check_path)\n\nprint(\"Checking test data ...\")\ndisplay.display(test_df['exists'].value_counts())\ntest_df = test_df[test_df['exists'] == True]\ntest_df.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:38.644502Z","iopub.execute_input":"2023-11-14T09:21:38.64487Z","iopub.status.idle":"2023-11-14T09:21:39.106816Z","shell.execute_reply.started":"2023-11-14T09:21:38.644841Z","shell.execute_reply":"2023-11-14T09:21:39.105861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_thumbnails = os.listdir(DATA_PATH/'test_thumbnails')\n# get_thumbnail = lambda thumbnail: int(thumbnail.split(\"_\")[0])\n# test_thumbnails = [get_thumbnail(t) for t in test_thumbnails]\n# test_df['is_tma'] = True\n# test_df.loc[test_df['image_id'].isin(test_thumbnails), 'is_tma'] = False\n# # test_df","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:39.108274Z","iopub.execute_input":"2023-11-14T09:21:39.108673Z","iopub.status.idle":"2023-11-14T09:21:39.113146Z","shell.execute_reply.started":"2023-11-14T09:21:39.108644Z","shell.execute_reply":"2023-11-14T09:21:39.112102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = train_df['label'].unique().tolist()\nid2label = {l:i for i, l in enumerate(labels)}\nlabel2id = {i:l for i, l in enumerate(labels)}\n\nprint(f\"id2label: {id2label}\")\nprint(f\"label2id: {label2id}\")\n\ntrain_df['target'] = train_df['label'].map(id2label)\n# test_df['target'] = 0\n# train_df['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:39.114406Z","iopub.execute_input":"2023-11-14T09:21:39.114746Z","iopub.status.idle":"2023-11-14T09:21:39.13015Z","shell.execute_reply.started":"2023-11-14T09:21:39.114717Z","shell.execute_reply":"2023-11-14T09:21:39.129148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def read_images(image_path, scale_factor=2):\n#     image = pyvips.Image.new_from_file(image_path, access='sequential')\n#     return image.resize(1.0/scale_factor).numpy()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:39.131495Z","iopub.execute_input":"2023-11-14T09:21:39.131833Z","iopub.status.idle":"2023-11-14T09:21:39.140007Z","shell.execute_reply.started":"2023-11-14T09:21:39.131806Z","shell.execute_reply":"2023-11-14T09:21:39.139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def vips_read_image(image_path, longest_edge):\n    \n    \"\"\"\n    Read image using libvips\n\n    Parameters\n    ----------\n    image_path: str\n        Path of the image\n\n    Returns\n    -------\n    image: numpy.ndarray of shape (height, width, 3)\n        Image array\n    \"\"\"\n    \n    image_thumbnail = pyvips.Image.thumbnail(image_path, longest_edge)\n\n    return np.ndarray(\n        buffer=image_thumbnail.write_to_memory(),\n        dtype=np.uint8,\n        shape=[image_thumbnail.height, image_thumbnail.width, image_thumbnail.bands]\n    )","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:39.143556Z","iopub.execute_input":"2023-11-14T09:21:39.144175Z","iopub.status.idle":"2023-11-14T09:21:39.151781Z","shell.execute_reply.started":"2023-11-14T09:21:39.144136Z","shell.execute_reply":"2023-11-14T09:21:39.150721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UBCDataset(Dataset):\n    \n    def __init__(self, data, transform):\n        self.data = data\n        self.transform = transform\n#         self.n_images, _, _, _ = data_path['images'].shape\n        self.n_images = len(data)\n    \n    def __len__(self):\n        return self.n_images\n    \n    \n    def __getitem__(self, idx):\n#         image = self.data.loc[idx, 'compressed_image_path']\n        image = self.data.loc[idx, 'image_path']\n        image = vips_read_image(image, longest_edge=5000)\n#         image = cv.imread(image)\n        image = image.astype(np.float32)/255\n#         image = cv.resize(image, (512, 512))\n            \n        image = self.transform(image=image)['image']\n        \n#         return {\"image\": image}\n        return {\"image\": image}","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:39.152922Z","iopub.execute_input":"2023-11-14T09:21:39.153238Z","iopub.status.idle":"2023-11-14T09:21:39.166621Z","shell.execute_reply.started":"2023-11-14T09:21:39.153211Z","shell.execute_reply":"2023-11-14T09:21:39.165763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(img_size):\n    train_tsfm = A.Compose(\n        [\n            A.Resize(height=img_size[0], width=img_size[1]),\n#             A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.05, rotate_limit=15, p=0.5),\n#             A.RGBShift(r_shift_limit=15, g_shift_limit=15, b_shift_limit=15, p=0.5),\n#             A.RandomBrightnessContrast(p=0.5),\n#             A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n            ToTensorV2(),\n        ]\n    )\n    \n    test_tsfm = A.Compose(\n        [\n            A.Resize(height=img_size[0], width=img_size[1]),\n            ToTensorV2(),\n        ]\n    )\n    \n    return {\"test\": test_tsfm}\n\n\ndef get_dataloaders(data, img_size, batch_size, num_workers, split='test'):\n    tsfm = get_transforms(img_size=img_size)\n    if split.lower() == 'test':\n        ds = UBCDataset(data, tsfm[split])\n        dls = DataLoader(ds, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True, drop_last=False)\n    else:\n        raise ValueError('Incorrect split format')\n    return dls","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:39.167887Z","iopub.execute_input":"2023-11-14T09:21:39.168342Z","iopub.status.idle":"2023-11-14T09:21:39.178416Z","shell.execute_reply.started":"2023-11-14T09:21:39.16831Z","shell.execute_reply":"2023-11-14T09:21:39.177553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config['img_size'][0] > 224:\n    model_id_timm = \"timm/tf_efficientnetv2_b3.in21k_ft_in1k\"\nelif config['img_size'][0] == 224:\n#     model_id_timm = \"timm/vit_base_patch32_clip_224.openai_ft_in1k\"\n    model_id_timm = \"timm/vit_base_patch8_224.dino\"\n# timm.list_pretrained(\"*swin*\")\nmodel = timm.create_model(model_id_timm, pretrained=False, num_classes=512)\n\nclass Regr(torch.nn.Module):\n    \n    def __init__(self, base_model, rate=0.0):\n        super(Regr, self).__init__()\n        self.base_model = base_model\n        self.base_model.classifier = nn.Identity()\n        self.base_model.global_pool = nn.Identity()\n        self.pooling = GeM()\n        \n        self.linear_model = torch.nn.Sequential(\n            nn.LazyBatchNorm1d(),\n            nn.LazyLinear(512),\n            nn.ReLU(),\n            nn.Dropout(rate),\n            nn.LazyBatchNorm1d(),\n            nn.LazyLinear(128),\n            nn.ReLU(),\n            nn.Dropout(rate),\n            nn.LazyLinear(5),\n            nn.Softmax(dim=1),\n        )\n        \n        \n    def forward(self, x):\n        x = self.base_model(x)\n        x = self.pooling(x)\n        x = x.view(x.size(0), -1)\n        x = self.linear_model(x)\n        return x\n\n\nclass LSTMClassifier(torch.nn.Module):\n\n    def __init__(self, base_model, rate=0.0):\n        super(LSTMClassifier, self).__init__()\n        self.base_model = base_model\n\n        self.lstm_model = nn.LSTM(input_size=512, hidden_size=256, num_layers=2, dropout=0.0, bidirectional=True, batch_first=True)\n\n        # Maintain the same output layer\n        self.output_layer = nn.Sequential(\n            nn.LazyLinear(128),\n            nn.LazyBatchNorm1d(),\n            nn.Dropout(rate),\n#             nn.LeakyReLU(0.1),\n            nn.ReLU(),\n            nn.LazyLinear(5),\n            nn.Softmax(dim=1),\n        )\n\n    def forward(self, x):\n        x = self.base_model(x)\n        x, _ = self.lstm_model(x)\n        x = x.view(x.size(0), -1)\n\n        x = self.output_layer(x)\n\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:39.180383Z","iopub.execute_input":"2023-11-14T09:21:39.18074Z","iopub.status.idle":"2023-11-14T09:21:39.780341Z","shell.execute_reply.started":"2023-11-14T09:21:39.180711Z","shell.execute_reply":"2023-11-14T09:21:39.779562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MetricMonitor:\n    def __init__(self, float_precision=4):\n        self.float_precision = float_precision\n        self.reset()\n\n    def reset(self):\n        self.metrics = defaultdict(lambda: {\"val\": 0, \"count\": 0, \"avg\": 0})\n\n    def update(self, metric_name, val):\n        metric = self.metrics[metric_name]\n\n        metric[\"val\"] += val\n        metric[\"count\"] += 1\n        metric[\"avg\"] = metric[\"val\"] / metric[\"count\"]\n\n    def __str__(self):\n        return \" | \".join(\n            [\n                \"{metric_name}: {avg:.{float_precision}f}\".format(\n                    metric_name=metric_name, avg=metric[\"avg\"], float_precision=self.float_precision\n                )\n                for (metric_name, metric) in self.metrics.items()\n            ]\n        )","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:39.781586Z","iopub.execute_input":"2023-11-14T09:21:39.781901Z","iopub.status.idle":"2023-11-14T09:21:39.789758Z","shell.execute_reply.started":"2023-11-14T09:21:39.781873Z","shell.execute_reply":"2023-11-14T09:21:39.788701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference(model, data_loader, device='cpu', epoch=1):\n    model.to(device)\n    model.eval()\n    embeds = list()\n    for i, batch in enumerate(tqdm(data_loader, total=len(data_loader))):\n        xb= batch['image']\n        xb = xb.to(device, non_blocking=True)\n        \n        with torch.autocast(device_type=device, dtype=torch.float16):\n            with torch.inference_mode():\n                outputs = model(xb)\n                embeds.append(outputs.detach().cpu().numpy())\n                \n    return embeds","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:39.791128Z","iopub.execute_input":"2023-11-14T09:21:39.791432Z","iopub.status.idle":"2023-11-14T09:21:39.803637Z","shell.execute_reply.started":"2023-11-14T09:21:39.791405Z","shell.execute_reply":"2023-11-14T09:21:39.802787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nname = model_id_timm.split('/')[-1]\nfold = 1\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f\"Using {device} ...\")\n\ntest_loader = get_dataloaders(test_df, img_size=config['img_size'], batch_size=config['batch_size'], num_workers=config['num_workers'])\n# train_loader = get_dataloaders(train_df, img_size=config['img_size'], batch_size=config['batch_size'])\n\ndef run_inference(fold, dataloader):\n#     run = wandb.init()\n    print(f\"Fold {fold} weights being downloaded ...\")\n    '/kaggle/input/ubc-tfeff3-training'\n    fold_path = Path(f\"../input/ubc-tfeff3-training/{name}_fold_{fold}.pth\")\n#     artifact = run.use_artifact(f'samu2505/cgair-pytorch-baseline/fold_{i}_weights:v0', type='model')\n#     artifact_dir = artifact.download()\n    WEIGHTS = torch.load(fold_path, map_location=torch.device('cpu'))\n#     regr_model = Regr(base_model=model, rate=0.0)\n    regr_model = LSTMClassifier(base_model=model, rate=0.0)\n    regr_model.load_state_dict(WEIGHTS)\n\n    TEST_PREDS = inference(regr_model, dataloader, device=device)\n    # TRAIN_PREDS = inference(regr_model, criterion, train_loader, device=device)\n\n    TEST_PREDS = np.concatenate(TEST_PREDS)\n    np.save(f\"test_preds_fold_{fold}_{config['img_size'][0]}.npy\", TEST_PREDS)\n    del dataloader, fold_path, WEIGHTS, regr_model, TEST_PREDS\n    flush()\n    \nrun_inference(fold=fold, dataloader=test_loader)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:21:39.804797Z","iopub.execute_input":"2023-11-14T09:21:39.805105Z","iopub.status.idle":"2023-11-14T09:22:23.188643Z","shell.execute_reply.started":"2023-11-14T09:21:39.805079Z","shell.execute_reply":"2023-11-14T09:22:23.187501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_PREDS = np.load(f'/kaggle/working/test_preds_fold_{fold}_512.npy')\nsample_df['preds'] = np.argmax(TEST_PREDS, axis=1)\nsample_df['label'] = sample_df['preds'].map(label2id)\nsample_df.drop('preds', axis=1, inplace=True)\nsample_df.to_csv('submission.csv', index=False)\npd.read_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-14T09:22:23.190352Z","iopub.execute_input":"2023-11-14T09:22:23.190785Z","iopub.status.idle":"2023-11-14T09:22:23.220026Z","shell.execute_reply.started":"2023-11-14T09:22:23.190747Z","shell.execute_reply":"2023-11-14T09:22:23.218906Z"},"trusted":true},"execution_count":null,"outputs":[]}]}