{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"},{"sourceId":6774553,"sourceType":"datasetVersion","datasetId":3898019},{"sourceId":6949713,"sourceType":"datasetVersion","datasetId":3974389},{"sourceId":7300822,"sourceType":"datasetVersion","datasetId":4235374},{"sourceId":157413798,"sourceType":"kernelVersion"}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# setup pyvips\n\n!ls /kaggle/input/pyvips-python-and-deb-package-gpu\n# intall the deb packages\n!yes | dpkg -i --force-depends /kaggle/input/pyvips-python-and-deb-package-gpu/linux_packages/archives/*.deb\n# install the python wrapper\n!pip install pyvips -f /kaggle/input/pyvips-python-and-deb-package-gpu/python_packages/ --no-index","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:48:11.869663Z","iopub.execute_input":"2024-03-15T14:48:11.869912Z","iopub.status.idle":"2024-03-15T14:49:16.027033Z","shell.execute_reply.started":"2024-03-15T14:48:11.869889Z","shell.execute_reply":"2024-03-15T14:49:16.025975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nimport os\nimport gc\nimport time\nfrom IPython import display\nimport glob\nimport random\nfrom joblib import Parallel, delayed\n\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms, datasets\nimport timm","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:49:26.011045Z","iopub.execute_input":"2024-03-15T14:49:26.011396Z","iopub.status.idle":"2024-03-15T14:49:29.397228Z","shell.execute_reply.started":"2024-03-15T14:49:26.01137Z","shell.execute_reply":"2024-03-15T14:49:29.396417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# explore devices\n\ndef try_gpu(i=0):\n    if torch.cuda.device_count()>=i+1:\n        return torch.device(f'cuda:{i}')\n    return torch.device('cpu')\n\ndef try_all_gpus():\n    devices=[torch.device(f'cuda:{i}') for i in range(torch.cuda.device_count())]\n    return devices if devices else [torch.device('cpu')]\n\nprint(try_all_gpus())","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:49:29.398805Z","iopub.execute_input":"2024-03-15T14:49:29.399099Z","iopub.status.idle":"2024-03-15T14:49:29.422807Z","shell.execute_reply.started":"2024-03-15T14:49:29.399075Z","shell.execute_reply":"2024-03-15T14:49:29.421921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_IMG_DIR = \"/kaggle/input/UBC-OCEAN/test_images/\"\nTEST_TBNLS_DIR = \"/kaggle/input/UBC-OCEAN/test_thumbnails/\"\nTEST_TILES_DIR = \"/kaggle/working/test_tiles/\"\n\ntest_df=pd.read_csv(\"/kaggle/input/UBC-OCEAN/test.csv\")\n\nprint('test dataset:')\nprint(test_df.shape)\nprint(test_df.head())\n\n# tiling helper func\n\nimport cv2\nimport pyvips\n\nos.environ['VIPS_CONCURRENCY'] = '4'\nos.environ['VIPS_DISC_THRESHOLD'] = '15gb' #use disk caching instead of memory when the image exceeds 15GB","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:49:29.423847Z","iopub.execute_input":"2024-03-15T14:49:29.424123Z","iopub.status.idle":"2024-03-15T14:49:29.967183Z","shell.execute_reply.started":"2024-03-15T14:49:29.424099Z","shell.execute_reply":"2024-03-15T14:49:29.966221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extract tiles from an image\n# critial to set up max_samples, because all samples will be used for predict\n\ndef extract_test_tiles(img_path, size=2048, scale=0.5, drop_thr=0.6, white_thr=240, max_samples=20):\n    \n    # print(f\"processing: {img_path}\")\n    im = pyvips.Image.new_from_file(img_path) #load image\n    \n    w=h=size\n    new_size = int(size * scale), int(size * scale)\n    \n    # https://stackoverflow.com/a/47581978/4521646\n    idxs = [(y, y + h, x, x + w) for y in range(0, im.height, h) for x in range(0, im.width, w)]\n    \n    # random subsample\n    max_samples = max_samples if isinstance(max_samples, int) else int(len(idxs) * max_samples)\n    random.shuffle(idxs)\n    \n    images = []\n    for y, y_, x, x_ in idxs:\n\n        tile = im.crop(x, y, min(w, im.width - x), min(h, im.height - y)).numpy()[..., :3] \n\n        # increase tile size to (h,w) for edge tiles\n        if tile.shape[:2] != new_size:\n            tile_ = tile\n            tile_size = (h, w) if tile.ndim == 2 else (h, w, tile.shape[2])\n            tile = np.zeros(tile_size, dtype=tile.dtype)\n            tile[:tile_.shape[0], :tile_.shape[1], ...] = tile_\n        \n        # reduce size\n        tile = np.array(Image.fromarray(tile).resize(new_size, Image.LANCZOS))\n        \n        # emptry ratio detection\n        black_bg = np.sum(tile, axis=2) == 0\n        tile[black_bg, :] = 255\n        mask_bg = np.mean(tile, axis=2) > white_thr\n        if np.sum(mask_bg) >= (np.prod(mask_bg.shape) * drop_thr):\n            #print(f\"skip almost empty tile: {k:06}_{int(x_ / w)}-{int(y_ / h)}\")\n            continue\n        # print(tile.shape, tile.dtype, tile.min(), tile.max())\n        \n        images.append(tile)\n        if len(images) >= max_samples:\n            break\n    return images","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:49:29.969403Z","iopub.execute_input":"2024-03-15T14:49:29.969823Z","iopub.status.idle":"2024-03-15T14:49:29.982075Z","shell.execute_reply.started":"2024-03-15T14:49:29.969788Z","shell.execute_reply":"2024-03-15T14:49:29.981122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# standard image augmentaion procedures and color normalizations\n\n#tile_color_mean=[0.8636166450980394, 0.7583822915468411, 0.8537942079084968]\n#tile_color_std=[0.06693396412705377, 0.09844775155547589, 0.0531690918923438]\n\nnet_input=1024\n\nimport albumentations as A\nfrom albumentations import Compose, CenterCrop, Normalize\nfrom albumentations.pytorch import ToTensorV2\n\n# use albumentations for augmentation\nval_transform = A.Compose([\n        CenterCrop(height=net_input, width=net_input),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()])\n","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:49:47.484361Z","iopub.execute_input":"2024-03-15T14:49:47.485042Z","iopub.status.idle":"2024-03-15T14:49:48.928726Z","shell.execute_reply.started":"2024-03-15T14:49:47.485013Z","shell.execute_reply":"2024-03-15T14:49:48.927905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# given one sample's img_path, __getitem__ outputs one tile at a time\nclass ImageTilesDataset(Dataset):\n\n    def __init__(self, img_path:str, size=2048, scale=0.25, drop_thr=0.6, max_samples=20, transform=None):\n        assert os.path.isfile(img_path)\n        self.transform = transform\n        self.tiles=extract_test_tiles(img_path, size=size, scale=scale, drop_thr=drop_thr, max_samples=max_samples)\n    \n    def __len__(self) -> int:\n        return len(self.tiles)\n    \n    def __getitem__(self, idx: int):\n        tile = self.tiles[idx]\n        if np.max(tile) < 1.5:\n            tile = np.clip(tile * 255, 0, 255).astype(np.uint8)\n        # augmentation\n        if self.transform:\n            augmented = self.transform(image=tile)  # 注意，这里输入的img是一个numpy数组\n            tile = augmented['image']\n        return tile\n","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:50:15.753401Z","iopub.execute_input":"2024-03-15T14:50:15.754532Z","iopub.status.idle":"2024-03-15T14:50:15.762498Z","shell.execute_reply.started":"2024-03-15T14:50:15.754493Z","shell.execute_reply":"2024-03-15T14:50:15.761692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# infer helper\n\nimport shutil\nfrom torch.utils.data import DataLoader\nimport torch.nn.functional as F\nfrom joblib.externals.loky.backend.context import get_context\n\nlabel_dict_reverse = {0: 'CC', 1: 'EC', 2: 'HGSC', 3: 'LGSC', 4: 'MC'}\n\ndef infer_single_image(idx_row, nets, device=try_gpu(), max_samples=20, threshold=10) -> dict:\n    row = dict(idx_row[1])\n    \n    # prepare data - cut and load tiles\n    img_path = os.path.join(TEST_IMG_DIR, f\"{str(row['image_id'])}.png\")\n    test_dataset = ImageTilesDataset(img_path, size=3072, scale=0.5, max_samples=max_samples, transform=val_transform)\n    if not len(test_dataset):\n        print (f\"seem no tiles were cut for `{row['image_id']}`\")\n        return row\n    test_iter = torch.utils.data.DataLoader(\n        test_dataset, batch_size=8, num_workers=2, shuffle=False,\n        # https://github.com/pytorch/pytorch/issues/44687#issuecomment-790842173\n        multiprocessing_context=get_context('loky'))\n\n    # iterate over images and collect predictions\n    preds1 = []\n    preds2 = []\n    preds3 = []\n    for imgs in test_iter:\n        #print(f\"{imgs.shape}\")\n        with torch.no_grad():\n            imgs = imgs.to(device).half()\n            pred = nets[0](imgs)\n            pred1 = F.softmax(pred, dim=1)\n            \n            pred = nets[1](imgs)\n            pred2 = F.softmax(pred, dim=1)\n            \n            pred = nets[2](imgs)\n            pred3 = F.softmax(pred, dim=1)\n        preds1 += pred1.cpu().numpy().tolist()\n        preds2 += pred2.cpu().numpy().tolist()\n        preds3 += pred3.cpu().numpy().tolist()\n    # print(f\"Sum contrinution from all tiles: {np.sum(preds, axis=0)}\")\n    # print(f\"Max contribution over all tiles: {np.max(preds, axis=0)}\")\n    \n    # decide label\n    preds1 = np.sum(preds1, axis=0)\n    preds2 = np.sum(preds2, axis=0)\n    preds3 = np.sum(preds3, axis=0)\n    \n    lb1 = np.argmax(preds1)\n    lb1_prob = np.max(preds1)\n    \n    lb2 = np.argmax(preds2)\n    lb2_prob = np.max(preds2)\n    lb3 = np.argmax(preds3)\n    lb3_prob = np.max(preds3)\n    \n    labels = [lb1, lb2, lb3]\n    print(labels, lb1_prob)\n    lb = labels[np.argmax([lb1_prob, lb2_prob, lb3_prob])]\n        \n    row['label'] = label_dict_reverse[lb]\n    \n    del test_iter, test_dataset\n    gc.collect()\n    \n    # print(row)\n    return row","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:50:40.545518Z","iopub.execute_input":"2024-03-15T14:50:40.546435Z","iopub.status.idle":"2024-03-15T14:50:40.561134Z","shell.execute_reply.started":"2024-03-15T14:50:40.54638Z","shell.execute_reply":"2024-03-15T14:50:40.56017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = timm.create_model('resnest200e', pretrained=0, num_classes=5)\nnet.load_state_dict(torch.load('/kaggle/input/ubc-final/UBC-models/resnest200e.in1k_bestAcc0.950_imgsize_1024_onlyRedMaskTileImg_stainmix.pt' ))\n\nnet2 = timm.create_model('tf_efficientnetv2_s', pretrained=0, num_classes=5)\nnet2.load_state_dict(torch.load( '/kaggle/input/ubc-final/UBC-models/tf_efficientnetv2_s.in21k_ft_in1k_swa_imgsize_1024_onlyMaskTileImg_stainmix.pt' )['state_dict'])\n\nnet3 = timm.create_model('tf_efficientnetv2_l', pretrained=0, num_classes=5)\nnet3.load_state_dict(torch.load( '/kaggle/input/ubc-final/UBC4/tf_efficientnetv2_l.in21k_ft_in1k_ep17_bestRecall0.741_imgsize_1024_onlyRedMaskTileImg_stainmix.pt' ))\n\n# infer\n\nfrom tqdm.auto import tqdm\nfrom joblib import Parallel, delayed\n\nnet = net.to(try_gpu()).half()\nnet.eval()\nnet2 = net2.to(try_gpu()).half()\nnet2.eval()\nnet3 = net3.to(try_gpu()).half()\nnet3.eval()","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:51:04.490192Z","iopub.execute_input":"2024-03-15T14:51:04.490595Z","iopub.status.idle":"2024-03-15T14:51:17.255441Z","shell.execute_reply.started":"2024-03-15T14:51:04.490563Z","shell.execute_reply":"2024-03-15T14:51:17.254471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nets = [net, net2, net3]\nmax_samples = 28\nthreshold = 0  # for identifying outliers\nsubmission = []\n\nsubmission = Parallel(n_jobs=2, backend='loky')(\n     delayed(infer_single_image)\n     (idx_row, nets=nets, device=try_gpu(), max_samples=max_samples, threshold=threshold)\n     for idx_row in tqdm(test_df.iterrows(), total=len(test_df))\n )\n\n# submission\n\noutput = pd.DataFrame(submission)[[\"image_id\", \"label\"]]\noutput.to_csv(\"submission.csv\", index=False)\nprint(output)\nprint(\"Your submission was successfully saved!\")\n","metadata":{"execution":{"iopub.status.busy":"2024-03-15T14:51:26.991356Z","iopub.execute_input":"2024-03-15T14:51:26.992295Z","iopub.status.idle":"2024-03-15T14:52:38.503871Z","shell.execute_reply.started":"2024-03-15T14:51:26.992264Z","shell.execute_reply":"2024-03-15T14:52:38.502612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}