{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook aims to provide baseline code for working with extracted features. For example with ``` efficientnet_b0 ``` \n\nJust replace ``` TRAIN_FEATURES_NUMPY ``` with your extracted fatures folder and adjust ``` FEATURES_SIZE ``` to be the size of last layer output in your backbone.","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os, gc, sys, random\nfrom tqdm.auto import tqdm\nimport pydicom\nimport glob\nimport cv2\nimport wandb\nimport warnings\nwarnings.simplefilter('ignore')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.backends.cudnn as cudnn\nfrom torch.utils.data import DataLoader, Dataset","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-09T08:12:15.680694Z","iopub.execute_input":"2022-09-09T08:12:15.681105Z","iopub.status.idle":"2022-09-09T08:12:17.537914Z","shell.execute_reply.started":"2022-09-09T08:12:15.681072Z","shell.execute_reply":"2022-09-09T08:12:17.53651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    DEBUG = True\n    USE_WANDB = False\n    ROOT_DIR = '/kaggle/input/rsna-2022-cervical-spine-fracture-detection'\n    WORKING_DIR = './'\n    \n    TRAIN_PATH = \"../input/rsna-2022-cervical-spine-fracture-detection/train.csv\"\n    TEST_PATH = \"../input/rsna-2022-cervical-spine-fracture-detection/test.csv\"\n\n    TRAIN_DIR = \"../input/rsna-2022-cervical-spine-fracture-detection/train_images\"\n    TEST_DIR = \"../input/rsna-2022-cervical-spine-fracture-detection/test_images\"\n    \n    TRAIN_FEATURES_NUMPY = '../input/rsna-2022-feature-extraction/train_features'\n    TEST_FEATURES_NUMPY = 'test_features'\n\n    SAMPLE_SUB = \"../input/rsna-2022-cervical-spine-fracture-detection/sample_submission.csv\"\n\n    LABELS_COLS = [\"patient_overall\", \"C1\", \"C2\", \"C3\", \"C4\", \"C5\", \"C6\", \"C7\"]\n\n    TO_EXCLUDE = \"1.2.826.0.1.3680043.20574\"\n    \n    BACKBONE = \"efficientnet_b0\"\n    MODEL_NAME = \"efficientnet_b0\"\n    VER = 1\n    \n    TRAIN_SIZE, VALID_SIZE, TEST_SIZE = [8, 1, 1] # ratio of splits\n\n    \n    SEED = 42\n    BATCH_SIZE = 32\n    NUM_WORKERS = 0\n    EPOCHS = 20\n    \n    FEATURES_SIZE = 2048\n    SEQ_LEN = 256 # (num_images_in_sample, FEATURES_SIZE) -> resite to -> (SEQ_LEN, FEATURES_SIZE)\n    LSTM_SIZE = 128\n    HIDDEN_SIZE = 128\n    POOL_SIZE = 16\n    DROPOUT=0.5\n    NUM_LAYERS=2\n    LR=3e-4\n    \ndef set_seed(seed=CFG.SEED):\n    cudnn.benchmark = True\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    random.seed(seed)\n    \ndef seed_worker(worker_id):\n    worker_seed = torch.initial_seed() % 2**32\n    np.random.seed(worker_seed)\n    random.seed(worker_seed)\n\nset_seed()\n    \ng = torch.Generator()\ng.manual_seed(CFG.SEED)    \n    \ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.DEBUG and CFG.USE_WANDB:\n    os.environ[\"WANDB_SILENT\"] = \"false\"\n    os.environ['WANDB_MODE'] = 'offline'\n\nif CFG.USE_WANDB:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    secret_value_0 = user_secrets.get_secret(\"wandb\")\n\n    wandb.login(key=secret_value_0)\n    run = wandb.init(project=\"RSNA 2022 Cervical Spine Feature Extraction\", entity=\"barteksadlej\", name=\"attention relu lstm\", config=CFG().__dict__)\n    wandb.config.update(CFG().__dict__)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RNSA2022Model(nn.Module):\n    \n    def __init__(self):\n        super().__init__()\n        self.lstm1 = nn.LSTM(CFG.FEATURES_SIZE, CFG.LSTM_SIZE, dropout=CFG.DROPOUT, num_layers=CFG.NUM_LAYERS, batch_first=True)\n        self.pool = nn.AdaptiveAvgPool2d(CFG.POOL_SIZE)\n        self.dropout = nn.Dropout(p=0.3)\n        self.fc = nn.Linear(CFG.POOL_SIZE*CFG.POOL_SIZE, len(CFG.LABELS_COLS))\n        \n    def forward(self, x):\n        lstm1_out, _ = self.lstm1(x)\n        pool_out = self.pool(lstm1_out)\n        pool_flatten = torch.flatten(pool_out, start_dim=1)\n        drop_pool_flatten = self.dropout(pool_flatten)\n        out = self.fc(drop_pool_flatten)\n        return out","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision.transforms import Resize\n\nclass FeaturesDataset():\n    \n    def __init__(self, df, training=False):\n        self.df = df\n        self.training = training\n        self.features_dir = CFG.TRAIN_FEATURES_NUMPY if training else CFG.TEST_FEATURES_NUMPY\n        self.transforms = Resize((CFG.SEQ_LEN, CFG.FEATURES_SIZE))\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        \n        item = self.df.iloc[index]\n        suid = item['StudyInstanceUID']\n        path = os.path.join(self.features_dir, f\"{suid}.npy\")\n        features = np.load(path)\n        features = torch.tensor(features, dtype=torch.float)\n        features = torch.unsqueeze(features, axis=-0)\n        features = self.transforms(features)\n        features = torch.squeeze(features, axis=-0)\n\n        data = {\n            'features' : features,\n            'StudyInstanceUID' : suid\n        }\n        \n        if self.training:\n            labels = torch.tensor(item[CFG.LABELS_COLS].values.astype(float), dtype=torch.float)\n            data['labels'] = labels\n            \n        return data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Strtify data taking into consideration all labels columns","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(CFG.TRAIN_PATH)\ntrain_df = train_df[train_df.StudyInstanceUID != CFG.TO_EXCLUDE]\n\nif CFG.DEBUG:\n    train_df = train_df.sample(20).reset_index(drop=True)\n    \nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\n\ntrain_df['multilabel'] = LabelEncoder().fit_transform([str(x) for x in train_df[CFG.LABELS_COLS].values])\ntrain_df.head()\nprint(train_df['multilabel'].unique().shape)\n\nskf = StratifiedKFold(n_splits = np.sum([CFG.TRAIN_SIZE, CFG.VALID_SIZE, CFG.TEST_SIZE]), shuffle=True, random_state=CFG.SEED)\n\ntrain_idx = []\nvalid_idx = []\ntest_idx = []\n\nfor split, (_, indexes) in enumerate(skf.split(train_df, train_df['multilabel'])):\n    \n    if split < CFG.TRAIN_SIZE:\n        train_idx.extend(indexes)\n    elif split < CFG.TRAIN_SIZE + CFG.VALID_SIZE:\n        valid_idx.extend(indexes)\n    else:\n        test_idx.extend(indexes)\n        \nprint(f\"train: {len(train_idx)}, valid: {len(valid_idx)}, test: {len(test_idx)}\")\ntrain_df.drop(['multilabel'], axis=1, inplace=True)\n\ntrain = train_df.iloc[train_idx]\nvalid = train_df.iloc[valid_idx]\ntest = train_df.iloc[test_idx]\n\nprint(f\"train: {len(train)}, valid: {len(valid)}, test: {len(test)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataloader(df, training=False):\n    \n    ds = FeaturesDataset(df, training)\n    dl = DataLoader(\n        ds,\n        batch_size=CFG.BATCH_SIZE,\n        num_workers=CFG.NUM_WORKERS,\n        shuffle=training)\n    \n    return dl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn = nn.BCEWithLogitsLoss(reduction='none')\n\ncompetition_weights = {\n    '-' : torch.tensor([7, 1, 1, 1, 1, 1, 1, 1], dtype=torch.float, device=device),\n    '+' : torch.tensor([14, 2, 2, 2, 2, 2, 2, 2], dtype=torch.float, device=device),\n}\n\ndef competiton_loss(y_hat, y):\n    loss = loss_fn(y_hat, y)\n    weights = y * competition_weights['+'] + (1 - y) * competition_weights['-']\n    loss = (loss * weights).sum(axis=1)\n    loss = loss / weights.sum(axis=1)\n    loss = loss.mean()\n    \n    return loss ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AvgMeter():\n    \n    def __init__(self, name=None):\n        self.reset()\n        self.name = name\n        \n    def reset(self):\n        self.count = 0\n        self.sum = 0\n        self.avg = 0\n        \n    def update(self, x, n=1):\n        n = np.float(n)\n        self.count +=  n\n        self.sum += x*n\n        self.avg = self.sum / self.count\n        \n        if CFG.USE_WANDB:\n            wandb.log({f\"{self.name}\" : x})\n        \n    def __str__(self):\n        return f\"{self.name} : {self.avg}\"\n    \n    def log(self):\n        if CFG.USE_WANDB:\n            wandb.log({f\"{self.name} epoch end\" : self.avg})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RNSA2022Model().to(device)\nif CFG.USE_WANDB:\n    wandb.watch(model)\noptimizer = torch.optim.Adam(model.parameters(), lr=CFG.LR)\nscheduler = None\n# scheduler= torch.optim.lr_scheduler.OneCycleLR(\n#     optimizer=optimizer, max_lr=CFG.LR, total_steps=len(train_dl) * CFG.EPOCHS,\n# )\ntrain_dl = get_dataloader(train, training=True)\n\ndef train_one_epoch(model, optimizer, train_dl, device, scheduler=None):\n    \n    train_loss = AvgMeter(\"train loss\")\n    model = model.train(True)\n    for batch in tqdm(train_dl):\n\n        X = batch['features'].to(device)\n        X = torch.nan_to_num(X)\n        target = batch[\"labels\"].to(device)\n        optimizer.zero_grad()\n        \n        y_hat = model(X)\n        loss = competiton_loss(y_hat, target)\n        train_loss.update(loss.item(), target.shape[0])\n        loss.backward()\n        optimizer.step()\n        \n        if scheduler is not None:\n            scheduler.step()\n            \n    return train_loss\n\n@torch.no_grad()\ndef eval_model(model, dl, device, loss_name=None):\n    \n    loss_meter = AvgMeter(loss_name)\n    model = model.train(False)\n    \n    for batch in tqdm(dl): \n        X = batch['features'].to(device)\n        X = torch.nan_to_num(X)\n        target = batch[\"labels\"].to(device)\n        y_hat = model(X)\n        loss = competiton_loss(y_hat, target)\n        loss_meter.update(loss.item(), target.shape[0])\n    \n    return loss_meter\n\nfrom collections import defaultdict\n\nclass ModelCheckpoint():\n    \n    def __init__(self, path=f\"{CFG.MODEL_NAME}_VER_{CFG.VER}.pt\"):\n        \n        self.best_values = defaultdict(lambda : np.inf)\n        self.path = path\n        \n    def make_checkpoint(self, model, values):\n        \n        for name, value in values:\n            \n            if value < self.best_values[name]:\n                \n                print(f\"new best {name}, saving model...\")\n                self.best_values[name] = value\n                torch.save(model.state_dict(), f\"{name}_{value:.6f}_{self.path}\")\n                print(\"model saved!\")\n                \nmckpt = ModelCheckpoint()\n\nfor epoch in range(CFG.EPOCHS):\n    \n    _ = gc.collect()\n    \n    print(\"=\" * 25)\n    print(f\"epoch [{epoch + 1}]\")\n    print(\"=\" * 25)\n    \n    \n    train_loss = train_one_epoch(model, optimizer, train_dl, device, scheduler)\n    print(train_loss)\n    train_loss.log()\n    _ = gc.collect()\n    \n    valid_dl = get_dataloader(valid, training=True)\n    valid_loss = eval_model(model, valid_dl, device, loss_name=f\"valid loss\")\n    del valid_dl\n    print(valid_loss)\n    valid_loss.log()\n    _ = gc.collect()\n    \n    test_dl = get_dataloader(test, training=True)\n    test_loss = eval_model(model, test_dl, device, loss_name=f\"test loss\")\n    del test_dl \n    print(test_loss)\n    test_loss.log()\n    _ = gc.collect()\n    \n    mckpt.make_checkpoint(model, list(map(lambda loss : (loss.name, loss.avg), [train_loss, valid_loss, test_loss])))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}