{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install iterative-stratification\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nimport pandas as pd\nmskf = MultilabelStratifiedKFold(n_splits=5, shuffle=True, random_state=2020)\n\ndf = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train.csv\")\n\n\n\nfor fold, ( _, val_) in enumerate(mskf.split(X=df, y=df[df.columns[1:]])):\n    df.loc[val_ , \"fold\"] = fold\n    \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-19T10:13:32.891933Z","iopub.execute_input":"2022-08-19T10:13:32.89243Z","iopub.status.idle":"2022-08-19T10:13:46.218572Z","shell.execute_reply.started":"2022-08-19T10:13:32.89235Z","shell.execute_reply":"2022-08-19T10:13:46.217317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys,gc,os,random,time,math\nimport matplotlib.pyplot as plt\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom collections import defaultdict, Counter\nfrom  torch.cuda.amp import autocast, GradScaler \nimport cv2\n\nfrom PIL import Image\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\nimport sklearn.metrics as metrics\nfrom sklearn.model_selection import StratifiedKFold,GroupKFold\nfrom sklearn.metrics import log_loss\nfrom functools import partial\nfrom tqdm import tqdm\nfrom sklearn.metrics import precision_score,recall_score,f1_score,log_loss\nfrom  sklearn.metrics import accuracy_score as acc\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD,AdamW\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau,CosineAnnealingWarmRestarts\nimport transformers as T\n\n\nfrom torch.optim.swa_utils import AveragedModel, SWALR\n\n###\nimport logging","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:13:46.221091Z","iopub.execute_input":"2022-08-19T10:13:46.22145Z","iopub.status.idle":"2022-08-19T10:13:48.355347Z","shell.execute_reply.started":"2022-08-19T10:13:46.221415Z","shell.execute_reply":"2022-08-19T10:13:48.354478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\n#### dataset ==============\nclass TrainDataset(torch.utils.data.Dataset):\n    def __init__(self, df,train=True):\n        self.df = df\n        self.cols = ['C1', 'C2', 'C3', 'C4', 'C5','C6', 'C7',\"patient_overall\"]\n\n        self.crop = 300\n        self.flip = 0.5\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        study = self.df[\"StudyInstanceUID\"].values[idx]\n        #seq = np.load(f\"/home/u094724e/rsna2022/dino/exp001/50epoch/{study}.npy\")#n_seq,2304\n        seq = np.load(f\"../input/rsna2022-get-embdino/3darray/{study}.npy\")\n        seq= seq[:,-384:]\n\n\n        if seq.shape[0]<self.crop:\n            x_input_base = np.zeros([self.crop,seq.shape[1]], np.float32)\n            x_input_base[:seq.shape[0],:] = seq\n            seq = x_input_base\n        else:\n            seq = seq[-self.crop:,:]#後ろ(頭)からとる\n\n        #if np.random.rand()<self.flip:#どちらが頭側か　の情報は大事っぽい\n        #    seq = np.copy(seq[::-1,:])\n        \n        label = self.df[self.cols].to_numpy()[idx]\n        seq = torch.from_numpy(seq).float()\n        label = torch.tensor(label).float()\n\n        return seq,label\n","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:13:48.356843Z","iopub.execute_input":"2022-08-19T10:13:48.357746Z","iopub.status.idle":"2022-08-19T10:13:48.368975Z","shell.execute_reply.started":"2022-08-19T10:13:48.357701Z","shell.execute_reply":"2022-08-19T10:13:48.367334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.nn import AdaptiveAvgPool2d, AdaptiveMaxPool2d\n\n\n# From: https://github.com/filipradenovic/cnnimageretrieval-pytorch/blob/master/cirtorch/layers/pooling.py\ndef gem_1d(x, p=3, eps=1e-6):\n    return F.avg_pool1d(x.clamp(min=eps).pow(p), (x.size(-1),)).pow(1./p)\n\n\ndef gem_2d(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\n\ndef gem_3d(x, p=3, eps=1e-6):\n    return F.avg_pool3d(x.clamp(min=eps).pow(p), (x.size(-3), x.size(-2), x.size(-1))).pow(1./p)\n\n\n_GEM_FN = {\n    1: gem_1d, 2: gem_2d, 3: gem_3d\n}\n\n\nclass GeM(nn.Module):\n\n    def __init__(self, p=3, eps=1e-6, dim=2):\n        super().__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n        self.dim = dim\n\n    def forward(self, x):\n        return _GEM_FN[self.dim](x, p=self.p, eps=self.eps)","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:13:48.37174Z","iopub.execute_input":"2022-08-19T10:13:48.372169Z","iopub.status.idle":"2022-08-19T10:13:48.390672Z","shell.execute_reply.started":"2022-08-19T10:13:48.372134Z","shell.execute_reply":"2022-08-19T10:13:48.38962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Noop(nn.Module):\n    def __init__(self,*args):\n        super(Noop, self).__init__()\n    def forward(self,x):\n        return x\n\nclass NoopAddDim(nn.Module):\n    def __init__(self):\n        super(NoopAddDim, self).__init__()\n    def forward(self,x):\n        return x.unsqueeze(-1)\n\ndef add_to_dim(x,num_dims,dim=0):\n    while len(x.shape)<num_dims:\n        x=x.unsqueeze(dim)\n    return x\nclass DummyEmbd(nn.Module):\n    def __init__(self,out_size,dtype=torch.float32):\n        super(DummyEmbd, self).__init__()\n        self.out_size=out_size\n        self.dtype=dtype\n    def forward(self,x):\n        return torch.zeros(x.shape+(self.out_size,),dtype=self.dtype,device=x.device)\n\nimport math\ndef calc_positional_encoder(d_model, max_seq_len = 32):\n        # create constant 'pe' matrix with values dependant on\n        # pos and i\n        pe = torch.zeros(max_seq_len, d_model)\n        for pos in range(max_seq_len):\n            for i in range(0, d_model, 2):\n                pe[pos, i] = \\\n                math.sin(pos / (10000 ** ((2 * i)/d_model)))\n                pe[pos, i + 1] = \\\n                math.cos(pos / (10000 ** ((2 * (i + 1))/d_model)))\n        return pe/(d_model**0.5)\n\nclass SEModule(nn.Module):\n\n    def __init__(self, channels, reduction):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n        self.fc1 = nn.Conv1d(channels, channels // reduction, kernel_size=1,\n                             padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv1d(channels // reduction, channels, kernel_size=1,\n                             padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x\n\nclass GaussianDropout(nn.Module):\n\n    def __init__(self, p: float):\n        \"\"\"\n        Multiplicative Gaussian Noise dropout with N(1, p/(1-p))\n        It is NOT (1-p)/p like in the paper, because here the\n        noise actually increases with p. (It can create the same\n        noise as the paper, but with reversed p values)\n\n        Source:\n        Dropout: A Simple Way to Prevent Neural Networks from Overfitting\n        https://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf\n\n        :param p: float - determines the the standard deviation of the\n        gaussian noise, where sigma = p/(1-p).\n        \"\"\"\n        super().__init__()\n        assert 0 <= p < 1\n        self.t_mean = torch.ones((0,))\n        self.shape = ()\n        self.p = p\n        self.t_std = self.compute_std()\n\n    def compute_std(self):\n        return self.p / (1 - self.p)\n\n    def forward(self, t_hidden):\n        if self.training and self.p > 0.:\n            if self.t_mean.shape != t_hidden.shape:\n                self.t_mean = torch.ones_like(input=t_hidden\n                                              , dtype=t_hidden.dtype\n                                              , device=t_hidden.device)\n            elif self.t_mean.device != t_hidden.device:\n                self.t_mean = self.t_mean.to(device=t_hidden.device, dtype=t_hidden.dtype)\n\n            t_gaussian_noise = torch.normal(self.t_mean, self.t_std)\n            t_hidden = t_hidden.mul(t_gaussian_noise)\n        return t_hidden\n\nclass TransformerModel(nn.Module):\n    def __init__(self,in_size=3,#d_model\n                 dim_feedforward=16,##hidden\n                 n_heads=4,\n                 n_encoders=4,\n                 num_outputs=8,\n                 use_age=False,\n                 max_site_num=7,\n                 use_sex=False,\n                 use_age_diff=False,\n                 use_position_enc=False,\n                 pool=\"avg\",\n                 embed=True):\n        super(TransformerModel, self).__init__()\n        self.in_size=in_size\n        self.do_embed = embed\n        self.encoder_layer =nn.TransformerEncoderLayer(in_size,#d_model(単語の次元..1280)\n                                                       n_heads,\n                                                       dim_feedforward=dim_feedforward)\n#        self.decoder_layer =nn.TransformerDecoderLayer(in_size, 4, dim_feedforward=in_size)\n        self.encoder=nn.TransformerEncoder(self.encoder_layer, n_encoders)\n#        self.decoder=nn.TransformerDecoder(self.decoder_layer, 2)\n        self.egg_emb=nn.Sequential(nn.Linear(12,16),nn.ReLU(),nn.Linear(16,in_size)) \n        ###meta_feature(csvから取ってくるやつ) ==========================\n        self.sex_embd=nn.Embedding(2,in_size) if  use_sex else DummyEmbd(in_size)\n        self.age_embd=nn.Sequential(NoopAddDim(),nn.Linear(1,16),nn.ReLU(),nn.Linear(16,in_size)) if use_age else DummyEmbd(in_size)\n        self.site_embd=nn.Embedding(max_site_num,in_size) if max_site_num>0 else DummyEmbd(in_size)\n        self.age_diff_embd=nn.Sequential(NoopAddDim(),nn.Linear(1,16),nn.ReLU(),nn.Linear(16,in_size)) if use_age else DummyEmbd(in_size)\n        # ==========================\n        if pool in ('avg','concat','gem','max'):\n            self.pool = GeM(dim=1)#nn.AdaptiveAvgPool1d(1)#\n            if pool == 'concat':\n                self.exam_classifier = nn.Linear(in_size*2, num_outputs)\n            else:\n                self.exam_classifier = nn.Linear(in_size, num_outputs)\n        else:\n            self.pool = None\n            self.exam_classifier = nn.Linear(in_size, num_outputs)\n        \n        #self.image_classifier = nn.Linear(in_size, num_outputs)\n        self.pos_embd=calc_positional_encoder(768,max_seq_len=300) if use_position_enc else None\n        self.layer1 = nn.Sequential(\n                nn.Conv1d(in_size, in_size, kernel_size=13, stride=1, padding=0, bias=False),\n                nn.BatchNorm1d(in_size),\n                #nn.ReLU(inplace=True),\n                nn.Mish(),\n                nn.Conv1d(in_size,in_size*2, kernel_size=7, stride=1, padding=0, bias=False),\n                nn.BatchNorm1d(in_size*2),\n                nn.Mish(),\n                #SEModule(in_size*2, in_size//2),#だめ\n                nn.Conv1d(in_size*2,in_size, kernel_size=7, stride=2, padding=0, bias=False),\n                nn.BatchNorm1d(in_size),\n                nn.Mish(),\n                #nn.Dropout(),#AUCはよい bceはだめ\n                #SEModule(dim, dim//4),\n        )\n        self.gru = nn.GRU(in_size, in_size//2, bidirectional=True, batch_first=True, num_layers=2)\n        self.gru_afetr = nn.GRU(in_size, in_size, bidirectional=False, batch_first=True, num_layers=2)\n        \n        self.dropout = nn.Dropout(0.2)\n        self.Gdropout = GaussianDropout(0.5)\n    #@time_function_execution\n    def forward(self, x,sex=None,age=None,site=None,age_diff=None,mask=None):\n        if self.pos_embd is not None:\n            if self.pos_embd.device!=x.device:\n                self.pos_embd = self.pos_embd.to(x.device)\n        ###meta_feature(csvから取ってくるやつ) ==========================...yuvalのsiimより??\n        #x = x if sex is None else x + self.sex_embd(sex)\n        #x = x if age is None else x + self.age_embd(age)\n        #x = x if site is None else x + self.site_embd(site)\n        #x = x if age_diff is None else x + self.age_diff_embd(age_diff)\n        # ==========================\n        #print(self.pos_embd[:x.shape[1]][None].shape,x.shape)\n        x = x if self.pos_embd is None else x + self.pos_embd[:x.shape[1]][None]\n        x = x if mask is None else x*mask.unsqueeze(-1)\n        #x = x if self.do_embed ==False else self.layer1(x.permute(0, 2, 1)).permute(0, 2, 1)#self.egg_emb(x)\n        x = x if self.do_embed ==False else self.gru(x)[0]#self.egg_emb(x)\n        #before 1dcnnの方がやはり良さそうだが、overfitしがち...1dcnn部分を薄くするかどうか..\n\n        x = self.encoder(x) #,mask=s_mask #bs,len,depth\n        #x = self.gru_afetr(x)[0]\n        #x = torch.stack([self.dropout(x) for _ in range(5)], 0).mean(0)\n        #x = self.Gdropout(x)#5e-3改善　どっちでも良さそう\n        if self.pool:\n            pooled = self.pool(x.permute(0,2,1))\n            #pooled = torch.stack([self.dropout(pooled) for _ in range(5)], 0).mean(0)\n            feature = pooled[:,:,0]\n            \n            exam = self.exam_classifier(feature)\n        else:\n            feature = x[:,0,:]\n            exam = self.exam_classifier(feature)##0番目の\"単語\"にクラス分類のための特徴が集約されるように学習\n        return exam\n\n\n\nclass gru_model(nn.Module):\n    def __init__(self,input_ch,hidden_size=256,pool=\"avg\"):\n        super().__init__()\n        \n        self.gru = nn.GRU(input_ch, hidden_size, bidirectional=True, batch_first=True, num_layers=2)\n        self.gru2 = nn.GRU(hidden_size*2, hidden_size, bidirectional=True, batch_first=True, num_layers=2)\n        \n        #if pool in ('avg','concat','gem','max'):\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        if pool == 'concat': hidden_size *= 2\n\n        self.exam_predictor = nn.Linear(hidden_size*2, 8)\n        \n    #@time_function_execution\n    def forward(self, embeds):\n        #embeds = embeds.permute(0, 2, 1)\n        embeds, _ = self.gru(embeds)\n        embeds, _ =self.gru2(embeds)\n        embeds = self.pool(embeds.permute(0,2,1))[:,:,0]\n        \n        exam_pred = self.exam_predictor(embeds)\n        return exam_pred\n\n\n#### model ================\n\n\n###  loss =============\n\nclass competiton_loss_row_norm(nn.Module):\n    \n    # https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/discussion/340392\n    def __init__(self,device):\n        super().__init__()\n        eacl_w = torch.full([7],2)\n        self.loss_fn = nn.BCEWithLogitsLoss(reduction=\"none\") \n        self.competition_weights = {\n    '-' : torch.tensor([1, 1, 1, 1, 1, 1, 1,7], dtype=torch.float, device=device),\n    '+' : torch.tensor([2, 2, 2, 2, 2, 2, 2,14], dtype=torch.float, device=device),\n}\n        \n        \n\n    def forward(self, y_hat, y):\n        loss = self.loss_fn(y_hat, y)\n        weights = y * self.competition_weights['+'] + (1 - y) * self.competition_weights['-']\n        loss = (loss * weights).sum(axis=1)\n        w_sum = weights.sum(axis=1)\n        loss = torch.div(loss, w_sum)\n        return loss.mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:13:48.392608Z","iopub.execute_input":"2022-08-19T10:13:48.392999Z","iopub.status.idle":"2022-08-19T10:13:48.43649Z","shell.execute_reply.started":"2022-08-19T10:13:48.392966Z","shell.execute_reply":"2022-08-19T10:13:48.43568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 10\nbatch_size = 32*2\nwarmup = 1\nlr = 2e-4\namp=True","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:13:48.438107Z","iopub.execute_input":"2022-08-19T10:13:48.438833Z","iopub.status.idle":"2022-08-19T10:13:48.449668Z","shell.execute_reply.started":"2022-08-19T10:13:48.438789Z","shell.execute_reply":"2022-08-19T10:13:48.448701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef train_fn(fold,folds):\n\n    cols = ['patient_overall', 'C1', 'C2', 'C3', 'C4', 'C5','C6', 'C7']\n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    print(f\"### fold: {fold} ###\")\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n\n    val_folds = folds.loc[val_idx].reset_index(drop=True)\n    tra_folds = folds.loc[trn_idx].reset_index(drop=True)\n\n   \n    train_dataset = TrainDataset(tra_folds)\n    valid_dataset = TrainDataset(val_folds)\n\n    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=12,pin_memory=True)\n    valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size*4, shuffle=False, num_workers=12,pin_memory=True)\n\n\n    ###  model select ============\n\n    #model = gru_model(2304).to(device)\n    model = TransformerModel(in_size=384,\n    dim_feedforward=256,#256 is ok\n    num_outputs=8,n_heads=4,\n    n_encoders=4,#4 is OK\n    pool =\"avg\",\n    #use_position_enc=True,\n    embed=True).to(device)\n\n    # ============\n\n\n    ###  optim select ============\n\n    \n    optimizer = AdamW(model.parameters(), lr=lr,weight_decay=5e-5)\n    # ============\n\n    ###  scheduler select ============\n    #if CFG.train.scheduler.name==\"cosine\":\n    #    scheduler = CosineAnnealingLR(optimizer, T_max=CFG.train.epochs, eta_min=CFG.train.scheduler.min_lr)\n    #elif CFG.train.scheduler.name==\"cosine_warmup\":\n    scheduler =T.get_cosine_schedule_with_warmup(optimizer,\n        num_warmup_steps=len(train_loader)*warmup,\n        num_training_steps=len(train_loader)*epochs)\n\n    # ============\n\n    ###  loss select ============\n    criterion=competiton_loss_row_norm(device)\n    val_criterion = competiton_loss_row_norm(torch.device(\"cpu\"))\n\n\n    print(criterion)\n    ###  loss select ============\n\n    sigmoid = nn.Sigmoid()\n    scaler = torch.cuda.amp.GradScaler()\n    best_score = 0\n    best_loss = np.inf\n    best_preds = None\n        \n    for epoch in range(epochs):\n        start_time = time.time()\n        model.train()\n        avg_loss = 0.\n        train_loss = []\n\n        tk0 = tqdm(enumerate(train_loader), total=len(train_loader))\n\n        for i, (images, labels) in tk0:\n            optimizer.zero_grad()\n            images = images.to(device)\n            labels = labels.to(device).float()\n            \n            rand = np.random.rand()\n            rand = 1\n            #if epoch>6:rand=1 #ないほうがよい\n            if 0.5>rand:\n                images, y_a, y_b, lam = mixup_data(images, labels)\n\n            with autocast(enabled=amp):\n\n                y_preds = model(images.float())\n                if 0.5>rand:\n                    loss = mixup_criterion(criterion, y_preds, y_a, y_b, lam)\n                else:\n\n                    loss = criterion(y_preds,labels)\n\n                \n                #loss = competiton_loss(y_preds,labels,device)\n\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n                scheduler.step()\n                        \n\n            avg_loss += loss.item() / len(train_loader)\n            loss_np = loss.detach().cpu().numpy()\n            train_loss.append(loss_np)\n        avg_loss = np.mean(train_loss)\n\n\n        model.eval()\n        avg_val_loss = 0.\n        LOGITS = []\n        TARGETS = []\n        preds = []\n        valid_labels = []\n        tk1 = tqdm(enumerate(valid_loader), total=len(valid_loader))\n\n        for i, (images, labels) in tk1:\n            images = images.to(device)\n            labels = labels.to(device).float()\n            with torch.no_grad():\n                logits = model(images.float())\n                LOGITS.append(logits.detach().cpu())\n                TARGETS.append(labels.detach().cpu())\n\n        \n            #valid_labels.append(labels.to('cpu').numpy())\n            #preds.append(y_preds.to('cpu').numpy())\n            #avg_val_loss += loss.item() / len(valid_loader)\n        #preds = np.concatenate(preds)\n        #valid_labels = np.concatenate(valid_labels)\n            \n        avg_val_loss = val_criterion(torch.cat(LOGITS), torch.cat(TARGETS)).numpy()\n        preds = torch.sigmoid(torch.cat(LOGITS)).numpy().squeeze()    \n        valid_labels = torch.cat(TARGETS).numpy()\n        \n        mean_pred = torch.stack([torch.from_numpy(valid_labels.mean(axis=0))]*valid_labels.shape[0]).logit()\n        #mean_pred = torch.full_like(torch.cat(TARGETS),0)\n        mean_loss = val_criterion(mean_pred, torch.cat(TARGETS)).numpy()\n        print(\"MEAN\",mean_loss)\n\n\n\n        print(preds.shape,valid_labels.shape)\n        print(valid_labels.mean(axis=0))\n\n        elapsed = time.time() - start_time\n\n\n        print(f'  Epoch {epoch+1} - avg_train_loss: {avg_loss:.6f}  avg_val_loss: {avg_val_loss:.6f}  time: {elapsed:.0f}s')\n\n        if best_loss>avg_val_loss:#pr_auc best\n            best_loss = avg_val_loss\n            best_preds = preds\n            print(f'  Epoch {epoch+1} - Save Best loss: {best_loss:.4f}')\n            torch.save(model.state_dict(), f'fold{fold}_002_dino_best_loss.pth')\n\n    for i in range(8):\n        col = f\"pred_{i}\"\n        val_folds[col]=best_preds[:,i]\n    del model,images,labels,avg_val_loss,avg_loss\n    \n\n    return best_preds, valid_labels,val_folds\n","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:13:48.452793Z","iopub.execute_input":"2022-08-19T10:13:48.453105Z","iopub.status.idle":"2022-08-19T10:13:48.476357Z","shell.execute_reply.started":"2022-08-19T10:13:48.45308Z","shell.execute_reply":"2022-08-19T10:13:48.475299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nvalid_labels = []\noof = pd.DataFrame()\nfor fold in range(5):\n    seed_torch(seed=42)\n    _preds, _valid_labels,_oof_val = train_fn(fold,df)\n    preds.append(_preds)\n    valid_labels.append(_valid_labels)\n    oof = pd.concat([oof,_oof_val])\npreds = np.concatenate(preds)\nvalid_labels = np.concatenate(valid_labels)\n\n\nval_criterion = competiton_loss_row_norm(torch.device(\"cpu\"))\nval_loss = val_criterion(torch.from_numpy(preds).logit(), torch.from_numpy(valid_labels)).numpy()\n\nprint(val_loss)","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:13:48.479093Z","iopub.execute_input":"2022-08-19T10:13:48.47969Z","iopub.status.idle":"2022-08-19T10:22:48.226512Z","shell.execute_reply.started":"2022-08-19T10:13:48.479655Z","shell.execute_reply":"2022-08-19T10:22:48.225226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"dim:256 0.47136664<br>\n→1dcnn:0.47522324<br>\n→msd :0.47203583<br>\n→add abs pos :0.47136256<br>\n→before GRUを2→4 layer :0.47333568<br>\n→before GRUを2→1 layer :0.47549278<br>\n→Gem :0.46634147\n\n→→DINO 768dim 0.4697205 3840.47772622:<br>\n","metadata":{}},{"cell_type":"code","source":"oof.to_csv(f\"oof_002_dino.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:22:48.22856Z","iopub.execute_input":"2022-08-19T10:22:48.228954Z","iopub.status.idle":"2022-08-19T10:22:48.274323Z","shell.execute_reply.started":"2022-08-19T10:22:48.228914Z","shell.execute_reply":"2022-08-19T10:22:48.273299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(torch.utils.data.Dataset):\n    def __init__(self, df,train=True):\n        self.df = df\n        self.cols = ['C1', 'C2', 'C3', 'C4', 'C5','C6', 'C7','patient_overall']\n\n        self.crop = 300\n        self.flip = 0.5\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        study = self.df[idx]\n\n        seq = np.load(f\"../input/rsna2022-get-embdino/3darray/{study}.npy\")#n_seq,2304            \n        seq= seq[:,-384:]\n\n        if seq.shape[0]<self.crop:\n            x_input_base = np.zeros([self.crop,seq.shape[1]], np.float32)\n            x_input_base[:seq.shape[0],:] = seq\n            seq = x_input_base\n        else:\n            seq = seq[-self.crop:,:]#後ろ(頭)からとる\n\n        #if np.random.rand()<self.flip:#どちらが頭側か　の情報は大事っぽい\n        #    seq = np.copy(seq[::-1,:])\n        \n        #label = self.df[self.cols].to_numpy()[idx]\n        seq = torch.from_numpy(seq).float()\n        #label = torch.tensor(label).float()\n\n        return seq#,label\ndef inference(model, test_loader,device):\n    model.eval()\n    probs = []\n    LOGITS = []\n\n    for i, images in tqdm(enumerate(test_loader), total=len(test_loader)):\n        images = images.to(device)\n        with torch.no_grad():\n            logits = model(images.float())\n            LOGITS.append(logits.detach().cpu())\n\n    PROBS = torch.sigmoid(torch.cat(LOGITS)).numpy().squeeze() \n    return PROBS\ndef get_oof():\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    \n    probs = []\n    all_study = []\n    for fold in range(5):\n        seed_torch(seed=42)\n        test_dataset = TestDataset(oof[oof[\"fold\"]==fold][\"StudyInstanceUID\"].unique())\n        test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=8,pin_memory=True)\n\n        weight_path = f\"fold{fold}_002_dino_best_loss.pth\"\n        model = TransformerModel(in_size=384,dim_feedforward=256,num_outputs=8,n_heads=4,n_encoders=4,pool =\"avg\",embed=True).to(device)\n        state_dict = torch.load(weight_path,map_location=device)\n        model.load_state_dict(state_dict)\n        preds = inference(model, test_loader,device)\n        probs.append(preds)\n        all_study.append(oof[oof[\"fold\"]==fold][\"StudyInstanceUID\"].unique())\n\n    preds = np.concatenate(probs, axis=0)\n    all_study = np.concatenate(all_study, axis=0)\n    return preds,all_study\n    \npreds,all_study = get_oof()\ncols = ['C1', 'C2', 'C3', 'C4', 'C5','C6', 'C7',\"patient_overall\"]\npred_df = dict(zip(all_study,preds))\ndef func_pred(study):\n    return pred_df[study]\n","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:22:48.278192Z","iopub.execute_input":"2022-08-19T10:22:48.280479Z","iopub.status.idle":"2022-08-19T10:22:57.605957Z","shell.execute_reply.started":"2022-08-19T10:22:48.280441Z","shell.execute_reply":"2022-08-19T10:22:57.60472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_cols = [f\"pred_{i}\" for i in range(8)]\npreds_oof = oof[pred_cols].to_numpy()\nval_loss = val_criterion(torch.from_numpy(preds_oof).logit(),torch.from_numpy(oof[cols].to_numpy()).float())\nprint(val_loss)","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:22:57.607604Z","iopub.execute_input":"2022-08-19T10:22:57.60829Z","iopub.status.idle":"2022-08-19T10:22:57.621206Z","shell.execute_reply.started":"2022-08-19T10:22:57.608241Z","shell.execute_reply":"2022-08-19T10:22:57.62006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_array = np.stack([func_pred(study) for study in oof[\"StudyInstanceUID\"].unique()])\nlabels_array= np.stack([oof[oof[\"StudyInstanceUID\"]==study][cols].to_numpy().squeeze() for study in oof[\"StudyInstanceUID\"].unique()]) \nval_loss = val_criterion(torch.from_numpy(predict_array).logit(),torch.from_numpy(labels_array).float())\nprint(val_loss)","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:22:57.62283Z","iopub.execute_input":"2022-08-19T10:22:57.623852Z","iopub.status.idle":"2022-08-19T10:22:59.699583Z","shell.execute_reply.started":"2022-08-19T10:22:57.623794Z","shell.execute_reply":"2022-08-19T10:22:59.698436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}