{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub = pd.read_csv('/kaggle/input/rsna-str-pulmonary-embolism-detection/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"commit_flag = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for commit\nif len(sample_sub)==152703:\n    sample_sub.to_csv('submission.csv', index=False)\n    commit_flag = True\n    sys.exit(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp ../input/gdcm-conda-install/gdcm.tar . \n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2\n!pip install /kaggle/input/software/timm-0.2.1-py3-none-any.whl > /dev/null\n!pip install /kaggle/input/software/omegaconf-2.0.2-py3-none-any.whl > /dev/null\n!pip install /kaggle/input/software/kornia-0.4.0-py2.py3-none-any.whl > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import gc\nimport math\nimport pickle\nimport numpy as np\nimport pandas as pd\nimport random\nimport pydicom\nimport cv2\nimport albumentations as A\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport timm\nimport kornia\nfrom omegaconf import OmegaConf\nfrom torch.nn import Parameter\nfrom torch.cuda.amp import autocast\nfrom torch.utils.data import Dataset, DataLoader\nfrom pathlib import Path\nfrom albumentations.pytorch import ToTensorV2\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 7\nrandom.seed(seed)\nnp.random.seed(seed)\ntorch.manual_seed(seed)\ntorch.cuda.manual_seed(seed)\ntorch.backends.cudnn.enabled = True\ntorch.backends.cudnn.benchmark = True\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_cols = [\n      \"pe_present_on_image\",\n      \"negative_exam_for_pe\",\n      \"indeterminate\",\n      \"chronic_pe\",\n      \"acute_and_chronic_pe\",\n      \"central_pe\",\n      \"leftsided_pe\",\n      \"rightsided_pe\",\n      \"rv_lv_ratio_gte_1\",\n      \"rv_lv_ratio_lt_1\",\n    ]\npred_cols = [f'{col}_pred' for col in label_cols]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def softmax(x, axis):\n    u = np.sum(np.exp(x), axis=axis, keepdims=True)\n    return np.exp(x)/u\n\ndef postprocess(x, s=2.0):\n    logit = np.log(x/(1 - x))\n    logit = logit + s\n    sigmoid = 1 / (1 + np.exp(-logit))\n    return sigmoid\n\ndef consistency_check(df):\n    df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'])['pe_present_on_image'].max())\n    df_pos = df.loc[df.positive_images_in_exam >  0.5]\n    df_neg = df.loc[df.positive_images_in_exam <= 0.5]\n    rule1a = df_pos.loc[((df_pos['rv_lv_ratio_lt_1']  >  0.5)  & \n                         (df_pos['rv_lv_ratio_gte_1'] >  0.5)) | \n                        ((df_pos['rv_lv_ratio_lt_1']  <= 0.5)  & \n                         (df_pos['rv_lv_ratio_gte_1'] <= 0.5))].reset_index(drop = True)\n    rule1a['broken_rule'] = '1a'\n    rule1b = df_pos.loc[(df_pos['central_pe']    <= 0.5) & \n                        (df_pos['rightsided_pe'] <= 0.5) & \n                        (df_pos['leftsided_pe']  <= 0.5)].reset_index(drop = True)\n    rule1b['broken_rule'] = '1b'\n\n    rule1c = df_pos.loc[(df_pos['acute_and_chronic_pe'] > 0.5) & \n                        (df_pos['chronic_pe']           > 0.5)].reset_index(drop = True)\n    rule1c['broken_rule'] = '1c'\n\n    rule1d = df_pos.loc[(df_pos['indeterminate']        > 0.5) | \n                        (df_pos['negative_exam_for_pe'] > 0.5)].reset_index(drop = True)\n    rule1d['broken_rule'] = '1d'\n    rule2a = df_neg.loc[((df_neg['indeterminate']        >  0.5)  & \n                         (df_neg['negative_exam_for_pe'] >  0.5)) | \n                        ((df_neg['indeterminate']        <= 0.5)  & \n                         (df_neg['negative_exam_for_pe'] <= 0.5))].reset_index(drop = True)\n    rule2a['broken_rule'] = '2a'\n\n    rule2b = df_neg.loc[(df_neg['rv_lv_ratio_lt_1']     > 0.5) | \n                        (df_neg['rv_lv_ratio_gte_1']    > 0.5) |\n                        (df_neg['central_pe']           > 0.5) | \n                        (df_neg['rightsided_pe']        > 0.5) | \n                        (df_neg['leftsided_pe']         > 0.5) |\n                        (df_neg['acute_and_chronic_pe'] > 0.5) | \n                        (df_neg['chronic_pe']           > 0.5)].reset_index(drop = True)\n    rule2b['broken_rule'] = '2b'\n    errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)\n    return errors['broken_rule'].value_counts()\n\ndef satisfy_label_consistency(df, delta=1):\n    rule_breaks = consistency_check(df).index\n    print(rule_breaks)\n    if len(rule_breaks) > 0:\n        df[\"positive_exam_for_pe\"] = 1 - df[\"negative_exam_for_pe\"]\n        df.loc[\n            df.query(\"positive_exam_for_pe <= pe_present_on_image\").index,\n            \"pe_present_on_image\",\n        ] = df.loc[\n            df.query(\"positive_exam_for_pe <= pe_present_on_image\").index,\n            \"positive_exam_for_pe\",\n        ]\n        rule_breaks = consistency_check(df).index\n        df[\"positive_images_in_exam\"] = df[\"StudyInstanceUID\"].map(\n            df.groupby([\"StudyInstanceUID\"])[\"pe_present_on_image\"].max()\n        )\n        df_pos = df.query(\"positive_images_in_exam > 0.5\")\n        df_neg = df.query(\"positive_images_in_exam <= 0.5\")\n        if \"1a\" in rule_breaks:\n            rv_filter = \"rv_lv_ratio_gte_1 > 0.5 & rv_lv_ratio_lt_1 > 0.5\"\n            while len(df_pos.query(rv_filter)) > 0:\n                df_pos.loc[df_pos.query(rv_filter).index, \"rv_min\"] = df_pos.query(\n                    rv_filter\n                )[label_cols[8:]].min(1)\n                for rv_col in label_cols[8:]:\n                    df_pos.loc[\n                        df_pos.query(rv_filter + f\" & {rv_col} == rv_min\").index, rv_col\n                    ] = postprocess(\n                        df_pos.query(rv_filter + f\" & {rv_col} == rv_min\")[\n                            rv_col\n                        ].values,\n                        s=-0.1,\n                    )\n            rv_filter = \"rv_lv_ratio_gte_1 <= 0.5 & rv_lv_ratio_lt_1 <= 0.5\"\n            while len(df_pos.query(rv_filter)) > 0:\n                df_pos.loc[df_pos.query(rv_filter).index, \"rv_max\"] = df_pos.query(\n                    rv_filter\n                )[label_cols[8:]].max(1)\n                for rv_col in label_cols[8:]:\n                    df_pos.loc[\n                        df_pos.query(rv_filter + f\" & {rv_col} == rv_max\").index, rv_col\n                    ] = postprocess(\n                        df_pos.query(rv_filter + f\" & {rv_col} == rv_max\")[\n                            rv_col\n                        ].values,\n                        s=0.1,\n                    )\n            df.loc[df_pos.index, label_cols[8:]] = df_pos[label_cols[8:]]\n        if \"1b\" in rule_breaks:\n            pe_filter = \" & \".join([f\"{col} <= 0.5\" for col in label_cols[5:8]])\n            while \"1b\" in consistency_check(df).index:\n                for col in label_cols[5:8]:\n                    df_pos.loc[df_pos.query(pe_filter).index, col] = postprocess(\n                        df_pos.loc[df_pos.query(pe_filter).index, col], s=0.1\n                    )\n                df.loc[df_pos.index, label_cols[5:8]] = df_pos[label_cols[5:8]].values\n        if \"1c\" in rule_breaks:\n            chronic_filter = \"chronic_pe > 0.5 & acute_and_chronic_pe > 0.5\"\n            df_pos.loc[df_pos.query(chronic_filter).index, label_cols[3:5]] = softmax(\n                df_pos.query(chronic_filter)[label_cols[3:5]].values, axis=1\n            )\n            df.loc[df_pos.index, label_cols[3:5]] = df_pos[label_cols[3:5]]\n        if \"1d\" in rule_breaks:\n            neg_filter = \"negative_exam_for_pe > 0.5 | indeterminate > 0.5\"\n            while \"1d\" in consistency_check(df).index:\n                for col in label_cols[1:3]:\n                    df_pos.loc[df_pos.query(neg_filter).index, col] = postprocess(\n                        df_pos.loc[df_pos.query(neg_filter).index, col], s=-0.1\n                    )\n                df.loc[df_pos.index, label_cols[1:3]] = df_pos[label_cols[1:3]].values\n        if \"2a\" in rule_breaks:\n            neg_filter = \"negative_exam_for_pe > 0.5 & indeterminate > 0.5\"\n            while len(df_neg.query(neg_filter)) > 0:\n                df_neg.loc[df_neg.query(neg_filter).index, \"neg_min\"] = df_neg.query(\n                    neg_filter\n                )[label_cols[1:3]].min(1)\n                for neg_col in label_cols[1:3]:\n                    df_neg.loc[\n                        df_neg.query(neg_filter + f\" & {neg_col} == neg_min\").index,\n                        neg_col,\n                    ] = postprocess(\n                        df_neg.query(neg_filter + f\" & {neg_col} == neg_min\")[\n                            neg_col\n                        ].values,\n                        s=-0.1,\n                    )\n            neg_filter = \"negative_exam_for_pe <= 0.5 & indeterminate <= 0.5\"\n            while len(df_neg.query(neg_filter)) > 0:\n                df_neg.loc[df_neg.query(neg_filter).index, \"neg_max\"] = df_neg.query(\n                    neg_filter\n                )[label_cols[1:3]].max(1)\n                for neg_col in label_cols[1:3]:\n                    df_neg.loc[\n                        df_neg.query(neg_filter + f\" & {neg_col} == neg_max\").index,\n                        neg_col,\n                    ] = postprocess(\n                        df_neg.query(neg_filter + f\" & {neg_col} == neg_max\")[\n                            neg_col\n                        ].values,\n                        s=0.1,\n                    )\n            df.loc[df_neg.index, label_cols[1:3]] = df_neg[label_cols[1:3]]\n        if \"2b\" in rule_breaks:\n            while \"2b\" in consistency_check(df).index:\n                for col in label_cols[3:]:\n                    df_neg.loc[df_neg.query(f\"{col} > 0.5\").index, col] = postprocess(\n                        df_neg.loc[df_neg.query(f\"{col} > 0.5\").index, col], s=-0.1\n                    )\n                df.loc[df_neg.index, label_cols[3:]] = df_neg[label_cols[3:]].values\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_scans(dcm_dir_path):\n    f = [pydicom.dcmread(file) for file in dcm_dir_path.glob('*.dcm')]\n    return f\n\ndef get_data(dcm_dir_path):\n    scans = load_scans(dcm_dir_path)\n    scans.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n    M = float(scans[0].RescaleSlope)\n    B = float(scans[0].RescaleIntercept)\n    meta_columns =  [\n      \"SOPInstanceUID\",\n      \"KVP\",\n      \"XRayTubeCurrent\",\n      \"Exposure\",\n      \"SliceThickness\",\n      \"ImagePositionPatient_x\",\n      \"ImagePositionPatient_y\",\n      \"ImagePositionPatient_z\",\n    ]\n    meta_data = []\n    image_data = []\n    for s in scans:\n        meta_data.append([s[col].value for col in meta_columns[: -3]])\n        meta_data[-1].extend([s['ImagePositionPatient'].value[i] for i in range(3)])\n        pixel_array = s.pixel_array * M + B\n        image_data.append(pixel_array)\n    return np.array(image_data), pd.DataFrame(np.array(meta_data), columns = meta_columns)\n\ndef window(img, WL=50, WW=350):\n    upper, lower = WL+WW//2, WL-WW//2\n    X = np.clip(img.copy(), lower, upper)\n    X = X - np.min(X)\n    X = X / np.max(X)\n    X = (X*255.0).astype('uint8')\n    return X\n\ndef convert_image(img, WL_list, WW_list):\n    return np.stack([window(img, WL, WW) for WL, WW in zip(WL_list, WW_list)], axis=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MODE = 'private'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_root = Path('/kaggle/input/rsna-str-pulmonary-embolism-detection')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(data_root / 'test.csv')\ntest_df = test_df[['StudyInstanceUID', 'SeriesInstanceUID']].drop_duplicates().reset_index(drop=True)\ntest_df['dcm_dir_path'] = test_df.apply(lambda x: data_root / 'test' / x[\"StudyInstanceUID\"] / x[\"SeriesInstanceUID\"], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"public_test_df = pd.read_csv('/kaggle/input/rsna2020-public-dataset/test.csv')\npublic_test_df = public_test_df[['StudyInstanceUID', 'SeriesInstanceUID']].drop_duplicates().reset_index(drop=True)\npublic_test_df['dcm_dir_path'] = public_test_df.apply(lambda x: data_root / 'test' / x[\"StudyInstanceUID\"] / x[\"SeriesInstanceUID\"], axis=1)\npublic_sample_sub = pd.read_csv('/kaggle/input/rsna2020-public-dataset/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if MODE == 'public':\n    test_df_ = public_test_df\nelif MODE == 'private':\n    ids = public_test_df['StudyInstanceUID'].unique()\n    test_df_ = test_df.query('StudyInstanceUID not in @ids').reset_index(drop=True)\nelse:\n    test_df_ = test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageDataset(Dataset):\n    def __init__(self, image):\n        self.image = image\n        \n    def __getitem__(self, index):\n        image = self.image[index].astype(np.float32) / 255\n        image = kornia.image_to_tensor(image)\n        out = {'image': image}\n        return out\n\n    def __len__(self):\n        return len(self.image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1. / p).squeeze(-1)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)\n    \nclass MeanMaxPooling(nn.Module):\n    def __init__(self):\n        super(MeanMaxPooling, self).__init__()\n        self.pool_mean = nn.AdaptiveAvgPool2d((1, 1))\n        self.pool_max = nn.AdaptiveMaxPool2d((1, 1))\n\n    def forward(self, x):\n        mean_x = self.pool_mean(x)\n        max_x = self.pool_max(x)\n        out = torch.cat((mean_x, max_x), dim=1)\n        return out\n\nclass Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ImageModel(nn.Module):\n    def __init__(self, cfg):\n        super(ImageModel, self).__init__()\n        self.arch = timm.create_model(cfg.model.name, pretrained=False)\n        out_channel = self.arch.num_features\n        self.pool = MeanMaxPooling()\n        if cfg.model.pool == 'MeanMax':\n            self.pool = MeanMaxPooling()\n            out_channel = out_channel * 2\n        elif cfg.model.pool == 'GeM':\n            self.pool = GeM()\n        else:\n            self.pool = nn.AdaptiveAvgPool2d((1, 1))\n        self.l0 = nn.Linear(out_channel, 256)\n        self.bn0 = nn.BatchNorm1d(256)\n        self.act = Swish()\n        self.l1 = nn.Linear(256, 10)\n\n    def forward(self, batch):\n        image = batch['image']\n        out = self.arch.forward_features(image)\n        out = self.pool(out)\n        out = out.view(out.shape[0], -1)\n        out = self.act(self.bn0(self.l0(out)))\n        out = self.l1(out)\n        return out\n\n    def get_feature(self, batch):\n        with torch.no_grad():\n            image = batch['image']\n            out = self.arch.forward_features(image)\n            out = self.pool(out)\n            out = out.view(out.shape[0], -1)\n            return out\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResNet(nn.Module):\n    def __init__(self, in_features, out_features, kernel_size, dropout=0.0):\n        super(ResNet, self).__init__()\n        assert kernel_size % 2 == 1\n        self.conv1 = nn.Sequential(\n            nn.Conv1d(\n                in_features,\n                out_features,\n                kernel_size,\n                stride=1,\n                padding=(kernel_size - 1) // 2,\n            ),\n            nn.BatchNorm1d(out_features),\n            nn.LeakyReLU(inplace=True),\n            nn.Dropout(dropout),\n        )\n        self.conv2 = nn.Sequential(\n            nn.Conv1d(\n                out_features,\n                out_features,\n                kernel_size,\n                stride=1,\n                padding=(kernel_size - 1) // 2,\n            ),\n            nn.BatchNorm1d(out_features),\n            nn.LeakyReLU(inplace=True),\n            nn.Dropout(dropout),\n        )\n\n    def forward(self, x):\n        x = self.conv1(x)\n        out = self.conv2(x) + x\n        return out\n\n    \nclass PositionalEncoding(nn.Module):\n    def __init__(self, d_model, dropout=0.1, max_len=201):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n\n        pe = torch.zeros(max_len, d_model)\n        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n        div_term = torch.exp(\n            torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)\n        )\n        pe[:, 0::2] = torch.sin(position * div_term)\n        if d_model % 2 == 0:\n            pe[:, 1::2] = torch.cos(position * div_term)\n        else:\n            pe[:, 1::2] = torch.cos(position * div_term)[:, : d_model // 2]\n        pe = pe.unsqueeze(0).transpose(0, 1)\n        self.register_buffer(\"pe\", pe)\n\n    def forward(self, x):\n        x = x + self.pe[: x.size(0), :]\n        return self.dropout(x)\n\n    \n    \nclass DeconvFeatureModel(nn.Module):\n    def __init__(self, model_config):\n        super(DeconvFeatureModel, self).__init__()\n        self.model_config = model_config\n        if model_config.backbone in [\"cnn\", \"cnn_rnn\"]:\n            self.resnet1 = ResNet(\n                model_config.num_feature, 512, kernel_size=3, dropout=model_config.dropout_rate\n            )\n            self.deconv1 = nn.ConvTranspose1d(\n                512, 512, kernel_size=3, stride=2, padding=1\n            )\n            self.resnet2 = ResNet(\n                512, 256, kernel_size=5, dropout=model_config.dropout_rate\n            )\n            self.deconv2 = nn.ConvTranspose1d(\n                256, 256, kernel_size=3, stride=2, padding=1\n            )\n            self.resnet3 = ResNet(\n                256, 128, kernel_size=7, dropout=model_config.dropout_rate\n            )\n            self.deconv3 = nn.ConvTranspose1d(\n                128, 128, kernel_size=3, stride=2, padding=1\n            )\n            self.resnet4 = ResNet(\n                128, 64, kernel_size=9, dropout=model_config.dropout_rate\n            )\n            self.deconv4 = nn.ConvTranspose1d(\n                64, 64, kernel_size=3, stride=2, padding=1\n            )\n            self.resnet5 = ResNet(\n                64, 32, kernel_size=11, dropout=model_config.dropout_rate\n            )\n            self.deconv5 = nn.ConvTranspose1d(\n                32, 32, kernel_size=3, stride=2, padding=1\n            )\n            out_features = 512 + 256 + 128 + 64 + 32\n        elif model_config.backbone == \"lstm\":\n            self.rnn1 = nn.LSTM(\n                model_config.num_feature,\n                512,\n                num_layers=2,\n                batch_first=True,\n                bidirectional=True,\n            )\n            self.deconv1 = nn.ConvTranspose1d(\n                1024, 512, kernel_size=3, stride=2, padding=1\n            )\n            self.rnn2 = nn.LSTM(\n                1024,\n                512,\n                num_layers=2,\n                batch_first=True,\n                bidirectional=True,\n            )\n            self.deconv2 = nn.ConvTranspose1d(\n                1024, 512, kernel_size=3, stride=2, padding=1\n            )\n            out_features = 1024\n        elif model_config.backbone == \"gru\":\n            self.rnn1 = nn.GRU(\n                model_config.num_feature,\n                512,\n                num_layers=2,\n                batch_first=True,\n                bidirectional=True,\n            )\n            self.deconv1 = nn.ConvTranspose1d(\n                1024, 512, kernel_size=3, stride=2, padding=1\n            )\n            self.rnn2 = nn.GRU(\n                1024,\n                512,\n                num_layers=2,\n                batch_first=True,\n                bidirectional=True,\n            )\n            self.deconv2 = nn.ConvTranspose1d(\n                1024, 512, kernel_size=3, stride=2, padding=1\n            )\n            out_features = 1024\n        elif model_config.backbone in [\"transformer\", \"transformer_rnn\"]:\n            self.linear = nn.Linear(model_config.num_feature, 2048)\n            self.scale = math.sqrt(2048)\n            self.pe = PositionalEncoding(2048, model_config.dropout_rate)\n            encoder_layer1 = nn.TransformerEncoderLayer(\n                2048,\n                nhead=8,\n                dim_feedforward=1024,\n                dropout=model_config.dropout_rate,\n                activation=\"gelu\",\n            )\n            self.transformer1 = nn.TransformerEncoder(encoder_layer1, 1)\n            self.deconv1 = nn.ConvTranspose1d(\n                2048, 1024, kernel_size=3, stride=2, padding=1\n            )\n            encoder_layer2 = nn.TransformerEncoderLayer(\n                1024,\n                nhead=8,\n                dim_feedforward=1024,\n                dropout=model_config.dropout_rate,\n                activation=\"gelu\",\n            )\n            self.transformer2 = nn.TransformerEncoder(encoder_layer2, 1)\n            self.deconv2 = nn.ConvTranspose1d(\n                2048, 1024, kernel_size=3, stride=2, padding=1\n            )\n            out_features = 2048\n        else:\n            raise NotImplementedError()\n        if model_config.backbone in [\"cnn_rnn\", \"transformer_rnn\"]:\n            self.rnn = nn.LSTM(\n                out_features,\n                512,\n                num_layers=2,\n                batch_first=True,\n                bidirectional=True,\n            )\n            out_features = 1024\n        self.exam_classfifier = nn.Linear(out_features, model_config.num_classes)\n        self.image_classfifier = nn.Linear(out_features, 1)\n\n    def pooling(self, feature, seq_lens):\n        pool_out = torch.stack(\n            [feature[idx, :seq_len].mean(0) for idx, seq_len in enumerate(seq_lens)]\n        )\n        # pool_out = feature.mean(1)\n        return pool_out\n\n    def forward(self, batch):\n        \"\"\"\n        Input:\n            data (torch.Tensor): shape [bs, seq_len, n_feature]\n        \"\"\"\n        feature = batch[\"feature\"].float()\n        meta_feature = batch[\"meta_feature\"].float()\n        feature = torch.cat([feature, meta_feature], dim=-1)\n        if self.model_config.backbone in [\"cnn\", \"cnn_rnn\"]:\n            feature = feature.permute(0, 2, 1)\n            outs = []\n            for i in range(1, 6):\n                feature = getattr(self, f\"resnet{i}\")(feature)\n                out = getattr(self, f\"deconv{i}\")(feature)\n                outs.append(out)\n            feature = torch.cat(outs, dim=1).permute(0, 2, 1)\n        elif self.model_config.backbone in [\"gru\", \"lstm\"]:\n            outs = []\n            feature, _ = self.rnn1(feature)\n            out = self.deconv1(feature.permute(0, 2, 1)).permute(0, 2, 1)\n            outs.append(out)\n            feature, _ = self.rnn2(feature)\n            out = self.deconv2(feature.permute(0, 2, 1)).permute(0, 2, 1)\n            outs.append(out)\n            feature = torch.cat(outs, dim=-1)\n        elif self.model_config.backbone in [\"transformer\", \"transformer_rnn\"]:\n            feature = torch.relu(self.linear(feature))\n            feature = self.pe(feature.permute(1, 0, 2))\n            outs = []\n            feature = self.transformer1(feature)\n            out = self.deconv1(feature.permute(1, 2, 0)).permute(2, 0, 1)\n            outs.append(out)\n            feature = self.transformer1(feature)\n            out = self.deconv2(feature.permute(1, 2, 0)).permute(2, 0, 1)\n            outs.append(out)\n            feature = torch.cat(outs, dim=-1).permute(1, 0, 2)\n        else:\n            pass\n        if self.model_config.backbone in [\"cnn_rnn\", \"transformer_rnn\"]:\n            feature, _ = self.rnn(feature)\n        # feature, _ = self.rnn(feature)\n        pool_out = self.pooling(feature, batch[\"seq_len\"])\n        exam_out = self.exam_classfifier(pool_out)\n        image_out = self.image_classfifier(feature)\n        return exam_out, image_out\n\n    \nclass StackingModel(nn.Module):\n    def __init__(self, model_config):\n        super(StackingModel, self).__init__()\n        self.model_config = model_config\n        in_features = 120\n        if model_config.backbone in [\"cnn\", \"cnn_rnn\"]:\n            self.resnet1 = ResNet(\n                in_features, 256, kernel_size=3, dropout=model_config.dropout_rate\n            )\n            self.resnet2 = ResNet(\n                256, 128, kernel_size=3, dropout=model_config.dropout_rate\n            )\n            self.resnet3 = ResNet(\n                128, 64, kernel_size=3, dropout=model_config.dropout_rate\n            )\n            self.resnet4 = ResNet(\n                64, 32, kernel_size=3, dropout=model_config.dropout_rate\n            )\n            self.resnet5 = ResNet(\n                32, 16, kernel_size=3, dropout=model_config.dropout_rate\n            )\n            out_features = 256 + 128 + 64 + 32 + 16\n        elif model_config.backbone == \"lstm\":\n            self.rnn = nn.LSTM(\n                in_features,\n                in_features,\n                num_layers=2,\n                batch_first=True,\n                bidirectional=True,\n            )\n            out_features = in_features * 2\n        elif model_config.backbone == \"gru\":\n            self.rnn = nn.GRU(\n                in_features,\n                in_features,\n                num_layers=2,\n                batch_first=True,\n                bidirectional=True,\n            )\n            out_features = in_features * 2\n        else:\n            pass\n        self.exam_classfifier = nn.Linear(out_features, model_config.num_classes)\n        self.image_classfifier = nn.Linear(out_features, 1)\n\n    def pooling(self, feature, seq_lens):\n        pool_out = torch.stack(\n            [feature[idx, :seq_len].mean(0) for idx, seq_len in enumerate(seq_lens)]\n        )\n        # pool_out = feature.mean(1)\n        return pool_out\n\n    def forward(self, batch):\n        \"\"\"\n        Input:\n            data (torch.Tensor): shape [bs, seq_len, n_feature]\n        \"\"\"\n        feature = batch[\"feature\"].float()\n        # feature = torch.cat([feature, meta_feature], dim=-1)\n        if self.model_config.backbone in [\"cnn\", \"cnn_rnn\"]:\n            feature = feature.permute(0, 2, 1)\n            outs = []\n            for i in range(1, 6):\n                feature = getattr(self, f\"resnet{i}\")(feature)\n                outs.append(feature)\n            feature = torch.cat(outs, dim=1).permute(0, 2, 1)\n        elif self.model_config.backbone in [\"gru\", \"lstm\"]:\n            feature, _ = self.rnn(feature)\n        else:\n            pass\n        pool_out = self.pooling(feature, batch[\"seq_len\"])\n        exam_out = self.exam_classfifier(pool_out)\n        image_out = self.image_classfifier(feature)\n        return exam_out, image_out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_image_512_model(cfg, data_path):\n    fold_num = 5\n    models = [ImageModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/image_model/fold_{}.ckpt'.format(i+1), map_location=device)['state_dict'] for i in range(fold_num)]\n    states = [{key[6:]: value for key, value in state.items()} for state in states]\n    [models[i].load_state_dict(states[i]) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_image_384_model(cfg, data_path):\n    fold_num = 5\n    models = [ImageModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/fold_{}.ckpt'.format(i+1), map_location=device)['state_dict'] for i in range(fold_num)]\n    states = [{key[6:]: value for key, value in state.items()} for state in states]\n    [models[i].load_state_dict(states[i]) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_seq1_512_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/tf_efficientnet_b5_ns_feature_deconv_cnn_scaler_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_seq2_512_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/tf_efficientnet_b5_ns_feature_deconv_rnn_scaler_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_seq3_512_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/tf_efficientnet_b5_ns_feature_deconv_gru_scaler_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\n\ndef load_seq4_512_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/tf_efficientnet_b5_ns_feature_deconv_cnn_rnn_scaler_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\n\ndef load_seq1_384_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/tf_efficientnet_b3_ns_feature_deconv_cnn_scaler_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_seq2_384_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/tf_efficientnet_b3_ns_feature_deconv_rnn_scaler_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_seq3_384_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/tf_efficientnet_b3_ns_feature_deconv_gru_scaler_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\n\ndef load_seq4_384_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/tf_efficientnet_b3_ns_feature_deconv_cnn_rnn_scaler_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_seq1_concat_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/concat_512_384_cnn_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_seq2_concat_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/concat_512_384_rnn_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_seq3_concat_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/concat_512_384_gru_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_seq4_concat_model(cfg, data_path):\n    fold_num = 5\n    models = [DeconvFeatureModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/concat_512_384_cnn_rnn_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\n\ndef load_stacking_cnn_model(cfg, data_path):\n    fold_num = 5\n    models = [StackingModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/stacking_cnn_final_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models\n\ndef load_stacking_rnn_model(cfg, data_path):\n    fold_num = 5\n    models = [StackingModel(cfg) for i in range(fold_num)]\n    [m.to(device) for m in models]\n    states = [torch.load(data_path + '/stacking_lstm_concat_fold{}_state_dict.ckpt'.format(i), map_location=device) for i in range(fold_num)]\n    [models[i].load_state_dict(states[i], strict=False) for i in range(fold_num)]\n    [m.eval() for m in models]\n    del states; gc.collect()\n    return models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ===== Image Model ======\n\ncfg_384_b3 =  '''\nmodel:\n  name: tf_efficientnet_b3_ns\n  pool: GeM\n  figsize: 384\n'''\ncfg_384_b3 = OmegaConf.create(cfg_384_b3)\n\n\ncfg_512_b5 =  '''\nmodel:\n  name: tf_efficientnet_b5_ns\n  pool: GeM\n  figsize: 512\n'''\ncfg_512_b5 = OmegaConf.create(cfg_512_b5)\n\n# ===== 512 Sequence Model ======\n\ncfg_cnn_512 =  '''\nmodel:\n  backbone: cnn\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 2055\n'''\ncfg_cnn_512 = OmegaConf.create(cfg_cnn_512)\n\n\ncfg_lstm_512 =  '''\nmodel:\n  backbone: lstm\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 2055\n'''\ncfg_lstm_512 = OmegaConf.create(cfg_lstm_512)\n\ncfg_gru_512 =  '''\nmodel:\n  backbone: gru\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 2055\n'''\ncfg_gru_512 = OmegaConf.create(cfg_gru_512)\n\ncfg_cnn_rnn_512 =  '''\nmodel:\n  backbone: cnn_rnn\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 2055\n'''\ncfg_cnn_rnn_512 = OmegaConf.create(cfg_cnn_rnn_512)\n\n\ncfg_transformer_512 =  '''\nmodel:\n  backbone: transformer\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 2055\n'''\ncfg_transformer_512 = OmegaConf.create(cfg_transformer_512)\n\n\n# ===== 384 Sequence Model ======\n\ncfg_cnn_384 =  '''\nmodel:\n  backbone: cnn\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 1543\n'''\ncfg_cnn_384 = OmegaConf.create(cfg_cnn_384)\n\n\ncfg_lstm_384 =  '''\nmodel:\n  backbone: lstm\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 1543\n'''\ncfg_lstm_384 = OmegaConf.create(cfg_lstm_384)\n\n\ncfg_gru_384 =  '''\nmodel:\n  backbone: gru\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 1543\n'''\ncfg_gru_384 = OmegaConf.create(cfg_gru_384)\n\n\ncfg_cnn_rnn_384 =  '''\nmodel:\n  backbone: cnn_rnn\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 1543\n'''\ncfg_cnn_rnn_384 = OmegaConf.create(cfg_cnn_rnn_384)\n\n\ncfg_transformer_384 =  '''\nmodel:\n  backbone: transformer\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 1543\n'''\ncfg_transformer_384 = OmegaConf.create(cfg_transformer_384)\n\n\n# ===== 512 + 384 Sequence Model ======\n\n\ncfg_cnn_concat =  '''\nmodel:\n  backbone: cnn\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 3591\n'''\ncfg_cnn_concat = OmegaConf.create(cfg_cnn_concat)\n\n\ncfg_lstm_concat =  '''\nmodel:\n  backbone: lstm\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 3591\n'''\ncfg_lstm_concat = OmegaConf.create(cfg_lstm_concat)\n\ncfg_gru_concat =  '''\nmodel:\n  backbone: gru\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 3591\n'''\ncfg_gru_concat = OmegaConf.create(cfg_gru_concat)\n\n\ncfg_cnn_rnn_concat =  '''\nmodel:\n  backbone: cnn_rnn\n  dropout_rate: 0.2\n  num_classes: 9\n  num_feature: 3591\n'''\ncfg_cnn_rnn_concat = OmegaConf.create(cfg_cnn_rnn_concat)\n\n\n# ===== Stacking Model ======\n\ncfg_stacking_cnn =  '''\nmodel:\n  backbone: cnn\n  dropout_rate: 0.2\n  num_classes: 9\n'''\ncfg_stacking_cnn = OmegaConf.create(cfg_stacking_cnn)\n\ncfg_stacking_lstm =  '''\nmodel:\n  backbone: lstm\n  dropout_rate: 0.2\n  num_classes: 9\n'''\ncfg_stacking_lstm = OmegaConf.create(cfg_stacking_lstm)\n\ncfg_stacking_gru =  '''\nmodel:\n  backbone: gru\n  dropout_rate: 0.2\n  num_classes: 9\n'''\ncfg_stacking_gru = OmegaConf.create(cfg_stacking_gru)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not commit_flag:\n    image_models_512 = load_image_512_model(cfg_512_b5, data_path = '/kaggle/input/tf-efficientnet-b5-ns-512')\n    image_models_384 = load_image_384_model(cfg_384_b3, data_path = '/kaggle/input/tf-efficientnet-b3-ns-384')\n\n    cnn_models_512 = load_seq1_512_model(cfg_cnn_512.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n    lstm_models_512 = load_seq2_512_model(cfg_lstm_512.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n    gru_models_512 = load_seq3_512_model(cfg_gru_512.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n    cnn_rnn_models_512 = load_seq4_512_model(cfg_cnn_rnn_512.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n    # transformer_models_512 = load_seq4_512_model(cfg_transformer_512.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models4')\n\n    cnn_models_384 = load_seq1_384_model(cfg_cnn_384.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n    lstm_models_384 = load_seq2_384_model(cfg_lstm_384.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n    gru_models_384 = load_seq3_384_model(cfg_gru_384.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n    cnn_rnn_models_384 = load_seq4_384_model(cfg_cnn_rnn_384.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n\n    cnn_models_concat = load_seq1_concat_model(cfg_cnn_concat.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n    lstm_models_concat = load_seq2_concat_model(cfg_lstm_concat.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n    gru_models_concat = load_seq3_concat_model(cfg_gru_concat.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')\n    cnn_rnn_models_concat = load_seq4_concat_model(cfg_cnn_rnn_concat.model, data_path = '/kaggle/input/rsna-shimacos-sequence-models-final')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lgb_dicts = []\nfor label_col in label_cols:\n    with open(f'/kaggle/input/stacking-models-final/stacking_{label_col}.pkl', 'rb') as f:\n        lgb_dicts.append(pickle.load(f))\nwith open(f'/kaggle/input/stacking-models-final/stacking_std.pkl', 'rb') as f:\n    stacking_std = pickle.load(f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stacking_cnn_models = load_stacking_cnn_model(cfg_stacking_cnn.model, data_path = '/kaggle/input/stacking-models-final')\nstacking_rnn_models = load_stacking_rnn_model(cfg_stacking_gru.model, data_path = '/kaggle/input/stacking-models-final')\n# stacking_rnn_models = load_stacking_rnn_model(cfg_stacking_lstm.model, data_path = '/kaggle/input/stacking-models-final')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean = torch.tensor([0.485, 0.456, 0.406])\nstd = torch.tensor([0.229, 0.224, 0.225])\ntransform_512_b5 = nn.Sequential(kornia.geometry.transform.Resize((512, 512)), \n                                 kornia.augmentation.Normalize(mean, std)\n                                )\ntransform_512_b5.to(device)\ntransform_384_b3 = nn.Sequential(kornia.geometry.transform.Resize((384, 384)), \n                                 kornia.augmentation.Normalize(mean, std))\ntransform_384_b3.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_slice_image(image_array):\n    instance_num_max = len(image_array)\n    seq_len = min(len(image_array), 401)\n    image = []\n    pe_array = np.array(\n        [window(i, 100, 700) for i in image_array]\n    )\n    for instance_num in range(seq_len)[0::2]:\n        if instance_num > 0:\n            if instance_num < instance_num_max:\n                idx_range = range(idx - 1, idx + 2)\n            else:\n                # instance_num_max の最小値は64\n                idx_range = range(idx - 2, idx + 1)\n        else:\n            idx_range = range(idx, idx + 3)\n        image.append(np.transpose(pe_array[idx_range], (1, 2, 0)))\n    image = np.array(image)\n    return image\n\n\ndef second_level_predict(models, cnn_feature_dict, meta_feature, seq_len):\n    preds = [\n        seq_model({\"feature\": cnn_feature_dict[i], 'meta_feature': meta_feature, \"seq_len\": [seq_len]})\n        for i, seq_model in enumerate(models)\n    ]\n    per_exam_xs, per_image_xs = zip(*preds)\n    per_exam_x = np.mean(\n        [\n            torch.sigmoid(per_exam_x.squeeze()).cpu().numpy()\n            for per_exam_x in per_exam_xs\n        ],\n        axis=0,\n    )\n    per_image_x = np.mean(\n        [\n            torch.sigmoid(per_image_x.squeeze()).cpu().numpy()\n            for per_image_x in per_image_xs\n        ],\n        axis=0,\n    )[:, None]\n    return per_exam_x, per_image_x\n\ndef stacking_predict(lgb_dict, feat):\n    return np.mean([lgb_dict[i].predict(feat) for i in range(5)], axis=0)\n\ndef inference_2_stage(image_array, meta_data):\n    N_MODEL = 12\n    WL_list = [-600, 100, 40]\n    WW_list = [1500, 700, 400]\n    meta_feature_cols = [\n        \"KVP\",\n        \"XRayTubeCurrent\",\n        \"Exposure\",\n        \"SliceThickness\",\n        \"ImagePositionPatient_x\",\n        \"ImagePositionPatient_y\",\n        \"ImagePositionPatient_z\",\n    ]\n    meta_mean = np.array(\n        [\n            114.08353157,\n            419.09953533,\n            108.65329098,\n            1.0090654,\n            -172.34524724,\n            -141.6034326,\n            -45.66431551,\n        ]\n    )\n    meta_std = np.array(\n        [\n            1.09305001e01,\n            1.92887125e02,\n            4.97377464e02,\n            2.65628402e-01,\n            2.62277483e01,\n            7.11791545e01,\n            4.41005297e02,\n        ]\n    )\n    seq_len = min(len(image_array), 401)\n    image = np.array(\n        [convert_image(i, WL_list, WW_list) for i in image_array[:401][0::2]]\n    )\n    dataset = ImageDataset(image)\n    dataloader = DataLoader(dataset, batch_size=32, drop_last=False, shuffle=False)\n    cnn_feature_dict_512 = {i: [] for i in range(len(cnn_models_512))}\n    cnn_feature_dict_384 = {i: [] for i in range(len(cnn_models_384))}\n    with torch.no_grad():\n        for batch in dataloader:\n            batch = {k: batch[k].to(device) for k in batch.keys()}\n            with autocast():\n                batch_512_image = transform_512_b5(batch[\"image\"])    \n                for i in range(len(image_models_512)):\n                    cnn_feature_dict_512[i].append(\n                        image_models_512[i].get_feature({\"image\": batch_512_image})\n                    )\n                batch_384_image = transform_384_b3(batch[\"image\"])\n                for i in range(len(image_models_384)):\n                    cnn_feature_dict_384[i].append(\n                        image_models_384[i].get_feature({\"image\": batch_384_image})\n                    )\n        cnn_feature_dict_512 = {\n            i: torch.cat(feature, dim=0).unsqueeze(0)\n            for i, feature in cnn_feature_dict_512.items()\n        }\n        cnn_feature_dict_512 = {\n            i: F.pad(feature, (0, 0, 0, 201 - len(image)))\n            for i, feature in cnn_feature_dict_512.items()\n        }\n        cnn_feature_dict_384 = {\n            i: torch.cat(feature, dim=0).unsqueeze(0)\n            for i, feature in cnn_feature_dict_384.items()\n        }\n        cnn_feature_dict_384 = {\n            i: F.pad(feature, (0, 0, 0, 201 - len(image)))\n            for i, feature in cnn_feature_dict_384.items()\n        }\n        cnn_feature_dict_concat = {\n            i: torch.cat([cnn_feature_dict_384[i], cnn_feature_dict_512[i]], dim=-1)\n            for i in range(5)\n        }\n\n        meta_feature = meta_data[meta_feature_cols].values[:401][0::2].astype(np.float32)\n        meta_feature = (meta_feature - meta_mean) / meta_std\n        meta_feature = F.pad(\n            torch.tensor(meta_feature, device=cnn_feature_dict_512[0].device), (0, 0, 0, 201 - len(image))\n        ).unsqueeze(0)\n        preds_512 = [second_level_predict(models, cnn_feature_dict_512, meta_feature, seq_len) for models in [cnn_models_512, lstm_models_512, gru_models_512, cnn_rnn_models_512]]\n        preds_384 = [second_level_predict(models, cnn_feature_dict_384, meta_feature, seq_len) for models in [cnn_models_384, lstm_models_384, gru_models_384, cnn_rnn_models_384]]\n        preds_concat = [second_level_predict(models, cnn_feature_dict_concat, meta_feature, seq_len) for models in [cnn_models_concat, lstm_models_concat, gru_models_concat, cnn_rnn_models_concat]]\n    exam_preds_512, image_preds_512 = zip(*preds_512)\n    exam_preds_384, image_preds_384 = zip(*preds_384)\n    exam_preds_concat, image_preds_concat = zip(*preds_concat)\n    exam_preds = exam_preds_512 + exam_preds_384 + exam_preds_concat\n    image_preds = np.concatenate(image_preds_512 + image_preds_384 + image_preds_concat, axis=1)[:seq_len]\n    image_feats = []\n    for i in range(N_MODEL):\n        image_feats.append(np.concatenate([image_preds[:, [i]], exam_preds[i][None, :].repeat(seq_len, axis=0)], axis=1))\n    image_feats = np.concatenate(image_feats, axis=1)\n    exam_preds = np.concatenate(exam_preds)\n    image_agg = pd.DataFrame(image_preds).agg(['mean', 'std', 'min', 'max'])\n    exam_feat = np.concatenate([exam_preds] + [image_agg[i].values for i in range(N_MODEL)])\n    \n    # Stacking prediction\n    lgb_image_pred = stacking_predict(lgb_dicts[0], image_feats)\n    lgb_exam_pred = np.squeeze(np.concatenate([stacking_predict(lgb_dict, exam_feat[None, :]) for lgb_dict in lgb_dicts[1:]]))\n    \n    \n    image_feats = stacking_std.transform(image_feats)\n    nn_feature = F.pad(torch.tensor(image_feats).cuda().unsqueeze(0), (0, 0, 0, 401 - seq_len))\n    cnn_exam_preds, cnn_image_preds = zip(*[model({'feature': nn_feature, 'seq_len': [seq_len]}) for model in stacking_cnn_models])\n    cnn_exam_pred = np.mean([torch.sigmoid(pred).detach().cpu().numpy().squeeze() for pred in cnn_exam_preds], axis=0)\n    cnn_image_pred = np.mean([torch.sigmoid(pred).detach().cpu().numpy().squeeze() for pred in cnn_image_preds], axis=0)[:seq_len]\n    \n    rnn_exam_preds, rnn_image_preds = zip(*[model({'feature': nn_feature, 'seq_len': [seq_len]}) for model in stacking_rnn_models])\n    rnn_exam_pred = np.mean([torch.sigmoid(pred).detach().cpu().numpy().squeeze() for pred in rnn_exam_preds], axis=0)\n    rnn_image_pred = np.mean([torch.sigmoid(pred).detach().cpu().numpy().squeeze() for pred in rnn_image_preds], axis=0)[:seq_len]\n\n    per_exam_x = np.average([lgb_exam_pred, cnn_exam_pred, rnn_exam_pred], axis=0, weights=[2, 1, 2])\n    per_image_x = np.average([lgb_image_pred, cnn_image_pred, rnn_image_pred], axis=0, weights=[2, 1, 2])\n    \n    # Mean Prediction\n#     per_image_x = lgb_image_pred\n#     per_exam_x = lgb_exam_pred\n    return per_exam_x, per_image_x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_dir(dcm_dir_path):\n    image_array, meta_data = get_data(dcm_dir_path)\n    per_exam_x, per_image_x = inference_2_stage(image_array, meta_data)\n    return per_exam_x, per_image_x, meta_data['SOPInstanceUID']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"per_exam_label_col = label_cols[1:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if not commit_flag:\n    submits = []\n    for idx, data in tqdm(test_df_.iterrows(), total=len(test_df_)):\n    #     for idx, data in tqdm(test_df_.query('StudyInstanceUID==\"84a57a6bc1b4\"').iterrows(), total=len(test_df_)):\n        per_exam_x, per_image_x, SOPInstanceUID = inference_dir(data['dcm_dir_path'])\n        if len(SOPInstanceUID) > 401:\n            out = np.zeros(len(SOPInstanceUID))\n            out[:401] = per_image_x\n            per_image_x = out\n        else:\n            per_image_x = per_image_x[:len(SOPInstanceUID)]\n        per_image_x = per_image_x[:]\n        StudyInstanceUID, SeriesInstanceUID = data['StudyInstanceUID'], data['SeriesInstanceUID']\n        tmp = pd.DataFrame({'SOPInstanceUID': SOPInstanceUID, 'pe_present_on_image': per_image_x})\n        for i, label_col in enumerate(per_exam_label_col):\n            tmp['StudyInstanceUID'] = StudyInstanceUID\n            tmp[label_col] = per_exam_x[i]\n        submits.append(tmp)\n    submits = pd.concat(submits).reset_index(drop=True)\n#     submits = satisfy_label_consistency(submits, delta=3)\n    exam_pred = submits[['StudyInstanceUID'] + per_exam_label_col].drop_duplicates()\n    submit_dict = {'id': [], 'label': []}\n    for label_col in per_exam_label_col:\n        submit_dict['id'].extend((exam_pred['StudyInstanceUID'] + f'_{label_col}').values.tolist())\n        submit_dict['label'].extend(exam_pred[label_col].values.tolist())\n    submit_dict['id'].extend(submits['SOPInstanceUID'].values.tolist())\n    submit_dict['label'].extend(submits['pe_present_on_image'].values.tolist())\n    submit = pd.DataFrame(submit_dict)\n\n    if MODE in ['public', 'private']:\n        submit = pd.merge(submit, sample_sub.rename(columns={'label': 'old_label'}), on='id', how='outer')\n        submit['label'] = submit['label'].fillna(submit['old_label'])\n        submit = submit[['id', 'label']]\n    submit.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}