{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Summary\n- this notebook remaster of great Egor Trushin work https://www.kaggle.com/code/egortrushin/gr-icrgw-pl-pipeline-improved at one of previous competitions.\n- Baseline written using Pytorch Lightning\n- resnet101 as encoder\n- pretrained imagenet weights\n- 700 input channels to 2D model. It is concatenating whole bunch of slices to numpy array for each patient.\n- Datasets are forming from original dicom files and saving to hard disk.\n- Option to change image size\n- Full precision training. Mixed precision training unstable(only useful with T4x2, on P100 this slows down training).\n- Multi and One GPU trainig. Training using 2 GPUs - is not efficient on kaggle but quite good locally.\n- dicomsdl used for faster files loading","metadata":{"_uuid":"0389e899-485f-4b05-8188-9e25c88560b3","_cell_guid":"9444f0b5-654f-43e7-896e-9fd5f9a3c633","trusted":true}},{"cell_type":"markdown","source":"### Training part","metadata":{"_uuid":"02e2d64a-973a-49e8-9657-7cc75f1370e7","_cell_guid":"5e6e9b40-6034-47d2-ba23-adc2a15dd1db","trusted":true}},{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/pretrained-models-pytorch\")\nsys.path.append(\"../input/efficientnet-pytorch\")\nsys.path.append(\"/kaggle/input/timm-pretrained-resnest/resnest/\")","metadata":{"_uuid":"54b3ea8a-8563-4339-a35f-111bb4b8204b","_cell_guid":"bb1272fb-2400-46e0-8d46-5c2c9a6985b2","collapsed":false,"_kg_hide-output":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-21T12:58:39.83956Z","iopub.execute_input":"2023-08-21T12:58:39.84001Z","iopub.status.idle":"2023-08-21T12:58:39.860031Z","shell.execute_reply.started":"2023-08-21T12:58:39.839969Z","shell.execute_reply":"2023-08-21T12:58:39.859003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/rsnaatd-output-public/dicomsdl-0.109.2-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl","metadata":{"_uuid":"7be9292c-99d2-4196-99c4-8d1ef1322ecf","_cell_guid":"f998ffc9-6d8d-479c-a4b0-7d5806accc36","collapsed":false,"_kg_hide-output":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-21T12:58:39.864735Z","iopub.execute_input":"2023-08-21T12:58:39.866181Z","iopub.status.idle":"2023-08-21T12:59:12.778814Z","shell.execute_reply.started":"2023-08-21T12:58:39.866146Z","shell.execute_reply":"2023-08-21T12:59:12.777654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nimport os\nimport glob\nimport tqdm\nos.makedirs(\"/tmp/output/\", exist_ok=True)\n\n####################################################################\n# enable this part if you want use already preprocessed numpy files\n# to be mentioned there are only 800 patients from 3k train set\n####################################################################\nfiles = glob.glob(\"/kaggle/input/data-rsnaatd-0400\" + '/npy_zip/*')\nfor src in tqdm.tqdm(files):\n    id_split = src.split(\"/\")[-1]\n    shutil.copyfile(src, \"/tmp/output/\" + id_split)\n\nfiles = glob.glob(\"/kaggle/input/data-rsnaatd-400800\" + '/npy_zip/*')\nfor src in tqdm.tqdm(files):\n    id_split = src.split(\"/\")[-1]\n    shutil.copyfile(src, \"/tmp/output/\" + id_split)\n\nfiles = glob.glob(\"/tmp/output/\" + '/*')\nprint (\"copyed files\",len(files))","metadata":{"execution":{"iopub.status.busy":"2023-08-21T12:59:12.7817Z","iopub.execute_input":"2023-08-21T12:59:12.782091Z","iopub.status.idle":"2023-08-21T13:07:37.66944Z","shell.execute_reply.started":"2023-08-21T12:59:12.782053Z","shell.execute_reply":"2023-08-21T13:07:37.668466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile config.yaml\n\ndata_path: \"/kaggle/input/rsna-2023-abdominal-trauma-detection/\"\noutput_dir: \"/kaggle/working/\"\ntemp_data_folder: \"/tmp/output/\"\nutils_path: \"/kaggle/input/rsnaatd-output-public/\"\n\nfolds:\n    n_splits: 4\n    random_state: 42\ntrain_folds: [0, 1, 2, 3]\nsamples_to_train: 800\n\nseed: 42\n\ntrain_bs: 4\nvalid_bs: 4\nworkers: 2\n    \nhealthy_cols: [\"bowel_healthy\",\"extravasation_healthy\",\"kidney_healthy\",\"liver_healthy\",\"spleen_healthy\"]\n\nprogress_bar_refresh_rate: 1\n\nearly_stop:\n    monitor: \"val_loss\"\n    mode: \"min\"\n    patience: 999\n    verbose: 1\n\ntrainer:\n    max_epochs: 4\n    min_epochs: 4\n    enable_progress_bar: True\n    precision: \"32-true\" #\"16-mixed\"\n    devices: 1\n    accumulate_grad_batches: 64\n\nmodel:\n    encoder_name: \"resnet101\"\n    max_slice_chnl: 700\n    loss_smooth: 1.0\n    image_size: 512\n    optimizer_params:\n        lr: 0.00015\n        weight_decay: 0.0\n    scheduler:\n        name: \"cosine_with_hard_restarts_schedule_with_warmup\"\n        params:\n              cosine_with_hard_restarts_schedule_with_warmup:\n                num_warmup_steps: 6 \n                num_training_steps: 20 \n                num_cycles: 1","metadata":{"_uuid":"51efa5b9-4147-423d-88f9-27ee154f7c47","_cell_guid":"ea40732e-6898-49fb-b3d8-5e020de75f80","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-21T13:07:37.672488Z","iopub.execute_input":"2023-08-21T13:07:37.673498Z","iopub.status.idle":"2023-08-21T13:07:37.681415Z","shell.execute_reply.started":"2023-08-21T13:07:37.673463Z","shell.execute_reply":"2023-08-21T13:07:37.680449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Implementation of the official score code","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport pandas.api.types\nimport sklearn.metrics\n\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n\ndef normalize_probabilities_to_one(df: pd.DataFrame, group_columns: list) -> pd.DataFrame:\n    # Normalize the sum of each row's probabilities to 100%.\n    # 0.75, 0.75 => 0.5, 0.5\n    # 0.1, 0.1 => 0.5, 0.5\n    row_totals = df[group_columns].sum(axis=1)\n    if row_totals.min() == 0:\n        row_totals[row_totals == 0] = 1\n        #raise ParticipantVisibleError('All rows must contain at least one non-zero prediction')\n    for col in group_columns:\n        df[col] /= row_totals\n    return df\n\n\ndef score(solution, submission, row_id_column_name: str, epoch) -> float:\n    \n    columns = {\"index\": \"patient_id\",\n                0: \"bowel_healthy\",\n                1: \"bowel_injury\",\n                2: \"extravasation_healthy\",\n                3: \"extravasation_injury\",\n                4: \"kidney_healthy\",\n                5: \"kidney_low\",\n                6: \"kidney_high\",\n                7: \"liver_healthy\",\n                8: \"liver_low\",\n                9: \"liver_high\",\n                10: \"spleen_healthy\",\n                11: \"spleen_low\",\n                12: \"spleen_high\"}\n\n    # convert python dict with patient_id and sigmoid(preds) do DataFrame\n    submission = pd.DataFrame.from_dict(submission).T.reset_index().rename(columns=columns)\n    solution = pd.DataFrame.from_dict(solution).T.reset_index().rename(columns=columns)\n\n    '''\n    Pseudocode:\n    1. For every label group (liver, bowel, etc):\n        - Normalize the sum of each row's probabilities to 100%.\n        - Calculate the sample weighted log loss.\n    2. Derive a new any_injury label by taking the max of 1 - p(healthy) for each label group\n    3. Calculate the sample weighted log loss for the new label group\n    4. Return the average of all of the label group log losses as the final score.\n    '''\n    del solution[row_id_column_name]\n    del submission[row_id_column_name]\n\n    # Run basic QC checks on the inputs\n    if not pandas.api.types.is_numeric_dtype(submission.values):\n        raise ParticipantVisibleError('All submission values must be numeric')\n\n    if not np.isfinite(submission.values).all():\n        raise ParticipantVisibleError('All submission values must be finite')\n\n    if solution.min().min() < 0:\n        raise ParticipantVisibleError('All labels must be at least zero')\n    if submission.min().min() < 0:\n        raise ParticipantVisibleError('All predictions must be at least zero')\n\n    # calculate weights \n    # original metric waits weigth weights in solution Dataframe as columns \n    # Here i calculate weights for every target row\n    solution[\"bowel_weight\"] = solution[\"bowel_healthy\"] * 1 + \\\n                               solution[\"bowel_injury\"] * 2  # (healthy, injury)\n    solution[\"extravasation_weight\"] = solution[\"extravasation_healthy\"] * 1 + \\\n                                       solution[\"extravasation_injury\"] * 6  # (healthy, injury)\n    solution[\"kidney_weight\"] = solution[\"kidney_healthy\"] * 1 + \\\n                                solution[\"kidney_low\"] * 2 + \\\n                                solution[\"kidney_high\"] * 4\n    solution[\"liver_weight\"] = solution[\"liver_healthy\"] * 1 + \\\n                               solution[\"liver_low\"] * 2 + \\\n                               solution[\"liver_high\"] * 4\n    solution[\"spleen_weight\"] = solution[\"spleen_healthy\"] * 1 + \\\n                                solution[\"spleen_low\"] * 2 + \\\n                                solution[\"spleen_high\"] * 4\n\n    # Calculate the label group log losses\n    binary_targets = ['bowel', 'extravasation']\n    triple_level_targets = ['kidney', 'liver', 'spleen']\n    all_target_categories = binary_targets + triple_level_targets\n\n    label_group_losses = []\n    for category in all_target_categories:\n        if category in binary_targets:\n            col_group = [f'{category}_healthy', f'{category}_injury']\n        else:\n            col_group = [f'{category}_healthy', f'{category}_low', f'{category}_high']\n\n        solution = normalize_probabilities_to_one(solution, col_group)\n\n        for col in col_group:\n            if col not in submission.columns:\n                raise ParticipantVisibleError(f'Missing submission column {col}')\n        submission = normalize_probabilities_to_one(submission, col_group)\n        label_group_losses.append(\n            sklearn.metrics.log_loss(\n                y_true=solution[col_group].values,\n                y_pred=submission[col_group].values,\n                sample_weight=solution[f'{category}_weight'],\n                eps=0.001\n            )\n        )\n\n    # Derive a new any_injury label by taking the max of 1 - p(healthy) for each label group\n    healthy_cols = [x + '_healthy' for x in all_target_categories]\n    any_injury_labels = (1 - solution[healthy_cols]).max(axis=1)\n    any_injury_predictions = (1 - submission[healthy_cols]).max(axis=1)\n    solution[\"any_injury_weight\"] = any_injury_labels * 6 + (1-any_injury_labels)*1\n\n    # Additional checks to work with small batches\n    if len(any_injury_labels.values)==1 or (any_injury_labels.sum()==0) or (any_injury_labels.sum()==len(any_injury_labels.values)):\n        any_injury_loss=0\n    else:\n        any_injury_loss = sklearn.metrics.log_loss(\n            y_true=any_injury_labels.values,\n            y_pred=any_injury_predictions.values,\n            sample_weight=solution['any_injury_weight'],\n            eps=0.001\n        )\n\n    label_group_losses.append(any_injury_loss)\n    return np.mean(label_group_losses)","metadata":{"execution":{"iopub.status.busy":"2023-08-21T13:07:37.682994Z","iopub.execute_input":"2023-08-21T13:07:37.683585Z","iopub.status.idle":"2023-08-21T13:07:38.898057Z","shell.execute_reply.started":"2023-08-21T13:07:37.683538Z","shell.execute_reply":"2023-08-21T13:07:38.897107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset","metadata":{}},{"cell_type":"code","source":"###################################################################################\n# Datasets are forming from original dicom files due to the first train epoch. \n# It takes about 2 hours. You can experiment with dicom preprocessing here. \n# Afterwards it cached to tmp folder as compressed numpy like \"patient_id.npz\".\n# It is reducing the training time for next epohes to minites per one.\n###################################################################################\n\n# Dataset\nimport gc\nimport os\nimport torch\nimport numpy as np\nimport torchvision.transforms as T\nimport dicomsdl\nimport cv2\nimport math\n\ndef standardize_pixel_array(dcm) -> np.ndarray:\n    \n    pixel_array = dcm.pixel_array\n    if dcm.PixelRepresentation == 1:\n        bit_shift = dcm.BitsAllocated - dcm.BitsStored\n        dtype = pixel_array.dtype\n        pixel_array = (pixel_array << bit_shift).astype(dtype) >>  bit_shift\n    return pixel_array\n\ndef preprocess(image, dicom):\n    \n    image = standardize_pixel_array(dicom)\n    \n    # Additional Checks\n    if 'WindowWidth' not in dicom.getPixelDataInfo() or 'WindowWidth' not in dicom.getPixelDataInfo():\n        return image\n\n    # Load only the variables we need\n    center = dicom['WindowCenter']\n    width = dicom['WindowWidth']\n    bits_stored = dicom['BitsStored']\n    voi_lut_function = dicom['VOILUTFunction']\n\n    # For sigmoid it's a list, otherwise a single value\n    if isinstance(center, list):\n        center = center[0]\n    if isinstance(width, list):\n        width = width[0]\n\n    # Set y_min, max & range\n    y_min = 0\n    y_max = float(2 ** bits_stored - 1)\n    y_range = y_max\n\n    # Checks width for < 1 (in our case not necessary, always >= 750)\n    center -= 0.5\n    width -= 1\n\n    below = image <= (center - width / 2)\n    above = image > (center + width / 2)\n    between = np.logical_and(~below, ~above)\n\n    image[below] = y_min\n    image[above] = y_max\n    if between.any():\n        image[between] = (\n                ((image[between] - center) / width + 0.5) * y_range + y_min\n        )\n\n    return image\n\ndef get_images(patient_id: str, parrent_folder: str) -> np.array:\n    \n    # we use cache to make dicom extraction once during training \n    path_to_save =  config[\"temp_data_folder\"] + str(patient_id)\n\n    # check if file already on disk\n    try:\n        imgs = np.load(path_to_save+ \".npz\")[\"data\"]\n    # if not make the new one\n    except Exception:\n        \n        # List of subfolders. Every patient can have one or to mri probes\n        scans_ids = os.listdir(parrent_folder+\"/\" + str(patient_id))\n        imgs = []\n        for scan_id in scans_ids:\n            # list of dicoms\n            img_ids = os.listdir(parrent_folder + \"/\" + str(patient_id)+\"/\"+scan_id)\n            for img_id in img_ids:\n                file_path = parrent_folder + \"/\" + str(patient_id)+\"/\"+scan_id +\"/\" + img_id\n                dicom = dicomsdl.open(file_path)\n                image = dicom.pixelData()\n                try:\n                    image = preprocess(image, dicom)\n                except:\n                    pass\n                # Some images may have 0 values as highest intensity and need to be inverted\n                if dicom.getPixelDataInfo()['PhotometricInterpretation'] == 'MONOCHROME1':\n                    image = np.max(image) - image\n                # Normalize [0,1] range\n                image = (image - image.min()) / (image.max() - image.min())\n                # Convert to uint8 image in range [0, 255]\n                image = (image * 255).astype(np.uint8)\n                image = cv2.resize(image, (config[\"model\"][\"image_size\"], config[\"model\"][\"image_size\"]), interpolation=cv2.INTER_AREA)\n                imgs.append(image)\n\n        # stack slices and save cache to hard disk\n        imgs = np.stack(imgs, axis=2)\n        if imgs.shape[2] > config[\"model\"][\"max_slice_chnl\"]:\n            imgs = imgs[:, :, :config[\"model\"][\"max_slice_chnl\"]]\n        np.savez_compressed(path_to_save, data=imgs)\n\n    # limit maximal array dimension with [512,512,700]\n    if imgs.shape[2] > config[\"model\"][\"max_slice_chnl\"]:\n        imgs = imgs[:, :, :config[\"model\"][\"max_slice_chnl\"]]\n\n    return imgs\n    \n    \nclass AtdDataset(torch.utils.data.Dataset):\n    def __init__(self, df_idx, parrent_folder: str, image_size=512, train=True, batch_size=3):\n        self.df_idx: pd.DataFrame = df_idx\n        self.parrent_folder: str = parrent_folder\n        self.batch_size = batch_size\n\n    def __len__(self):\n        return math.floor(len(self.df_idx)/self.batch_size) # for the batch trick\n\n    def __getitem__(self, idx):\n\n        #################################################################################\n        # Here we use a little trick to pass samples of different sizes into one batch.\n        # Patients have number of slices ~ 150-1500. Padding with zeros slices used to make an input with same size.\n        # We return several images at one request as a sequence (1,2,3,4...).\n        # If you don't do this, the pythorch will try to merge several different tensors and will crash.\n        # The whole trick is needed to do padding on GPU, numpy is 10 times slower. Pytorch lightning does not\n        # allow you to move tensors on gpu inside the dataset. The reason of that is in case of ddp training\n        # inside a dataset class we don't know on which GPU to send the sample to. And I don't want to rewrite ddp:)\n        # Batching is done in train_step val_step.\n        # We pass it as tuple to return so that the standard dataloader can read it.\n        ##################################################################################\n\n        imgs, patients, labels  = [],[],[]\n        for m_idx in range(0,self.batch_size):\n            patient_id = self.df_idx.iloc[self.batch_size*idx + m_idx]['patient_id']\n            imgs.append(torch.tensor(get_images(patient_id, self.parrent_folder)))\n            patients.append(patient_id)\n            labels.append(torch.tensor(self.df_idx.iloc[self.batch_size*idx + m_idx][1:-1].values).to(torch.float32))\n        imgs.extend(labels)\n        imgs.extend(patients)\n        values = tuple(item for item in imgs)\n\n        return values\n","metadata":{"_uuid":"54351e98-a913-4c65-8d85-5b3fe11aa3ee","_cell_guid":"93646514-5e7c-456e-8d40-b3c012941823","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-21T13:07:38.899593Z","iopub.execute_input":"2023-08-21T13:07:38.899942Z","iopub.status.idle":"2023-08-21T13:07:42.888422Z","shell.execute_reply.started":"2023-08-21T13:07:38.899909Z","shell.execute_reply":"2023-08-21T13:07:42.887431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lightning module\n\nimport torch\nimport torch.nn as nn\nfrom torchmetrics.functional import dice\nimport pytorch_lightning as pl\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau\nfrom transformers import get_cosine_with_hard_restarts_schedule_with_warmup\nfrom torch.optim import AdamW\nimport timm\nfrom timm.models.resnet import _cfg\n\n\nclassification_models = {\n    \"resnet50d\": {\"model\":timm.models.resnet.resnet50, \"pretrain_path\":'/kaggle/input/resnet-pretrained-models/resnet_pretrained/resnet50_d-f39db8af.pth'},\n    \"resnet101\": {\"model\":timm.models.resnet.resnet101, \"pretrain_path\":'/kaggle/input/resnet-pretrained-models/resnet_pretrained/resnet101_a1_0-cdcb52a9.pth'}\n}\n\n\nclass LightningModule(pl.LightningModule):\n    def __init__(self, config):\n        super().__init__()\n        self.config = config\n        \n        pretrained_config = _cfg(url='', file=classification_models[config[\"encoder_name\"]][\"pretrain_path\"])\n        self.model = classification_models[config[\"encoder_name\"]][\"model\"](pretrained=True, pretrained_cfg=pretrained_config, in_chans=config[\"max_slice_chnl\"], num_classes=14, global_pool='max')\n        # accumulating batch preds to on_epoch_end scoring\n        self.val_step_outputs = {}\n        self.val_step_labels = {}\n\n        # weight for loss function\n        weights = torch.tensor([    1,  # \"bowel_healthy\",\n                                    2,  # \"bowel_injury\",\n                                    1,  # \"extravasation_healthy\",\n                                    6,  # \"extravasation_injury\",\n                                    1,  # \"kidney_healthy\",\n                                    2,  # \"kidney_low\",\n                                    4,  # \"kidney_high\"\n                                    1,  # \"liver_healthy\",\n                                    2,  # \"liver_low\",\n                                    4,  # \"liver_high\",\n                                    1,  # \"spleen_healthy\",\n                                    2,  # \"spleen_low\",\n                                    4,  # \"spleen_high\"\n                                    6,  # any injury\n                                     ])\n\n        self.sigmoid = torch.sigmoid\n        self.loss_module = nn.BCEWithLogitsLoss(weight=weights)\n        \n        \n    def gpu_padding(self, imgs):\n        # normalisation and padding on GPU 10 times faster\n        imgs = imgs.to(torch.float32)\n        imgs = (255 - imgs) / 255\n        imgs = (imgs - 0.485) / 0.229\n        # padding\n        if imgs.size()[3] < self.config[\"max_slice_chnl\"]:\n            pad_chanells = self.config[\"max_slice_chnl\"] - imgs.size()[3]\n            imgs = torch.nn.functional.pad(imgs, (0, pad_chanells), \"constant\", 0)\n        imgs = imgs.permute(0, 3, 1, 2)\n\n        return imgs\n\n\n    def forward(self, batch):\n        imgs = batch\n        preds = self.model(imgs)\n        return preds\n\n    def configure_optimizers(self):\n        optimizer = AdamW(self.parameters(), **self.config[\"optimizer_params\"])\n\n        if self.config[\"scheduler\"][\"name\"] == \"CosineAnnealingLR\":\n            scheduler = CosineAnnealingLR(\n                optimizer,\n                **self.config[\"scheduler\"][\"params\"][\"CosineAnnealingLR\"],\n            )\n            lr_scheduler_dict = {\"scheduler\": scheduler, \"interval\": \"step\"}\n            return {\"optimizer\": optimizer, \"lr_scheduler\": lr_scheduler_dict}\n        elif self.config[\"scheduler\"][\"name\"] == \"ReduceLROnPlateau\":\n            scheduler = ReduceLROnPlateau(\n                optimizer,\n                **self.config[\"scheduler\"][\"params\"][\"ReduceLROnPlateau\"],\n            )\n            lr_scheduler = {\"scheduler\": scheduler, \"monitor\": \"val_loss\"}\n            return {\"optimizer\": optimizer, \"lr_scheduler\": lr_scheduler}\n        elif self.config[\"scheduler\"][\"name\"] == \"cosine_with_hard_restarts_schedule_with_warmup\":\n            scheduler = get_cosine_with_hard_restarts_schedule_with_warmup(\n                optimizer,\n                **self.config[\"scheduler\"][\"params\"][self.config[\"scheduler\"][\"name\"]],\n            )\n            lr_scheduler_dict = {\"scheduler\": scheduler, \"interval\": \"step\"}\n            return {\"optimizer\": optimizer, \"lr_scheduler\": lr_scheduler_dict}\n\n    def training_step(self, batch, batch_idx):\n        \n        data = list(batch)\n        imgs, labels, patient_ids = [], [], []\n        for i in range(0,config[\"train_bs\"]):\n            imgs.append(self.gpu_padding(data[i]))\n            labels.append(data[config[\"train_bs\"] + i])\n            patient_ids.append(data[ 2* config[\"train_bs\"] + i])\n        \n        # make a batch\n        imgs = torch.cat(imgs, dim=0)\n        labels = torch.cat(labels, dim=0)\n        patient_id = torch.cat(patient_ids, dim=0)\n\n        # forward and loss\n        preds = self.model(imgs)\n        loss = self.loss_module(preds,labels)\n\n        self.log(\"train_loss\", loss, on_step=True, on_epoch=True, prog_bar=True, batch_size=16)\n        for param_group in self.trainer.optimizers[0].param_groups:\n            lr = param_group[\"lr\"]\n        self.log(\"lr\", lr, on_step=True, on_epoch=False, prog_bar=True)\n\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        \n        data = list(batch)\n        imgs, labels, patient_ids = [], [], []\n        for i in range(0,config[\"valid_bs\"]):\n            imgs.append(self.gpu_padding(data[i]))\n            labels.append(data[config[\"train_bs\"] + i])\n            patient_ids.append(data[ 2* config[\"train_bs\"] + i])\n\n        # make a batch\n        imgs = torch.cat(imgs, dim=0)\n        labels = torch.cat(labels, dim=0)\n        patient_id = torch.cat(patient_ids, dim=0)\n\n        # forward and loss\n        preds = self.model(imgs)\n        loss = self.loss_module(preds,labels)\n\n        # accumulate data to epoch_end\n        label_dict, pred_dict = {}, {}\n        for i in range(0,config[\"valid_bs\"]):\n            self.val_step_labels[patient_id[i].item()]= labels[i].cpu().detach().numpy()\n            self.val_step_outputs[patient_id[i].item()]= self.sigmoid(preds[i]).cpu().detach().numpy()\n            label_dict[patient_id[i].item()] = labels[i].cpu().detach().numpy()\n            pred_dict[patient_id[i].item()] = self.sigmoid(preds[i]).cpu().detach().numpy()\n\n        val_kaggle_score = score(label_dict, pred_dict,\"patient_id\", self.current_epoch)\n        self.log(\"kaggle_val_score\", val_kaggle_score, on_step=False, on_epoch=True, prog_bar=True)\n        self.log(\"val_loss\", loss, on_step=False, on_epoch=True, prog_bar=True)\n   \n\n    def on_validation_epoch_end(self):\n\n        # it should called at ones for only one parralel thread\n        if self.trainer.global_rank == 0:\n            val_kaggle_score = score(self.val_step_labels, self.val_step_outputs, \"patient_id\", self.current_epoch)\n            self.log(\"kaggle_val_score\", val_kaggle_score, on_step=False, on_epoch=True, prog_bar=True)\n            self.val_step_outputs.clear()\n            self.val_step_labels.clear()\n            print(f\"\\nEpoch {self.current_epoch}\",f\"kaggle_val_score {val_kaggle_score}\", flush=True)\n","metadata":{"_uuid":"d587834a-d24c-4589-bf79-43d6e128211c","_cell_guid":"f6629bfe-49ec-4272-aa85-29f0d07dea1a","collapsed":false,"_kg_hide-output":true,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-21T13:07:42.890017Z","iopub.execute_input":"2023-08-21T13:07:42.89066Z","iopub.status.idle":"2023-08-21T13:07:55.849246Z","shell.execute_reply.started":"2023-08-21T13:07:42.890625Z","shell.execute_reply":"2023-08-21T13:07:55.847359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Actual training","metadata":{}},{"cell_type":"code","source":"import warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport torch\nimport yaml\nimport glob\nimport tqdm\nimport pandas as pd\nimport pytorch_lightning as pl\nfrom pprint import pprint\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, TQDMProgressBar\nfrom torch.utils.data import DataLoader\nfrom sklearn.model_selection import KFold\nfrom pytorch_lightning.loggers import CSVLogger\n\n\nwith open(\"config.yaml\", \"r\") as file_obj:\n    config = yaml.safe_load(file_obj)\nprint(\"config\", config)\n\ntorch.set_float32_matmul_precision(\"medium\")\npl.seed_everything(config[\"seed\"])\ngc.enable()\n\ndf = pd.read_csv(config[\"data_path\"] + \"train.csv\")\ndf2 = pd.read_csv(config[\"utils_path\"] + \"img_sizes.csv\")\n\n###############################################################################################################\n# generate any_injury class to target\n# we taking samples only with <700 slices and number of samples [:800]. It can be more and limited by disk space\n################################################################################################################\nany_injury_labels = (1 - df[config[\"healthy_cols\"]]).max(axis=1)\ndf[\"any_injury\"] = any_injury_labels\ndf = df[df2[\"img_size\"] < config[\"model\"][\"max_slice_chnl\"]].reset_index(drop=True)[:config[\"samples_to_train\"]]\n\n\n############################################################################\n# use only one fold fo faster train feel free to change \"for fold in [0]\"\n###########################################################################\n\nFold = KFold(shuffle=True, **config[\"folds\"])\nfor n, (trn_index, val_index) in enumerate(Fold.split(df)):\n    df.loc[val_index, \"kfold\"] = int(n)\ndf[\"kfold\"] = df[\"kfold\"].astype(int)\n\nfor fold in [0]: #config[\"train_folds\"]: \n    print(f\"\\n###### Fold {fold}\")\n    trn_df = df[df.kfold != fold].reset_index(drop=True)\n    vld_df = df[df.kfold == fold].reset_index(drop=True)\n\n    dataset_train = AtdDataset(trn_df, config[\"data_path\"] + \"train_images\", image_size=config[\"model\"][\"image_size\"], train=True, batch_size = config[\"train_bs\"])\n    dataset_validation = AtdDataset(vld_df,config[\"data_path\"] + \"train_images\", image_size=config[\"model\"][\"image_size\"], train=False, batch_size = config[\"valid_bs\"])\n\n    data_loader_train = DataLoader(\n        dataset_train,\n        batch_size=1,\n        shuffle=True,\n        num_workers=config[\"workers\"]\n    )\n    data_loader_validation = DataLoader(\n        dataset_validation,\n        batch_size=1,\n        shuffle=False,\n        num_workers=config[\"workers\"]\n    )\n\n    checkpoint_callback = ModelCheckpoint(\n        save_weights_only=True,\n        monitor=\"kaggle_val_score\",\n        dirpath=config[\"output_dir\"],\n        mode=\"min\",\n        filename=f\"ddp-f{str(config['model']['image_size'])+ config['model']['encoder_name']}-model-f{fold}-{{kaggle_val_score:.4f}}\",\n        save_top_k=1,\n        verbose=1\n    )\n\n    progress_bar_callback = TQDMProgressBar(\n        refresh_rate=config[\"progress_bar_refresh_rate\"]\n    )\n\n    early_stop_callback = EarlyStopping(**config[\"early_stop\"])\n\n\n    trainer = pl.Trainer(\n        callbacks=[checkpoint_callback, early_stop_callback, progress_bar_callback],\n        logger=CSVLogger(save_dir=f'/kaggle/working/logs/logs_f{fold}/'),\n        **config[\"trainer\"],\n    )\n    \n    model = LightningModule(config[\"model\"])\n\n    trainer.fit(model, data_loader_train, data_loader_validation)\n\n    del (\n        dataset_train,\n        dataset_validation,\n        data_loader_train,\n        data_loader_validation,\n        model,\n        trainer,\n        checkpoint_callback,\n        progress_bar_callback,\n        early_stop_callback,\n    )\n    torch.cuda.empty_cache()\n    gc.collect()","metadata":{"_uuid":"e763fa4c-cb2e-4c5d-a33c-b60c22f3b6eb","_cell_guid":"e1d6e561-057d-46b6-aa5c-c1470ff82424","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-21T13:07:55.850953Z","iopub.execute_input":"2023-08-21T13:07:55.851295Z","iopub.status.idle":"2023-08-21T13:17:56.003018Z","shell.execute_reply.started":"2023-08-21T13:07:55.851261Z","shell.execute_reply":"2023-08-21T13:17:56.001809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission part","metadata":{"_uuid":"03ac4b6c-a73f-4cb2-8f3c-d61a87184516","_cell_guid":"7fe8c7af-4afc-4adf-a8f4-034ea51dfc05","trusted":true}},{"cell_type":"code","source":"class LightningModule(pl.LightningModule):\n    def __init__(self, config):\n        super().__init__()\n        \n        self.model = timm.create_model(config[\"model\"][\"encoder_name\"], pretrained=False, in_chans=config[\"model\"][\"max_slice_chnl\"], num_classes=14)\n        weights = torch.tensor([1, 2, 1, 6, 1, 2, 4, 1, 2, 4, 1, 2, 4, 6,])\n        self.loss_module = nn.BCEWithLogitsLoss(weight=weights)\n        \n\n    def forward(self, batch):\n        imgs = batch\n        preds = self.model(imgs)\n        preds= torch.sigmoid(preds)\n        return preds","metadata":{"execution":{"iopub.status.busy":"2023-08-21T13:17:56.006405Z","iopub.execute_input":"2023-08-21T13:17:56.00673Z","iopub.status.idle":"2023-08-21T13:17:56.016006Z","shell.execute_reply.started":"2023-08-21T13:17:56.006698Z","shell.execute_reply":"2023-08-21T13:17:56.015026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nMODEL_PATH = \"/kaggle/working/\"\nprefix = \"\"\n\nwith open(MODEL_PATH+\"config.yaml\", \"r\") as file_obj:\n    ddp10config = yaml.safe_load(file_obj)\n    \ntarget_col = [\"patient_id\",\"bowel_injury\", \"extravasation_injury\", \"kidney_healthy\", \"kidney_low\",\n              \"kidney_high\", \"liver_healthy\", \"liver_low\", \"liver_high\",\n              \"spleen_healthy\", \"spleen_low\", \"spleen_high\"]\n\ntest_df = pd.read_csv('/kaggle/input/rsna-2023-abdominal-trauma-detection/sample_submission.csv')\nprint(test_df)\n\n# remove old temp files\nfiles = glob.glob(config[\"temp_data_folder\"] + '*')\nfor f in files:\n    os.remove(f)\nprint (\"removed files\",len(files))","metadata":{"execution":{"iopub.status.busy":"2023-08-21T13:20:54.503682Z","iopub.execute_input":"2023-08-21T13:20:54.504675Z","iopub.status.idle":"2023-08-21T13:20:54.532979Z","shell.execute_reply.started":"2023-08-21T13:20:54.504641Z","shell.execute_reply":"2023-08-21T13:20:54.532039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gpu_padding(imgs):\n\n    imgs = imgs.to(torch.float32)\n    imgs = (255 - imgs) / 255\n    imgs = (imgs - 0.485) / 0.229\n    if imgs.size()[2] < config[\"model\"][\"max_slice_chnl\"]:\n        pad_chanells = config[\"model\"][\"max_slice_chnl\"] - imgs.size()[2]\n        imgs = torch.nn.functional.pad(imgs, (0, pad_chanells), \"constant\", 0)\n    imgs = torch.unsqueeze(imgs.permute(2, 0, 1),dim=0)\n\n    return imgs","metadata":{"execution":{"iopub.status.busy":"2023-08-21T13:17:57.394271Z","iopub.execute_input":"2023-08-21T13:17:57.394656Z","iopub.status.idle":"2023-08-21T13:17:57.405868Z","shell.execute_reply.started":"2023-08-21T13:17:57.394623Z","shell.execute_reply":"2023-08-21T13:17:57.404018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\nmodels_list = glob.glob(MODEL_PATH+ prefix + '*.ckpt')\nnumber_of_models = len(models_list)\npreds = {}\n\nfor i, model_path in enumerate(models_list):\n    print(model_path)\n    path_split = model_path.split(\"/\")[-1]\n    print(path_split)\n    print(config[\"model\"])\n    model = LightningModule(config).load_from_checkpoint(model_path, config=config)\n    model.to(device)\n    model.eval()\n\n    # Iterate over all patient_id\n    for idx, row in test_df.iterrows():\n        # make try on case of broken files in test set\n        try:\n            patient_id = int(row['patient_id'])\n            images = torch.tensor(get_images(patient_id, \"/kaggle/input/rsna-2023-abdominal-trauma-detection/test_images\")).to(device)\n            images = gpu_padding(images)             \n            with torch.no_grad():\n                pred = model(images)\n            pred = pred.cpu().detach().numpy()\n\n        except Exception:\n\n            # fill with means from  https://www.kaggle.com/code/vishakkbhat/rsna23-weighted-mean-baseline/output\n            pred = [np.array([\n            0.9796631712742294,\n            0.06101048617731173,\n            0.936447410231967,\n            1.0803940260565619,\n            0.94216714331109,\n            0.109628217349857,\n            0.14903082300603748,\n            0.897998093422307,\n            0.2469018112488084,\n            0.13790911979663173,\n            0.8875119161105816,\n            0.18970448045757865,\n            0.3447727994915793,\n            4.618684461391801\n            ])]\n\n        # averaging preds right here\n        if patient_id in preds:\n            preds[patient_id] = preds[patient_id] + pred[0]/number_of_models\n        else:\n            preds[patient_id]  = pred[0]/number_of_models\n    \n    del model    \n    torch.cuda.empty_cache()\n    gc.collect() \n    print(\"################################################################\")\n    print()\n    \n\nsubmission = pd.DataFrame.from_dict(preds).T.reset_index()\nsubmission = submission.rename(columns={\"index\": \"patient_id\",\n                                    0: \"bowel_healthy\",\n                                    1: \"bowel_injury\",\n                                    2: \"extravasation_healthy\",\n                                    3: \"extravasation_injury\",\n                                    4: \"kidney_healthy\",\n                                    5: \"kidney_low\",\n                                    6: \"kidney_high\",\n                                    7: \"liver_healthy\",\n                                    8: \"liver_low\",\n                                    9: \"liver_high\",\n                                    10: \"spleen_healthy\",\n                                    11: \"spleen_low\",\n                                    12: \"spleen_high\"})\n\n# Save Submission!\nsubmission.to_csv('submission.csv', index=False)\nprint(submission)","metadata":{"execution":{"iopub.status.busy":"2023-08-21T13:17:57.408958Z","iopub.execute_input":"2023-08-21T13:17:57.411057Z","iopub.status.idle":"2023-08-21T13:18:00.411072Z","shell.execute_reply.started":"2023-08-21T13:17:57.411024Z","shell.execute_reply":"2023-08-21T13:18:00.410049Z"},"trusted":true},"execution_count":null,"outputs":[]}]}