{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.8.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport os\nfrom tqdm import tqdm\nimport glob\nimport matplotlib.pyplot as plt\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport sklearn\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nimport tensorflow_probability as tfp\nimport math","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q keras-cv-attention-models\n# !pip install -qU scikit-learn\n# !pip install -q seabornhttps://pip.pypa.io/warnings/venv%3C/span%3E%3Cspan","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/awsaf49/rsna-atd-cnn-tpu-train\n\nclass CFG:\n    competition   = 'rsna-atd' \n    \n    debug         = False\n    comment       = 'EfficientNetV1B0-256x256-low_lr-vflip'\n    exp_name      = 'baseline-v4: new_ds + multi_head' # name of the experiment, folds will be grouped using 'exp_name'\n    \n    # use verbose=0 for silent, vebose=1 for interactive,\n    verbose      = 0\n    display_plot = True\n\n    # device\n    device = \"TPU-VM\" #or \"GPU\"\n\n    model_name = 'EfficientNetV1B0'\n\n    # seed for data-split, layer init, augs\n    seed = 42\n\n    # number of folds for data-split\n    folds = 4\n    \n    # which folds to train\n    selected_folds = [0, 1, 2]\n\n    # size of the image\n    img_size = [256, 256]\n#     eq_dim = np.prod(img_size)**0.5\n\n    # batch_size and epochs\n    batch_size = 32\n    epochs = 10\n\n    # loss\n    loss      = 'BCE & CCE'  # BCE, Focal\n    \n    # optimizer\n    optimizer = 'Adam'\n\n    # augmentation\n    augment   = True\n\n    # scale-shift-rotate-shear\n    transform = 0.90  # transform prob\n    fill_mode = 'constant'\n    rot    = 2.0\n    shr    = 2.0\n    hzoom  = 50.0\n    wzoom  = 50.0\n    hshift = 10.0\n    wshift = 10.0\n\n    # flip\n    hflip = True\n    vflip = True\n\n    # clip\n    clip = False\n\n    # lr-scheduler\n    scheduler   = 'cosine' # cosine\n\n    # dropout\n    drop_prob   = 0.6\n    drop_cnt    = 5\n    drop_size   = 0.05\n    \n    # cut-mix-up\n    mixup_prob = 0.0\n    mixup_alpha = 0.5\n    \n    cutmix_prob = 0.0\n    cutmix_alpha = 2.5\n\n    # pixel-augment\n    pixel_aug = 0.90  # prob of pixel_aug\n    sat  = [0.7, 1.3]\n    cont = [0.8, 1.2]\n    bri  = 0.15\n    hue  = 0.05\n\n    # test-time augs\n    tta = 1\n    \n    # target column\n    target_col  = [ \"bowel_injury\", \"extravasation_injury\", \"kidney_healthy\", \"kidney_low\",\n                   \"kidney_high\", \"liver_healthy\", \"liver_low\", \"liver_high\",\n                   \"spleen_healthy\", \"spleen_low\", \"spleen_high\"] # not using \"bowel_healthy\" & \"extravasation_healthy\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_DIR = '/tmp/Dataset/'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r {IMG_DIR}\nos.makedirs(f'{IMG_DIR}', exist_ok = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:#023e8a;font-size:24px\">Unzip the png files</span>","metadata":{}},{"cell_type":"code","source":"%%time\nimport zipfile\nwith zipfile.ZipFile('/kaggle/input/generate-png-dataset/rsna.zip', 'r') as zip_ref:\n    zip_ref.extractall(f'{IMG_DIR}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **<a id=\"Content\" style=\"color:#023e8a;\">Idea Implementation</a>**\n* **<span style=\"color:#023e8a;\">1. read in data directly from PNG</span>**\n* **<span style=\"color:#023e8a;\">2. decode and preprocess all samples to tensorflow dataset</span>**\n* **<span style=\"color:#023e8a;\">3. data augmentation</span>**\n* **<span style=\"color:#023e8a;\">4. Split data into batches</span>**\n* **<span style=\"color:#023e8a;\">5. feed training data into model</span>**","metadata":{}},{"cell_type":"markdown","source":"<span style=\"color:#023e8a;font-size:24px\">Get all file paths and merge with Target columns</span>","metadata":{}},{"cell_type":"code","source":"# target \ntrain = pd.read_csv('/kaggle/input/rsna-2023-abdominal-trauma-detection/train.csv')\ndirectory =  f'{IMG_DIR}/train_images'\npatient_ids = []\nseries_ids = []\nfor patient_id in tqdm(os.listdir(directory)):\n    for series_id in os.listdir(os.path.join(directory,patient_id)):\n        patient_ids.append(patient_id)\n        series_ids.append(series_id)\n        \n        \ndf = pd.DataFrame({\n    'patient_id': patient_ids,\n    'series_id': series_ids\n})\ndf['patient_id'] = df['patient_id'].astype('int64')\ndf['series_id'] = df['series_id'].astype('int64')\n\ndf_train = df.merge(train, how = 'left', left_on = 'patient_id', right_on = 'patient_id')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<span style=\"color:#023e8a;font-size:24px\">split data into train and test set and drop 80% of healthy patient to manually treat the imbalance in dataset</span>","metadata":{}},{"cell_type":"code","source":"# List of target columns\ntarget_cols = ['bowel_healthy', 'bowel_injury', 'extravasation_healthy', \n               'extravasation_injury', 'kidney_healthy', 'kidney_low', \n               'kidney_high', 'liver_healthy', 'liver_low', 'liver_high',\n               'spleen_healthy', 'spleen_low', 'spleen_high', 'any_injury']\n\n# Convert target columns to string and concatenate\ndf_train['stratify_col'] = df_train[target_cols].astype(str).apply(''.join, axis=1)\n\n# remove the ones samples with only 1 in target cols concatetation, or else cause problem in stratifying\nclass_counts = df_train['stratify_col'].value_counts()\nto_remove = class_counts[class_counts == 1].index\ndf_train_stratify = df_train[~df_train['stratify_col'].isin(to_remove)]\n\n# Filtering rows with stratify_col == '10101001001000' aka healthy person\nrows_to_drop = df_train_stratify[df_train_stratify['stratify_col'] == '10101001001000']\n\n# Randomly selecting 80% of those rows\ndrop_indices = rows_to_drop.sample(frac=0.7, random_state=42).index\n\n# Dropping those rows from the original dataframe\ndf_train_stratify = df_train_stratify.drop(drop_indices)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(df_train_stratify[['patient_id', 'series_id']], \n                                                  df_train_stratify[CFG.target_col], \n                                                  stratify=df_train_stratify['stratify_col'], \n                                                  test_size=0.2)  # small training size to see if this will work","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths = []\nlabels = []\n\nfor i in tqdm(range(X_train.shape[0])):\n    for path in sorted(glob.glob(directory + f'/{X_train.iloc[i,0]}/{X_train.iloc[i,1]}/*.png'))[::2]:\n        paths.append(path)\n        labels.append(y_train.iloc[i,:].values)\n\nprint(f\"we have {len(paths)} images in training \")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_paths = []\nval_labels = []\n\nfor i in tqdm(range(X_val.shape[0])):\n    for path in sorted(glob.glob(directory + f'/{X_val.iloc[i,0]}/{X_val.iloc[i,1]}/*.png'))[::2]:\n        val_paths.append(path)\n        val_labels.append(y_val.iloc[i,:].values)\n\nprint(f\"we have {len(val_paths)} images in validation \")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n<span style=\"color:#023e8a;font-size:24px\">Build Dataset Functions</span>","metadata":{}},{"cell_type":"code","source":"def build_decoder(with_labels=True, target_size=CFG.img_size, ext='png'):\n    def decode_image(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3, dtype=tf.uint8)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n        img = tf.image.resize(img, target_size, method='bilinear')\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.reshape(img, [*target_size, 3])\n\n        return img\n    \n    def decode_label(label):\n        label = tf.cast(label, tf.float32)\n        return (label[0:1], label[1:2], label[2:5], label[5:8], label[8:11])\n    \n    def decode_with_labels(path, label):\n        return decode_image(path), decode_label(label)\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True, dim=CFG.img_size):\n    def augment(img, dim=dim):\n        if random_float() < CFG.transform:\n            img = transform(img,DIM=dim)\n        img = tf.image.random_flip_left_right(img) if CFG.hflip else img\n        img = tf.image.random_flip_up_down(img) if CFG.vflip else img\n#         if random_float() < CFG.pixel_aug:\n#             img = tf.image.random_hue(img, CFG.hue)\n#             img = tf.image.random_saturation(img, CFG.sat[0], CFG.sat[1])\n#             img = tf.image.random_contrast(img, CFG.cont[0], CFG.cont[1])\n#             img = tf.image.random_brightness(img, CFG.bri)\n#         img = tf.clip_by_value(img, 0, 1)  if CFG.clip else img         \n        img = tf.reshape(img, [*dim, 3])\n        return img\n    \n    def augment_with_labels(img, label):    \n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, batch_size=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\", drop_remainder=False):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    ds = tf.data.Dataset.from_tensor_slices(slices)\n    ds = ds.map(decode_fn, num_parallel_calls=AUTO)\n    ds = ds.cache(cache_dir) if cache else ds\n    ds = ds.repeat() if repeat else ds\n    if shuffle: \n        ds = ds.shuffle(shuffle, seed=CFG.seed)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n    ds = ds.map(augment_fn, num_parallel_calls=AUTO) if augment else ds\n    if augment and labels is not None:\n        ds = ds.map(lambda img, label: (dropout(img, \n                                               DIM=CFG.img_size, \n                                               PROBABILITY=CFG.drop_prob, \n                                               CT=CFG.drop_cnt,\n                                               SZ=CFG.drop_size), label),num_parallel_calls=AUTO)\n    ds = ds.batch(batch_size, drop_remainder=drop_remainder)\n    if augment and labels is not None:\n        if CFG.cutmix_prob:\n            ds = ds.map(get_cutmix(alpha=CFG.cutmix_alpha,prob=CFG.cutmix_prob),num_parallel_calls=AUTO)\n        if CFG.mixup_prob:\n            ds = ds.map(get_mixup(alpha=CFG.mixup_alpha,prob=CFG.mixup_prob),num_parallel_calls=AUTO)\n    ds = ds.prefetch(AUTO)\n    return ds\n\ndef random_int(shape=[], minval=0, maxval=1):\n    return tf.random.uniform(\n        shape=shape, minval=minval, maxval=maxval, dtype=tf.int32)\n\n\ndef random_float(shape=[], minval=0.0, maxval=1.0):\n    rnd = tf.random.uniform(\n        shape=shape, minval=minval, maxval=maxval, dtype=tf.float32)\n    return rnd\n\n# mixup\ndef get_mixup(alpha=0.2, prob=0.5):\n    @tf.function\n    def mixup(images, labels, alpha=alpha, prob=prob):\n        if random_float() > prob:\n            return images, labels\n\n        image_shape = tf.shape(images)\n        label_shape = tf.shape(labels)\n\n        beta = tfp.distributions.Beta(alpha, alpha)\n        lam = beta.sample(1)[0]\n\n        images = lam * images + (1.0 - lam) * tf.roll(images, shift=1, axis=0)\n        labels = lam * labels + (1.0 - lam) * tf.roll(labels, shift=1, axis=0)\n\n        images = tf.reshape(images, image_shape)\n        labels = tf.reshape(labels, label_shape)\n        return images, labels\n    return mixup\n\n# cutmix\ndef get_cutmix(alpha, prob=0.5):\n    @tf.function\n    def cutmix(images, labels, alpha=alpha, prob=prob):\n        if random_float() > prob:\n            return images, labels\n        image_shape = tf.shape(images)\n        label_shape = tf.shape(labels)\n        \n        W = tf.cast(image_shape[2], tf.int32)\n        H = tf.cast(image_shape[1], tf.int32)\n\n        beta = tfp.distributions.Beta(alpha, alpha)\n        lam = beta.sample(1)[0]\n\n        images_rolled = tf.roll(images, shift=1, axis=0)\n        labels_rolled = tf.roll(labels, shift=1, axis=0)\n\n        r_x = random_int([], minval=0, maxval=W)\n        r_y = random_int([], minval=0, maxval=H)\n        r = 0.5 * tf.math.sqrt(1.0 - lam)\n        r_w_half = tf.cast(r * tf.cast(W, tf.float32), tf.int32)\n        r_h_half = tf.cast(r * tf.cast(H, tf.float32), tf.int32)\n\n        x1 = tf.cast(tf.clip_by_value(r_x - r_w_half, 0, W), tf.int32)\n        x2 = tf.cast(tf.clip_by_value(r_x + r_w_half, 0, W), tf.int32)\n        y1 = tf.cast(tf.clip_by_value(r_y - r_h_half, 0, H), tf.int32)\n        y2 = tf.cast(tf.clip_by_value(r_y + r_h_half, 0, H), tf.int32)\n\n        # outer-pad patch -> [0, 0, 1, 1, 0, 0]\n        patch1 = images[:, y1:y2, x1:x2, :]  # [batch, height, width, channel]\n        patch1 = tf.pad(\n            patch1, [[0, 0], [y1, H - y2], [x1, W - x2], [0, 0]])  # outer-pad\n\n        # inner-pad patch -> [1, 1, 0, 0, 1, 1]\n        patch2 = images_rolled[:, y1:y2, x1:x2, :]\n        patch2 = tf.pad(\n            patch2, [[0, 0], [y1, H - y2], [x1, W - x2], [0, 0]])  # outer-pad\n        patch2 = images_rolled - patch2  # inner-pad = img - outer-pad\n\n        images = patch1 + patch2  # cutmix img\n\n        lam = tf.cast((1.0 - (x2 - x1) * (y2 - y1) / (W * H)), tf.float32)  # no H as (y1 - y2)/H = 1\n        labels = lam * labels + (1.0 - lam) * labels_rolled  # cutmix label\n\n        images = tf.reshape(images, image_shape)\n        labels = tf.reshape(labels, label_shape)\n\n        return images, labels\n\n    return cutmix\n\ndef get_mat(shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    #rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n#     c1   = tf.math.cos(rotation)\n#     s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n#     rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n#                                    -s1,  c1,   zero, \n#                                    zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                               zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n\n    return  K.dot(shear_matrix,K.dot(zoom_matrix, shift_matrix)) #K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))                  \n\ndef transform(image, DIM=CFG.img_size):#[rot,shr,h_zoom,w_zoom,h_shift,w_shift]):\n    if DIM[0]>DIM[1]:\n        diff  = (DIM[0]-DIM[1])\n        pad   = [diff//2, diff//2 + diff%2]\n        image = tf.pad(image, [[0, 0], [pad[0], pad[1]],[0, 0]])\n        NEW_DIM = DIM[0]\n    elif DIM[0]<DIM[1]:\n        diff  = (DIM[1]-DIM[0])\n        pad   = [diff//2, diff//2 + diff%2]\n        image = tf.pad(image, [[pad[0], pad[1]], [0, 0],[0, 0]])\n        NEW_DIM = DIM[1]\n    \n    rot = CFG.rot * tf.random.normal([1], dtype='float32')\n    shr = CFG.shr * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / CFG.hzoom\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / CFG.wzoom\n    h_shift = CFG.hshift * tf.random.normal([1], dtype='float32') \n    w_shift = CFG.wshift * tf.random.normal([1], dtype='float32') \n    \n    transformation_matrix=tf.linalg.inv(get_mat(shr,h_zoom,w_zoom,h_shift,w_shift))\n    \n    flat_tensor=tfa.image.transform_ops.matrices_to_flat_transforms(transformation_matrix)\n    \n    image=tfa.image.transform(image,flat_tensor, fill_mode=CFG.fill_mode)\n    \n    rotation = math.pi * rot / 180.\n    \n    image=tfa.image.rotate(image,-rotation, fill_mode=CFG.fill_mode)\n    \n    if DIM[0]>DIM[1]:\n        image=tf.reshape(image, [NEW_DIM, NEW_DIM,3])\n        image = image[:, pad[0]:-pad[1],:]\n    elif DIM[1]>DIM[0]:\n        image=tf.reshape(image, [NEW_DIM, NEW_DIM,3])\n        image = image[pad[0]:-pad[1],:,:]\n    image = tf.reshape(image, [*DIM, 3])    \n    return image\n\ndef dropout(image,DIM=CFG.img_size, PROBABILITY = 0.6, CT = 5, SZ = 0.1):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image with CT squares of side size SZ*DIM removed\n    \n    # DO DROPOUT WITH PROBABILITY DEFINED ABOVE\n    P = tf.cast( tf.random.uniform([],0,1)<PROBABILITY, tf.int32)\n    if (P==0)|(CT==0)|(SZ==0): \n        return image\n    \n    for k in range(CT):\n        # CHOOSE RANDOM LOCATION\n        x = tf.cast( tf.random.uniform([],0,DIM[1]),tf.int32)\n        y = tf.cast( tf.random.uniform([],0,DIM[0]),tf.int32)\n        # COMPUTE SQUARE \n        WIDTH = tf.cast( SZ*min(DIM),tf.int32) * P\n        ya = tf.math.maximum(0,y-WIDTH//2)\n        yb = tf.math.minimum(DIM[0],y+WIDTH//2)\n        xa = tf.math.maximum(0,x-WIDTH//2)\n        xb = tf.math.minimum(DIM[1],x+WIDTH//2)\n        # DROPOUT IMAGE\n        one = image[ya:yb,0:xa,:]\n        two = tf.zeros([yb-ya,xb-xa,3], dtype = image.dtype) \n        three = image[ya:yb,xb:DIM[1],:]\n        middle = tf.concat([one,two,three],axis=1)\n        image = tf.concat([image[0:ya,:,:],middle,image[yb:DIM[0],:,:]],axis=0)\n        image = tf.reshape(image,[*DIM,3])\n\n#     image = tf.reshape(image,[*DIM,3])\n    return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = build_dataset(paths, labels, cache=True, batch_size=CFG.batch_size,\n                   repeat=False, shuffle=True, augment=True)\n# ds = ds.apply(tf.data.experimental.prefetch_to_device(\"/gpu:0\"))\n# ds = ds.unbatch().batch(20)\n# batch = next(iter(ds))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_ds = build_dataset(val_paths, val_labels, cache=True, batch_size=CFG.batch_size,\n                    repeat=False, shuffle=False, augment=False)\n# # val_ds = val_ds.unbatch().batch(20)\n# val_batch = next(iter(val_ds))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_batch(batch, size=2):\n    if isinstance(batch, tuple):\n        imgs, tars = batch\n        tars = tf.concat(tars,axis=-1).numpy()\n    else:\n        imgs = batch\n        tars = None\n    plt.figure(figsize=(size*5, 10))\n    for img_idx in range(size):\n        plt.subplot(1, size, img_idx+1)\n        if tars is not None:\n            plt.title(f'{tars[img_idx].round(2)}', fontsize=12)\n        img = imgs[img_idx,]\n        plt.imshow(img)\n        plt.xticks([]); plt.yticks([])\n    plt.tight_layout()\n    plt.show() ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display_batch(batch, 10);","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A modified model here","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import regularizers\nimport tensorflow as tf\nfrom keras_cv_attention_models import efficientnet\n\n\ndef build_model(model_name=CFG.model_name,\n                loss_name=CFG.loss,\n                dim=CFG.img_size,\n                compile_model=True,\n                include_top=False):\n    \n    # Define backbone\n    base = getattr(efficientnet, model_name)(input_shape=(*dim, 3),\n                                             pretrained='imagenet',\n                                             num_classes=0)\n    \n    inp = base.inputs\n    x = base.output\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    \n    # Define 'necks' for each head with added regularization and batch normalization\n    def build_neck(x):\n        x_neck = tf.keras.layers.Dense(32, activation=None, kernel_regularizer=regularizers.l1_l2(l1=1e-5, l2=1e-4))(x)\n#         x_neck = tf.keras.layers.Dense(32, activation=None)(x)\n        x_neck = tf.keras.layers.Activation('silu')(x_neck)\n        x_neck = tf.keras.layers.Dropout(0.03)(x_neck)\n        return x_neck\n    \n    x_bowel = build_neck(x)\n    x_extra = build_neck(x)\n    x_liver = build_neck(x)\n    x_kidney = build_neck(x)\n    x_spleen = build_neck(x)\n    \n    # Define heads\n    out_bowel = tf.keras.layers.Dense(1, name='bowel', activation='sigmoid')(x_bowel)\n    out_extra = tf.keras.layers.Dense(1, name='extra', activation='sigmoid')(x_extra)\n    out_liver = tf.keras.layers.Dense(3, name='liver', activation='softmax')(x_liver)\n    out_kidney = tf.keras.layers.Dense(3, name='kidney', activation='softmax')(x_kidney)\n    out_spleen = tf.keras.layers.Dense(3, name='spleen', activation='softmax')(x_spleen)\n    \n    # Combine outputs\n    out = [out_bowel, out_extra, out_liver, out_kidney, out_spleen]\n    \n    # Create model\n    model = tf.keras.Model(inputs=inp, outputs=out)\n    \n    if compile_model:\n        # optimizer\n        opt = tf.keras.optimizers.Adam(learning_rate=0.0003)\n        \n        # loss\n        loss = {\n            'bowel': tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05),\n            'extra': tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05),\n            'liver': tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),\n            'kidney': tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),\n            'spleen': tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),\n        }\n        \n        # metric\n        metrics = {\n            'bowel': ['accuracy'],\n            'extra': ['accuracy'],\n            'liver': ['accuracy'],\n            'kidney': ['accuracy'],\n            'spleen': ['accuracy'],\n        }\n        \n        # compile\n        model.compile(optimizer=opt,\n                      loss=loss,\n                      metrics=metrics)\n    \n    return model\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from keras_cv_attention_models import efficientnet\n\n# def build_model(model_name=CFG.model_name,\n#                 loss_name=CFG.loss,\n#                 dim=CFG.img_size,\n#                 compile_model=True,\n#                 include_top=False):         \n    \n#     # Define backbone\n#     base = getattr(efficientnet, model_name)(input_shape=(*dim,3),\n#                                     pretrained='imagenet',\n#                                     num_classes=0) # get base model (efficientnet), use imgnet weights\n\n#     inp = base.inputs\n#     x = base.output\n#     x = tf.keras.layers.GlobalAveragePooling2D()(x) # use GAP to get pooling result form conv outputs\n\n#     # Define 'necks' for each head\n#     x_bowel = tf.keras.layers.Dense(32, activation='silu')(x)\n#     x_extra = tf.keras.layers.Dense(32, activation='silu')(x)\n#     x_liver = tf.keras.layers.Dense(32, activation='silu')(x)\n#     x_kidney = tf.keras.layers.Dense(32, activation='silu')(x)\n#     x_spleen = tf.keras.layers.Dense(32, activation='silu')(x)\n\n#     # Define heads\n#     out_bowel = tf.keras.layers.Dense(1, name='bowel', activation='sigmoid')(x_bowel) # use sigmoid to convert predictions to [0-1]\n#     out_extra = tf.keras.layers.Dense(1, name='extra', activation='sigmoid')(x_extra) # use sigmoid to convert predictions to [0-1]\n#     out_liver = tf.keras.layers.Dense(3, name='liver', activation='softmax')(x_liver) # use softmax for the liver head\n#     out_kidney = tf.keras.layers.Dense(3, name='kidney', activation='softmax')(x_kidney) # use softmax for the kidney head\n#     out_spleen = tf.keras.layers.Dense(3, name='spleen', activation='softmax')(x_spleen) # use softmax for the spleen head\n\n#     # Combine outputs\n# #     out = tf.keras.layers.Concatenate()([out_bowel, out_extra, \n# #                                          out_liver, out_kidney, out_spleen])\n#     out = [out_bowel, out_extra, out_liver, out_kidney, out_spleen]\n\n#     # Create model\n#     model = tf.keras.Model(inputs=inp, outputs=out)\n\n    \n#     if compile_model:\n#         # optimizer\n#         opt = tf.keras.optimizers.Adam(learning_rate=0.0001)\n#         # loss\n#         loss = {\n#             'bowel':tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05),\n#             'extra':tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05),\n#             'liver':tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),\n#             'kidney':tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),\n#             'spleen':tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.05),\n#         }\n#         # metric\n#         metrics = {\n#             'bowel':['accuracy'],\n#             'extra':['accuracy'],\n#             'liver':['accuracy'],\n#             'kidney':['accuracy'],\n#             'spleen':['accuracy'],\n#         }\n#         # compile\n#         model.compile(optimizer=opt,\n#                       loss=loss,\n#                       metrics=metrics)\n#     return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if \"TPU\" in CFG.device:\n    tpu = 'local' if CFG.device=='TPU-VM' else None\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=tpu)\n        strategy = tf.distribute.TPUStrategy(tpu)\n    except:\n        print('change to GPU')\n        CFG.device = \"GPU\"\n        \nif CFG.device == \"GPU\" or CFG.device==\"CPU\":\n    ngpu = len(tf.config.experimental.list_physical_devices('GPU'))\n    if ngpu>1:\n        print(\"Using multi GPU\")\n        strategy = tf.distribute.MirroredStrategy()\n    elif ngpu==1:\n        print(\"Using single GPU\")\n        strategy = tf.distribute.get_strategy()\n    else:\n        print(\"Using CPU\")\n        strategy = tf.distribute.get_strategy()\n        CFG.device = \"CPU\"\n\nif CFG.device == \"GPU\":\n    print(\"Num GPUs Available: \", ngpu)\n    \n\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = build_model(CFG.model_name, dim=CFG.img_size, compile_model=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callbacks = []\nmodel.fit(\n        ds, \n        epochs=5, \n        callbacks = callbacks, \n        steps_per_epoch=math.ceil(len(paths)/CFG.batch_size/REPLICAS),\n        validation_data=val_ds, \n        validation_steps=math.ceil(len(val_paths)/CFG.batch_size/REPLICAS),\n        verbose=1\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('/kaggle/working/CNN_9_5.h5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_pred = model.predict(val_ds, steps = math.ceil(len(val_paths)/CFG.batch_size/REPLICAS), verbose = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_pred = np.concatenate(val_pred, axis = -1)\nval_pred = pd.DataFrame(val_pred,columns = CFG.target_col)\nval_pred['bowel_healthy'] = 1 - val_pred['bowel_injury']\nval_pred['extravasation_healthy'] = 1 - val_pred['extravasation_injury']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_cols = ['bowel_healthy', 'bowel_injury', 'extravasation_healthy', \n               'extravasation_injury', 'kidney_healthy', 'kidney_low', \n               'kidney_high', 'liver_healthy', 'liver_low', 'liver_high',\n               'spleen_healthy', 'spleen_low', 'spleen_high']\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create Submission\nval_pred_df = pd.DataFrame({'patient_id':X_val.iloc[:,0].values,})\nval_pred_df[target_cols] = val_pred[target_cols].astype('float64')\nval_pred_df = val_pred_df.groupby('patient_id').mean().reset_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_pred_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_training_solution(y_train):\n    sol_train = y_train.copy()\n    \n    # bowel healthy|injury sample weight = 1|2\n    sol_train['bowel_weight'] = np.where(sol_train['bowel_injury'] == 1, 2, 1)\n    \n    # extravasation healthy/injury sample weight = 1|6\n    sol_train['extravasation_weight'] = np.where(sol_train['extravasation_injury'] == 1, 6, 1)\n    \n    # kidney healthy|low|high sample weight = 1|2|4\n    sol_train['kidney_weight'] = np.where(sol_train['kidney_low'] == 1, 2, np.where(sol_train['kidney_high'] == 1, 4, 1))\n    \n    # liver healthy|low|high sample weight = 1|2|4\n    sol_train['liver_weight'] = np.where(sol_train['liver_low'] == 1, 2, np.where(sol_train['liver_high'] == 1, 4, 1))\n    \n    # spleen healthy|low|high sample weight = 1|2|4\n    sol_train['spleen_weight'] = np.where(sol_train['spleen_low'] == 1, 2, np.where(sol_train['spleen_high'] == 1, 4, 1))\n    \n    # any healthy|injury sample weight = 1|6\n    sol_train['any_injury_weight'] = np.where(sol_train['any_injury'] == 1, 6, 1)\n    return sol_train","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# solution\nval_pred_df_label = pd.concat([X_val.iloc[:,0], y_val], axis=1).groupby('patient_id').mean().reset_index()\n\nval_pred_df_label['bowel_healthy'] = 1 - val_pred_df_label['bowel_injury']\nval_pred_df_label['extravasation_healthy'] = 1 - val_pred_df_label['extravasation_injury']\n\n# solution\nval_pred_df_label = pd.concat([val_pred_df_label['patient_id'], val_pred_df_label[target_cols]], axis=1)\n# Identify injury columns\ninjury_columns = [col for col in val_pred_df_label.columns if \"healthy\" not in col]\n\n# Create 'any_injury' column based on injury columns\nval_pred_df_label['any_injury'] = val_pred_df_label[injury_columns].sum(axis=1).apply(lambda x: 1 if x > 0 else 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_pred_df_label = create_training_solution(val_pred_df_label)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pandas.api.types\nimport sklearn.metrics\n\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n\ndef normalize_probabilities_to_one(df: pd.DataFrame, group_columns: list) -> pd.DataFrame:\n    # Normalize the sum of each row's probabilities to 100%.\n    # 0.75, 0.75 => 0.5, 0.5\n    # 0.1, 0.1 => 0.5, 0.5\n    row_totals = df[group_columns].sum(axis=1)\n    if row_totals.min() == 0:\n        raise ParticipantVisibleError('All rows must contain at least one non-zero prediction')\n    for col in group_columns:\n        df[col] /= row_totals\n    return df\n\n\ndef score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str) -> float:\n    '''\n    Pseudocode:\n    1. For every label group (liver, bowel, etc):\n        - Normalize the sum of each row's probabilities to 100%.\n        - Calculate the sample weighted log loss.\n    2. Derive a new any_injury label by taking the max of 1 - p(healthy) for each label group\n    3. Calculate the sample weighted log loss for the new label group\n    4. Return the average of all of the label group log losses as the final score.\n    '''\n    del solution[row_id_column_name]\n    del submission[row_id_column_name]\n\n    # Run basic QC checks on the inputs\n    if not pandas.api.types.is_numeric_dtype(submission.values):\n        raise ParticipantVisibleError('All submission values must be numeric')\n\n    if not np.isfinite(submission.values).all():\n        raise ParticipantVisibleError('All submission values must be finite')\n\n    if solution.min().min() < 0:\n        raise ParticipantVisibleError('All labels must be at least zero')\n    if submission.min().min() < 0:\n        raise ParticipantVisibleError('All predictions must be at least zero')\n\n    # Calculate the label group log losses\n    binary_targets = ['bowel', 'extravasation']\n    triple_level_targets = ['kidney', 'liver', 'spleen']\n    all_target_categories = binary_targets + triple_level_targets\n\n    label_group_losses = []\n    for category in all_target_categories:\n        if category in binary_targets:\n            col_group = [f'{category}_healthy', f'{category}_injury']\n        else:\n            col_group = [f'{category}_healthy', f'{category}_low', f'{category}_high']\n\n        solution = normalize_probabilities_to_one(solution, col_group)\n\n        for col in col_group:\n            if col not in submission.columns:\n                raise ParticipantVisibleError(f'Missing submission column {col}')\n        submission = normalize_probabilities_to_one(submission, col_group)\n        label_group_losses.append(\n            sklearn.metrics.log_loss(\n                y_true=solution[col_group].values,\n                y_pred=submission[col_group].values,\n                sample_weight=solution[f'{category}_weight'].values\n            )\n        )\n\n    # Derive a new any_injury label by taking the max of 1 - p(healthy) for each label group\n    healthy_cols = [x + '_healthy' for x in all_target_categories]\n    any_injury_labels = (1 - solution[healthy_cols]).max(axis=1)\n    any_injury_predictions = (1 - submission[healthy_cols]).max(axis=1)\n    any_injury_loss = sklearn.metrics.log_loss(\n        y_true=any_injury_labels.values,\n        y_pred=any_injury_predictions.values,\n        sample_weight=solution['any_injury_weight'].values\n    )\n\n    label_group_losses.append(any_injury_loss)\n    return np.mean(label_group_losses)","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score(val_pred_df_label,val_pred_df,'patient_id')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}