{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"},{"sourceId":4619805,"sourceType":"datasetVersion","datasetId":2688675}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport os\nimport random\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, models\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.utils import plot_model, array_to_img, img_to_array, load_img\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import SparseCategoricalCrossentropy\nimport cv2 as cv\nimport gc\nfrom IPython.display import clear_output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:23:45.315747Z","iopub.execute_input":"2025-01-03T02:23:45.31608Z","iopub.status.idle":"2025-01-03T02:23:53.998288Z","shell.execute_reply.started":"2025-01-03T02:23:45.31605Z","shell.execute_reply":"2025-01-03T02:23:53.997539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.optim import Adam\nfrom torch.utils.data import DataLoader, Dataset, WeightedRandomSampler\nfrom ignite.engine import Events, create_supervised_trainer, create_supervised_evaluator\nfrom ignite.metrics import Accuracy, Loss, RunningAverage\nfrom ignite.contrib.handlers import ProgressBar\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models, transforms","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:23:53.999433Z","iopub.execute_input":"2025-01-03T02:23:54.000052Z","iopub.status.idle":"2025-01-03T02:23:58.997091Z","shell.execute_reply.started":"2025-01-03T02:23:54.00002Z","shell.execute_reply":"2025-01-03T02:23:58.996236Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install pydicom -q","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:23:58.99863Z","iopub.execute_input":"2025-01-03T02:23:58.999152Z","iopub.status.idle":"2025-01-03T02:24:03.394131Z","shell.execute_reply.started":"2025-01-03T02:23:58.999128Z","shell.execute_reply":"2025-01-03T02:24:03.393052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pydicom\nimport numpy as np\nimport cv2\n\ndef read_xray(file_path, img_size=None):\n    \"\"\"\n    Read the DICOM data and get the image.\n    Args:\n        file_path: The path of the DICOM file.\n        img_size: Size of the output image as a tuple (width, height).\n    Returns:\n        Preprocessed image as a NumPy array.\n    \"\"\"\n\n    # Read the DICOM file\n    dicom = pydicom.dcmread(file_path)\n\n    # Extract the pixel array from the DICOM file\n    img = dicom.pixel_array\n\n    # Check if the image is monochrome and invert if necessary\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        img = np.max(img) - img  # Invert pixel values\n        # Inverting is necessary because MONOCHROME1 represents higher values as darker,\n        # while MONOCHROME2 represents higher values as brighter.\n\n    # Resize the image if a target size is specified\n    if img_size:\n        img = cv2.resize(img, img_size, interpolation=cv2.INTER_AREA)\n\n    # Add a channel dimension at the first axis (required for many ML models)\n    img = img[np.newaxis, ...]\n\n    # Normalize the image to the range [0, 1]\n    img = img / np.max(img)\n\n    # Convert to float32\n    img = img.astype(np.float32)\n\n    return img","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:24:03.395847Z","iopub.execute_input":"2025-01-03T02:24:03.396189Z","iopub.status.idle":"2025-01-03T02:24:03.83287Z","shell.execute_reply.started":"2025-01-03T02:24:03.396156Z","shell.execute_reply":"2025-01-03T02:24:03.831996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def patchify(batch, patch_size):\n    b, h, w, c = batch.shape\n    ph, pw = patch_size\n\n    # Calculate required padding for height and width\n    pad_h = (ph - h % ph) % ph\n    pad_w = (pw - w % pw) % pw\n\n    # Pad the batch along height and width dimensions\n    batch = nn.functional.pad(batch, (0, 0, 0, pad_w, 0, pad_h, 0, 0))  # Padding format: (dim3-right, dim3-left, dim2-right, dim2-left, ...)\n    \n    nh, nw = (h + pad_h) // ph, (w + pad_w) // pw\n\n    # Reshape and permute to get patches\n    batch_patches = batch.reshape(b, nh, ph, nw, pw, c)\n    batch_patches = batch_patches.permute(0, 1, 3, 2, 4, 5)\n\n    return batch_patches","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:24:03.833972Z","iopub.execute_input":"2025-01-03T02:24:03.83428Z","iopub.status.idle":"2025-01-03T02:24:03.839111Z","shell.execute_reply.started":"2025-01-03T02:24:03.834251Z","shell.execute_reply":"2025-01-03T02:24:03.838275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"FILE_PATH = ('/kaggle/input/rsna-breast-cancer-detection/'\n             'train_images/10006/1459541791.dcm')\n\nimg = read_xray(FILE_PATH, img_size=(512, 512))\n\nbatch = torch.tensor(img[None])\npatch_size = (16, 16)\nbatch_patches = patchify(batch, patch_size)\n\npatches = batch_patches[0]\nc, nh, nw, ph, pw = patches.shape\n\nplt.figure(figsize=(5, 5))\nplt.imshow(img[0], cmap=\"gray\")\nplt.axis(\"off\")\n\nplt.figure(figsize=(5, 5))\nfor i in range(nh):\n    for j in range(nw):\n        plt.subplot(nh, nw, i * nw + j + 1)\n        plt.imshow(patches[0, i, j], cmap=\"gray\")\n        plt.axis(\"off\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:24:03.839968Z","iopub.execute_input":"2025-01-03T02:24:03.84025Z","iopub.status.idle":"2025-01-03T02:24:19.896048Z","shell.execute_reply.started":"2025-01-03T02:24:03.840221Z","shell.execute_reply":"2025-01-03T02:24:19.895301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_mlp(in_features, hidden_units, out_features):\n    \"\"\"\n    Returns a Multi-Layer Perceptron (MLP) head.\n\n    Args:\n        in_features: Number of input features.\n        hidden_units: List of integers, representing the number of units in hidden layers.\n        out_features: Number of output features.\n\n    Returns:\n        A Sequential model with Linear and ReLU layers.\n    \"\"\"\n    dims = [in_features] + hidden_units + [out_features]\n    layers = []\n\n    # Add Linear + ReLU for hidden layers\n    for dim1, dim2 in zip(dims[:-2], dims[1:-1]):\n        layers.append(nn.Linear(dim1, dim2))\n        layers.append(nn.ReLU())\n\n    # Add the final Linear layer\n    layers.append(nn.Linear(dims[-2], dims[-1]))\n\n    # Return the Sequential model\n    return nn.Sequential(*layers)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:24:19.896869Z","iopub.execute_input":"2025-01-03T02:24:19.897203Z","iopub.status.idle":"2025-01-03T02:24:19.901987Z","shell.execute_reply.started":"2025-01-03T02:24:19.897171Z","shell.execute_reply":"2025-01-03T02:24:19.901168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Img2Seq(nn.Module):\n    \"\"\"\n    This layer takes a batch of images as input and\n    returns a batch of sequences.\n\n    Shape:\n        input: (b, h, w, c)\n        output: (b, s, d)\n    \"\"\"\n    def __init__(self, img_size, patch_size, n_channels, d_model):\n        super().__init__()\n        self.patch_size = patch_size\n        self.img_size = img_size\n\n        nh, nw = img_size[0] // patch_size[0], img_size[1] // patch_size[1]\n        n_tokens = nh * nw\n\n        # Dimension of each patch flattened\n        token_dim = patch_size[0] * patch_size[1] * n_channels\n        self.linear = nn.Linear(token_dim, d_model)\n\n        # Learnable parameters: CLS token and positional embeddings\n        self.cls_token = nn.Parameter(torch.randn(1, 1, d_model))\n        self.pos_emb = nn.Parameter(torch.randn(n_tokens, d_model))\n\n    def __call__(self, batch):\n        # Patchify the input batch\n        batch = patchify(batch, self.patch_size)\n\n        # Get the dimensions of the patched batch\n        b, nh, nw, ph, pw, c = batch.shape\n\n        # Flatten the patches and permute the dimensions\n        batch = batch.permute(0, 1, 2, 5, 3, 4).reshape(b, nh * nw, -1)\n\n        # Apply the linear transformation to project into d_model dimensions\n        batch = self.linear(batch)\n\n        # Expand cls_token to match the batch size\n        cls = self.cls_token.expand(b, -1, -1)\n\n        # Add positional embeddings\n        emb = batch + self.pos_emb\n\n        # Concatenate the cls_token and the embeddings\n        return torch.cat([cls, emb], axis=1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:24:19.904037Z","iopub.execute_input":"2025-01-03T02:24:19.90428Z","iopub.status.idle":"2025-01-03T02:24:19.921412Z","shell.execute_reply.started":"2025-01-03T02:24:19.90426Z","shell.execute_reply":"2025-01-03T02:24:19.920522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ViT(nn.Module):\n    def __init__(\n        self,\n        img_size,\n        patch_size,\n        n_channels,\n        d_model,\n        nhead,\n        dim_feedforward,\n        blocks,\n        mlp_head_units,\n        n_classes,\n    ):\n        super().__init__()\n        \"\"\"\n        Args:\n            img_size: Size of the image\n            patch_size: Size of the patch\n            n_channels: Number of image channels\n            d_model: The number of features in the transformer encoder\n            nhead: The number of heads in the multiheadattention models\n            dim_feedforward: The dimension of the feedforward network model in the encoder\n            blocks: The number of sub-encoder-layers in the encoder\n            mlp_head_units: The hidden units of mlp_head\n            n_classes: The number of output classes\n        \"\"\"\n        self.img2seq = Img2Seq(img_size, patch_size, n_channels, d_model)\n        \n        # Create an encoder layer for the transformer using GELU activation\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=d_model,\n            nhead=nhead,\n            dim_feedforward=dim_feedforward,\n            activation=\"gelu\",\n        )\n        \n        # Create an encoder block with the specified number of layers\n        self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=blocks)\n        \n        # Define the MLP head for classification\n        self.mlp = get_mlp(d_model, mlp_head_units, n_classes)\n        \n        # Define the output activation function (sigmoid for binary classification, softmax for multi-class)\n        self.output = nn.Sigmoid() if n_classes == 1 else nn.Softmax(dim=1)\n\n    def forward(self, batch):\n        # Convert the input images into sequences\n        batch = self.img2seq(batch)\n        \n        # Pass the sequences through the transformer encoder\n        batch = self.transformer_encoder(batch)\n        \n        # Use only the [CLS] token's output for classification\n        batch = batch[:, 0, :]\n        \n        # Pass through the MLP head\n        batch = self.mlp(batch)\n        \n        # Apply the output activation function\n        output = self.output(batch)\n        return output\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:24:19.922589Z","iopub.execute_input":"2025-01-03T02:24:19.92287Z","iopub.status.idle":"2025-01-03T02:24:19.93852Z","shell.execute_reply.started":"2025-01-03T02:24:19.922838Z","shell.execute_reply":"2025-01-03T02:24:19.937663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ViT(nn.Module):\n    def __init__(self, img_size, patch_size, n_channels, d_model, nhead, dim_feedforward, blocks, mlp_head_units, n_classes):\n        super(ViT, self).__init__()\n\n        self.img_size = img_size\n        self.patch_size = patch_size\n        self.n_channels = n_channels\n        self.d_model = d_model\n        self.nhead = nhead\n        self.dim_feedforward = dim_feedforward\n        self.blocks = blocks\n        self.mlp_head_units = mlp_head_units\n        self.n_classes = n_classes\n\n        # Define patch embedding\n        self.patch_embedding = nn.Conv2d(n_channels, d_model, kernel_size=patch_size, stride=patch_size)\n        \n        # Define transformer encoder\n        self.encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, nhead=nhead, dim_feedforward=dim_feedforward)\n        self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=blocks)\n        \n        # MLP head\n        self.mlp_head = nn.Sequential(\n            nn.Linear(d_model, mlp_head_units[0]),\n            nn.ReLU(),\n            nn.Linear(mlp_head_units[0], mlp_head_units[1]),\n            nn.ReLU(),\n            nn.Linear(mlp_head_units[1], n_classes)\n        )\n\n    def img2seq(self, batch):\n        \"\"\"\n        Converts the image batch into a sequence of patches.\n        \"\"\"\n        batch_size, _, height, width = batch.shape\n\n        # Apply patch embedding\n        patches = self.patch_embedding(batch)\n        patches = patches.flatten(2).transpose(1, 2)  # Flatten patches and transpose for sequence input\n        \n        return patches\n\n    def forward(self, batch):\n        \"\"\"\n        Forward pass through the model.\n        \"\"\"\n        # Convert the input images into sequences (patches)\n        batch = self.img2seq(batch)\n\n        # Check the shape of batch after img2seq for debugging\n        # print(f\"Shape of batch after img2seq: {batch.shape}\")  # Debug print\n\n        # Pass the sequence through the transformer encoder\n        batch = self.transformer_encoder(batch)\n\n        # Reshape the output before passing through the final MLP head\n        # batch has shape (batch_size, seq_length, d_model)\n        batch = batch.mean(dim=1)  # Global average pooling (or any other pooling)\n\n        # Pass the output through the MLP head to get predictions\n        output = self.mlp_head(batch)\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:34:56.371305Z","iopub.execute_input":"2025-01-03T02:34:56.371622Z","iopub.status.idle":"2025-01-03T02:34:56.379098Z","shell.execute_reply.started":"2025-01-03T02:34:56.371592Z","shell.execute_reply":"2025-01-03T02:34:56.377989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:35:00.536029Z","iopub.execute_input":"2025-01-03T02:35:00.536346Z","iopub.status.idle":"2025-01-03T02:35:00.540207Z","shell.execute_reply.started":"2025-01-03T02:35:00.536317Z","shell.execute_reply":"2025-01-03T02:35:00.539283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:35:02.429811Z","iopub.execute_input":"2025-01-03T02:35:02.430236Z","iopub.status.idle":"2025-01-03T02:35:02.435359Z","shell.execute_reply.started":"2025-01-03T02:35:02.430199Z","shell.execute_reply":"2025-01-03T02:35:02.434653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class RSNADataset(Dataset):\n\n    def __init__(self, df, img_path, device):\n        \"\"\"\n        Args:\n            df: DataFrame containing patient_id, image_id, and cancer labels.\n            img_path: Path to the directory containing image files.\n            device: Torch device (e.g., 'cuda' or 'cpu').\n        \"\"\"\n        self.df = df\n        self.img_path = img_path\n        self.device = device\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        # Extract patient_id, image_id, and cancer label from the DataFrame\n        patient_id, image_id, cancer = self.df.iloc[idx][['patient_id', 'image_id', 'cancer']]\n\n        # Construct the full file path for the image\n        file_path = os.path.join(self.img_path, f'{patient_id}_{image_id}.png')\n\n        # Read the image file\n        file = cv2.imread(file_path, cv2.IMREAD_GRAYSCALE)\n        if file is None:\n            raise FileNotFoundError(f\"Image file not found at path: {file_path}\")\n\n        # Apply CLAHE (Contrast Limited Adaptive Histogram Equalization)\n        clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n        file = clahe.apply(file)\n\n        # Normalize the image to the range [0, 1]\n        file = file / 255.0\n\n        # Convert the image to a torch tensor with an additional channel dimension\n        X = torch.tensor(file[np.newaxis, :, :].astype('float32')).to(self.device)\n\n        # Convert the label to a torch tensor\n        y = torch.tensor([cancer]).float().to(self.device)\n\n        return X, y\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:35:03.852892Z","iopub.execute_input":"2025-01-03T02:35:03.853187Z","iopub.status.idle":"2025-01-03T02:35:03.859229Z","shell.execute_reply.started":"2025-01-03T02:35:03.853163Z","shell.execute_reply":"2025-01-03T02:35:03.858422Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Read the dataframe from CSV\ndf = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\n\n# Apply value_counts to get class distribution in the 'cancer' column\ncounts = df['cancer'].value_counts()\n\n# Apply weights based on class distribution\ndf['weights'] = df['cancer'].apply(lambda x: 1 / counts[x])\n\n# Split the data into train and validation sets, stratifying by the 'cancer' column\ntrain_df, val_df = train_test_split(df, test_size=0.25, stratify=df['cancer'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:35:08.408246Z","iopub.execute_input":"2025-01-03T02:35:08.408566Z","iopub.status.idle":"2025-01-03T02:35:08.640961Z","shell.execute_reply.started":"2025-01-03T02:35:08.408535Z","shell.execute_reply":"2025-01-03T02:35:08.63998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_path = '/kaggle/input/rsna-breast-cancer-512-pngs'\n\ntrain_samples = 1000  # Number of samples for the training set\nval_samples = 500     # Number of samples for the validation set\n\n# Create the datasets with the device argument\ntrain_ds = RSNADataset(train_df, img_path, device)\nval_ds = RSNADataset(val_df, img_path, device)\n\n# WeightedRandomSampler for training set\ntrain_sampler = WeightedRandomSampler(\n    train_df['weights'].values, \n    train_samples\n)\n\n# DataLoader for training set\ntrain_loader = DataLoader(\n    train_ds, \n    batch_size=8, \n    sampler=train_sampler\n)\n\n# WeightedRandomSampler for validation set\nval_sampler = WeightedRandomSampler(\n    val_df['weights'].values, \n    val_samples\n)\n\n# DataLoader for validation set\nval_loader = DataLoader(\n    val_ds, \n    batch_size=8, \n    sampler=val_sampler\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:35:10.363724Z","iopub.execute_input":"2025-01-03T02:35:10.364073Z","iopub.status.idle":"2025-01-03T02:35:10.36965Z","shell.execute_reply.started":"2025-01-03T02:35:10.36404Z","shell.execute_reply":"2025-01-03T02:35:10.368552Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize model\nmodel = ViT(\n    img_size = (512, 512),\n    patch_size = (16, 16),\n    n_channels = 1,\n    d_model = 1024,\n    nhead = 4,\n    dim_feedforward = 1024,\n    blocks = 8,\n    mlp_head_units = [512, 512],\n    n_classes = 1,\n).to(device)\n\n# Set up the optimizer (Adam optimizer for the model)\noptimizer = Adam(model.parameters(), lr=1e-4)\n\n# Set the loss function (BCE Loss for binary classification)\ncriterion = nn.BCEWithLogitsLoss()\n\n# Create the trainer\ntrainer = create_supervised_trainer(model, optimizer, criterion, device=device)\n\n# Define evaluation metrics\nval_metrics = {\n    \"bce\": Loss(criterion)\n}\n\n# Create the evaluator\nevaluator = create_supervised_evaluator(model, metrics=val_metrics, device=device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:35:14.364939Z","iopub.execute_input":"2025-01-03T02:35:14.365223Z","iopub.status.idle":"2025-01-03T02:35:14.578966Z","shell.execute_reply.started":"2025-01-03T02:35:14.365199Z","shell.execute_reply":"2025-01-03T02:35:14.578019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"log_interval = 10\nmax_epochs = 5\nbest_loss = float('inf')\n\nRunningAverage(output_transform=lambda x: x).attach(trainer, 'loss')\n\npbar = ProgressBar()\npbar.attach(trainer, ['loss'])\n\n@trainer.on(Events.EPOCH_COMPLETED)\ndef log_validation_results(trainer):\n    global best_loss\n    evaluator.run(val_loader)\n    loss = evaluator.state.metrics['bce']\n    \n    # Save the best model if the validation loss improves\n    if loss < best_loss:\n        best_loss = loss  # Update the best_loss with the new lower loss\n        torch.save(model.state_dict(), 'best_model_vit.pt')  # Save the model's state_dict\n\n    print(f\"Validation Results - Epoch: {trainer.state.epoch} Avg loss: {loss:.2f}\")\n\noutput_state = trainer.run(train_loader, max_epochs=max_epochs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-03T02:35:16.485677Z","iopub.execute_input":"2025-01-03T02:35:16.486034Z","iopub.status.idle":"2025-01-03T02:46:31.289174Z","shell.execute_reply.started":"2025-01-03T02:35:16.486002Z","shell.execute_reply":"2025-01-03T02:46:31.288368Z"}},"outputs":[],"execution_count":null}]}