{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpu1vmV38","dataSources":[{"sourceId":18647,"databundleVersionId":1126921,"sourceType":"competition"}],"dockerImageVersionId":30777,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"colab":{"provenance":[],"gpuType":"T4"},"accelerator":"GPU"},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom PIL import Image","metadata":{"id":"hpIMg089QCyD","execution":{"iopub.status.busy":"2024-10-06T10:32:59.665445Z","iopub.execute_input":"2024-10-06T10:32:59.665712Z","iopub.status.idle":"2024-10-06T10:33:01.493982Z","shell.execute_reply.started":"2024-10-06T10:32:59.665684Z","shell.execute_reply":"2024-10-06T10:33:01.49306Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#  Data Import","metadata":{"id":"VtDW8KWkQCyE"}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/prostate-cancer-grade-assessment/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/prostate-cancer-grade-assessment/test.csv\")\ntrain.head()","metadata":{"id":"rOnAKZZgQCyF","outputId":"6a176385-1295-4872-ad2e-1631586b9162","execution":{"iopub.status.busy":"2024-10-06T10:33:01.495493Z","iopub.execute_input":"2024-10-06T10:33:01.495861Z","iopub.status.idle":"2024-10-06T10:33:01.548289Z","shell.execute_reply.started":"2024-10-06T10:33:01.495833Z","shell.execute_reply":"2024-10-06T10:33:01.547417Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Info\n\n- image_id: ID code for the image.\n\n- data_provider: The name of the institution that provided the data. Both the Karolinska Institute and Radboud University Medical Center contributed data. They used different scanners with slightly different maximum microscope resolutions and worked with different pathologists for labeling their images.\n\n- isup_grade: Train only. The target variable. The severity of the cancer on a 0-5 scale.\n\n- gleason_score: Train only. An alternate cancer severity rating system with more levels than the ISUP scale. For details on how the gleason and ISUP systems compare, see the Additional Resources tab.","metadata":{"id":"ThY1XES-QCyG"}},{"cell_type":"code","source":"# openslide resorces (if you are working with a TPU on)\n!pip install openslide-python\n!apt-get update\n!apt-get install -y openslide-tools\n\nimport openslide","metadata":{"id":"pclp7l1PQCyG","outputId":"504b20f6-be70-4654-8ffb-9e1fbc5ab55c","execution":{"iopub.status.busy":"2024-10-06T10:33:01.54933Z","iopub.execute_input":"2024-10-06T10:33:01.54956Z","iopub.status.idle":"2024-10-06T10:33:11.777135Z","shell.execute_reply.started":"2024-10-06T10:33:01.549536Z","shell.execute_reply":"2024-10-06T10:33:11.776177Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define directories\ninput_dir = \"/kaggle/input/prostate-cancer-grade-assessment/train_images/\"\noutput_dir = \"/kaggle/working/preprocessed-images\"\nos.makedirs(output_dir, exist_ok=True)\n\n# Function to preprocess a single image\ndef preprocess_image(image_path, output_path, target_size=(512, 512)):\n    slide = openslide.OpenSlide(image_path)\n    thumbnail = slide.get_thumbnail(target_size)\n    thumbnail = thumbnail.convert(\"RGB\")\n    thumbnail.save(output_path, \"JPEG\")\n\n# Calculate the number of images to process (25% of total images)\ntotal_images = len(os.listdir(input_dir))\nimages_to_process = int(total_images * 0.25)\n\n# Process 30% of images in the directory\nprocessed_count = 0\nfor image_name in os.listdir(input_dir):\n    if image_name.endswith(\".tiff\"):\n        input_path = os.path.join(input_dir, image_name)\n        output_path = os.path.join(output_dir, image_name.replace(\".tiff\", \".jpg\"))\n        preprocess_image(input_path, output_path)\n        processed_count += 1\n        if processed_count >= images_to_process:\n            break\n\nprint(f\"Processed {processed_count} images out of {total_images} ({(processed_count/total_images)*100:.2f}% of the dataset).\")","metadata":{"id":"vERUg5DHQCyH","outputId":"c7d070d5-acee-408e-de35-a50dc6388e87","execution":{"iopub.status.busy":"2024-10-06T10:33:11.779696Z","iopub.execute_input":"2024-10-06T10:33:11.780006Z","iopub.status.idle":"2024-10-06T10:39:47.053349Z","shell.execute_reply.started":"2024-10-06T10:33:11.779976Z","shell.execute_reply":"2024-10-06T10:39:47.052618Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch  # Import torch for handling tensors\nfrom torchvision import transforms\n\n# Define a normalization transform\nnormalize_transform = transforms.Compose([\n    transforms.ToTensor(),  # Convert image to a PyTorch tensor\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],  # Normalization values typically used for pre-trained models\n                         std=[0.229, 0.224, 0.225])\n])\n\n# Function to load and normalize an image\ndef load_and_normalize_image(image_path):\n    image = Image.open(image_path).convert(\"RGB\")\n    image = normalize_transform(image)\n    return image\n\n# Apply normalization to all preprocessed images and save them as tensors (optional step)\nnormalized_output_dir = \"/kaggle/working/normalized-images\"\nos.makedirs(normalized_output_dir, exist_ok=True)\n\nfor image_name in os.listdir(output_dir):\n    if image_name.endswith(\".jpg\"):\n        input_path = os.path.join(output_dir, image_name)\n        normalized_image = load_and_normalize_image(input_path)\n        normalized_image_path = os.path.join(normalized_output_dir, image_name.replace(\".jpg\", \".pt\"))\n        torch.save(normalized_image, normalized_image_path)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:39:47.054348Z","iopub.execute_input":"2024-10-06T10:39:47.054617Z","iopub.status.idle":"2024-10-06T10:40:23.663918Z","shell.execute_reply.started":"2024-10-06T10:39:47.05459Z","shell.execute_reply":"2024-10-06T10:40:23.662938Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Load labels from the CSV file\nlabels_df = pd.read_csv(\"/kaggle/input/prostate-cancer-grade-assessment/train.csv\")\n\n# Create a DataFrame that combines image file paths with labels\nlabels_df['image_path'] = labels_df['image_id'].apply(lambda x: os.path.join(normalized_output_dir, f\"{x}.pt\"))\n\n# Filter the DataFrame to include only images that were successfully preprocessed\nlabels_df = labels_df[labels_df['image_id'].apply(lambda x: os.path.exists(os.path.join(normalized_output_dir, f\"{x}.pt\")))]\n\n# Split the data into training and validation sets\ntrain_df, val_df = train_test_split(labels_df, test_size=0.2, stratify=labels_df['isup_grade'], random_state=42)\n\n# Optionally, save the splits to CSV files for future use\ntrain_df.to_csv(\"/kaggle/working/train_split.csv\", index=False)\nval_df.to_csv(\"/kaggle/working/val_split.csv\", index=False)\n\nprint(\"Training and validation sets created:\")\nprint(f\"Training set size: {len(train_df)}\")\nprint(f\"Validation set size: {len(val_df)}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:23.665097Z","iopub.execute_input":"2024-10-06T10:40:23.665538Z","iopub.status.idle":"2024-10-06T10:40:25.272572Z","shell.execute_reply.started":"2024-10-06T10:40:23.665506Z","shell.execute_reply":"2024-10-06T10:40:25.271905Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport torch\n\n# Function to display images with their labels\ndef display_samples(dataframe, n=5):\n    plt.figure(figsize=(15, 5))\n    for i, row in enumerate(dataframe.sample(n).iterrows()):\n        image_tensor = torch.load(row[1]['image_path'])\n        image = transforms.ToPILImage()(image_tensor)  # Convert tensor to PIL image\n        plt.subplot(1, n, i + 1)\n        plt.imshow(image)\n        plt.title(f\"ISUP Grade: {row[1]['isup_grade']}\")\n        plt.axis('off')\n    plt.show()\n\n# Display 5 random samples from the training set\ndisplay_samples(train_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:25.273503Z","iopub.execute_input":"2024-10-06T10:40:25.273772Z","iopub.status.idle":"2024-10-06T10:40:26.679746Z","shell.execute_reply.started":"2024-10-06T10:40:25.273746Z","shell.execute_reply":"2024-10-06T10:40:26.679064Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot the distribution of ISUP grades\nplt.figure(figsize=(10, 6))\ntrain_df['isup_grade'].value_counts().sort_index().plot(kind='bar')\nplt.title('Distribution of ISUP Grades')\nplt.xlabel('ISUP Grade')\nplt.ylabel('Count')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:26.680685Z","iopub.execute_input":"2024-10-06T10:40:26.681052Z","iopub.status.idle":"2024-10-06T10:40:26.832867Z","shell.execute_reply.started":"2024-10-06T10:40:26.681025Z","shell.execute_reply":"2024-10-06T10:40:26.832209Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate basic statistics for a subset of images\nmeans = []\nstds = []\n\nfor image_path in train_df['image_path'].sample(100):\n    image_tensor = torch.load(image_path)\n    means.append(image_tensor.mean().item())\n    stds.append(image_tensor.std().item())\n\nprint(f\"Mean pixel value: {np.mean(means):.4f}\")\nprint(f\"Standard deviation of pixel values: {np.mean(stds):.4f}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:26.833797Z","iopub.execute_input":"2024-10-06T10:40:26.834064Z","iopub.status.idle":"2024-10-06T10:40:32.763916Z","shell.execute_reply.started":"2024-10-06T10:40:26.834039Z","shell.execute_reply":"2024-10-06T10:40:32.763026Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the directory where label masks are stored\nlabel_masks_dir = \"/kaggle/input/prostate-cancer-grade-assessment/train_label_masks\"\n\n# Function to display images with their corresponding masks\ndef display_samples_with_masks(dataframe, n=5):\n    plt.figure(figsize=(15, 10))\n    for i, row in enumerate(dataframe.sample(n).iterrows()):\n        image_tensor = torch.load(row[1]['image_path'])\n        image = transforms.ToPILImage()(image_tensor)\n        mask_path = os.path.join(label_masks_dir, f\"{row[1]['image_id']}_mask.tiff\")\n        if os.path.exists(mask_path):\n            mask = openslide.OpenSlide(mask_path).get_thumbnail(image.size)\n            mask = mask.convert(\"L\")  # Convert mask to grayscale\n            plt.subplot(n, 2, 2*i + 1)\n            plt.imshow(image)\n            plt.title(f\"Image: {row[1]['image_id']}\")\n            plt.axis('off')\n            plt.subplot(n, 2, 2*i + 2)\n            plt.imshow(mask, cmap='jet', alpha=0.5)\n            plt.title(f\"Mask: {row[1]['image_id']}\")\n            plt.axis('off')\n    plt.show()\n\n# Display 5 random samples with masks from the training set\ndisplay_samples_with_masks(train_df)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:32.767294Z","iopub.execute_input":"2024-10-06T10:40:32.767902Z","iopub.status.idle":"2024-10-06T10:40:38.001794Z","shell.execute_reply.started":"2024-10-06T10:40:32.767865Z","shell.execute_reply":"2024-10-06T10:40:38.001079Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torchvision.models as models\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms\nimport numpy as np\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:38.002718Z","iopub.execute_input":"2024-10-06T10:40:38.002974Z","iopub.status.idle":"2024-10-06T10:40:40.000877Z","shell.execute_reply.started":"2024-10-06T10:40:38.00295Z","shell.execute_reply":"2024-10-06T10:40:40.000051Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ProstateCancerDataset(Dataset):\n    def __init__(self, dataframe, transform=None, target_size=(512, 512)):\n        self.dataframe = dataframe\n        self.transform = transform\n        self.target_size = target_size\n\n    def __len__(self):\n        return len(self.dataframe)\n\n    def __getitem__(self, idx):\n        image_path = self.dataframe.iloc[idx]['image_path']\n        image = torch.load(image_path)\n\n        # Ensure image is resized to the target size if needed\n        if image.size()[1:] != self.target_size:  # Check if resizing is needed\n            image = transforms.functional.resize(image, self.target_size)\n        \n        if self.transform:\n            image = self.transform(image)\n            \n        label = self.dataframe.iloc[idx]['isup_grade']\n        return image, label\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:40.001807Z","iopub.execute_input":"2024-10-06T10:40:40.002048Z","iopub.status.idle":"2024-10-06T10:40:40.233324Z","shell.execute_reply.started":"2024-10-06T10:40:40.002025Z","shell.execute_reply":"2024-10-06T10:40:40.232706Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the DenseNet121 model pre-trained on ImageNet\ndensenet = models.densenet121(pretrained=True)\n\n# Remove the classification layer to use the model as a feature extractor\ndensenet_features = nn.Sequential(*list(densenet.children())[:-1])\n\n# Set the model to evaluation mode\ndensenet_features.eval()\n\n# If a GPU is available, move the model to GPU\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndensenet_features = densenet_features.to(device)\n\n# Load the VGG16 model pre-trained on ImageNet\nvgg16 = models.vgg16(pretrained=True)\n\n# Remove the classification layer to use the model as a feature extractor\nvgg16_features = nn.Sequential(*list(vgg16.children())[:-1])\n\n# Set the model to evaluation mode\nvgg16_features.eval()\n\n# Move the model to GPU if available\nvgg16_features = vgg16_features.to(device)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:40.234071Z","iopub.execute_input":"2024-10-06T10:40:40.234285Z","iopub.status.idle":"2024-10-06T10:40:45.010862Z","shell.execute_reply.started":"2024-10-06T10:40:40.234261Z","shell.execute_reply":"2024-10-06T10:40:45.010018Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torchvision import transforms\n\n# Define a transform with resizing and normalization (remove ToTensor)\ntransform = transforms.Compose([\n    transforms.Resize((512, 512)),  # Resize images to 512x512\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # Normalize images\n])\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:45.011793Z","iopub.execute_input":"2024-10-06T10:40:45.012045Z","iopub.status.idle":"2024-10-06T10:40:45.016154Z","shell.execute_reply.started":"2024-10-06T10:40:45.012021Z","shell.execute_reply":"2024-10-06T10:40:45.015463Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_features(model, dataloader):\n    features = []\n    labels = []\n    model.eval()  # Set the model to evaluation mode\n    with torch.no_grad():\n        for images, label in dataloader:\n            images = images.to(device)  # Move images to GPU if available\n            output = model(images)\n            features.append(output.squeeze().cpu().numpy())\n            labels.append(label.cpu().numpy())\n    return np.concatenate(features), np.concatenate(labels)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:45.017066Z","iopub.execute_input":"2024-10-06T10:40:45.017303Z","iopub.status.idle":"2024-10-06T10:40:45.033626Z","shell.execute_reply.started":"2024-10-06T10:40:45.017279Z","shell.execute_reply":"2024-10-06T10:40:45.033053Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create dataset and dataloader with the updated transform\ntrain_dataset = ProstateCancerDataset(train_df, transform=transform)\ntrain_loader = DataLoader(train_dataset, batch_size=16, shuffle=False, num_workers=2)\n\nval_dataset = ProstateCancerDataset(val_df, transform=transform)\nval_loader = DataLoader(val_dataset, batch_size=16, shuffle=False, num_workers=2)\n\n# Extract features using DenseNet121 for the training set\ntrain_features, train_labels = extract_features(densenet_features, train_loader)\n\n# Extract features using VGG16 for the validation set\nval_features, val_labels = extract_features(vgg16_features, val_loader)\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:40:45.034408Z","iopub.execute_input":"2024-10-06T10:40:45.034626Z","iopub.status.idle":"2024-10-06T10:50:36.494216Z","shell.execute_reply.started":"2024-10-06T10:40:45.034604Z","shell.execute_reply":"2024-10-06T10:50:36.492807Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install scikit-optimize\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:50:36.495824Z","iopub.execute_input":"2024-10-06T10:50:36.496132Z","iopub.status.idle":"2024-10-06T10:50:40.522362Z","shell.execute_reply.started":"2024-10-06T10:50:36.496104Z","shell.execute_reply":"2024-10-06T10:50:40.521406Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install torch-xla torch-xla-core\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:50:40.523748Z","iopub.execute_input":"2024-10-06T10:50:40.524042Z","iopub.status.idle":"2024-10-06T10:50:41.511275Z","shell.execute_reply.started":"2024-10-06T10:50:40.524012Z","shell.execute_reply":"2024-10-06T10:50:41.510311Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets, transforms\nfrom skopt import gp_minimize\nfrom skopt.space import Real, Integer\nfrom skopt.utils import use_named_args\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\n\n# Define the search space for hyperparameters\nspace = [\n    Real(1e-6, 1e-1, name='learning_rate'),\n    Integer(64, 512, name='batch_size'),  # Increased upper limit for TPU\n    Integer(2, 5, name='num_layers'),\n    Real(0.0, 0.5, name='dropout_rate')\n]\n\n# Example dataset and data loading\ntransform = transforms.Compose([\n    transforms.Resize((32, 32)),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\n# Replace CIFAR-10 with your dataset as needed\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\nval_dataset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)\n\n# Define the model class\nclass CustomModel(nn.Module):\n    def __init__(self, input_size, num_layers, dropout_rate):\n        super(CustomModel, self).__init__()\n        self.flatten = nn.Flatten()\n        self.fc_layers = nn.ModuleList()\n        \n        # Build fully connected layers\n        for i in range(num_layers):\n            if i == 0:\n                self.fc_layers.append(nn.Linear(input_size, 128))\n            else:\n                self.fc_layers.append(nn.Linear(128, 128))\n            self.fc_layers.append(nn.ReLU())\n            self.fc_layers.append(nn.Dropout(p=dropout_rate))\n        self.fc_layers.append(nn.Linear(128, 1))\n        self.fc_layers.append(nn.Sigmoid())  # For binary classification\n\n    def forward(self, x):\n        x = self.flatten(x)\n        for layer in self.fc_layers:\n            x = layer(x)\n        return x\n\n# Define the objective function for Bayesian Optimization\n@use_named_args(space)\ndef objective(**params):\n    # Set the device to TPU\n    device = xm.xla_device()\n\n    # Extract hyperparameters\n    learning_rate = params['learning_rate']\n    batch_size = int(params['batch_size'])\n    num_layers = params['num_layers']\n    dropout_rate = params['dropout_rate']\n    \n    # Determine input size with a dummy input\n    dummy_input = torch.randn(1, 3, 32, 32).to(device)  # Assuming CIFAR-10 size\n    flattened_size = torch.flatten(dummy_input, start_dim=1).shape[1]\n\n    # Initialize the model\n    model = CustomModel(input_size=flattened_size, num_layers=num_layers, dropout_rate=dropout_rate).to(device)\n    \n    # Define loss function and optimizer\n    criterion = nn.BCELoss()  # Assuming binary classification\n    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n\n    # Create DataLoader for the current batch size\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=8, pin_memory=True)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=8, pin_memory=True)\n    \n    # Training loop\n    model.train()\n    for epoch in range(10):  # Adjust number of epochs as needed\n        running_loss = 0.0\n        para_train_loader = pl.ParallelLoader(train_loader, [device]).per_device_loader(device)\n        for images, labels in para_train_loader:\n            images, labels = images.to(device), labels.to(device)\n            optimizer.zero_grad()\n\n            outputs = model(images)\n            loss = criterion(outputs, labels.view(-1, 1).float())\n            \n            loss.backward()\n            xm.optimizer_step(optimizer, barrier=True)\n            running_loss += loss.item()\n    \n    # Evaluate on validation set\n    model.eval()\n    val_loss = 0.0\n    para_val_loader = pl.ParallelLoader(val_loader, [device]).per_device_loader(device)\n    with torch.no_grad():\n        for images, labels in para_val_loader:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            val_loss += criterion(outputs, labels.view(-1, 1).float()).item()\n    \n    return val_loss\n\n# Perform Bayesian Optimization\nres_gp = gp_minimize(objective, space, n_calls=10, random_state=42)\n\n# Print the best hyperparameters found\nprint(\"Best hyperparameters found:\")\nprint(f\"Learning Rate: {res_gp.x[0]}\")\nprint(f\"Batch Size: {res_gp.x[1]}\")\nprint(f\"Number of Layers: {res_gp.x[2]}\")\nprint(f\"Dropout Rate: {res_gp.x[3]}\")\n\n# Train the final model using these hyperparameters\nbest_learning_rate = res_gp.x[0]\nbest_batch_size = res_gp.x[1]\nbest_num_layers = res_gp.x[2]\nbest_dropout_rate = res_gp.x[3]\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T10:50:41.512763Z","iopub.execute_input":"2024-10-06T10:50:41.51304Z","iopub.status.idle":"2024-10-06T11:11:52.318809Z","shell.execute_reply.started":"2024-10-06T10:50:41.513012Z","shell.execute_reply":"2024-10-06T11:11:52.316663Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, SubsetRandomSampler\nfrom torchvision import datasets, transforms\nfrom sklearn.model_selection import KFold\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport numpy as np\n\n# Simplified settings\nk_folds = 4  # Reduce folds\nbest_batch_size = 128  # Smaller batch size\nepochs = 15  # Only 1 epoch\nnum_samples = 1000  # Small dataset for testing\n\n# Define device (TPU)\ndevice = xm.xla_device()\n\n# Simplified model\nclass SimpleModel(nn.Module):\n    def __init__(self):\n        super(SimpleModel, self).__init__()\n        self.fc = nn.Linear(32*32*3, 10)  # Simple single-layer model\n\n    def forward(self, x):\n        x = x.view(x.size(0), -1)  # Flatten the input\n        return self.fc(x)\n\n# Initialize the K-Fold cross-validator\nkfold = KFold(n_splits=k_folds, shuffle=True, random_state=42)\n\n# Use a small dataset subset\ntransform = transforms.Compose([\n    transforms.Resize((32, 32)),\n    transforms.ToTensor(),\n])\n\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\nval_dataset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)\n\n# Subsample the dataset\ntrain_subset_indices = np.random.choice(len(train_dataset), num_samples, replace=False)\nval_subset_indices = np.random.choice(len(val_dataset), num_samples, replace=False)\ntrain_subset = torch.utils.data.Subset(train_dataset, train_subset_indices)\nval_subset = torch.utils.data.Subset(val_dataset, val_subset_indices)\n\n# Start the K-Fold cross-validation process\nfor fold, (train_ids, val_ids) in enumerate(kfold.split(train_subset)):\n\n    print(f'FOLD {fold}')\n    print('--------------------------------')\n\n    train_subsampler = SubsetRandomSampler(train_ids)\n    val_subsampler = SubsetRandomSampler(val_ids)\n\n    # Define data loaders for training and validation\n    train_loader = DataLoader(train_subset, batch_size=best_batch_size, sampler=train_subsampler, num_workers=0, pin_memory=False)\n    val_loader = DataLoader(val_subset, batch_size=best_batch_size, sampler=val_subsampler, num_workers=0, pin_memory=False)\n\n    # Initialize the model for this fold\n    model = SimpleModel().to(device)\n\n    # Initialize optimizer and loss function\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.001)\n\n    # Training loop for each fold\n    model.train()\n    for epoch in range(epochs):\n        running_loss = 0.0\n        for images, labels in train_loader:\n            images, labels = images.to(device), labels.to(device)\n            optimizer.zero_grad()\n\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            xm.optimizer_step(optimizer)\n            running_loss += loss.item()\n\n        print(f'Fold {fold}, Epoch {epoch+1}, Loss: {running_loss/len(train_loader)}')\n\n    # Validation loop for each fold\n    model.eval()\n    val_loss = 0.0\n    with torch.no_grad():\n        for images, labels in val_loader:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            val_loss += criterion(outputs, labels).item()\n\n    print(f'Fold {fold}, Validation Loss: {val_loss/len(val_loader)}')","metadata":{"execution":{"iopub.status.busy":"2024-10-06T11:11:52.322455Z","iopub.execute_input":"2024-10-06T11:11:52.324051Z","iopub.status.idle":"2024-10-06T11:18:04.335041Z","shell.execute_reply.started":"2024-10-06T11:11:52.323957Z","shell.execute_reply":"2024-10-06T11:18:04.334056Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torchvision import datasets, transforms\nfrom torch.utils.data import DataLoader\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nimport numpy as np\n\n# Enhanced CNN Model\nclass EnhancedCNN(nn.Module):\n    def __init__(self):\n        super(EnhancedCNN, self).__init__()\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, padding=1)\n        self.conv2 = nn.Conv2d(64, 128, kernel_size=3, padding=1)\n        self.conv3 = nn.Conv2d(128, 256, kernel_size=3, padding=1)\n        self.pool = nn.MaxPool2d(2, 2)\n        self.fc1 = nn.Linear(256 * 4 * 4, 512)\n        self.fc2 = nn.Linear(512, 10)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(0.5)\n        self.batch_norm1 = nn.BatchNorm2d(64)\n        self.batch_norm2 = nn.BatchNorm2d(128)\n        self.batch_norm3 = nn.BatchNorm2d(256)\n\n    def forward(self, x):\n        x = self.pool(self.relu(self.batch_norm1(self.conv1(x))))\n        x = self.pool(self.relu(self.batch_norm2(self.conv2(x))))\n        x = self.pool(self.relu(self.batch_norm3(self.conv3(x))))\n        x = x.view(-1, 256 * 4 * 4)\n        x = self.dropout(self.relu(self.fc1(x)))\n        x = self.fc2(x)\n        return x\n\n# Define transformations and data loaders\ntransform = transforms.Compose([\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation(10),\n    transforms.RandomCrop(32, padding=4),\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\ntrain_dataset = datasets.CIFAR10(root='./data', train=True, download=True, transform=transform)\ntest_dataset = datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)\n\ntrain_loader = DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=4, pin_memory=True)\ntest_loader = DataLoader(test_dataset, batch_size=128, shuffle=False, num_workers=4, pin_memory=True)\n\n# Initialize the model, loss function, and optimizer\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = EnhancedCNN().to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001)\n\n# Training loop\nfor epoch in range(50):  # Increased epochs for further training\n    running_loss = 0.0\n    for images, labels in train_loader:\n        images, labels = images.to(device), labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()\n\n    print(f'Epoch {epoch + 1}, Loss: {running_loss / len(train_loader)}')\n\n# Evaluate on the test set\nmodel.eval()\ncorrect = 0\ntotal = 0\nall_labels = []\nall_probs = []\n\nwith torch.no_grad():\n    for images, labels in test_loader:\n        images, labels = images.to(device), labels.to(device)\n        outputs = model(images)\n        _, predicted = torch.max(outputs.data, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n\n        # Store predictions and labels for ROC-AUC calculation\n        all_probs.append(outputs.softmax(dim=1).cpu().numpy())\n        all_labels.append(labels.cpu().numpy())","metadata":{"execution":{"iopub.status.busy":"2024-10-06T11:18:04.337098Z","iopub.execute_input":"2024-10-06T11:18:04.337933Z","iopub.status.idle":"2024-10-06T11:41:38.906156Z","shell.execute_reply.started":"2024-10-06T11:18:04.337893Z","shell.execute_reply":"2024-10-06T11:41:38.904726Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate accuracy\naccuracy = 100 * correct / total\nprint(f'Test Accuracy: {accuracy:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2024-10-06T11:41:59.826539Z","iopub.execute_input":"2024-10-06T11:41:59.827431Z","iopub.status.idle":"2024-10-06T11:41:59.831729Z","shell.execute_reply.started":"2024-10-06T11:41:59.8274Z","shell.execute_reply":"2024-10-06T11:41:59.830877Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare for ROC-AUC calculation\nall_probs = np.vstack(all_probs)\nall_labels = np.hstack(all_labels)\n\n# Calculate ROC-AUC\nroc_auc = roc_auc_score(all_labels, all_probs, multi_class='ovr')\nprint(f'ROC-AUC: {roc_auc:.2f}')","metadata":{"execution":{"iopub.status.busy":"2024-10-06T11:41:38.914937Z","iopub.execute_input":"2024-10-06T11:41:38.915173Z","iopub.status.idle":"2024-10-06T11:41:38.969553Z","shell.execute_reply.started":"2024-10-06T11:41:38.91515Z","shell.execute_reply":"2024-10-06T11:41:38.968622Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, roc_auc_score, roc_curve, auc\nfrom itertools import cycle\n\n# Plot ROC curve for each class\nfpr = dict()\ntpr = dict()\nroc_auc = dict()\nn_classes = 10\n\nfor i in range(n_classes):\n    fpr[i], tpr[i], _ = roc_curve(all_labels, all_probs[:, i], pos_label=i)\n    roc_auc[i] = auc(fpr[i], tpr[i])\n\n# Plot ROC curve for each class\ncolors = cycle(['aqua', 'darkorange', 'cornflowerblue', 'green', 'red', 'purple', 'brown', 'pink', 'gray', 'olive'])\nplt.figure(figsize=(10, 8))\nfor i, color in zip(range(n_classes), colors):\n    plt.plot(fpr[i], tpr[i], color=color, lw=2,\n             label=f'Class {i} (AUC = {roc_auc[i]:.2f})')\n\nplt.plot([0, 1], [0, 1], 'k--', lw=2)\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('ROC Curve for Each Class')\nplt.legend(loc='lower right')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-06T11:42:05.332876Z","iopub.execute_input":"2024-10-06T11:42:05.333345Z","iopub.status.idle":"2024-10-06T11:42:05.90511Z","shell.execute_reply.started":"2024-10-06T11:42:05.333313Z","shell.execute_reply":"2024-10-06T11:42:05.904138Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.graph_objects as go\n\n# Example accuracy data\nepochs = list(range(1, 11))  # Assuming 10 epochs for the sake of example\npaper_2_accuracy = [60, 80, 84,90 , 97, 95, 97, 99.5, 99.8, 99.9]  # Example accuracy values for Paper 2\npaper_3_accuracy = [60, 80, 84,89 , 94, 97, 98, 99.8, 99.9, 99.9]  # Example accuracy values for Paper 3\n\n# Create a line plot for both Paper 2 and Paper 3\nfig = go.Figure()\n\n# Paper 2 accuracy line\nfig.add_trace(go.Scatter(\n    x=epochs, y=paper_2_accuracy,\n    mode='lines+markers',\n    name='Paper 2 Accuracy',\n    line=dict(color='blue'),\n    marker=dict(color='blue', size=8)\n))\n\n# Paper 3 accuracy line\nfig.add_trace(go.Scatter(\n    x=epochs, y=paper_3_accuracy,\n    mode='lines+markers',\n    name='Paper 3 Accuracy',\n    line=dict(color='red'),\n    marker=dict(color='red', size=8)\n))\n\n# Update layout\nfig.update_layout(\n    title='Training Accuracy Comparison between Paper 2 and Paper 3',\n    xaxis_title='Epochs',\n    yaxis_title='Accuracy (%)',\n    legend_title='Papers',\n    template='plotly_white'\n)\n\n# Show plot\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T11:45:49.469438Z","iopub.execute_input":"2024-10-06T11:45:49.469851Z","iopub.status.idle":"2024-10-06T11:45:49.507876Z","shell.execute_reply.started":"2024-10-06T11:45:49.469818Z","shell.execute_reply":"2024-10-06T11:45:49.507077Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.graph_objects as go\n\n# Example accuracy data\npaper_names = ['Paper 2', 'Paper 3']  # Names of the papers\naccuracy_values = [84.7, 84.9]  # Example final accuracy values for Paper 2 and Paper 3\n\n# Create a line chart\nfig = go.Figure()\n\n# Add a line plot\nfig.add_trace(go.Scatter(\n    x=paper_names, \n    y=accuracy_values,\n    mode='lines+markers',\n    name='Accuracy',\n    line=dict(color='blue')\n))\n\n# Update layout\nfig.update_layout(\n    title='Final Testing Accuracy Comparison between Paper 2 and Paper 3',\n    xaxis_title='Papers',\n    yaxis_title='Accuracy (%)',\n    template='plotly_white'\n)\n\n# Show plot\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-06T11:45:21.202935Z","iopub.execute_input":"2024-10-06T11:45:21.203338Z","iopub.status.idle":"2024-10-06T11:45:21.558956Z","shell.execute_reply.started":"2024-10-06T11:45:21.203303Z","shell.execute_reply":"2024-10-06T11:45:21.558125Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}