{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":52254,"databundleVersionId":9674523,"sourceType":"competition"},{"sourceId":6523471,"sourceType":"datasetVersion","datasetId":3771357},{"sourceId":6524344,"sourceType":"datasetVersion","datasetId":3771912},{"sourceId":7015603,"sourceType":"datasetVersion","datasetId":4033648},{"sourceId":7432254,"sourceType":"datasetVersion","datasetId":4325089},{"sourceId":7665407,"sourceType":"datasetVersion","datasetId":4470305}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install nibabel","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-02T10:41:48.649588Z","iopub.execute_input":"2024-10-02T10:41:48.650168Z","iopub.status.idle":"2024-10-02T10:42:01.512681Z","shell.execute_reply.started":"2024-10-02T10:41:48.650133Z","shell.execute_reply":"2024-10-02T10:42:01.511719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torch.nn as nn\n# import torch_xla.core.xla_model as xm\n# import torch_xla.distributed.parallel_loader as pl\n# import torch_xla.distributed.xla_multiprocessing as xmp\n\n# import torch\n# import torch.nn.functional as F\n# import torch_xla.core.xla_model as xm\n# from torch.nn import Transformer\n\n# def xla_linear(input, weight, bias=None):\n#     if isinstance(input, torch.Tensor) and input.device.type == 'xla':\n# #         print(\"input\", input.shape)\n# #         print(\"************************************************************************\")\n# #         print(\"weight\", weight)\n# #         print(\"$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\")\n# #         print(\"bias\", bias)\n# #         return torch.nn.functional.linear(input, weight, bias)\n#         return torch.matmul(input.to(xm.xla_device()), weight.to(xm.xla_device()).t()) + bias.to(xm.xla_device())\n#     else:\n#         input_xla = input.to(xm.xla_device())\n#         weight_xla = weight.to(xm.xla_device())\n#         if bias is not None:\n#             bias_xla = bias.to(xm.xla_device())\n#         else:\n#             bias_xla = None\n# #         return torch.nn.functional.linear(input_xla, weight_xla, bias_xla)\n#         return torch.matmul(input_xla, weight_xla.t()) + bias_xla\n    \n# # Override the torch.nn.functional.linear function with the XLA version\n# # F.linear = xla_linear\n\n# def xla_layer_norm(input, normalized_shape, weight=None, bias=None, eps=1e-5):\n#     if input.device.type == 'xla':\n#         # Calculate the mean and variance along the last dimension\n#         mean = input.mean(dim=-1, keepdim=True)\n#         var = input.var(dim=-1, unbiased=False, keepdim=True)\n        \n#         # Reshape weight and bias to match the shape of input\n#         if weight is not None:\n#             weight = weight.view(*input.shape[-len(normalized_shape):])\n#         if bias is not None:\n#             bias = bias.view(*input.shape[-len(normalized_shape):])\n        \n#         # Normalize the input\n#         input = (input - mean) / torch.sqrt(var + eps)\n        \n#         # Apply weight and bias\n#         if weight is not None:\n#             input = input * weight\n#         if bias is not None:\n#             input = input + bias\n#         print(input.shape)\n#         return input\n#     else:\n#         # Fall back to PyTorch's layer normalization\n#         return F.layer_norm(input, normalized_shape, weight, bias, eps)\n\n# # Override the torch.nn.functional.layer_norm function with the XLA version\n# # F.layer_norm = xla_layer_norm\n\n# input_shape = (128, 128, 128)  # Depth x Height x Width\n# num_classes = 14  # Number of classes for classification\n\n# class Transformer3DClassifier(nn.Module):\n#     def __init__(self, input_shape, num_classes, num_layers=6, d_model=16, nhead=8, dim_feedforward=2048, dropout=0.1):\n#         super(Transformer3DClassifier, self).__init__()\n        \n#         # Initialize d_model\n#         self.d_model = d_model\n        \n#         # Calculate the input size for the transformer\n#         d_in = input_shape[0] * input_shape[1] * input_shape[2]  # Depth x Height x Width\n#         self.embedding = nn.Linear(d_in, d_model)\n        \n#         self.transformer = Transformer(\n#             d_model=d_model,\n#             nhead=nhead,\n#             num_encoder_layers=num_layers,\n#             dim_feedforward=dim_feedforward,\n#             dropout=dropout\n#         )\n        \n#         self.fc = nn.Linear(d_model, num_classes)\n   \n\n#     def forward(self, x):\n#         # Flatten the input and apply linear embedding\n#         x = x.view(x.size(0), -1)\n#         print(\"Before embedding x.shape is \", x.shape)\n#         x = self.embedding(x)\n#         print(\"After embedding x.shape is \", x.shape)\n        \n#         # Reshape to add a third dimension (seq_len)\n#         x = x.unsqueeze(0)\n#         print(\"x shape after unsqueeze\", x.shape)\n#         # Create a dummy target tensor (you can adjust its size if needed)\n#         tgt = torch.zeros(1, x.size(1), self.d_model).to(x.device)\n#         print(\"tgt shape\", tgt.shape)\n        \n#         # Transformer encoder\n#         output = self.transformer(x, tgt)\n#         print(\"Output shape after transformer\", output.shape)\n\n#         # Remove the added dimension\n# #         output = output.squeeze(0)\n# #         print(\"Output shape after squeeze\", output.shape)\n        \n# #         # Global average pooling\n# #         output = output.mean(dim=1)\n# #         print(\"Output shape after global average pooling\", output.shape)\n\n#         # Classification layer\n#         logits = self.fc(output)\n        \n#         # Add batch dimension to logits\n#         logits = logits.unsqueeze(0)\n        \n        \n#         return logits\n\n# # Define XLA tensors for input and hidden layer sizes\n# # input_size = torch.tensor(32, device=xm.xla_device())\n# input_size = 32 # Adjust the dimensions as needed\n# hidden_size = 16  # Adjust the dimensions as needed\n# # hidden_size = torch.tensor(16, device=xm.xla_device())\n# import torch\n# import torch.nn as nn\n\n\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n\n# class UNet(nn.Module):\n#     def __init__(self, in_channels=1, out_channels=5):\n#         super(UNet, self).__init__()\n\n#         # Contracting path\n#         self.encoder = nn.Sequential(\n#             nn.Conv3d(in_channels, 64, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.MaxPool3d(2),\n#             nn.Conv3d(64, 128, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.MaxPool3d(2),\n#         )\n\n#         # Bottleneck\n#         self.bottleneck = nn.Sequential(\n#             nn.Conv3d(128, 256, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#         )\n\n#         # Expanding path (decoder)\n#         self.decoder = nn.Sequential(\n#             nn.ConvTranspose3d(256, 128, kernel_size=3, stride=2, padding=1, output_padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.ConvTranspose3d(128, 64, kernel_size=3, stride=2, padding=1, output_padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(64, 64, kernel_size=3, padding=1),  # Additional layer for symmetry\n#             nn.ReLU(inplace=True),\n#         )\n\n#         # Output layer\n#         self.output_layer = nn.Conv3d(64, out_channels, kernel_size=1)\n#     def forward(self, x):\n#         x = x.to(xm.xla_device())\n#         print(x.shape, \"before encode\")\n#         x = self.encoder(x)\n#         print(x.shape, \"after encode\")\n#         x = self.bottleneck(x)\n#         print(x.shape, \"after bottleneck\")\n#         x = self.decoder(x)\n#         print(x.shape, \"after decoder\")\n#         x = self.output_layer(x)\n#         return x\n\n# in_channels = 3  # Adjust based on your input channels\n# out_channels = 5  # Number of classes\n# class Custom3DViTModelTPU(nn.Module):\n#     def __init__(self, in_channels, num_classes,batch_size):\n#         super(Custom3DViTModelTPU, self).__init__()\n#         self.batch_size = batch_size\n#         self.num_classes = num_classes\n        \n# #         self.vit_backbone = VisionTransformer3DBackboneTPU(\n# #             in_channels=in_channels,\n# #             embedding_dim=32,  # Adjust the embedding dimension as needed\n# #             num_heads=2,       # Number of attention heads\n# #             num_layers=2       # Number of transformer layers\n# #         )\n        \n#         self.vit_backbone = Transformer3DClassifier(\n#             input_shape,\n#             num_classes\n#         )\n\n#         self.classification_head = nn.Sequential(\n# #             nn.Linear(batch_size, 16),\n# #             nn.ReLU(inplace=True),\n# #             nn.Linear(16, num_classes),\n# #             nn.Sigmoid()\n#             nn.Linear(self.vit_backbone.d_model, num_classes)\n#         )\n\n       \n        \n# #     def print_weights(self):\n# #         for name, param in self.named_parameters():\n# #             print(f\"Layer: {name}, Size: {param.size()}\")\n# #             print(param)\n\n#     def forward(self, x):\n#         print(\"x shape and segmentation_mask shape\", x.shape)\n        \n#         # Move input tensors to XLA devices\n#         x = x.to(xm.xla_device())\n        \n\n#         features = self.vit_backbone(x)\n#         features = features.to(xm.xla_device())\n#         print(\"features shape\", features.shape)\n        \n#         #classification_output = self.classification_head(features)\n#         # Reshape it to (32, 10)\n#         classification_output = features.view(self.batch_size, self.num_classes)\n        \n#         print(\"classification_output\", classification_output.shape)\n        \n#         return classification_output\n\n\n# batch_size = 32\n\n# # Move the entire model to XLA devices\n# def get_model():\n#     return Custom3DViTModelTPU(1, 14, batch_size),UNet(1,5)\n# # Modify the run function to accept the process index\n# def run(index):\n#     print(\"^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\")\n# #     l_in = torch.randn(10, device=xm.xla_device())\n# #     linear = torch.nn.Linear(10, 20).to(xm.xla_device())\n# #     l_out = linear(l_in)\n# #     print(l_out)\n    \n#     model_class = get_model()[0]\n#     model_seg=get_model()[1]\n# #     model.print_weights()\n#     model_class = model_class.to(xm.xla_device())\n#     model_seg = model_seg.to(xm.xla_device())\n#     print(\">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\")\n#     # Create sample input tensors (modify this according to your data)\n#     batch_images = torch.randn(32, 1, 128, 128, 128)  # Example input shape\n#     batch_segmentation_masks = torch.randn(32, 1, 128, 128, 128)  # Example mask shape\n\n#     batch_images = batch_images.to(xm.xla_device())  # Move input tensors to XLA device\n#     batch_segmentation_masks = batch_segmentation_masks.to(xm.xla_device())\n#     print(\"*****************************************************************************\")\n\n#     # Forward pass\n#     classification_outputs = model_class(batch_images)\n#     segmentation_outputs=model_seg(batch_segmentation_masks)\n    \n# # # Use XLA multiprocessing to distribute across TPUs\n# if __name__ == '__main__':\n#      xmp.spawn(run, nprocs=1, start_method='fork')\n","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-10-02T10:42:01.51477Z","iopub.execute_input":"2024-10-02T10:42:01.515074Z","iopub.status.idle":"2024-10-02T10:42:01.530096Z","shell.execute_reply.started":"2024-10-02T10:42:01.515046Z","shell.execute_reply":"2024-10-02T10:42:01.529259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class UNet(nn.Module):\n#     def __init__(self, in_channels=1, out_channels=5):\n#         super(UNet, self).__init__()\n\n#         # Contracting path\n#         self.encoder = nn.Sequential(\n#             nn.Conv3d(in_channels, 64, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(64, 64, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.MaxPool3d(2),\n#             nn.Conv3d(64, 128, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(128, 128, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.MaxPool3d(2),\n#             nn.Conv3d(128, 256, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(256, 256, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.MaxPool3d(2),\n#         )\n#         self.bottleneck = nn.Sequential(\n#             nn.Conv3d(256, 256, kernel_size=3, padding=1),  # Corrected input channels\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(256, 512, kernel_size=3, padding=1),  # Corrected input channels\n#             nn.ReLU(inplace=True),\n#         )\n        \n\n#         # Expanding path\n#         self.decoder = nn.Sequential(\n#             nn.ConvTranspose3d(512, 256, kernel_size=3, stride=2, padding=1, output_padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(256, 256, kernel_size=3, padding=1),  # Corrected input channels\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(256, 256, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.ConvTranspose3d(256, 128, kernel_size=3, stride=2, padding=1, output_padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(128, 128, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(128, 128, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.ConvTranspose3d(128, 64, kernel_size=3, stride=2, padding=1, output_padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(64, 64, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#             nn.Conv3d(64, 64, kernel_size=3, padding=1),\n#             nn.ReLU(inplace=True),\n#         )\n\n#         # Output layer\n#         self.output_layer = nn.Conv3d(64, out_channels, kernel_size=1)\n\n#     def forward(self, x):\n#         x = x.to(xm.xla_device())\n#         print(x.shape, \"before encode\")\n#         x = self.encoder(x)\n#         print(x.shape, \"after encode\")\n#         x = self.bottleneck(x)\n#         print(x.shape, \"after bottleneck\")\n#         x = self.decoder(x)\n#         print(x.shape, \"after decoder\")\n#         x = self.output_layer(x)\n#         return x","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-10-02T10:42:01.531293Z","iopub.execute_input":"2024-10-02T10:42:01.531584Z","iopub.status.idle":"2024-10-02T10:42:01.544311Z","shell.execute_reply.started":"2024-10-02T10:42:01.53156Z","shell.execute_reply":"2024-10-02T10:42:01.543407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torch.nn as nn\n# import torch_xla.core.xla_model as xm\n# import torch_xla.distributed.parallel_loader as pl\n# import torch_xla.distributed.xla_multiprocessing as xmp\n\n# import torch\n# import torch.nn.functional as F\n# import torch_xla.core.xla_model as xm\n# from torch.nn import Transformer\n\n# def xla_linear(input, weight, bias=None):\n#     if isinstance(input, torch.Tensor) and input.device.type == 'xla':\n# #         print(\"input\", input.shape)\n# #         print(\"************************************************************************\")\n# #         print(\"weight\", weight)\n# #         print(\"$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\")\n# #         print(\"bias\", bias)\n# #         return torch.nn.functional.linear(input, weight, bias)\n#         return torch.matmul(input.to(xm.xla_device()), weight.to(xm.xla_device()).t()) + bias.to(xm.xla_device())\n#     else:\n#         input_xla = input.to(xm.xla_device())\n#         weight_xla = weight.to(xm.xla_device())\n#         if bias is not None:\n#             bias_xla = bias.to(xm.xla_device())\n#         else:\n#             bias_xla = None\n# #         return torch.nn.functional.linear(input_xla, weight_xla, bias_xla)\n#         return torch.matmul(input_xla, weight_xla.t()) + bias_xla\n    \n# # Override the torch.nn.functional.linear function with the XLA version\n# # F.linear = xla_linear\n\n# def xla_layer_norm(input, normalized_shape, weight=None, bias=None, eps=1e-5):\n#     if input.device.type == 'xla':\n#         # Calculate the mean and variance along the last dimension\n#         mean = input.mean(dim=-1, keepdim=True)\n#         var = input.var(dim=-1, unbiased=False, keepdim=True)\n        \n#         # Reshape weight and bias to match the shape of input\n#         if weight is not None:\n#             weight = weight.view(*input.shape[-len(normalized_shape):])\n#         if bias is not None:\n#             bias = bias.view(*input.shape[-len(normalized_shape):])\n        \n#         # Normalize the input\n#         input = (input - mean) / torch.sqrt(var + eps)\n        \n#         # Apply weight and bias\n#         if weight is not None:\n#             input = input * weight\n#         if bias is not None:\n#             input = input + bias\n#         print(input.shape)\n#         return input\n#     else:\n#         # Fall back to PyTorch's layer normalization\n#         return F.layer_norm(input, normalized_shape, weight, bias, eps)\n\n# # Override the torch.nn.functional.layer_norm function with the XLA version\n# # F.layer_norm = xla_layer_norm\n\n# input_shape = (128, 128, 128)  # Depth x Height x Width\n# num_classes = 14  # Number of classes for classification\n# import torch\n# import torch.nn as nn\n\n\n\n# class Transformer(nn.Module):\n#     def __init__(self, input_shape=(128, 128, 128), num_classes=14, num_layers=6, d_model=16, nhead=8, dim_feedforward=2048, dropout=0.1):\n#         super(Transformer, self).__init__()\n        \n#         # Initialize d_model\n#         self.d_model = d_model\n        \n#         # Calculate the input size for the transformer\n#         d_in = input_shape[0] * input_shape[1] * input_shape[2]  # Depth x Height x Width\n#         self.embedding = nn.Linear(d_in, d_model)\n        \n#         self.transformer = Transformer(\n#             d_model=d_model,\n#             nhead=nhead,\n#             num_encoder_layers=num_layers,\n#             dim_feedforward=dim_feedforward,\n#             dropout=dropout\n#         )\n        \n#         self.fc = nn.Linear(d_model, num_classes)\n   \n\n#     def forward(self, x):\n#         # Flatten the input and apply linear embedding\n#         x = x.view(x.size(0), -1)\n#         print(\"Before embedding x.shape is \", x.shape)\n#         x = self.embedding(x)\n#         print(\"After embedding x.shape is \", x.shape)\n        \n#         # Reshape to add a third dimension (seq_len)\n#         x = x.unsqueeze(0)\n#         print(\"x shape after unsqueeze\", x.shape)\n#         # Create a dummy target tensor (you can adjust its size if needed)\n#         tgt = torch.zeros(1, x.size(1), self.d_model).to(x.device)\n#         print(\"tgt shape\", tgt.shape)\n        \n#         # Transformer encoder\n#         output = self.transformer(x, tgt)\n#         print(\"Output shape after transformer\", output.shape)\n\n#         # Remove the added dimension\n# #         output = output.squeeze(0)\n# #         print(\"Output shape after squeeze\", output.shape)\n        \n# #         # Global average pooling\n# #         output = output.mean(dim=1)\n# #         print(\"Output shape after global average pooling\", output.shape)\n\n#         # Classification layer\n#         logits = self.fc(output)\n        \n#         # Add batch dimension to logits\n#         logits = logits.unsqueeze(0)\n        \n        \n#         return logits\n\n# # Define XLA tensors for input and hidden layer sizes\n# # input_size = torch.tensor(32, device=xm.xla_device())\n# input_size = 32 # Adjust the dimensions as needed\n# hidden_size = 16  # Adjust the dimensions as needed\n# # hidden_size = torch.tensor(16, device=xm.xla_device())\n#   # Number of classes\n# import torch\n# import torch.nn as nn\n# import torch_xla.core.xla_model as xm\n\n# # Assuming you have a Transformer class implementation\n\n\n# # Modified version of the Convolutional Vision Transformer\n# class ConvViTBackbone(nn.Module):\n#     def __init__(self, in_channels, embedding_dim, num_heads, num_layers, patch_size, image_size):\n#         super(ConvViTBackbone, self).__init__()\n\n#         # Convolutional layer as the initial layer\n#         self.conv_layer = nn.Conv2d(in_channels, embedding_dim, kernel_size=patch_size, stride=patch_size)\n        \n#         # Adjust image size based on patch size\n#         image_size //= patch_size\n\n#         # Flatten the output\n#         self.flatten = nn.Flatten()\n\n#         # Transformer\n#         self.transformer = Transformer(\n#             d_model=embedding_dim,\n#             nhead=num_heads,\n#             num_encoder_layers=num_layers,\n#             dim_feedforward=embedding_dim * 4,  # Adjust as needed\n#             dropout=0.1\n#         )\n\n#     def forward(self, x):\n#         # Apply the convolutional layer\n#         x = self.conv_layer(x)\n\n#         # Flatten the output\n#         x = self.flatten(x)\n\n#         # Reshape to add a third dimension (seq_len)\n#         x = x.unsqueeze(0)\n\n#         # Create a dummy target tensor\n#         tgt = torch.zeros(1, x.size(1), x.size(2)).to(x.device)\n\n#         # Transformer encoder\n#         output = self.transformer(x, tgt)\n\n#         return output\n\n# # Assuming your Custom3DViTModelTPU class remains the same\n# class Custom3DViTModelTPU(nn.Module):\n#     def __init__(self, in_channels, num_classes, batch_size):\n#         super(Custom3DViTModelTPU, self).__init__()\n#         self.batch_size = batch_size\n#         self.num_classes = num_classes\n\n#         # Assuming input size of 128x128x128 and patch size of 32\n#         self.vit_backbone = ConvViTBackbone(\n#             in_channels=in_channels,\n#             embedding_dim=32,\n#             num_heads=8,\n#             num_layers=6,\n#             patch_size=32,\n#             image_size=128\n#         )\n\n#         self.classification_head = nn.Sequential(\n#             nn.Linear(self.vit_backbone.transformer.d_model, num_classes)\n#         )\n\n#     def forward(self, x):\n#         print(\"x shape\", x.shape)\n\n#         # Move input tensors to XLA devices\n#         x = x.to(xm.xla_device())\n\n#         features = self.vit_backbone(x)\n#         features = features.to(xm.xla_device())\n#         print(\"features shape\", features.shape)\n\n#         # Reshape it to (32, 10)\n#         classification_output = features.view(self.batch_size, self.num_classes)\n\n#         print(\"classification_output\", classification_output.shape)\n\n#         return classification_output\n\n# batch_size = 32\n\n# # Move the entire model to XLA devices\n# def get_model():\n#     return Custom3DViTModelTPU(1, 14, batch_size),UNet(1,5)\n# # Modify the run function to accept the process index\n# def run(index):\n#     print(\"^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\")\n# #     l_in = torch.randn(10, device=xm.xla_device())\n# #     linear = torch.nn.Linear(10, 20).to(xm.xla_device())\n# #     l_out = linear(l_in)\n# #     print(l_out)\n    \n#     model_class = get_model()[0]\n#     model_seg=get_model()[1]\n# #     model.print_weights()\n#     model_class = model_class.to(xm.xla_device())\n#     model_seg = model_seg.to(xm.xla_device())\n#     print(\">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\")\n#     # Create sample input tensors (modify this according to your data)\n#     batch_images = torch.randn(32, 1, 128, 128, 128)  # Example input shape\n#     batch_segmentation_masks = torch.randn(32, 1, 128, 128, 128)  # Example mask shape\n\n#     batch_images = batch_images.to(xm.xla_device())  # Move input tensors to XLA device\n#     batch_segmentation_masks = batch_segmentation_masks.to(xm.xla_device())\n#     print(\"*****************************************************************************\")\n\n#     # Forward pass\n#     classification_outputs = model_class(batch_images)\n#     segmentation_outputs=model_seg(batch_segmentation_masks)\n    \n# # # Use XLA multiprocessing to distribute across TPUs\n# if __name__ == '__main__':\n#      xmp.spawn(run, nprocs=1, start_method='fork')\n","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-10-02T10:42:01.548034Z","iopub.execute_input":"2024-10-02T10:42:01.548553Z","iopub.status.idle":"2024-10-02T10:42:01.561816Z","shell.execute_reply.started":"2024-10-02T10:42:01.54852Z","shell.execute_reply":"2024-10-02T10:42:01.56095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.nn import TransformerEncoder, TransformerEncoderLayer\n\nclass PatchEmbed3D(nn.Module):\n    def __init__(self, in_channels=1, patch_size=(16,16,16), embed_dim=768):\n        super(PatchEmbed3D, self).__init__()\n        self.patch_size = patch_size\n        self.embed_dim = embed_dim\n        self.conv = nn.Conv3d(in_channels, embed_dim, kernel_size=patch_size, stride=patch_size)\n    \n    def forward(self, x):\n        x = self.conv(x)  # [batch_size, embed_dim, D_patch, H_patch, W_patch]\n        x = x.flatten(2)  # [batch_size, embed_dim, num_patches]\n        x = x.transpose(1, 2)  # [batch_size, num_patches, embed_dim]\n        return x\n\nclass Transformer3DClassifier(nn.Module):\n    def __init__(self, \n                 in_channels=1, \n                 num_classes=14,  # Đảm bảo num_classes phù hợp với dữ liệu của bạn\n                 patch_size=(16,16,16), \n                 embed_dim=768, \n                 num_heads=12, \n                 num_layers=12, \n                 dim_feedforward=3072, \n                 dropout=0.1):\n        super(Transformer3DClassifier, self).__init__()\n        self.patch_embed = PatchEmbed3D(in_channels, patch_size, embed_dim)\n        num_patches = (128 // patch_size[0]) * (128 // patch_size[1]) * (128 // patch_size[2])  # Ví dụ: (128/16)^3=512\n        \n        # [CLS] token\n        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))\n        \n        # Positional encoding\n        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim))\n        self.pos_drop = nn.Dropout(p=dropout)\n        \n        # Transformer Encoder với batch_first=True\n        encoder_layers = TransformerEncoderLayer(\n            d_model=embed_dim, \n            nhead=num_heads, \n            dim_feedforward=dim_feedforward, \n            dropout=dropout,\n            batch_first=True  # Thiết lập batch_first=True\n        )\n        self.transformer_encoder = TransformerEncoder(encoder_layers, num_layers)\n        \n        # Classification head\n        self.norm = nn.LayerNorm(embed_dim)\n        self.fc = nn.Linear(embed_dim, num_classes)\n        \n        # Initialize weights\n        self._init_weights()\n    \n    def _init_weights(self):\n        nn.init.trunc_normal_(self.pos_embed, std=0.02)\n        nn.init.trunc_normal_(self.cls_token, std=0.02)\n        nn.init.xavier_uniform_(self.fc.weight)\n        if self.fc.bias is not None:\n            nn.init.zeros_(self.fc.bias)\n    \n    def forward(self, x):\n        batch_size = x.size(0)\n        x = self.patch_embed(x)  # [batch_size, num_patches, embed_dim]\n        \n        # Prepend [CLS] token vào mỗi mẫu\n        cls_tokens = self.cls_token.expand(batch_size, -1, -1)  # [batch_size, 1, embed_dim]\n        x = torch.cat((cls_tokens, x), dim=1)  # [batch_size, 1 + num_patches, embed_dim]\n        \n        # Thêm positional encoding\n        x = x + self.pos_embed[:, :x.size(1), :]\n        x = self.pos_drop(x)\n        \n        # Không cần transpose vì batch_first=True\n        x = self.transformer_encoder(x)  # [batch_size, 1 + num_patches, embed_dim]\n        \n        # Lấy output của [CLS] token để phân loại\n        cls_output = x[:,0]  # [batch_size, embed_dim]\n        cls_output = self.norm(cls_output)\n        logits = self.fc(cls_output)  # [batch_size, num_classes]\n        return logits\n","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:42:01.562807Z","iopub.execute_input":"2024-10-02T10:42:01.563053Z","iopub.status.idle":"2024-10-02T10:42:04.48436Z","shell.execute_reply.started":"2024-10-02T10:42:01.563031Z","shell.execute_reply":"2024-10-02T10:42:04.483414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torch.nn as nn\n\n# class ConvolutionalVisionTransformer(nn.Module):\n#     def __init__(self, in_channels, num_classes, patch_size=16, dim=16, num_layers=6, num_heads=8, dim_feedforward=2048, dropout=0.1):\n#         super(ConvolutionalVisionTransformer, self).__init__()\n\n#         # Patch embedding layer\n#         self.patch_embedding = nn.Conv3d(in_channels, dim, kernel_size=patch_size, stride=patch_size)\n\n#         # Calculate number of patches\n#         self.num_patches = (int((128 - patch_size) / patch_size) + 1) ** 3\n\n#         # Positional embedding (corrected shape)\n#         self.positional_embedding = nn.Parameter(torch.zeros(1, self.num_patches, dim))  # Removed extra dimension\n\n\n#         # Transformer encoder\n#         self.transformer = nn.TransformerEncoder(\n#             nn.TransformerEncoderLayer(d_model=dim, nhead=num_heads, dim_feedforward=dim_feedforward, dropout=dropout),\n#             num_layers=num_layers\n#         )\n\n#         # Classification head\n#         self.classification_head = nn.Linear(dim, num_classes)\n\n#     def forward(self, x):\n#         # Patchify and flatten\n#         x = self.patch_embedding(x)\n#         x = x.flatten(2).transpose(1, 2)\n\n#         # Add positional embedding\n#         x = x + self.positional_embedding[:, :x.size(1)]  # Align to the number of extracted patches\n\n#         # Transformer encoding\n#         x = self.transformer(x)\n\n#         # Classification head\n#         logits = self.classification_head(x[:, 0, :])  # Use output from the first token\n\n#         return logits\n\n# # Define the model\n# model = ConvolutionalVisionTransformer(1, 14)\n# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# model = model.to(device)\n\n# # Create sample input tensors\n# batch_images = torch.randn(32, 1, 128, 128, 128).to(device)\n\n# # Forward pass\n# classification_outputs = model(batch_images)\n# print(classification_outputs.shape)  # Output shape: (32, 14)\n# print(classification_outputs)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-10-02T10:42:04.485558Z","iopub.execute_input":"2024-10-02T10:42:04.485951Z","iopub.status.idle":"2024-10-02T10:42:04.491551Z","shell.execute_reply.started":"2024-10-02T10:42:04.485925Z","shell.execute_reply":"2024-10-02T10:42:04.490672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install einops\n","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:42:04.492726Z","iopub.execute_input":"2024-10-02T10:42:04.493209Z","iopub.status.idle":"2024-10-02T10:42:16.27717Z","shell.execute_reply.started":"2024-10-02T10:42:04.493184Z","shell.execute_reply":"2024-10-02T10:42:16.276228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%pip install --upgrade jupyter jupyterlab\n","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:42:16.278691Z","iopub.execute_input":"2024-10-02T10:42:16.279014Z","iopub.status.idle":"2024-10-02T10:42:35.045536Z","shell.execute_reply.started":"2024-10-02T10:42:16.278984Z","shell.execute_reply":"2024-10-02T10:42:35.044467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# import torch.nn as nn\n\n# class ConvolutionalVisionTransformer(nn.Module):\n#     def __init__(self, in_channels, num_classes, patch_size=16, dim=16, num_layers=6, num_heads=8, dim_feedforward=2048, dropout=0.1):\n#         super(ConvolutionalVisionTransformer, self).__init__()\n\n#         # Patch embedding layer\n#         self.patch_embedding = nn.Conv2d(in_channels, dim, kernel_size=patch_size, stride=patch_size)\n\n#         # Calculate number of patches\n#         self.num_patches = (int((128 - patch_size) / patch_size) + 1) ** 2\n\n#         # Positional embedding\n#         self.positional_embedding = nn.Parameter(torch.zeros(1, self.num_patches, dim))\n\n#         # Convolutional layers\n#         self.conv_layers = nn.Sequential(\n#             nn.Conv2d(dim, dim, kernel_size=3, stride=1, padding=1),\n#             nn.BatchNorm2d(dim),\n#             nn.ReLU(),\n#             nn.Conv2d(dim, dim, kernel_size=3, stride=1, padding=1),\n#             nn.BatchNorm2d(dim),\n#             nn.ReLU(),\n#             nn.Conv2d(dim, dim, kernel_size=3, stride=1, padding=1),\n#             nn.BatchNorm2d(dim),\n#             nn.ReLU()\n#         )\n\n#         # Transformer encoder\n#         self.transformer = nn.TransformerEncoder(\n#             nn.TransformerEncoderLayer(d_model=dim, nhead=num_heads, dim_feedforward=dim_feedforward, dropout=dropout),\n#             num_layers=num_layers\n#         )\n\n#         # Classification head\n#         self.classification_head = nn.Linear(dim, num_classes)\n\n#     def forward(self, x):\n#         # Patchify\n#         x = self.patch_embedding(x)\n\n#         # Add positional embedding\n#         x = x + self.positional_embedding[:, :x.size(2), :x.size(3)]\n\n#         # Convolutional layers\n#         x = self.conv_layers(x)\n\n#         # Flatten and transpose\n#         x = x.flatten(2).transpose(1, 2)\n\n#         # Transformer encoding\n#         x = self.transformer(x)\n\n#         # Classification head\n#         logits = self.classification_head(x[:, 0, :])  # Use output from the first token\n\n#         return logits\n\n# # Define the model\n# model = ConvolutionalVisionTransformer(1, 14)\n# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# model = model.to(device)\n\n# # Create sample input tensors\n# batch_images = torch.randn(32, 1, 128, 128,128).to(device)\n\n# # Forward pass\n# classification_outputs = model(batch_images)\n# print(classification_outputs.shape)  # Output shape: (32, 14)\n# print(classification_outputs)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-10-02T10:42:35.047044Z","iopub.execute_input":"2024-10-02T10:42:35.047375Z","iopub.status.idle":"2024-10-02T10:42:35.054731Z","shell.execute_reply.started":"2024-10-02T10:42:35.047328Z","shell.execute_reply":"2024-10-02T10:42:35.053737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport nibabel as nib\nfrom scipy.ndimage import zoom\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import KFold\nfrom torchvision import transforms\nfrom tqdm import tqdm\n\n# 1. Định Nghĩa CustomDataset\nclass CustomDataset(Dataset):\n    def __init__(self, file_paths, labels, transform=None, desired_shape=(128, 128, 128)):\n        \"\"\"\n        Args:\n            file_paths (list): Danh sách đường dẫn đến các file .npy đã được tiền xử lý.\n            labels (numpy.ndarray): Mảng nhãn tương ứng với các file.\n            transform (callable, optional): Biến đổi sẽ được áp dụng lên ảnh.\n            desired_shape (tuple, optional): Kích thước mong muốn của ảnh sau khi thay đổi kích thước.\n        \"\"\"\n        self.file_paths = file_paths\n        self.labels = labels\n        self.transform = transform\n        self.desired_shape = desired_shape\n\n    def __len__(self):\n        return len(self.file_paths)\n\n    def __getitem__(self, idx):\n        file_path = self.file_paths[idx]\n        label = self.labels[idx]\n\n        # Tải ảnh đã được tiền xử lý từ file .npy\n        image = np.load(file_path)\n        \n        # Nếu cần thay đổi kích thước (mặc định đã được thay đổi trong tiền xử lý)\n        if self.desired_shape and image.shape != self.desired_shape:\n            factors = (\n                self.desired_shape[0] / image.shape[0],\n                self.desired_shape[1] / image.shape[1],\n                self.desired_shape[2] / image.shape[2]\n            )\n            image = zoom(image, factors, order=3)\n            image = image.astype(np.float32)\n\n        # Chuyển đổi numpy array thành tensor và thêm chiều kênh\n        image = torch.from_numpy(image).float().unsqueeze(0)  # [1, D, H, W]\n\n        # Áp dụng biến đổi nếu có\n        if self.transform:\n            image = self.transform(image)\n\n        # Chuyển đổi nhãn thành tensor\n        label = torch.from_numpy(label).float()\n\n        return image, label\n\n# 2. Định Nghĩa Simple3DModel\nclass Simple3DModel(nn.Module):\n    def __init__(self, num_classes=14, input_shape=(128, 128, 128)):\n        super(Simple3DModel, self).__init__()\n        self.conv1 = nn.Conv3d(in_channels=1, out_channels=16, kernel_size=3, padding=1)\n        self.pool = nn.MaxPool3d(kernel_size=2)\n        self.conv2 = nn.Conv3d(in_channels=16, out_channels=32, kernel_size=3, padding=1)\n        # Tính kích thước sau các lớp convolution và pooling\n        self.fc1 = nn.Linear(32 * (input_shape[0] // 4) * (input_shape[1] // 4) * (input_shape[2] // 4), 128)\n        self.fc2 = nn.Linear(128, num_classes)\n\n    def forward(self, x):\n        x = torch.relu(self.conv1(x))  # [batch, 16, 128, 128, 128]\n        x = self.pool(x)               # [batch, 16, 64, 64, 64]\n        x = torch.relu(self.conv2(x))  # [batch, 32, 64, 64, 64]\n        x = self.pool(x)               # [batch, 32, 32, 32, 32]\n        x = x.view(x.size(0), -1)      # [batch, 32*32*32*32]\n        x = torch.relu(self.fc1(x))    # [batch, 128]\n        x = self.fc2(x)                # [batch, num_classes]\n        return x\n\n# 3. Định Nghĩa train_validate_fold\ndef train_validate_fold(fold, train_loader, val_loader, model, criterion, optimizer, device, num_epochs):\n    \"\"\"\n    Huấn luyện và đánh giá mô hình cho một fold cụ thể.\n    \n    Args:\n        fold (int): Số thứ tự fold.\n        train_loader (DataLoader): DataLoader cho tập huấn luyện.\n        val_loader (DataLoader): DataLoader cho tập kiểm tra.\n        model (nn.Module): Mô hình neural network.\n        criterion (nn.Module): Hàm loss.\n        optimizer (torch.optim.Optimizer): Optimizer.\n        device (torch.device): Thiết bị (CPU/GPU).\n        num_epochs (int): Số epoch huấn luyện.\n    \n    Returns:\n        dict: Các metric của fold này.\n    \"\"\"\n    metrics = {'train_loss': [], 'val_loss': [], 'train_accuracy': [], 'val_accuracy': []}\n\n    for epoch in range(num_epochs):\n        # Giai đoạn Huấn luyện\n        model.train()\n        running_loss = 0.0\n        correct = 0\n        total = 0\n\n        for images, labels in train_loader:\n            images, labels = images.to(device), labels.to(device)\n\n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n\n            running_loss += loss.item()\n\n            preds = (torch.sigmoid(outputs) > 0.5).float()\n            correct += (preds == labels).sum().item()\n            total += labels.numel()\n\n        train_loss = running_loss / len(train_loader)\n        train_accuracy = correct / total\n        metrics['train_loss'].append(train_loss)\n        metrics['train_accuracy'].append(train_accuracy)\n\n        # Giai đoạn Đánh giá\n        model.eval()\n        val_loss = 0.0\n        val_correct = 0\n        val_total = 0\n\n        with torch.no_grad():\n            for val_images, val_labels in val_loader:\n                val_images, val_labels = val_images.to(device), val_labels.to(device)\n                val_outputs = model(val_images)\n                loss = criterion(val_outputs, val_labels)\n                val_loss += loss.item()\n\n                val_preds = (torch.sigmoid(val_outputs) > 0.5).float()\n                val_correct += (val_preds == val_labels).sum().item()\n                val_total += val_labels.numel()\n\n        val_loss /= len(val_loader)\n        val_accuracy = val_correct / val_total\n        metrics['val_loss'].append(val_loss)\n        metrics['val_accuracy'].append(val_accuracy)\n\n        print(f\"Fold {fold + 1}, Epoch [{epoch + 1}/{num_epochs}] | \"\n              f\"Train Loss: {train_loss:.4f}, Train Acc: {train_accuracy:.4f} | \"\n              f\"Val Loss: {val_loss:.4f}, Val Acc: {val_accuracy:.4f}\")\n\n    return {\n        'train_loss': np.mean(metrics['train_loss']),\n        'val_loss': np.mean(metrics['val_loss']),\n        'train_accuracy': np.mean(metrics['train_accuracy']),\n        'val_accuracy': np.mean(metrics['val_accuracy']),\n    }\n\n# 4. Định Nghĩa preprocess_files\ndef preprocess_files(file_path, preprocessed_data_dir, desired_shape=(128, 128, 128)):\n    \"\"\"\n    Tiền xử lý một file NIfTI và lưu nó dưới dạng file .npy.\n    \n    Args:\n        file_path (str): Đường dẫn đến file NIfTI.\n        preprocessed_data_dir (str): Đường dẫn đến thư mục lưu trữ các file đã được tiền xử lý.\n        desired_shape (tuple, optional): Kích thước mong muốn của ảnh sau khi thay đổi kích thước.\n    \"\"\"\n    if not os.path.exists(preprocessed_data_dir):\n        os.makedirs(preprocessed_data_dir, exist_ok=True)\n\n    file_name = os.path.basename(file_path)\n    preprocessed_file_path = os.path.join(preprocessed_data_dir, file_name.replace('.nii.gz', '.npy').replace('.nii', '.npy'))\n\n    if os.path.exists(preprocessed_file_path):\n        # File đã được tiền xử lý, bỏ qua\n        return\n\n    try:\n        image = nib.load(file_path).get_fdata()\n        factors = (\n            desired_shape[0] / image.shape[0],\n            desired_shape[1] / image.shape[1],\n            desired_shape[2] / image.shape[2]\n        )\n        resized_image = zoom(image, factors, order=3)\n        resized_image = resized_image.astype(np.float32)\n        np.save(preprocessed_file_path, resized_image)\n    except Exception as e:\n        print(f\"Error processing {file_name}: {e}\")\n\n# 5. Định Nghĩa hàm main\ndef main():\n    # Đường dẫn và cài đặt\n    csv_file = '/kaggle/input/unhealthy-csv-file/combined_data (6).csv'\n    raw_data_dir = '/kaggle/input/abdominal-trauma-nii-csv'\n    preprocessed_data_dir = '/kaggle/working/preprocessed_data/'\n    desired_shape = (128, 128, 128)\n    K = 4  # Số lượng folds\n    num_epochs = 100\n    batch_size = 16\n    num_workers = 4\n    num_classes = 14\n\n    # Định nghĩa các biến đổi (transform)\n    transform = transforms.Compose([\n        transforms.Normalize(mean=[0.5], std=[0.5])  # Ví dụ: Chuẩn hóa dữ liệu\n        # Bạn có thể thêm các biến đổi khác nếu cần\n    ])\n\n    # Đọc file CSV\n    data = pd.read_csv(csv_file)\n    data.columns = data.columns.str.strip()\n    file_paths = data['file_path'].values\n    labels = data[['bowel_healthy','bowel_injury','extravasation_healthy','extravasation_injury',\n                  'kidney_healthy','kidney_low','kidney_high','liver_healthy','liver_low',\n                  'liver_high','spleen_healthy','spleen_low','spleen_high','any_injury']].values\n\n    # Kiểm tra số lượng file trong thư mục gốc\n    raw_files = os.listdir(raw_data_dir)\n    print(f\"Số lượng file trong thư mục dữ liệu gốc: {len(raw_files)}\")\n\n    # Tiền xử lý dữ liệu: tiền xử lý từng file theo file_paths trong CSV\n    print(\"Bắt đầu tiền xử lý dữ liệu...\")\n    for file_path in tqdm(file_paths, desc=\"Preprocessing files\"):\n        full_file_path = os.path.join(raw_data_dir, file_path)\n        if not os.path.exists(full_file_path):\n            print(f\"File không tồn tại: {full_file_path}\")\n            continue\n        preprocess_files(full_file_path, preprocessed_data_dir, desired_shape)\n    print(\"Tiền xử lý dữ liệu hoàn thành.\")\n\n    # Cập nhật đường dẫn file_paths sau khi tiền xử lý\n    preprocessed_paths = [\n        os.path.join(preprocessed_data_dir, os.path.basename(fp).replace('.nii.gz', '.npy').replace('.nii', '.npy'))\n        for fp in file_paths\n    ]\n\n    # Kiểm tra sự tồn tại của các file đã tiền xử lý\n    missing_files = [fp for fp in preprocessed_paths if not os.path.exists(fp)]\n    if missing_files:\n        print(f\"Có {len(missing_files)} file chưa được tiền xử lý hoặc không tồn tại.\")\n        # Loại bỏ các file thiếu\n        preprocessed_paths = [fp for fp in preprocessed_paths if os.path.exists(fp)]\n        labels = labels[:len(preprocessed_paths)]\n    else:\n        print(\"Tất cả các file đã được tiền xử lý.\")\n\n    # Kiểm tra số lượng file đã tiền xử lý\n    print(f\"Số lượng file đã được tiền xử lý: {len(preprocessed_paths)}\")\n\n    if len(preprocessed_paths) == 0:\n        print(\"Không có file đã được tiền xử lý. Vui lòng kiểm tra lại.\")\n        return\n\n    # Điều chỉnh K dựa trên số lượng file\n    K = min(K, len(preprocessed_paths))  \n    kf = KFold(n_splits=K, shuffle=True, random_state=42)\n\n    # Khởi tạo danh sách để lưu trữ metrics cho mỗi fold\n    fold_metrics = []\n\n    # Khởi tạo thiết bị (CPU hoặc GPU)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    if device.type == 'cpu':\n        print(\"CUDA is not available. Using CPU.\")\n\n    # Chia dữ liệu thành K fold và huấn luyện mô hình\n    for fold, (train_idx, val_idx) in enumerate(kf.split(preprocessed_paths)):\n        print(f\"\\n=== Fold {fold + 1}/{K} ===\")\n\n        # Lấy đường dẫn và nhãn cho fold hiện tại\n        train_paths = [preprocessed_paths[i] for i in train_idx]\n        train_labels = labels[train_idx]\n        val_paths = [preprocessed_paths[i] for i in val_idx]\n        val_labels = labels[val_idx]\n\n        # Tạo Dataset và DataLoader cho tập huấn luyện và kiểm tra\n        train_dataset = CustomDataset(train_paths, train_labels, transform=transform, desired_shape=desired_shape)\n        val_dataset = CustomDataset(val_paths, val_labels, transform=transform, desired_shape=desired_shape)\n        \n        train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\n        val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\n        # Initialize model, criterion, optimizer\n        model = Simple3DModel(num_classes=num_classes, input_shape=desired_shape).to(device)\n        criterion = nn.BCEWithLogitsLoss()\n        optimizer = optim.Adam(model.parameters(), lr=0.001)\n\n        # Huấn luyện và đánh giá cho fold hiện tại\n        metrics = train_validate_fold(\n            fold=fold,\n            train_loader=train_loader,\n            val_loader=val_loader,\n            model=model,\n            criterion=criterion,\n            optimizer=optimizer,\n            device=device,\n            num_epochs=num_epochs\n        )\n        fold_metrics.append(metrics)\n\n    # Tính toán và in ra kết quả trung bình của K-Fold Cross-Validation\n    avg_train_loss = np.mean([fold['train_loss'] for fold in fold_metrics])\n    avg_val_loss = np.mean([fold['val_loss'] for fold in fold_metrics])\n    avg_train_accuracy = np.mean([fold['train_accuracy'] for fold in fold_metrics])\n    avg_val_accuracy = np.mean([fold['val_accuracy'] for fold in fold_metrics])\n\n    print(\"\\n=== K-Fold Cross-Validation Results ===\")\n    print(f\"Average Train Loss: {avg_train_loss:.4f}\")\n    print(f\"Average Validation Loss: {avg_val_loss:.4f}\")\n    print(f\"Average Train Accuracy: {avg_train_accuracy:.4f}\")\n    print(f\"Average Validation Accuracy: {avg_val_accuracy:.4f}\")\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-02T10:42:35.057575Z","iopub.execute_input":"2024-10-02T10:42:35.057897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_average_accuracy(fold_metrics, num_epochs, K):\n    epochs = range(1, num_epochs + 1)\n    \n    train_acc_all_folds = np.array([fold_metrics[fold]['train_accuracy'] for fold in range(K)])\n    val_acc_all_folds = np.array([fold_metrics[fold]['val_accuracy'] for fold in range(K)])\n    \n    avg_train_acc = np.mean(train_acc_all_folds, axis=0)\n    avg_val_acc = np.mean(val_acc_all_folds, axis=0)\n    \n    plt.figure(figsize=(12, 6))\n    plt.plot(epochs, avg_train_acc, label='Average Train Accuracy', marker='o')\n    plt.plot(epochs, avg_val_acc, label='Average Validation Accuracy', marker='o')\n    \n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.title('Average Train and Validation Accuracy per Epoch')\n    plt.legend()\n    plt.grid(True)\n    plt.show()\n\n# Sau khi huấn luyện xong, gọi hàm vẽ đồ thị trung bình:\nplot_average_accuracy(fold_metrics, num_epochs, K)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}