{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"},{"sourceId":6984590,"sourceType":"datasetVersion","datasetId":4014175},{"sourceId":7280378,"sourceType":"datasetVersion","datasetId":4221234},{"sourceId":7289631,"sourceType":"datasetVersion","datasetId":4227609},{"sourceId":7289632,"sourceType":"datasetVersion","datasetId":4227610},{"sourceId":7312403,"sourceType":"datasetVersion","datasetId":4229785},{"sourceId":157189984,"sourceType":"kernelVersion"}],"dockerImageVersionId":30626,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport torch\nfrom torch.utils.data import Dataset, random_split, DataLoader\nfrom torchvision import transforms\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch.optim as optim\n\nfrom PIL import Image, ImageChops, ImageDraw\nImage.MAX_IMAGE_PIXELS = None\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\n\nimport os\n\nimport shutil\n\nimport re","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:08.188158Z","iopub.execute_input":"2023-12-30T10:42:08.189065Z","iopub.status.idle":"2023-12-30T10:42:10.926101Z","shell.execute_reply.started":"2023-12-30T10:42:08.189022Z","shell.execute_reply":"2023-12-30T10:42:10.924606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    is_submission = True\n    \n    # Training\n    train_csv_path = \"/kaggle/input/UBC-OCEAN/train.csv\"\n    train_thumbnail_paths = \"/kaggle/input/UBC-OCEAN/train_thumbnails\"\n    train_image_paths = \"/kaggle/input/ubc-train-images-4096x4096\"\n    train_resized_part1 = '/kaggle/input/ubc-train-imgs-8192x8192-part1'\n    train_resized_part2 = '/kaggle/input/ubc-train-imgs-8192x8192-part2'\n    train_from_mask_paths = '/kaggle/input/ubc-data-from-masks'\n    batch_size = 8\n    learning_rate = 1e-4\n    epochs = 10\n    SEED = 42\n    \n    # Inference\n    test_csv_path = \"/kaggle/input/UBC-OCEAN/test.csv\"\n    test_thumbnail_paths = \"/kaggle/input/UBC-OCEAN/test_thumbnails\"\n    test_images_paths = \"/kaggle/input/UBC-OCEAN/test_images\"\n    test_image_parts = \"/kaggle/working/image_parts\"\n    \n    # Labels encoding\n    label_mapping = {'HGSC': 0, 'LGSC': 1, 'EC': 2, 'CC': 3, 'MC': 4, 'Other': 5}\n    label_remapping = {0: 'HGSC', 1: 'LGSC', 2: 'EC', 3: 'CC', 4: 'MC', 5: 'Other'}\n\n\nconfig = Config()","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:10.928899Z","iopub.execute_input":"2023-12-30T10:42:10.929705Z","iopub.status.idle":"2023-12-30T10:42:10.94067Z","shell.execute_reply.started":"2023-12-30T10:42:10.929653Z","shell.execute_reply":"2023-12-30T10:42:10.939363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not config.is_submission:\n    df = pd.read_csv(config.train_csv_path)\n    \n    # Create the thumbnail df where is_tma == False\n#     df = df[df[\"is_tma\"] == False]\n    \n    # Create train df\n    train_df = df.copy()\n    \n    # Get the thumbnail image paths\n    train_df[\"image_path\"] = \"\"\n#     train_df[\"image_path\"] = train_df.apply(lambda row: f\"{config.train_thumbnail_paths}/{row['image_id']}_thumbnail.png\" if row['is_tma'] == False \n#                                             else f\"{config.train_image_paths}/{row['image_id']}.png\", axis=1)\n#     train_df[\"image_path\"] = train_df.apply(lambda row: f\"{config.train_image_paths}/{row['image_id']}.png\", axis=1)\n    train_df[\"image_path\"] = train_df.apply(lambda row: f\"{config.train_resized_part1}/{row['image_id']}.png\" if os.path.exists(f\"{config.train_resized_part1}/{row['image_id']}.png\")\n                                            else f\"{config.train_resized_part2}/{row['image_id']}.png\" if os.path.exists(f\"{config.train_resized_part2}/{row['image_id']}.png\")\n                                            else f\"{config.train_image_paths}/{row['image_id']}.png\"\n                                            , axis=1)\n    \n    # Get the thumbnail image paths\n#     train_df[\"mask_path\"] = \"\"\n#     train_df[\"mask_path\"] = train_df.apply(lambda row: f\"{config.mask_paths}/{row['image_id']}.png\", axis=1)\n#     # Display data\n#     value_counts = train_df['label'].value_counts()\n#     plt.bar(value_counts.index, value_counts.values)\n#     plt.xlabel('Label')\n#     plt.ylabel('Occurrences')\n#     plt.title('Occurrences of unique values in labels')\n#     plt.grid()\n#     plt.show()\n    \n#     # Display random image\n#     random_row = train_df.sample(1)\n#     image_path = random_row[\"image_path\"].values[0]\n#     image = Image.open(image_path)\n#     plt.imshow(image)\n#     plt.title(f'image_id: {random_row[\"image_id\"].tolist()[0]}, label: {random_row[\"label\"].tolist()[0]}')\n#     plt.axis('off')\n#     plt.show()\n    \n    # Map label\n    train_df['label'] = train_df['label'].map(config.label_mapping)\n\n    print(train_df.head())","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:10.942769Z","iopub.execute_input":"2023-12-30T10:42:10.943606Z","iopub.status.idle":"2023-12-30T10:42:10.955957Z","shell.execute_reply.started":"2023-12-30T10:42:10.943557Z","shell.execute_reply":"2023-12-30T10:42:10.954583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Your custom transform\n# class SingleTransform(object):\n#     def __init__(self, transform):\n#         self.transform = transform\n\n#     def __call__(self, img_path):\n#         img = Image.open(img_path).convert('L')\n#         return self.transform(img)\n\n# # Display image\n# def show_img(img):\n#     plt.imshow(img, cmap='gray')\n#     plt.axis('off')\n#     plt.show()\n#     return img\n\n# # Display image\n# def edit_img(img):\n#     threshold = 150\n    \n#     img = np.array(img).astype('uint8')\n\n#         # Find columns where all cells are below the threshold\n#     columns_to_remove = np.all(img < threshold, axis=0)\n\n#     # Remove the columns\n#     cropped_img = img[:, ~columns_to_remove]\n\n#     img = Image.fromarray(cropped_img)\n#     return img\n\n# # Define transformations\n# transform = transforms.Compose([\n#     # Use the custom function in the Lambda transform\n#     transforms.Lambda(lambda img: show_img(img)),\n#     transforms.Lambda(lambda img: edit_img(img)),\n    \n#     transforms.Resize((256,256)),\n# #     transforms.Lambda(lambda img: show_img(img)),\n    \n#     # Use the custom function in the Lambda transform\n# #     transforms.Lambda(lambda img: show_img(img)),\n    \n#     transforms.ToTensor(),\n#     transforms.Normalize(mean=[0.5], std=[0.5]),\n# ])\n# # 45990\n# # Assuming img_path is the path to your image\n# condition = train_df[\"image_id\"]==10143\n# img_path  = train_df.loc[condition, \"image_path\"].tolist()[0]\n# print(img_path)\n# # Create a single-element dataset and apply the transform\n# single_element_dataset = SingleTransform(transform)\n# resulting_tensor = single_element_dataset(img_path)\n\n# # Print or use the resulting tensor as needed\n# print(resulting_tensor.shape)","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:10.959399Z","iopub.execute_input":"2023-12-30T10:42:10.960801Z","iopub.status.idle":"2023-12-30T10:42:10.975132Z","shell.execute_reply.started":"2023-12-30T10:42:10.960741Z","shell.execute_reply":"2023-12-30T10:42:10.973778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomTrainDataset(Dataset):\n    def __init__(self, file_paths, labels, transform=None):\n        self.file_paths = file_paths\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.file_paths)\n\n    def __getitem__(self, idx):\n        img_path = self.file_paths[idx]\n        image = Image.open(img_path).convert('L')  # Convert to grayscale\n\n        if self.transform:\n            image = self.transform(image)\n\n        label = self.labels[idx]\n        return image, label\n    \nclass CustomTestDataset(Dataset):\n    def __init__(self, file_paths, transform=None):\n        self.file_paths = file_paths\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.file_paths)\n\n    def __getitem__(self, idx):\n        img_path = self.file_paths[idx]\n        image = Image.open(img_path).convert('L')  # Convert to grayscale\n\n        if self.transform:\n            image = self.transform(image)\n\n        return image\n\n# Remove empty rows$cols from image\ndef remove_empty_lines(img):\n    threshold = 150\n    \n    img = np.array(img).astype('uint8')\n\n    # Find columns where all cells are below the threshold\n    columns_to_remove = np.all(img < threshold, axis=0)\n    rows_to_remove = np.all(img < threshold, axis=1)\n\n    # Remove the columns\n    img = img[:, ~columns_to_remove]\n\n    # Remove the rows\n    img = img[~rows_to_remove, :]\n\n    return Image.fromarray(img)\n\n# Define transformations\ntransform = transforms.Compose([\n    # Use the custom function in the Lambda transform\n    transforms.Lambda(lambda img: remove_empty_lines(img)),\n    transforms.Resize((256,256)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.5], std=[0.5]),\n])","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:10.977346Z","iopub.execute_input":"2023-12-30T10:42:10.978316Z","iopub.status.idle":"2023-12-30T10:42:10.999657Z","shell.execute_reply.started":"2023-12-30T10:42:10.978262Z","shell.execute_reply":"2023-12-30T10:42:10.998323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create train dataset loader\nif not config.is_submission:\n#     file_paths = train_df['image_path'].tolist()\n#     labels = train_df['label'].tolist()\n    file_paths = []\n    labels = []\n    \n    # append additional files from masks\n    path_necrosis = os.path.join(config.train_from_mask_paths, 'necrosis')\n    files_necrosis = os.listdir(path_necrosis)\n    files_paths_necrosis = [os.path.join(path_necrosis, file) for file in files_necrosis]\n    file_paths = file_paths + files_paths_necrosis\n    labels = labels + [5 for file in files_necrosis]\n    \n    path_stroma = os.path.join(config.train_from_mask_paths, 'stroma')\n    files_stroma = os.listdir(path_stroma)\n    files_paths_stroma = [os.path.join(path_stroma, file) for file in files_stroma]\n    file_paths = file_paths + files_paths_stroma\n    labels = labels + [5 for file in files_stroma]\n    \n    path_tumor = os.path.join(config.train_from_mask_paths, 'tumor')\n    files_tumor = os.listdir(path_tumor)\n    files_paths_tumor = [os.path.join(path_tumor, file) for file in files_tumor]\n    file_paths = file_paths + files_paths_tumor\n    labels = labels + [train_df[train_df['image_id'] == int(os.path.basename(file).split('_')[0])].reset_index().at[0, 'label'] for file in files_tumor]\n\n    # Create the custom dataset\n    train_dataset = CustomTrainDataset(file_paths=file_paths, labels=labels, transform=transform)\n\n    # Example of accessing data\n    for i in range(min(2, len(train_dataset))):\n        sample, label = train_dataset[i]\n        print(f\"Sample {i + 1}: {sample.shape}, Label: {label}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:11.001967Z","iopub.execute_input":"2023-12-30T10:42:11.002941Z","iopub.status.idle":"2023-12-30T10:42:11.020314Z","shell.execute_reply.started":"2023-12-30T10:42:11.002887Z","shell.execute_reply":"2023-12-30T10:42:11.018736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dropout added\nclass Model(nn.Module):\n    def __init__(self, num_classes, dropout_prob=0.5):\n        super(Model, self).__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, stride=1, padding=1)\n        self.bn1 = nn.BatchNorm2d(16)\n        self.pool = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=3, stride=1, padding=1)\n        self.bn2 = nn.BatchNorm2d(32)\n        \n        # Adjusted the input size of the linear layer based on 256x256 image size\n        self.fc1 = nn.Linear(32 * 64 * 64, 128)\n        self.bn_fc1 = nn.BatchNorm1d(128)\n        self.dropout = nn.Dropout(p=dropout_prob)\n        self.fc2 = nn.Linear(128, num_classes)\n\n    def forward(self, x):\n        x = self.pool(F.relu(self.bn1(self.conv1(x))))\n        x = self.pool(F.relu(self.bn2(self.conv2(x))))\n        \n        # Adjusted the view size dynamically based on the output size of convolutions\n        x = x.view(-1, 32 * 64 * 64)\n        \n        x = F.relu(self.bn_fc1(self.fc1(x)))\n        x = self.dropout(x)\n        x = self.fc2(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:11.022504Z","iopub.execute_input":"2023-12-30T10:42:11.023032Z","iopub.status.idle":"2023-12-30T10:42:11.038784Z","shell.execute_reply.started":"2023-12-30T10:42:11.022982Z","shell.execute_reply":"2023-12-30T10:42:11.037679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate class weight func\ndef calculate_class_weights(data_loader, num_classes):\n    class_counts = torch.zeros(num_classes)\n    total_samples = 0\n\n    for _, labels in data_loader:\n        class_counts += torch.bincount(labels, minlength=num_classes)\n        total_samples += len(labels)\n\n    class_weights = total_samples / (num_classes * class_counts)\n\n    return class_weights","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:11.040264Z","iopub.execute_input":"2023-12-30T10:42:11.041778Z","iopub.status.idle":"2023-12-30T10:42:11.057271Z","shell.execute_reply.started":"2023-12-30T10:42:11.041726Z","shell.execute_reply":"2023-12-30T10:42:11.055945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not config.is_submission:\n    # Adjust the train_size and random seed\n    train_size = 0.8\n    random_seed = config.SEED\n    batch_size = config.batch_size\n    epochs = config.epochs\n    num_classes = len(config.label_mapping.keys())\n\n    # Calculate the sizes of the training and testing sets\n    train_size = int(train_size * len(train_dataset))\n    test_size = len(train_dataset) - train_size\n\n    # Use random_split to split the dataset\n    train_data, test_data = random_split(train_dataset, [train_size, test_size], generator=torch.Generator().manual_seed(random_seed))\n\n    # Create a DataLoader for the training dataset\n    batch_size = config.batch_size\n    train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)\n    test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False)\n    \n    # Calculate class weights for training set & normalize\n#     train_class_weights = calculate_class_weights(train_loader, num_classes)\n#     train_class_weights = torch.tensor(train_class_weights) / sum(train_class_weights)\n\n    # Initialize the model, loss function, and optimizer\n    model = Model(num_classes=num_classes)\n    #     criterion = nn.CrossEntropyLoss()\n    criterion = nn.CrossEntropyLoss()#weight=torch.FloatTensor(train_class_weights))\n    optimizer = optim.Adam(model.parameters(), lr=config.learning_rate)\n\n    for epoch in range(epochs):\n        model.train()\n        for batch_X, batch_y in train_loader:\n            optimizer.zero_grad()\n\n            if batch_X.size(0) != batch_y.size(0):\n                continue\n\n            output = model(batch_X)\n            loss = criterion(output, batch_y)\n\n            loss.backward()\n            optimizer.step()\n\n        # Validation loop\n        model.eval()\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for batch_X, batch_y in test_loader:\n                if batch_X.size(0) != batch_y.size(0):\n                    continue\n\n                output = model(batch_X)\n                _, predicted = torch.max(output.data, 1)\n                total += batch_y.size(0)\n                correct += (predicted == batch_y).sum().item()\n\n        accuracy = correct / total\n        print(f'Epoch {epoch + 1}/{epochs}, Loss: {loss.item()}, Accuracy: {accuracy * 100:.2f}%')\n\n    # Save the trained model if needed\n    torch.save(model.state_dict(), 'image_classifier_model.pth')","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:11.059282Z","iopub.execute_input":"2023-12-30T10:42:11.060021Z","iopub.status.idle":"2023-12-30T10:42:11.078405Z","shell.execute_reply.started":"2023-12-30T10:42:11.059976Z","shell.execute_reply":"2023-12-30T10:42:11.077384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create quaters and center of original img\n\ndef get_parts_dimensions(image, rows=4, cols=4):\n    width, height = image.size\n\n    parts = []\n    for row in range(rows):\n        for col in range(cols):\n            left = col * (width // cols)\n            upper = row * (height // rows)\n            right = (col + 1) * (width // cols)\n            lower = (row + 1) * (height // rows)\n\n            part = (left, upper, right, lower)\n            parts.append(part)\n\n    return parts\n\ndef create_parts_and_center(input_path, output_folder):\n    # Open the image\n    with Image.open(input_path) as img:\n        width, height = img.size\n\n        # Calculate the coordinates for the four quarters\n        parts = get_parts_dimensions(img, 3, 3)\n#         [\n#             (0, 0, width // 2, height // 2),  # Top-left quarter\n#             (width // 2, 0, width, height // 2),  # Top-right quarter\n#             (0, height // 2, width // 2, height),  # Bottom-left quarter\n#             (width // 2, height // 2, width, height)  # Bottom-right quarter\n#         ]\n\n        # Extract the filename (without extension) from the path\n        filename = os.path.splitext(os.path.basename(input_path))[0]\n\n        # Create output folder if it doesn't exist\n        if not os.path.exists(output_folder):\n            os.makedirs(output_folder)\n\n        # Save each quarter as a separate PNG\n        for i, (left, top, right, bottom) in enumerate(parts, start=1):\n            part_img = img.crop((left, top, right, bottom)).resize((512,512))\n            part_output_path = os.path.join(output_folder, f\"{filename}_part_{i}.png\")\n            \n            part_img.save(part_output_path)\n\n#             # Display the saved quarter image\n#             plt.imshow(part_img)\n#             plt.title(f\"Saved part {i}\")\n#             plt.show()\n\n        # Calculate the coordinates for the center with a size equal to a quarter\n#         center_left = width // 4\n#         center_top = height // 4\n#         center_right = width * 3 // 4\n#         center_bottom = height * 3 // 4\n\n#         # Save the center image\n#         center_img = img.crop((center_left, center_top, center_right, center_bottom)).resize((512,512))\n#         center_output_path = os.path.join(output_folder, f\"{filename}_center.png\")\n#         center_img.save(center_output_path)\n\n#         # Display the saved center image\n#         plt.imshow(center_img)\n#         plt.title(\"Saved Center Image\")\n#         plt.show()\n\nif config.is_submission:\n    df_parts = pd.read_csv(config.test_csv_path)\n    df_parts[\"image_path\"] = df_parts[\"image_id\"].apply(lambda x: f\"{config.test_thumbnail_paths}/{x}_thumbnail.png\")\n#     df_parts = pd.read_csv(config.train_csv_path)\n#     df_parts = df_parts[df_parts['is_tma'] == False]\n#     df_parts[\"image_path\"] = df_parts[\"image_id\"].apply(lambda x: f\"{config.train_thumbnail_paths}/{x}_thumbnail.png\")\n\n    # Assuming df is your DataFrame with a 'path' column\n    for index, row in df_parts.iterrows():\n        input_path = row['image_path']\n        output_root_folder = config.test_image_parts  \n        create_parts_and_center(input_path, output_root_folder)","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:11.083184Z","iopub.execute_input":"2023-12-30T10:42:11.084153Z","iopub.status.idle":"2023-12-30T10:42:12.456652Z","shell.execute_reply.started":"2023-12-30T10:42:11.084099Z","shell.execute_reply":"2023-12-30T10:42:12.45529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Make predictions on test data\nif config.is_submission:\n    # read images from csv\n    test_df = pd.read_csv(config.test_csv_path)\n    test_df[\"image_path\"] = test_df[\"image_id\"].apply(lambda x: f\"{config.test_thumbnail_paths}/{x}_thumbnail.png\")\n    \n    # Drop all rows\n    test_df = test_df.drop(test_df.index)\n    \n    for i in range(16):\n        # read & merge images from main img\n        df_part = pd.read_csv(config.test_csv_path)\n        df_part[\"image_path\"] = df_part[\"image_id\"].apply(lambda x: f\"{config.test_image_parts}/{x}_thumbnail_part_{i+1}.png\")\n        test_df = pd.concat([test_df, df_part], ignore_index=True)\n\n    # Save test df\n#     test_df = pd.concat([df, df_q1, df_q2, df_q3, df_q4, df_q5, df_q6, df_q7, df_q8, df_q9, df_center], ignore_index=True)\n#     test_df = pd.concat([df_q1, df_q2, df_q3, df_q4, df_q5, df_q6, df_q7, df_q8, df_q9, df_center], ignore_index=True)\n    test_df = test_df.query('image_path.apply(@os.path.exists)', engine='python')\n    \n    # Load the pre-trained model\n    model_path = '/kaggle/input/ubc-test-notebook/image_classifier_model.pth'\n    num_classes = len(config.label_mapping.keys())\n    model = Model(num_classes=num_classes)\n    model.load_state_dict(torch.load(model_path))\n    model.eval()\n\n    # Save list of image paths\n    test_file_paths = test_df[\"image_path\"].tolist()\n\n    # Create the test dataset\n    test_dataset = CustomTestDataset(file_paths=test_file_paths, transform=transform)\n\n    # Create a DataLoader for the test dataset\n    batch_size = config.batch_size\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n\n    # Make predictions\n    predictions = []\n\n    # Set label 'Other' if prediction below threshold\n    threshold = 0.3  \n    default_label = 5  \n\n    with torch.no_grad():\n        for batch_X in test_loader:\n            # Assuming batch_X is a batch of test images\n            output = model(batch_X)\n            probabilities = F.softmax(output, dim=1)\n\n#             # Check if all predicted probabilities are below the threshold\n#             if all(p < threshold for p in probabilities.max(dim=1).values):\n#                 # Use the default label\n#                 predicted_labels = torch.tensor([default_label])\n#             else:\n#                 # Use the predicted labels\n#                 _, predicted_labels = torch.max(probabilities, 1)\n\n            # Use the predicted labels\n            _, predicted_labels = torch.max(probabilities, 1)\n\n            predictions.extend(predicted_labels.cpu().numpy())\n\n        # Now, 'predictions' contains the predicted labels, with default labels assigned when uncertain\n        print(predictions[:10])\n        print(len(predictions))","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:12.458535Z","iopub.execute_input":"2023-12-30T10:42:12.458961Z","iopub.status.idle":"2023-12-30T10:42:14.510583Z","shell.execute_reply.started":"2023-12-30T10:42:12.458926Z","shell.execute_reply":"2023-12-30T10:42:14.509613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if config.is_submission:\n    # remap text labels from integers\n    test_df['label'] = predictions\n    test_df = test_df.groupby('image_id')['label'].apply(lambda x: x.mode().iloc[0]).reset_index()\n    test_df['label'] = test_df['label'].map(config.label_remapping)\n    # Create the submission\n    submission_df = test_df[[\"image_id\", \"label\"]]\n    submission_df.to_csv(\"submission.csv\", index=False)\n    \n    # clear output\n    shutil.rmtree('/kaggle/working/image_parts') if os.path.exists('/kaggle/working/image_parts') else None\n    \n    # Save the trained model for later\n    torch.save(model.state_dict(), 'image_classifier_model.pth')","metadata":{"execution":{"iopub.status.busy":"2023-12-30T10:42:35.255547Z","iopub.execute_input":"2023-12-30T10:42:35.256061Z","iopub.status.idle":"2023-12-30T10:42:35.385093Z","shell.execute_reply.started":"2023-12-30T10:42:35.256025Z","shell.execute_reply":"2023-12-30T10:42:35.384134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}