{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-22T21:50:14.456592Z","iopub.execute_input":"2023-12-22T21:50:14.456969Z","iopub.status.idle":"2023-12-22T21:50:14.463396Z","shell.execute_reply.started":"2023-12-22T21:50:14.456931Z","shell.execute_reply":"2023-12-22T21:50:14.462408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import librarys","metadata":{}},{"cell_type":"code","source":"import os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nimport os\n%matplotlib inline\nfrom PIL import Image\nimport cv2\nImage.MAX_IMAGE_PIXELS = 10000000000\n\nfrom torch.utils.data import Dataset, DataLoader\nimport torch\nfrom torchvision import transforms, models\nimport torchvision\nimport os\nimport torch.nn as nn\nfrom torch import optim\nfrom torch.autograd import Variable\n\nfrom torchinfo import summary","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:14.465509Z","iopub.execute_input":"2023-12-22T21:50:14.46585Z","iopub.status.idle":"2023-12-22T21:50:18.822227Z","shell.execute_reply.started":"2023-12-22T21:50:14.465818Z","shell.execute_reply":"2023-12-22T21:50:18.821305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Select Device","metadata":{}},{"cell_type":"code","source":"device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:18.823461Z","iopub.execute_input":"2023-12-22T21:50:18.823909Z","iopub.status.idle":"2023-12-22T21:50:18.860143Z","shell.execute_reply.started":"2023-12-22T21:50:18.823861Z","shell.execute_reply":"2023-12-22T21:50:18.858412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# File paths","metadata":{}},{"cell_type":"code","source":"test_path = '/kaggle/input/UBC-OCEAN/test.csv'\ntrain_path = '/kaggle/input/UBC-OCEAN/train.csv'\n\n#学習する画像のパス\n# main_pazth = '/kaggle/input/UBC-OCEAN/train_images/'\nmain_path = '/kaggle/input/UBC-OCEAN/train_thumbnails/'","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:18.861508Z","iopub.execute_input":"2023-12-22T21:50:18.861838Z","iopub.status.idle":"2023-12-22T21:50:18.879137Z","shell.execute_reply.started":"2023-12-22T21:50:18.861811Z","shell.execute_reply":"2023-12-22T21:50:18.878056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read csvfile","metadata":{}},{"cell_type":"code","source":"def read_csv(path):\n    df = pd.read_csv(path)\n    print(df.head())\n    print(df.shape)\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:18.882307Z","iopub.execute_input":"2023-12-22T21:50:18.882668Z","iopub.status.idle":"2023-12-22T21:50:18.889603Z","shell.execute_reply.started":"2023-12-22T21:50:18.882634Z","shell.execute_reply":"2023-12-22T21:50:18.888685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = read_csv(train_path)\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:18.891075Z","iopub.execute_input":"2023-12-22T21:50:18.891397Z","iopub.status.idle":"2023-12-22T21:50:18.932753Z","shell.execute_reply.started":"2023-12-22T21:50:18.891365Z","shell.execute_reply":"2023-12-22T21:50:18.931885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_des = train.describe() \nprint(train_des)\n\nwidth_max = int(train_des['image_width'].max())\nheight_max = int(train_des['image_height'].max())\nprint('width_max : ', width_max)\nprint('height_max : ', height_max)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:18.934113Z","iopub.execute_input":"2023-12-22T21:50:18.934387Z","iopub.status.idle":"2023-12-22T21:50:18.952657Z","shell.execute_reply.started":"2023-12-22T21:50:18.934361Z","shell.execute_reply":"2023-12-22T21:50:18.951714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Adjust image size due to different image size.","metadata":{}},{"cell_type":"code","source":"class Adjust_image_size():\n    '''\n    画像サイズを調整\n    '''\n    def __init__(self, target_size, height_max, width_max):\n        self.target_size = target_size\n        if height_max > width_max:\n            self.max_size = height_max\n            self.max_direction = 'heigth'\n        else:\n            self.max_size = width_max\n            self.max_direction = 'width'\n            \n    def __call__(self, image:Image.Image):\n        width, height = image.size\n        if self.max_direction == 'width':\n            s = self.target_size / width\n            target_height = int(s*height)\n            image = image.resize((self.target_size, target_height))\n            img = Image.new(mode='RGB', size=(self.target_size, self.target_size), color=(0,0,0))\n        elif self.max_direction == 'heigth':\n            s = self.target_size / height\n            target_width = int(s*width)\n            image = image.resize((target_width, self.target_size))\n            img = Image.new(mode='RGB', size=(self.target_size, self.target_size), color=(0,0,0))\n        img.paste(im=image)\n        return img","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:18.954366Z","iopub.execute_input":"2023-12-22T21:50:18.954753Z","iopub.status.idle":"2023-12-22T21:50:18.964981Z","shell.execute_reply.started":"2023-12-22T21:50:18.954717Z","shell.execute_reply":"2023-12-22T21:50:18.96361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_size = 512\nadjust = Adjust_image_size(target_size, height_max, width_max)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:18.966662Z","iopub.execute_input":"2023-12-22T21:50:18.966989Z","iopub.status.idle":"2023-12-22T21:50:18.973028Z","shell.execute_reply.started":"2023-12-22T21:50:18.966947Z","shell.execute_reply":"2023-12-22T21:50:18.972217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create a path to load images","metadata":{}},{"cell_type":"code","source":"thumbnail_paths = []\nfor dirname, _, filenames in os.walk(main_path):\n    for filename in filenames:\n        thumbnail_paths.append(os.path.join(dirname, filename))\n        \nthumbnail_paths[1]","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:18.974888Z","iopub.execute_input":"2023-12-22T21:50:18.975291Z","iopub.status.idle":"2023-12-22T21:50:19.022409Z","shell.execute_reply.started":"2023-12-22T21:50:18.975259Z","shell.execute_reply":"2023-12-22T21:50:19.021509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(train)):\n    thum_path = os.path.join(main_path,str(train.loc[i, 'image_id'])+'_thumbnail.png')\n    if thum_path in thumbnail_paths:\n        train.loc[i,'path'] = os.path.join(main_path, str(train.loc[i, 'image_id'])+'_thumbnail.png')\n    else:\n        train.loc[i, 'path']=\"No_files\"\n# train = train.dropna()\ntrain = train[train['path'] !='No_files']\ntrain = train.reset_index()\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.02363Z","iopub.execute_input":"2023-12-22T21:50:19.023992Z","iopub.status.idle":"2023-12-22T21:50:19.153445Z","shell.execute_reply.started":"2023-12-22T21:50:19.023954Z","shell.execute_reply":"2023-12-22T21:50:19.152601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_labels = pd.DataFrame(train['label'].unique(), columns=['label'])\nfor i in range(len(df_labels)):\n    df_labels.loc[i, 'label_count'] = len(train[train['label']== df_labels.iloc[i,0]])\ndf_labels['label_No'] = df_labels.index\nprint(df_labels)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.1552Z","iopub.execute_input":"2023-12-22T21:50:19.155536Z","iopub.status.idle":"2023-12-22T21:50:19.171656Z","shell.execute_reply.started":"2023-12-22T21:50:19.155504Z","shell.execute_reply":"2023-12-22T21:50:19.170609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.pie(df_labels['label_count'], labels=df_labels['label'], autopct='%.1f%%')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.173277Z","iopub.execute_input":"2023-12-22T21:50:19.173607Z","iopub.status.idle":"2023-12-22T21:50:19.361122Z","shell.execute_reply.started":"2023-12-22T21:50:19.173573Z","shell.execute_reply":"2023-12-22T21:50:19.359828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_labels.loc[0,'label']","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.375967Z","iopub.execute_input":"2023-12-22T21:50:19.379801Z","iopub.status.idle":"2023-12-22T21:50:19.393402Z","shell.execute_reply.started":"2023-12-22T21:50:19.379731Z","shell.execute_reply":"2023-12-22T21:50:19.392031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(df_labels['label'])):\n    train.loc[train['label']==df_labels.loc[i,'label'], 'label_No'] = df_labels['label_No'][i]\ntrain['label_No'] = train['label_No'].astype('int')\n# train = train.dropna()\nntrain = len(train)\nprint(ntrain)\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.400595Z","iopub.execute_input":"2023-12-22T21:50:19.4044Z","iopub.status.idle":"2023-12-22T21:50:19.436937Z","shell.execute_reply.started":"2023-12-22T21:50:19.404343Z","shell.execute_reply":"2023-12-22T21:50:19.435982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load images for train","metadata":{}},{"cell_type":"code","source":"batch_size = 5\nnum_class = len(df_labels)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.43833Z","iopub.execute_input":"2023-12-22T21:50:19.439167Z","iopub.status.idle":"2023-12-22T21:50:19.442911Z","shell.execute_reply.started":"2023-12-22T21:50:19.439131Z","shell.execute_reply":"2023-12-22T21:50:19.442043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LoadFromFolder(Dataset):\n    def __init__(self, train:pd.DataFrame, transform = None):\n        self.train = train\n        self.transform = transform\n    \n    def __getitem__(self,index):\n        filepath = self.train['path'][index]\n        label = self.train['label_No'][index]\n        img = Image.open(filepath).convert('RGB')\n        imglow = self.transform(img)\n        return imglow, label\n        \n    def __len__(self):\n        return len(self.train)\n        ","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.444059Z","iopub.execute_input":"2023-12-22T21:50:19.445022Z","iopub.status.idle":"2023-12-22T21:50:19.453766Z","shell.execute_reply.started":"2023-12-22T21:50:19.444984Z","shell.execute_reply":"2023-12-22T21:50:19.45267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n            transforms.Lambda(adjust),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ])","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.455273Z","iopub.execute_input":"2023-12-22T21:50:19.455665Z","iopub.status.idle":"2023-12-22T21:50:19.463778Z","shell.execute_reply.started":"2023-12-22T21:50:19.45563Z","shell.execute_reply":"2023-12-22T21:50:19.462964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = LoadFromFolder(train, transform)\nlen(dataset)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.465133Z","iopub.execute_input":"2023-12-22T21:50:19.465378Z","iopub.status.idle":"2023-12-22T21:50:19.476358Z","shell.execute_reply.started":"2023-12-22T21:50:19.465356Z","shell.execute_reply":"2023-12-22T21:50:19.475366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datalen = len(dataset)\ntrain_len = int(0.8 * datalen)\nval_len = int(datalen - train_len) \nprint(datalen, train_len, val_len)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.477911Z","iopub.execute_input":"2023-12-22T21:50:19.478231Z","iopub.status.idle":"2023-12-22T21:50:19.486592Z","shell.execute_reply.started":"2023-12-22T21:50:19.478201Z","shell.execute_reply":"2023-12-22T21:50:19.485031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset, val_dataset  = torch.utils.data.random_split(dataset, lengths=[train_len, val_len], generator = torch.Generator().manual_seed(42))","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.488145Z","iopub.execute_input":"2023-12-22T21:50:19.48846Z","iopub.status.idle":"2023-12-22T21:50:19.513035Z","shell.execute_reply.started":"2023-12-22T21:50:19.488427Z","shell.execute_reply":"2023-12-22T21:50:19.512263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True)\nval_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True, drop_last=True)\nlen(train_dataloader)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.514286Z","iopub.execute_input":"2023-12-22T21:50:19.515156Z","iopub.status.idle":"2023-12-22T21:50:19.523027Z","shell.execute_reply.started":"2023-12-22T21:50:19.515121Z","shell.execute_reply":"2023-12-22T21:50:19.52201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Model Definition","metadata":{}},{"cell_type":"code","source":"class DenseResidualBlock(nn.Module):\n    def __init__(self, filters, res_scale = 1):\n        super(DenseResidualBlock, self).__init__()\n        self.res_scale = res_scale\n        def block(in_features, non_linearity=True):\n            layers = [nn.Conv2d(in_features, filters, 1, 1, bias=True)]\n            if non_linearity:\n                layers += [nn.LeakyReLU()]\n            return nn.Sequential(*layers)\n        self.b1 = block(in_features=1 * filters)\n        self.b2 = block(in_features=2 * filters)\n        self.b3 = block(in_features=3 * filters)\n        self.b4 = block(in_features=4 * filters)\n        self.b5 = block(in_features=5 * filters, non_linearity=False)\n        self.blocks =[self.b1, self.b2, self.b3, self.b4, self.b5]\n\n    def forward(self, x):\n        inputs = x\n#         print('x.size:', x.size())\n        for block in self.blocks:\n            out = block(inputs)\n            inputs = torch.cat([inputs, out], 1)\n        return out.mul(self.res_scale) + x\n\nclass ResidualInResidualDenseBlock(nn.Module):\n    \"\"\"\n    ResidualInResidualDenseBlockのクラス\n    \"\"\"\n    def __init__(self, filters, res_scale=1):\n        super(ResidualInResidualDenseBlock, self).__init__()\n        self.res_scale = res_scale\n        self.dense_blocks = nn.Sequential(\n            DenseResidualBlock(filters), \n            DenseResidualBlock(filters), \n            DenseResidualBlock(filters),\n        )\n    \n    def forward(self, x):\n        return self.dense_blocks(x).mul(self.res_scale) + x\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.524678Z","iopub.execute_input":"2023-12-22T21:50:19.525015Z","iopub.status.idle":"2023-12-22T21:50:19.54027Z","shell.execute_reply.started":"2023-12-22T21:50:19.524983Z","shell.execute_reply":"2023-12-22T21:50:19.539396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Model_CNN(nn.Module):\n    def __init__(self, filters, num_class) -> None:\n        super(Model_CNN, self).__init__()\n        \n        self.res_block = nn.Sequential(ResidualInResidualDenseBlock(filters))\n        self.main = nn.Sequential(\n            nn.Conv2d(3, 512, kernel_size=19, stride=11, padding=1, bias=False),\n            nn.BatchNorm2d(512),\n            nn.LeakyReLU(0.2),\n\n            nn.Conv2d(512, 256, kernel_size=13, stride=11, padding=1, bias=False),\n            nn.BatchNorm2d(256),\n            nn.LeakyReLU(0.2),\n\n            nn.Conv2d(256, 128, kernel_size=4, stride=1, bias=False),\n            nn.BatchNorm2d(128),\n            nn.LeakyReLU(0.2),\n        )\n        self.classifier = nn.Sequential(nn.Linear(128, num_class))\n                                \n        \n    def forward(self, x):\n        out = self.res_block(x)\n        out = self.main(out)\n        out = self.main(x)\n        out = out.view(out.size(0), -1)\n        out = self.classifier(out)\n#         length = len(x)\n#         out = torch.tensor([[0.11, 0.12, 0.13, 0.14, 0.15]] * length).to(device)\n        return out","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.541539Z","iopub.execute_input":"2023-12-22T21:50:19.541902Z","iopub.status.idle":"2023-12-22T21:50:19.554536Z","shell.execute_reply.started":"2023-12-22T21:50:19.54187Z","shell.execute_reply":"2023-12-22T21:50:19.553533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class Model_CNN(nn.Module):\n#     def __init__(self, filters, num_class) -> None:\n#         super(Model_CNN, self).__init__()\n        \n#         self.conv1 = nn.Conv2d(3, filters, kernel_size=3, stride=1, padding=1)\n#         self.res_block = nn.Sequential(ResidualInResidualDenseBlock(filters))\n#         self.main = nn.Sequential(\n#             nn.Conv2d(3, 1024, kernel_size=11, stride=5, padding=1, bias=False),\n#             nn.BatchNorm2d(1024),\n#             nn.LeakyReLU(0.2),\n            \n#             nn.Conv2d(1024, 512, kernel_size=11, stride=5, padding=1, bias=False),\n#             nn.BatchNorm2d(512),\n#             nn.LeakyReLU(0.2),\n            \n#             nn.Conv2d(512, 256, kernel_size=11, stride=5, padding=1, bias=False),\n#             nn.BatchNorm2d(256),\n#             nn.LeakyReLU(0.2),\n\n#             nn.Conv2d(256, 128, kernel_size=7, stride=1, bias=False),\n#             nn.BatchNorm2d(128),\n#             nn.LeakyReLU(0.2),\n#         )\n#         self.classifier = nn.Sequential(nn.Linear(128, num_class))\n                                \n        \n#     def forward(self, x):\n#         out = self.res_block(x)\n#         out = self.main(out)\n#         out = out.view(out.size(0), -1)\n#         out = self.classifier(out)\n#         return out","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.555867Z","iopub.execute_input":"2023-12-22T21:50:19.556173Z","iopub.status.idle":"2023-12-22T21:50:19.56816Z","shell.execute_reply.started":"2023-12-22T21:50:19.556143Z","shell.execute_reply":"2023-12-22T21:50:19.567347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_Test = Model_CNN(filters = 3, num_class=num_class)\n# model = models.resnet50(weights = models.ResNet50_Weights.IMAGENET1K_V1)\nsummary(model_Test, (batch_size, 3, target_size, target_size), verbose=2)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:19.569279Z","iopub.execute_input":"2023-12-22T21:50:19.571267Z","iopub.status.idle":"2023-12-22T21:50:27.787326Z","shell.execute_reply.started":"2023-12-22T21:50:19.571235Z","shell.execute_reply":"2023-12-22T21:50:27.786364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model_CNN(filters = 3, num_class=num_class).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=5e-4)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:27.788622Z","iopub.execute_input":"2023-12-22T21:50:27.788975Z","iopub.status.idle":"2023-12-22T21:50:28.060916Z","shell.execute_reply.started":"2023-12-22T21:50:27.78894Z","shell.execute_reply":"2023-12-22T21:50:28.059879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 30\ntrain_loss_list = []\ntrain_acc_list = []\nval_loss_list = []\nval_acc_list = []","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:28.062242Z","iopub.execute_input":"2023-12-22T21:50:28.062593Z","iopub.status.idle":"2023-12-22T21:50:28.067389Z","shell.execute_reply.started":"2023-12-22T21:50:28.062539Z","shell.execute_reply":"2023-12-22T21:50:28.066425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_dataloader))\nprint(len(train_dataloader.dataset))\nprint(len(val_dataloader.dataset))","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:28.068704Z","iopub.execute_input":"2023-12-22T21:50:28.06906Z","iopub.status.idle":"2023-12-22T21:50:28.079384Z","shell.execute_reply.started":"2023-12-22T21:50:28.069024Z","shell.execute_reply":"2023-12-22T21:50:28.078425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    model.train()\n    train_loss = 0\n    train_acc = 0\n    before_acc = 0\n    for imgs, labels in train_dataloader:\n#         print(imgs.size())\n        imgs = Variable(imgs.to(device))\n        labels = Variable(labels.to(device))\n        \n        optimizer.zero_grad()\n        outputs = model(imgs)\n        loss = criterion(outputs, labels)\n        train_loss += loss.item()\n        train_loss_list.append(loss.item())\n        acc = (outputs.max(1)[1] == labels).sum().item()\n        train_acc += acc\n        train_acc_list.append(acc)\n        loss.backward()\n        optimizer.step()\n        \n        ave_train_loss = train_loss / len(train_dataloader)\n        ave_train_acc = train_acc / len(train_dataloader)\n        if before_acc < acc:\n            w_save_path = \"/kaggle/working/best_weight.pth\"\n#             os.path.exists(w_save_path)\n            torch.save(model.state_dict(), w_save_path)\n        before_acc = acc\n    print(f'train epoch_{epoch}:train_loss {loss}, ave_train_loss {ave_train_loss}, train_acc {acc}/{batch_size}, ave_train_acc {ave_train_acc}')\n\n\n    # ここからバリデーションループを追加\n    model.eval()  # モデルを評価モードに設定\n    val_loss = 0\n    val_acc = 0\n    with torch.no_grad():  # 勾配計算を無効化\n        for imgs, labels in val_dataloader:  # バリデーションデータローダーを使用\n            imgs = imgs.to(device)\n            labels = labels.to(device)\n\n            outputs = model(imgs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            val_loss_list.append(loss.item())\n            val_acc = (outputs.max(1)[1] == labels).sum().item()\n            val_acc += val_acc\n            val_acc_list.append(val_acc)\n\n    ave_val_loss = val_loss / len(val_dataloader)\n    ave_val_acc = val_acc / len(val_dataloader)\n    print(f'val epoch_{epoch}: val_loss {val_loss}, ave_val_loss {ave_val_loss}, val_acc {val_acc}/{batch_size}, ave_val_acc {ave_val_acc}')\n","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:50:28.080857Z","iopub.execute_input":"2023-12-22T21:50:28.081229Z","iopub.status.idle":"2023-12-22T21:53:05.301476Z","shell.execute_reply.started":"2023-12-22T21:50:28.081195Z","shell.execute_reply":"2023-12-22T21:53:05.300558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_loss)\nprint(train_acc)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:53:05.302622Z","iopub.execute_input":"2023-12-22T21:53:05.302875Z","iopub.status.idle":"2023-12-22T21:53:05.307813Z","shell.execute_reply.started":"2023-12-22T21:53:05.302852Z","shell.execute_reply":"2023-12-22T21:53:05.306928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(train_loss_list)\nplt.plot(train_acc_list)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:53:05.308968Z","iopub.execute_input":"2023-12-22T21:53:05.309222Z","iopub.status.idle":"2023-12-22T21:53:05.579074Z","shell.execute_reply.started":"2023-12-22T21:53:05.309199Z","shell.execute_reply":"2023-12-22T21:53:05.578164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(val_loss_list)\nplt.plot(val_acc_list)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:53:05.580146Z","iopub.execute_input":"2023-12-22T21:53:05.580495Z","iopub.status.idle":"2023-12-22T21:53:05.820992Z","shell.execute_reply.started":"2023-12-22T21:53:05.580452Z","shell.execute_reply":"2023-12-22T21:53:05.820122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test = read_csv(test_path)\ntest_des = test.describe() \n\ntest_width_max = int(test_des['image_width'].max())\ntest_height_max = int(test_des['image_height'].max())","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:53:05.822083Z","iopub.execute_input":"2023-12-22T21:53:05.822347Z","iopub.status.idle":"2023-12-22T21:53:05.840185Z","shell.execute_reply.started":"2023-12-22T21:53:05.822323Z","shell.execute_reply":"2023-12-22T21:53:05.839397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdir_path = '/kaggle/input/UBC-OCEAN/test_thumbnails/'\nthumbnail_paths = []\nfor dirname, _, filenames in os.walk(testdir_path):\n    for filename in filenames:\n        thumbnail_paths.append(os.path.join(dirname, filename))\nthumbnail_paths","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:53:05.841214Z","iopub.execute_input":"2023-12-22T21:53:05.841475Z","iopub.status.idle":"2023-12-22T21:53:05.852494Z","shell.execute_reply.started":"2023-12-22T21:53:05.841451Z","shell.execute_reply":"2023-12-22T21:53:05.851596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(test)):\n    thum_path = os.path.join(testdir_path,str(test.loc[i, 'image_id'])+'_thumbnail.png')\n    if thum_path in thumbnail_paths:\n        test.loc[i,'path'] = os.path.join('/kaggle/input/UBC-OCEAN/test_thumbnails', str(test.loc[i, 'image_id'])+'_thumbnail.png')\n    else:\n        test.loc[i, 'path']=\"No_files\"\n# train = train.dropna()\ntest = test[test['path'] !='No_files']\ntest = test.reset_index()\ntest","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:53:05.853585Z","iopub.execute_input":"2023-12-22T21:53:05.853912Z","iopub.status.idle":"2023-12-22T21:53:05.866739Z","shell.execute_reply.started":"2023-12-22T21:53:05.853888Z","shell.execute_reply":"2023-12-22T21:53:05.865775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestLoadFromFolder(Dataset):\n    def __init__(self, train:pd.DataFrame, transform = None):\n        self.train = train\n        self.transform = transform\n    \n    def __getitem__(self,index):\n        filepath = self.train['path'][index]\n        img = Image.open(filepath).convert('RGB')\n        imglow = self.transform(img)\n        return imglow\n        \n    def __len__(self):\n        return len(self.train)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:53:05.86783Z","iopub.execute_input":"2023-12-22T21:53:05.868149Z","iopub.status.idle":"2023-12-22T21:53:05.874791Z","shell.execute_reply.started":"2023-12-22T21:53:05.868122Z","shell.execute_reply":"2023-12-22T21:53:05.873919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_adjust = Adjust_image_size(target_size, test_height_max, test_width_max)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:53:05.87582Z","iopub.execute_input":"2023-12-22T21:53:05.876076Z","iopub.status.idle":"2023-12-22T21:53:05.885591Z","shell.execute_reply.started":"2023-12-22T21:53:05.876053Z","shell.execute_reply":"2023-12-22T21:53:05.884645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_transform = transforms.Compose([\n            transforms.Lambda(test_adjust),\n            transforms.ToTensor(),\n        ])","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:53:05.886652Z","iopub.execute_input":"2023-12-22T21:53:05.886973Z","iopub.status.idle":"2023-12-22T21:53:05.894565Z","shell.execute_reply.started":"2023-12-22T21:53:05.886942Z","shell.execute_reply":"2023-12-22T21:53:05.893722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestLoadFromFolder(test, test_transform)\nlen(test_dataset)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:54:13.190924Z","iopub.execute_input":"2023-12-22T21:54:13.191852Z","iopub.status.idle":"2023-12-22T21:54:13.19802Z","shell.execute_reply.started":"2023-12-22T21:54:13.191816Z","shell.execute_reply":"2023-12-22T21:54:13.197046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\nlen(testdataloader)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:54:13.204029Z","iopub.execute_input":"2023-12-22T21:54:13.204615Z","iopub.status.idle":"2023-12-22T21:54:13.210873Z","shell.execute_reply.started":"2023-12-22T21:54:13.204575Z","shell.execute_reply":"2023-12-22T21:54:13.21005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_list = []\n\nmodel.load_state_dict(torch.load(w_save_path))\nmodel.eval()\nwith torch.no_grad():\n    for imgs in testdataloader:\n#         if len(imgs) == 3:\n#             imgs = imgs[np.newaxis, :, :, :]\n        imgs = Variable(imgs.to(device))\n        pred = model(imgs)\n        pred_list.append(int(pred.max(1)[1]))","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:54:13.212448Z","iopub.execute_input":"2023-12-22T21:54:13.212766Z","iopub.status.idle":"2023-12-22T21:54:13.651852Z","shell.execute_reply.started":"2023-12-22T21:54:13.212741Z","shell.execute_reply":"2023-12-22T21:54:13.650841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_list","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:54:13.653098Z","iopub.execute_input":"2023-12-22T21:54:13.65347Z","iopub.status.idle":"2023-12-22T21:54:13.661748Z","shell.execute_reply.started":"2023-12-22T21:54:13.653435Z","shell.execute_reply":"2023-12-22T21:54:13.660586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"label_No\"] = pred_list\nfor i in range(len(df_labels['label'])):\n    # print(labels['label_No'][i])\n    test.loc[test['label_No']==df_labels.loc[i,'label_No'], 'label'] = df_labels['label'][i]\nsubmission = test[['image_id', 'label']]\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:54:13.664645Z","iopub.execute_input":"2023-12-22T21:54:13.665043Z","iopub.status.idle":"2023-12-22T21:54:13.685036Z","shell.execute_reply.started":"2023-12-22T21:54:13.665008Z","shell.execute_reply":"2023-12-22T21:54:13.683632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 重みの削除　エラーが出るため\nos.remove(w_save_path)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:54:13.686529Z","iopub.execute_input":"2023-12-22T21:54:13.686866Z","iopub.status.idle":"2023-12-22T21:54:13.704777Z","shell.execute_reply.started":"2023-12-22T21:54:13.686836Z","shell.execute_reply":"2023-12-22T21:54:13.703886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=None)","metadata":{"execution":{"iopub.status.busy":"2023-12-22T21:54:13.705911Z","iopub.execute_input":"2023-12-22T21:54:13.706448Z","iopub.status.idle":"2023-12-22T21:54:13.713965Z","shell.execute_reply.started":"2023-12-22T21:54:13.706416Z","shell.execute_reply":"2023-12-22T21:54:13.712988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}