{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # for google colab\n# from kaggle_datasets import KaggleDatasets\n# KaggleDatasets().get_gcs_path()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-11T15:26:20.743358Z","iopub.execute_input":"2022-08-11T15:26:20.743924Z","iopub.status.idle":"2022-08-11T15:26:20.76872Z","shell.execute_reply.started":"2022-08-11T15:26:20.743804Z","shell.execute_reply":"2022-08-11T15:26:20.767329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:20.771693Z","iopub.execute_input":"2022-08-11T15:26:20.772479Z","iopub.status.idle":"2022-08-11T15:26:33.683158Z","shell.execute_reply.started":"2022-08-11T15:26:20.77242Z","shell.execute_reply":"2022-08-11T15:26:33.681718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport zipfile\nimport torch\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\n\nimport cv2\n# from efficientnet_pytorch import EfficientNet\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import lr_scheduler\nfrom torchvision import models\nimport torchvision\nimport torch.nn as nn\nimport timm\nfrom torchvision import transforms\nfrom timm.data import resolve_data_config\nfrom timm.data.transforms_factory import create_transform","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:33.686254Z","iopub.execute_input":"2022-08-11T15:26:33.686703Z","iopub.status.idle":"2022-08-11T15:26:37.055907Z","shell.execute_reply.started":"2022-08-11T15:26:33.686653Z","shell.execute_reply":"2022-08-11T15:26:37.054575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:37.058822Z","iopub.execute_input":"2022-08-11T15:26:37.060003Z","iopub.status.idle":"2022-08-11T15:26:37.14172Z","shell.execute_reply.started":"2022-08-11T15:26:37.059945Z","shell.execute_reply":"2022-08-11T15:26:37.140744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"args = {}\nmodel_name = 'tf_efficientnetv2_m'\ndata_config = timm.data.resolve_data_config({}, model=model_name, verbose=True)\nprint(data_config[\"mean\"], data_config[\"std\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:37.14587Z","iopub.execute_input":"2022-08-11T15:26:37.148043Z","iopub.status.idle":"2022-08-11T15:26:37.163723Z","shell.execute_reply.started":"2022-08-11T15:26:37.148003Z","shell.execute_reply":"2022-08-11T15:26:37.162415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N = 16","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:37.165716Z","iopub.execute_input":"2022-08-11T15:26:37.166441Z","iopub.status.idle":"2022-08-11T15:26:37.171861Z","shell.execute_reply.started":"2022-08-11T15:26:37.166376Z","shell.execute_reply":"2022-08-11T15:26:37.170716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_class = 2\nbackbone = timm.create_model(model_name, pretrained=True, num_classes=0).to(device)\nmodel = nn.Sequential(\n    backbone,\n    nn.Dropout(0.2),\n    nn.Linear(backbone.num_features, n_class)\n).to(device)\n# params_to_update = []\n# update_param_names = [\"effnet.classifier.weight\", \"effnet.classifier.bias\"]\n# for name, param in model.named_parameters():\n#     if name in update_param_names:\n#         param.requires_grad = True\n#         params_to_update.append(param)\n#     else:\n#         param.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:37.173389Z","iopub.execute_input":"2022-08-11T15:26:37.174018Z","iopub.status.idle":"2022-08-11T15:26:41.292835Z","shell.execute_reply.started":"2022-08-11T15:26:37.173982Z","shell.execute_reply":"2022-08-11T15:26:41.29175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/train.csv')\nsum(train[\"image_id\"] == \"2c3c06_0\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:41.294423Z","iopub.execute_input":"2022-08-11T15:26:41.294799Z","iopub.status.idle":"2022-08-11T15:26:41.320283Z","shell.execute_reply.started":"2022-08-11T15:26:41.294757Z","shell.execute_reply":"2022-08-11T15:26:41.319371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This block is reffered from https://www.kaggle.com/code/yasufuminakama/mayo-train-images-size-1024-n-16-1/notebook\ntrain = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/train.csv')\n# train = train[train[\"image_id\"] != \"2c3c06_0\"]\n\ntrain['image_dir'] = ''\n# train.loc[:100,'image_dir'] = '/kaggle/input/mayo-train-images-size1024-n16/train_images_1/'\n# train.loc[100:200,'image_dir'] = '/kaggle/input/mayo-train-images-size1024-n16/train_images_2/'\n# train.loc[200:300,'image_dir'] = '/kaggle/input/mayo-train-images-size1024-n16/train_images_3/'\n# train.loc[300:400,'image_dir'] = '/kaggle/input/mayo-train-images-size1024-n16/train_images_4/'\n# train.loc[400:500,'image_dir'] = '/kaggle/input/mayo-train-images-size1024-n16/train_images_5/'\n# train.loc[500:600,'image_dir'] = '/kaggle/input/mayo-train-images-size1024-n16/train_images_6/'\n# train.loc[600:700,'image_dir'] = '/kaggle/input/mayo-train-images-size1024-n16/train_images_7/'\n# train.loc[700:,'image_dir'] = '/kaggle/input/mayo-train-images-size1024-n16/train_images_8/'\n\ntrain.loc[:100,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_1/'\ntrain.loc[100:200,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_2/'\ntrain.loc[200:300,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_3/'\ntrain.loc[300:400,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_4/'\ntrain.loc[400:500,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_5/'\ntrain.loc[500:600,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_6/'\ntrain.loc[600:700,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_7/'\ntrain.loc[700:,'image_dir'] = '/kaggle/input/mayo-tiled-16-384x384/train_images/train_images/train_images_8/'\n\n\ntarget_mapper = {\"CE\": 0, \"LAA\": 1}\n\ntrain[\"target\"] = train[\"label\"].map(lambda x: target_mapper[x])\n\ntrain_full = pd.DataFrame()\nfor i in range(N):\n    train[\"path\"] = train[\"image_dir\"] + train[\"image_id\"] + f'_{i}.jpg'\n    if i == 0:\n        train_full = train\n    else:\n        train_full = pd.concat([train_full, train], axis=0)\n\ntrain_full.reset_index(inplace=True)\n\nclass TrainDataset(Dataset):\n    def __init__(self, cfg, df, transform=None, aug=True):\n        self.cfg = cfg\n        self.image_ids = df['image_id'].values\n        self.image_dirs = df['image_dir'].values\n        self.image_path = df[\"path\"].values\n        self.labels = df['target'].values\n        self.transform = transform\n        self.aug = aug\n\n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, idx):\n#         image_id = self.image_ids[idx]\n#         image_dir = self.image_dirs[idx]\n#         images = []\n#         for i in range(16):\n#             path = image_dir + image_id + f'_{i}.jpg'\n#             image = cv2.imread(path)\n#             image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n#             image = self.transform(image)\n# #             image = self.transform(image=image)[\"image\"]            \n#             images.append(image)\n#         images = torch.stack(images, dim=0)\n        image = cv2.imread(self.image_path[idx])\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n#         image = image.transpose((2, 0, 1)) \n        image = transforms.ToPILImage()(image)\n        if self.aug:\n            image = torchvision.transforms.RandAugment()(image)\n        image = self.transform(image)\n        label = torch.tensor(self.labels[idx]).long()\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:41.32381Z","iopub.execute_input":"2022-08-11T15:26:41.324078Z","iopub.status.idle":"2022-08-11T15:26:41.391943Z","shell.execute_reply.started":"2022-08-11T15:26:41.324054Z","shell.execute_reply":"2022-08-11T15:26:41.390953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transform = transforms.Compose([\n#                     transforms.ToTensor(),\n#                     transforms.Normalize([0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])\n#                     ])\nconfig = resolve_data_config({}, model=backbone)\ntransform = create_transform(**config)\ntransform","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:41.393276Z","iopub.execute_input":"2022-08-11T15:26:41.393637Z","iopub.status.idle":"2022-08-11T15:26:41.403868Z","shell.execute_reply.started":"2022-08-11T15:26:41.393601Z","shell.execute_reply":"2022-08-11T15:26:41.402691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_full[\"path\"]:\n    if not os.path.exists(i):\n        print(i+ \" does not exist!\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:41.405616Z","iopub.execute_input":"2022-08-11T15:26:41.406322Z","iopub.status.idle":"2022-08-11T15:26:51.940228Z","shell.execute_reply.started":"2022-08-11T15:26:41.406283Z","shell.execute_reply":"2022-08-11T15:26:51.939258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_set = TrainDataset(None,df=train_full, transform=transform, aug=False)\ntraining_loader = torch.utils.data.DataLoader(training_set, shuffle=True, num_workers=2, batch_size=16)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:51.942418Z","iopub.execute_input":"2022-08-11T15:26:51.943043Z","iopub.status.idle":"2022-08-11T15:26:51.950389Z","shell.execute_reply.started":"2022-08-11T15:26:51.943001Z","shell.execute_reply":"2022-08-11T15:26:51.949278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn = nn.CrossEntropyLoss().cuda()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=0.0001)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:51.952074Z","iopub.execute_input":"2022-08-11T15:26:51.952847Z","iopub.status.idle":"2022-08-11T15:26:51.974197Z","shell.execute_reply.started":"2022-08-11T15:26:51.952803Z","shell.execute_reply":"2022-08-11T15:26:51.973008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.parameters()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:51.975763Z","iopub.execute_input":"2022-08-11T15:26:51.976418Z","iopub.status.idle":"2022-08-11T15:26:51.991252Z","shell.execute_reply.started":"2022-08-11T15:26:51.976379Z","shell.execute_reply":"2022-08-11T15:26:51.990243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(model, device, train_loader, optimizer, epoch, loss_fn):\n    running_loss = 0.\n    last_loss = 0.\n    model.train(True)\n\n    # Here, we use enumerate(training_loader) instead of\n    # iter(training_loader) so that we can track the batch\n    # index and do some intra-epoch reporting\n    for i, (data, target) in enumerate(train_loader):\n        # Every data instance is an input + label pair\n        inputs, labels = data.to(device), target.to(device)\n\n        # Zero your gradients for every batch!\n        optimizer.zero_grad()\n\n        # Make predictions for this batch\n        outputs = model(inputs)\n\n        # Compute the loss and its gradients\n        loss = loss_fn(outputs, labels)\n        loss.backward()\n\n        # Adjust learning weights\n        optimizer.step()\n\n        # Gather data and report\n        running_loss += loss.item()\n        if i % 100 == 99:\n            last_loss = running_loss / 100 # loss per batch\n            print('  batch {} loss: {}'.format(i + 1, last_loss))\n            tb_x = epoch * len(train_loader) + i + 1\n            running_loss = 0.\n\n    return last_loss","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:51.992739Z","iopub.execute_input":"2022-08-11T15:26:51.993258Z","iopub.status.idle":"2022-08-11T15:26:52.004093Z","shell.execute_reply.started":"2022-08-11T15:26:51.993222Z","shell.execute_reply":"2022-08-11T15:26:52.003208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(model)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:52.005626Z","iopub.execute_input":"2022-08-11T15:26:52.00651Z","iopub.status.idle":"2022-08-11T15:26:52.016195Z","shell.execute_reply.started":"2022-08-11T15:26:52.006443Z","shell.execute_reply":"2022-08-11T15:26:52.015247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initializing in a separate cell so we can easily add more epochs to the same run\nfrom datetime import datetime\ntimestamp = datetime.now().strftime('%Y%m%d_%H%M%S')\n# writer = SummaryWriter('runs/fashion_trainer_{}'.format(timestamp))\nepoch_number = 0\n\nEPOCHS = 5\n\nfor epoch in range(EPOCHS):\n    print('EPOCH {}:'.format(epoch_number + 1))\n\n    avg_loss = train_one_epoch(model=model, device=device, train_loader=training_loader, optimizer=optimizer, epoch=epoch, loss_fn=loss_fn)\n\n    epoch_number += 1","metadata":{"execution":{"iopub.status.busy":"2022-08-11T15:26:52.017665Z","iopub.execute_input":"2022-08-11T15:26:52.018363Z","iopub.status.idle":"2022-08-11T16:02:58.005527Z","shell.execute_reply.started":"2022-08-11T15:26:52.018327Z","shell.execute_reply":"2022-08-11T16:02:58.004194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(model.state_dict(), \"best_model_384.pth\")","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:02:58.008582Z","iopub.execute_input":"2022-08-11T16:02:58.008983Z","iopub.status.idle":"2022-08-11T16:02:58.46956Z","shell.execute_reply.started":"2022-08-11T16:02:58.008935Z","shell.execute_reply":"2022-08-11T16:02:58.468517Z"},"trusted":true},"execution_count":null,"outputs":[]}]}