{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"How to improve this code?  \n1. Use different models\n    - https://paperswithcode.com/area/computer-vision\n2. Regularization and dropout\n3. More Augmentation\n4. More epochs\n5. Using big images(I'm using thumbnails to work faster, just for the start)\n6. weight balancing(or any other method to use for unbalanced data)\n7. learning rate scheduler(a lot of finetuning can be done here)\n8. Ensemble Methods\n    - both for imbalanced data and for better results\n    - a lot of methods for ensembling, I'll eventually go down this rabbithole\n9. Calculate different mean and std maybe?\n1. Early stopping? Interferes with Learning rate scheduler, I think...\n\n\nsuggestions are appreciated, I'm a beginner after all","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\nimport cv2\n\n# from skimage import io\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torch.nn as nn\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nfrom torch.optim.lr_scheduler import OneCycleLR\nfrom torch.optim.lr_scheduler import StepLR\n\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\n\nimport torchmetrics\nfrom torchmetrics.classification import MulticlassF1Score\n\nfrom torchvision import transforms\nfrom torchvision import models\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer\nfrom pytorch_lightning.callbacks import ModelCheckpoint\nfrom pytorch_lightning.loggers import TensorBoardLogger\nfrom pytorch_lightning.callbacks import EarlyStopping\nfrom pytorch_lightning  import Trainer, seed_everything\nseed_everything(42, workers=True)\n\nimport timm\nfrom PIL import Image","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-31T08:42:47.657677Z","iopub.execute_input":"2023-10-31T08:42:47.657957Z","iopub.status.idle":"2023-10-31T08:43:02.653676Z","shell.execute_reply.started":"2023-10-31T08:42:47.657932Z","shell.execute_reply":"2023-10-31T08:43:02.652882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/UBC-OCEAN/train.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-31T08:43:02.655258Z","iopub.execute_input":"2023-10-31T08:43:02.655534Z","iopub.status.idle":"2023-10-31T08:43:02.682852Z","shell.execute_reply.started":"2023-10-31T08:43:02.65551Z","shell.execute_reply":"2023-10-31T08:43:02.681962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\nencoder.fit(df['label'])\ndf['label'] = encoder.transform(df['label'])\n\nduplicate_list = [281,706,1252,1295,1660,1666,1943,2391,2706,3055,3092,3098,\n                  3264,3672,4827,5251,5264,5307,5852,5992,6175,6449,6793,6843,\n                  7955,8130,8985,9341,10548,10642,11559,12222,12442,13526,13987,14039,\n                  14312,14542,15221,15293,15470,15486,15912,16042,16494,17067,17416,18014,\n                  18196,18810,18896,18914,19157,19512,20670,20858,21232,21260,21910,22290,22425,    \n                  22654,22924,23523,24023,24759,25256,25561,25792,26190,26603,26644,26862,27851,\n                  27950,28121,28519,28562,28603,29240,29331,29915,30508,30515,30539,30738,30792,\n                  30868,31333,31473,32112,32596,32636,34277,34845,35592,35652,35792,35953,36008,\n                  36204,37190,38041,38097,38118,38585,38687,38959,39144,39208,39297,39365,40129,\n                  40503,41361,41801,42296,43280,43796,43875,43998,44804,44962,44976,45104,45254,\n                  45578,45725,45990,46543,46736,46769,46793,47020,47911,47984,48502,48861,48973,\n                  50048,50246,50712,51021,51346,51832,52259,52375,52420,52461,52612,52752,52931,\n                  53059,53900,54473,54506,54825,54990,55279,55281,56500,56843,57100,58974,59900,\n                  60287,60685,60928,61033,61100,61689,62828,63367,63429,64188,64824,64950,65300]\n# complex_images = [46543,22924,20858,34845,26862,39144,54506,17416,48502,50246,30868]\nTRAIN_THUMBNAILS = '/kaggle/input/UBC-OCEAN/train_thumbnails'\nTRAIN_IMAGES = '/kaggle/input/UBC-OCEAN/train_images'\nDUPLICATE_PATH = '/kaggle/input/cropped-duplicates' # only for thumbnails that have duplicate issue\n\ndef get_file_path(image_id):\n    if os.path.exists(f\"{TRAIN_THUMBNAILS}/{image_id}_thumbnail.png\"):\n        if image_id in duplicate_list:\n            return f\"{DUPLICATE_PATH}/{image_id}_thumbnail.png\"\n        else:\n            return f\"{TRAIN_THUMBNAILS}/{image_id}_thumbnail.png\"\n    else:\n        return f\"{TRAIN_IMAGES}/{image_id}.png\"","metadata":{"execution":{"iopub.status.busy":"2023-10-31T08:43:02.684338Z","iopub.execute_input":"2023-10-31T08:43:02.684631Z","iopub.status.idle":"2023-10-31T08:43:02.698974Z","shell.execute_reply.started":"2023-10-31T08:43:02.684606Z","shell.execute_reply":"2023-10-31T08:43:02.69803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['file_path'] = df['image_id'].apply(get_file_path)\n# df['label'] = 0","metadata":{"execution":{"iopub.status.busy":"2023-10-31T08:45:07.18686Z","iopub.execute_input":"2023-10-31T08:45:07.187789Z","iopub.status.idle":"2023-10-31T08:45:09.558984Z","shell.execute_reply.started":"2023-10-31T08:45:07.187732Z","shell.execute_reply":"2023-10-31T08:45:09.557994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-31T08:45:12.524793Z","iopub.execute_input":"2023-10-31T08:45:12.525687Z","iopub.status.idle":"2023-10-31T08:45:12.537537Z","shell.execute_reply.started":"2023-10-31T08:45:12.525652Z","shell.execute_reply":"2023-10-31T08:45:12.536504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Initialize sums for mean and std calculation\n# sums = np.zeros(3)\n# sums_squared = np.zeros(3)\n# normalizer = 0\n\n# for idx, row in df.iterrows():\n#     img = cv2.imread(row['file_path'])\n#     # Convert to float and normalize\n#     img = img.astype(np.float32) / 255.0\n#     # Resize image to fit VGG16 input size\n#     img_resized = cv2.resize(img, (224, 224))\n    \n#     for i in range(3):  # Assuming 3 channels: R, G, B\n#         sums[i] += np.sum(img_resized[:, :, i])\n#         sums_squared[i] += np.sum(np.square(img_resized[:, :, i]))\n#     normalizer += img_resized[:, :, 0].size  # size of one channel\n    \n# # Calculate mean and std\n# mean = sums / normalizer\n# std = np.sqrt((sums_squared / normalizer) - np.square(mean))","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-10-31T06:01:32.458235Z","iopub.execute_input":"2023-10-31T06:01:32.45853Z","iopub.status.idle":"2023-10-31T06:01:32.468837Z","shell.execute_reply.started":"2023-10-31T06:01:32.458504Z","shell.execute_reply":"2023-10-31T06:01:32.467754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UBCDataset(Dataset):\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_path = self.df.iloc[idx]['file_path']  \n        label = self.df.iloc[idx]['label']\n        \n        img = Image.open(img_path).convert(\"RGB\")\n        \n        if self.transforms:\n            img = self.transforms(img)\n\n        return img, label","metadata":{"execution":{"iopub.status.busy":"2023-10-31T08:46:05.229182Z","iopub.execute_input":"2023-10-31T08:46:05.229537Z","iopub.status.idle":"2023-10-31T08:46:05.236481Z","shell.execute_reply.started":"2023-10-31T08:46:05.22951Z","shell.execute_reply":"2023-10-31T08:46:05.235328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UBCModel(pl.LightningModule):\n\n    def __init__(self):\n        super(UBCModel, self).__init__()\n        self.num_classes = 5\n        \n        self.model = timm.create_model('resnext50_32x4d', pretrained=False)\n        self.model.load_state_dict(torch.load('/kaggle/input/resnext50-32x4d/resnext50_32x4d.pth'))\n        \n        self.dropout = nn.Dropout(0.5)\n        self.model.fc= torch.nn.Linear(in_features=2048, out_features=self.num_classes, bias=True)\n        self.criterion = nn.CrossEntropyLoss()\n        \n        self.f1 = MulticlassF1Score(num_classes=self.num_classes, average='macro')\n        self.accuracy = torchmetrics.Accuracy(num_classes=self.num_classes, task='multiclass')\n        self.precision = torchmetrics.Precision(average='macro', num_classes=self.num_classes, task='multiclass')\n        self.recall = torchmetrics.Recall(average='macro', num_classes=self.num_classes, task='multiclass')\n        \n    def forward(self, x):\n        x = self.model(x)\n        x = self.dropout(x)\n        return x\n    \n    def training_step(self, batch, batch_idx):\n        x, y = batch\n        y_pred = self(x)\n        loss = self.criterion(y_pred, y)\n        self.log('train_loss', loss)\n        self.log('train_f1', self.f1(y_pred, y))\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        y_pred = self(x)\n        loss = self.criterion(y_pred, y)\n        self.log('val_loss', loss)\n        self.log('val_f1', self.f1(y_pred, y))\n        self.log('val_acc', self.accuracy(y_pred, y))\n        self.log('val_precision', self.precision(y_pred, y))\n        self.log('val_recall', self.recall(y_pred, y))\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=1e-3, weight_decay=1e-5)\n        scheduler = StepLR(optimizer, step_size=10, gamma=0.3)\n        return {\n            'optimizer': optimizer,\n            'lr_scheduler': {\n                'scheduler': scheduler,\n                'interval': 'epoch',\n            }\n        }","metadata":{"execution":{"iopub.status.busy":"2023-10-31T08:46:13.730082Z","iopub.execute_input":"2023-10-31T08:46:13.730452Z","iopub.status.idle":"2023-10-31T08:46:13.743943Z","shell.execute_reply.started":"2023-10-31T08:46:13.730423Z","shell.execute_reply":"2023-10-31T08:46:13.742973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean = [0.48828688, 0.42932517, 0.49162089]\nstd = [0.41380908, 0.37492874, 0.41795654]\n\n# Define transformations for training and test datasets\ntrain_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomRotation(degrees=(-25, 25)),\n    \n    transforms.ToTensor(),\n    transforms.Normalize(mean, std)\n])\n\nval_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean, std)  # Use your own calculated mean and std\n])","metadata":{"execution":{"iopub.status.busy":"2023-10-31T08:46:14.569298Z","iopub.execute_input":"2023-10-31T08:46:14.569663Z","iopub.status.idle":"2023-10-31T08:46:14.576973Z","shell.execute_reply.started":"2023-10-31T08:46:14.569633Z","shell.execute_reply":"2023-10-31T08:46:14.575604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X = df['image_id'].to_numpy()\n# y = df['label'].to_numpy()  # Assuming 'file_path' is the label column\n\n# skf = StratifiedKFold(n_splits=4)\n\n# for fold, (train_index, val_index) in enumerate(skf.split(X, y)):\n#     # Create dataframes for this fold\n#     train_df_fold = df.iloc[train_index]\n#     val_df_fold = df.iloc[val_index]\n\n#     # Create DataLoaders for this fold\n#     train_dataset = UBCDataset(df=train_df_fold, transforms=train_transform)\n#     train_loader = DataLoader(train_dataset, batch_size=12, shuffle=True, num_workers=2, pin_memory=True)\n#     steps_per_epoch = len(train_loader)\n\n#     val_dataset = UBCDataset(df=val_df_fold, transforms=val_transform)\n#     val_loader = DataLoader(val_dataset, batch_size=12, shuffle=False, num_workers=2, pin_memory=True)\n\n#     # Initialize and train your model\n#     model = UBCModel(steps_per_epoch=steps_per_epoch)\n\n#     checkpoint_callback = ModelCheckpoint(\n#         dirpath=f'/kaggle/working/fold_{fold + 1}/',\n#         monitor='val_f1',\n#         save_top_k=10,\n#         mode='max',\n#     )\n#     early_stop_callback = EarlyStopping(\n#         monitor='val_f1',\n#         patience=5,\n#         verbose=True,\n#         mode='max',\n#     )\n#     logger = TensorBoardLogger(f\"tb_logs/fold_{fold + 1}\", name=\"my_model\")\n\n#     trainer = Trainer(\n#         deterministic=True,\n#         log_every_n_steps=3,\n#         max_epochs=30,\n#         logger=logger,\n#         callbacks=[checkpoint_callback]\n#     )\n    \n#     print(f\"Training for fold {fold+1}...\")\n#     trainer.fit(model, train_loader, val_loader)","metadata":{"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-10-31T08:46:15.113222Z","iopub.execute_input":"2023-10-31T08:46:15.113553Z","iopub.status.idle":"2023-10-31T08:46:15.119419Z","shell.execute_reply.started":"2023-10-31T08:46:15.113529Z","shell.execute_reply":"2023-10-31T08:46:15.118465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, val_df = train_test_split(df, test_size=0.2, random_state=42)\n\ntrain_dataset = UBCDataset(df=train_df, transforms=train_transform)\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=2, pin_memory=True)\n\nval_dataset = UBCDataset(df=val_df, transforms=val_transform)\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False, num_workers=2, pin_memory=True)\n\n# Initialize and train your model\nmodel = UBCModel()\n\ncheckpoint_callback = ModelCheckpoint(\n    dirpath=f'/kaggle/working/',\n    monitor='val_loss',\n    save_top_k=1,\n    mode='max',\n)\n# early_stop_callback = EarlyStopping(\n#     monitor='val_loss',\n#     patience=5,\n#     verbose=True,\n#     mode='max',\n# )\nlogger = TensorBoardLogger(f\"logs/\", name=\"my_model\")\n\ntrainer = Trainer(\n    deterministic=True,\n    log_every_n_steps=1,\n    max_epochs=25,\n    logger=logger,\n    callbacks=[checkpoint_callback]\n)\n    \ntrainer.fit(model, train_loader, val_loader)","metadata":{"execution":{"iopub.status.busy":"2023-10-31T08:46:16.863308Z","iopub.execute_input":"2023-10-31T08:46:16.864013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ncheckpoint_path = '/kaggle/working/epoch=18-step=266.ckpt'\nmodel = UBCModel.load_from_checkpoint(checkpoint_path, map_location=device)\nmodel = model.eval()","metadata":{"execution":{"iopub.status.busy":"2023-10-31T06:33:12.205363Z","iopub.execute_input":"2023-10-31T06:33:12.2058Z","iopub.status.idle":"2023-10-31T06:33:13.232424Z","shell.execute_reply.started":"2023-10-31T06:33:12.205748Z","shell.execute_reply":"2023-10-31T06:33:13.231234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nlabels = []\n\n# Assuming val_loader is your validation DataLoader\nwith torch.no_grad():\n    for data, label in val_loader:\n        data = data.float()\n        data = data.to(device)\n        label = label.long()\n        \n        output = model(data)\n        pred = torch.argmax(output, dim=1)  # For multi-class classification\n        preds.extend(pred.cpu().numpy())\n        labels.extend(label.cpu().numpy())\n\npreds = torch.tensor(preds)\nlabels = torch.tensor(labels).long()","metadata":{"execution":{"iopub.status.busy":"2023-10-31T06:33:18.177852Z","iopub.execute_input":"2023-10-31T06:33:18.178297Z","iopub.status.idle":"2023-10-31T06:33:33.9236Z","shell.execute_reply.started":"2023-10-31T06:33:18.178262Z","shell.execute_reply":"2023-10-31T06:33:33.92233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = torchmetrics.Accuracy(num_classes=5, task='multiclass')(preds, labels)\nprecision = torchmetrics.Precision(average='macro', num_classes=5, task='multiclass')(preds, labels)\nrecall = torchmetrics.Recall(average='macro', num_classes=5, task='multiclass')(preds, labels)\ncm = torchmetrics.ConfusionMatrix(num_classes=5, task='multiclass')(preds, labels)\n\nprint(f'Val Accuracy {acc}')\nprint(f'Val Precision {precision}')\nprint(f'Val Recall {recall}')\nprint(f'Val ConfusionMatrix {cm}')\n\n# something is really wrong here lol, definitely overfitting on train and val data,\n# I assume it's because of stratified k folding","metadata":{"execution":{"iopub.status.busy":"2023-10-31T06:33:33.925778Z","iopub.execute_input":"2023-10-31T06:33:33.926224Z","iopub.status.idle":"2023-10-31T06:33:33.956544Z","shell.execute_reply.started":"2023-10-31T06:33:33.926187Z","shell.execute_reply":"2023-10-31T06:33:33.955008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"authtoken = ''\n# get ur auth token here https://dashboard.ngrok.com/get-started/your-authtoken\n# u need an account","metadata":{"execution":{"iopub.status.busy":"2023-10-31T06:33:38.477291Z","iopub.execute_input":"2023-10-31T06:33:38.477654Z","iopub.status.idle":"2023-10-31T06:33:38.482704Z","shell.execute_reply.started":"2023-10-31T06:33:38.477626Z","shell.execute_reply":"2023-10-31T06:33:38.48162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n!unzip -o ngrok-stable-linux-amd64.zip\n\nos.system(\"./ngrok authtoken \"+ authtoken)\n\n# Run tensorboard as well as Ngrox (for tunneling as non-blocking processes)\nimport multiprocessing\n\npool = multiprocessing.Pool(processes = 10)\nresults_of_processes = [pool.apply_async(os.system, args=(cmd, ), callback = None )\n                        for cmd in [\n                        f\"tensorboard --logdir /kaggle/working/logs/my_model --host 0.0.0.0 --port 6008 &\",\n                        \"./ngrok http 6008 &\"\n                        ]]","metadata":{"execution":{"iopub.status.busy":"2023-10-31T06:33:38.738015Z","iopub.execute_input":"2023-10-31T06:33:38.738419Z","iopub.status.idle":"2023-10-31T06:33:42.863362Z","shell.execute_reply.started":"2023-10-31T06:33:38.738389Z","shell.execute_reply":"2023-10-31T06:33:42.858808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! curl -s http://localhost:4040/api/tunnels | python3 -c \\\n    \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\"","metadata":{"execution":{"iopub.status.busy":"2023-10-31T06:33:43.75287Z","iopub.execute_input":"2023-10-31T06:33:43.753388Z","iopub.status.idle":"2023-10-31T06:33:45.118927Z","shell.execute_reply.started":"2023-10-31T06:33:43.753348Z","shell.execute_reply":"2023-10-31T06:33:45.117407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}