{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Ovarian Cancer Cropped Data Pytorch Lightning CNN\nhttps://www.kaggle.com/code/stpeteishii/ovarian-cancer-subtype-pytorch-lightning-cnn<br/>\nhttps://www.kaggle.com/code/stpeteishii/cropped-train-test-datasets","metadata":{}},{"cell_type":"markdown","source":"# CAUTION!\n# Notebook Threw Exception\nYour notebook hit an unhandled error while rerunning your code. Note that the hidden dataset can be larger/smaller/different than the public dataset.","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import random_split\nfrom torch.utils.data import DataLoader, Dataset, Subset\nfrom torch.utils.data import random_split, SubsetRandomSampler\nfrom torchvision import datasets, transforms, models \nfrom torchvision.datasets import ImageFolder\nfrom torchvision.transforms import ToTensor\nfrom torchvision.utils import make_grid\nfrom pytorch_lightning import LightningModule\nfrom pytorch_lightning import Trainer\nimport pytorch_lightning as pl\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2023-04-25T13:11:17.858505Z","iopub.execute_input":"2023-04-25T13:11:17.859355Z","iopub.status.idle":"2023-04-25T13:11:27.854739Z","shell.execute_reply.started":"2023-04-25T13:11:17.859298Z","shell.execute_reply":"2023-04-25T13:11:27.853219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform=transforms.Compose([\n        transforms.RandomRotation(10),      # rotate +/- 10 degrees\n        transforms.RandomHorizontalFlip(),  # reverse 50% of images\n        transforms.Resize(224),             # resize shortest side to 224 pixels\n        transforms.CenterCrop(224),         # crop longest side to 224 pixels at center\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n])","metadata":{"papermill":{"duration":0.035423,"end_time":"2021-10-16T13:11:35.650718","exception":false,"start_time":"2021-10-16T13:11:35.615295","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-25T13:11:27.856923Z","iopub.execute_input":"2023-04-25T13:11:27.857379Z","iopub.status.idle":"2023-04-25T13:11:27.865377Z","shell.execute_reply.started":"2023-04-25T13:11:27.857344Z","shell.execute_reply":"2023-04-25T13:11:27.863809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset0=datasets.ImageFolder(root=\"/kaggle/input/cropped-train-test-datasets/train\",transform=None)\n\nclass_names=dataset0.classes\nprint(class_names)\nprint(len(class_names))\nN=list(range(len(class_names)))\nnormal_mapping=dict(zip(class_names,N)) \nreverse_mapping=dict(zip(N,class_names))       \n","metadata":{"papermill":{"duration":6.427069,"end_time":"2021-10-16T13:11:42.192371","exception":false,"start_time":"2021-10-16T13:11:35.765302","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-04-25T13:11:27.867461Z","iopub.execute_input":"2023-04-25T13:11:27.868396Z","iopub.status.idle":"2023-04-25T13:11:28.55757Z","shell.execute_reply.started":"2023-04-25T13:11:27.86834Z","shell.execute_reply":"2023-04-25T13:11:28.556446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataModule(pl.LightningDataModule):\n    \n    def __init__(self, transform=transform, batch_size=32):\n        super().__init__()\n        self.root_dir = \"/kaggle/input/cropped-train-test-datasets/train\"\n        self.transform = transform\n        self.batch_size = batch_size\n\n    def setup(self, stage=None):\n        dataset = datasets.ImageFolder(root=self.root_dir, transform=self.transform)\n        \n        n_data = len(dataset)\n        n_train = int(0.7 * n_data)\n        n_valid = int(0.15 * n_data)\n        n_test = n_data - n_train - n_valid\n        \n        trainset, validset,testset = torch.utils.data.random_split(dataset, [n_train, n_valid, n_test])\n        \n        self.train_dataset = DataLoader(trainset, batch_size=self.batch_size, shuffle=True)\n        self.valid_dataset = DataLoader(validset, batch_size=self.batch_size)\n        self.test_dataset = DataLoader(testset, batch_size=self.batch_size)\n\n    def train_dataloader(self):\n        return self.train_dataset\n\n    def valid_dataloader(self):\n        return self.valid_dataset\n    \n    def test_dataloader(self):\n        return self.test_dataset\n\n","metadata":{"execution":{"iopub.status.busy":"2023-04-25T13:11:28.560195Z","iopub.execute_input":"2023-04-25T13:11:28.560833Z","iopub.status.idle":"2023-04-25T13:11:28.569769Z","shell.execute_reply.started":"2023-04-25T13:11:28.560792Z","shell.execute_reply":"2023-04-25T13:11:28.56857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvolutionalNetwork(LightningModule):\n    \n    def __init__(self):\n        super(ConvolutionalNetwork, self).__init__()\n        self.conv1 = nn.Conv2d(3, 6, 3, 1)\n        self.conv2 = nn.Conv2d(6, 16, 3, 1)\n        self.fc1 = nn.Linear(16 * 54 * 54, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84, 20)\n        self.fc4 = nn.Linear(20, len(class_names))\n\n    def forward(self, X):\n        X = F.relu(self.conv1(X))\n        X = F.max_pool2d(X, 2, 2)\n        X = F.relu(self.conv2(X))\n        X = F.max_pool2d(X, 2, 2)\n        X = X.view(-1, 16 * 54 * 54)\n        X = F.relu(self.fc1(X))\n        X = F.relu(self.fc2(X))\n        X = F.relu(self.fc3(X))\n        X = self.fc4(X)\n        return F.log_softmax(X, dim=1)\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=0.001)\n        return optimizer\n\n    def training_step(self, train_batch, batch_idx):\n        X, y = train_batch\n        y_hat = self(X)\n        loss = F.cross_entropy(y_hat, y)\n        pred = y_hat.argmax(dim=1, keepdim=True)\n        acc = pred.eq(y.view_as(pred)).sum().item() / y.shape[0]\n        self.log(\"train_loss\", loss)\n        self.log(\"train_acc\", acc)\n        return loss\n\n    def validation_step(self, val_batch, batch_idx):\n        X, y = val_batch\n        y_hat = self(X)\n        loss = F.cross_entropy(y_hat, y)\n        pred = y_hat.argmax(dim=1, keepdim=True)\n        acc = pred.eq(y.view_as(pred)).sum().item() / y.shape[0]\n        self.log(\"val_loss\", loss)\n        self.log(\"val_acc\", acc)\n\n    def test_step(self, test_batch, batch_idx):\n        X, y = test_batch\n        y_hat = self(X)\n        loss = F.cross_entropy(y_hat, y)\n        pred = y_hat.argmax(dim=1, keepdim=True)\n        acc = pred.eq(y.view_as(pred)).sum().item() / y.shape[0]\n        self.log(\"test_loss\", loss)\n        self.log(\"test_acc\", acc)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T13:11:28.571625Z","iopub.execute_input":"2023-04-25T13:11:28.572361Z","iopub.status.idle":"2023-04-25T13:11:28.590454Z","shell.execute_reply.started":"2023-04-25T13:11:28.572322Z","shell.execute_reply":"2023-04-25T13:11:28.58907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    datamodule = DataModule()\n    datamodule.setup()\n    model = ConvolutionalNetwork()\n    trainer = pl.Trainer(max_epochs=100)\n    trainer.fit(model, datamodule)\n    datamodule.setup(stage='test')\n    test_loader = datamodule.test_dataloader()\n    trainer.test(dataloaders=test_loader)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T13:12:33.34668Z","iopub.execute_input":"2023-04-25T13:12:33.347122Z","iopub.status.idle":"2023-04-25T13:12:33.530305Z","shell.execute_reply.started":"2023-04-25T13:12:33.347085Z","shell.execute_reply":"2023-04-25T13:12:33.528592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for images, labels in datamodule.train_dataloader():\n    break\nim=make_grid(images,nrow=16)\n\nplt.figure(figsize=(12,12))\nplt.imshow(np.transpose(im.numpy(),(1,2,0)))\n\ninv_normalize=transforms.Normalize(mean=[-0.485/0.229,-0.456/0.224,-0.406/0.225],\n                                   std=[1/0.229,1/0.224,1/0.225])\nim=inv_normalize(im)\n\nplt.figure(figsize=(12,12))\nplt.imshow(np.transpose(im.numpy(),(1,2,0)))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T13:11:28.962203Z","iopub.status.idle":"2023-04-25T13:11:28.963638Z","shell.execute_reply.started":"2023-04-25T13:11:28.963287Z","shell.execute_reply":"2023-04-25T13:11:28.963322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cpu\")   #\"cuda:0\"\n\nmodel.eval()\ny_true=[]\ny_pred=[]\nwith torch.no_grad():\n    for test_data in datamodule.test_dataloader():\n        test_images, test_labels = test_data[0].to(device), test_data[1].to(device)\n        pred = model(test_images).argmax(dim=1)\n        for i in range(len(pred)):\n            y_true.append(test_labels[i].item())\n            y_pred.append(pred[i].item())\n\nprint(classification_report(y_true,y_pred,target_names=class_names,digits=4))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T13:11:28.965146Z","iopub.status.idle":"2023-04-25T13:11:28.96576Z","shell.execute_reply.started":"2023-04-25T13:11:28.965423Z","shell.execute_reply":"2023-04-25T13:11:28.965455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Process Test Image","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport numpy as np\nimport pandas as pd \n\nimport matplotlib.pyplot as plt\nimport cv2\nfrom PIL import Image\nImage.MAX_IMAGE_PIXELS = 10000000000","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir test","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tpaths=[]\nfor dirname, _, filenames in os.walk('/kaggle/input/UBC-OCEAN/test_images'):\n    for filename in filenames:\n        tpaths+=[(os.path.join(dirname, filename))]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_width,new_height = 6224,6224\ncrop_width,crop_height = 224,224\nrange_width = new_width-crop_width\nrange_height = new_height-crop_height","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(tpaths)):\n    file=tpaths[i].split('/')[-1]\n    path=os.path.join('/kaggle/input/UBC-OCEAN/test_images',file)\n    img0 = Image.open(path)\n    img = img0.resize((new_width,new_height))\n    X = random.sample(range(range_width),16)\n    Y = random.sample(range(range_height),16)\n    for j in range(16):\n        x,y=X[j],Y[j]\n        topath=os.path.join('test',file[0:-4]+'_'+str(j).zfill(2)+'.png')  \n        cropped = img.crop((x,y,x+crop_width,y+crop_height))\n        channel_zero_count = 0\n        for yi in range(crop_height):\n            for xi in range(crop_width):\n                r,g,b = cropped.getpixel((xi,yi))\n                if (r==0 and g==0 and b==0) or (r>220 and g>220 and b>220):\n                    channel_zero_count += 1\n        if channel_zero_count <= 1000:\n            cropped.save(topath)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t2paths=[]\nfor dirname, _, filenames in os.walk('test'):\n    for filename in filenames:\n        t2paths+=[(os.path.join(dirname, filename))]\ntest_df=pd.DataFrame(columns=['path'],data=t2paths)\ntest_df['image_id']=test_df['path'].apply(lambda x: int(x.split('/')[-1].split('_')[0]))\nprint(len(t2paths))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"class CustomTestDataset(Dataset):\n    def __init__(self, transform=transform, batch_size=16):\n        self.root_dir =  \"test\"\n        self.transform = transform\n        self.image_files = os.listdir(self.root_dir)\n\n    def __len__(self):\n        return len(self.image_files)\n\n    def __getitem__(self, idx):\n        image_path = os.path.join(self.root_dir, self.image_files[idx])\n        image = Image.open(image_path)\n        if self.transform:\n            image = self.transform(image)\n\n        return image","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"custom_test_dataset = CustomTestDataset()\ntest_loader = DataLoader(custom_test_dataset, batch_size=32, shuffle=False)\n\nfor batch in test_loader:\n    pred = model(batch)\n    pred_np = pred.detach().numpy() \n    print(pred_np.shape)\n\npred_df=pd.DataFrame(data=pred_np)\ntest_df=pd.concat([test_df,pred_df],axis=1)\ndisplay(test_df)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df2=test_df.groupby('image_id',as_index=False).mean().reset_index(drop=True)\ndisplay(test_df2)\n\ndef map_row_to_label(row):\n    return reverse_mapping[row.idxmax()]\n\ntest_df2['label'] = test_df2.iloc[:,1:].apply(map_row_to_label, axis=1)\ndisplay(test_df2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit=test_df2[['image_id','label']]\ndisplay(submit)\nsubmit.to_csv('submission.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CAUTION!\n# Notebook Threw Exception\nYour notebook hit an unhandled error while rerunning your code. Note that the hidden dataset can be larger/smaller/different than the public dataset.","metadata":{}},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.040346,"end_time":"2021-10-16T14:03:19.432209","exception":false,"start_time":"2021-10-16T14:03:19.391863","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}