{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Links","metadata":{}},{"cell_type":"markdown","source":"Jirka's links:\n* [Spine🦴Fracture: EDA🔎 & loading DICOM & 3D browse](https://www.kaggle.com/code/jirkaborovec/spine-fracture-eda-loading-dicom-3d-browse)\n* [Spine🦴Fracture: convert🤖 DICOM imgs -> 3D volume](https://www.kaggle.com/code/jirkaborovec/spine-fracture-convert-dicom-imgs-3d-volume)\n* [Spine🦴Fracture: convert🤖 DICOM -> equalized PNG](https://www.kaggle.com/code/jirkaborovec/spine-fracture-convert-dicom-equalized-png)\n* [SpineFrac🦴Classif: e2e ~ Lightning⚡MONAI⚕️3D](https://www.kaggle.com/code/jirkaborovec/spinefrac-classif-e2e-lightning-monai-3d/notebook)\n* [Cervical Spine Fracture Detection: 3D volumes](https://www.kaggle.com/datasets/jirkaborovec/cervical-spine-fracture-detection-npz-3d-volumes)\n* [Cervical Spine Fracture Detection: equalized PNG](https://www.kaggle.com/datasets/jirkaborovec/cervical-spine-fracture-detection-equalized-png)\n\nSam's links:\n\n* [🦴 RSNA Fracture Detection - in-depth EDA](https://www.kaggle.com/code/samuelcortinhas/rsna-fracture-detection-in-depth-eda)\n* [Extracting Vertebrae C1, ..., C7](https://www.kaggle.com/code/samuelcortinhas/extracting-vertebrae-c1-c7)\n* [RSNA - CT gifs](https://www.kaggle.com/code/samuelcortinhas/rsna-ct-gifs)\n* [RSNA 2022 Spine Fracture Detection - Metadata](https://www.kaggle.com/datasets/samuelcortinhas/rsna-2022-spine-fracture-detection-metadata)\n* [RSNA - 3D train tensors [first half]](https://www.kaggle.com/datasets/samuelcortinhas/rsna-3d-train-tensors-first-half)\n* [RSNA - 3D train tensors [second half]](https://www.kaggle.com/datasets/samuelcortinhas/rsna-3d-train-tensors-second-half)","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"!pip install -qU ../input/for-pydicom/python_gdcm-3.0.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl ../input/for-pydicom/pylibjpeg-1.4.0-py3-none-any.whl --find-links frozen_packages --no-index","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-28T15:34:41.1085Z","iopub.execute_input":"2022-08-28T15:34:41.109355Z","iopub.status.idle":"2022-08-28T15:34:58.627534Z","shell.execute_reply.started":"2022-08-28T15:34:41.109221Z","shell.execute_reply":"2022-08-28T15:34:58.625636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q kaggle_vol3d_classify -f ../input/cervical-spine-fracture-detection-npz-3d-volumes/frozen_packages --no-index\n# !pip install -qU \"pytorch-lightning>1.5.0\" --no-index\n#!pip uninstall -y torchtext\n#!pip list | grep -e lightning -e kaggle -e monai","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-28T15:34:58.630953Z","iopub.execute_input":"2022-08-28T15:34:58.631638Z","iopub.status.idle":"2022-08-28T15:35:18.844332Z","shell.execute_reply.started":"2022-08-28T15:34:58.631571Z","shell.execute_reply":"2022-08-28T15:35:18.842512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport matplotlib.patches as patches\nimport seaborn as sns\nsns.set(style='darkgrid', font_scale=1.6)\nimport cv2\nimport os\nfrom os import listdir\nimport re\nimport gc\nimport random\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom tqdm.auto import tqdm\nfrom pprint import pprint\nfrom time import time\nimport itertools\nfrom skimage import measure\nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection\nimport nibabel as nib\nfrom glob import glob\nimport warnings\n#warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n#warnings.filterwarnings(\"ignore\", category=UserWarning)\n#warnings.filterwarnings(\"ignore\", category=FutureWarning)\nimport zipfile\nfrom scipy import ndimage\nfrom sklearn.model_selection import train_test_split\nfrom joblib import Parallel, delayed\nfrom PIL import Image\nfrom dipy.denoise.nlmeans import nlmeans\nfrom dipy.denoise.noise_estimate import estimate_sigma\nfrom kaggle_volclassif.utils import interpolate_volume\nfrom skimage import exposure\n\n# Pytorch\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.optim.lr_scheduler as lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nimport torch.nn.functional as F\nimport kornia\nimport kornia.augmentation as augmentation","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-28T15:35:18.848234Z","iopub.execute_input":"2022-08-28T15:35:18.848858Z","iopub.status.idle":"2022-08-28T15:35:20.979151Z","shell.execute_reply.started":"2022-08-28T15:35:18.848802Z","shell.execute_reply":"2022-08-28T15:35:20.977428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reproducibility","metadata":{}},{"cell_type":"code","source":"# Set random seeds\ndef set_seed(seed=0):\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.manual_seed(seed)\nset_seed()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T15:36:00.891488Z","iopub.execute_input":"2022-08-28T15:36:00.891978Z","iopub.status.idle":"2022-08-28T15:36:00.911561Z","shell.execute_reply.started":"2022-08-28T15:36:00.891943Z","shell.execute_reply":"2022-08-28T15:36:00.9096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"# Hyperparameters\nBATCH_SIZE = 4\nLEARNING_RATE = 0.0001\nN_EPOCHS = 20\nPATIENCE = 3\nEXPERIMENTAL = False\nAUGMENTATIONS = True\n\n# Config device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-08-28T15:36:03.978034Z","iopub.execute_input":"2022-08-28T15:36:03.978959Z","iopub.status.idle":"2022-08-28T15:36:03.99338Z","shell.execute_reply.started":"2022-08-28T15:36:03.978911Z","shell.execute_reply":"2022-08-28T15:36:03.991537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"markdown","source":"### Load tables","metadata":{}},{"cell_type":"code","source":"# Load metadata\ntrain_df = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train.csv\")\ntrain_bbox = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train_bounding_boxes.csv\")\ntest_df = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/test.csv\")\nss = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/sample_submission.csv\")\n\n# Print dataframe shapes\nprint('train shape:', train_df.shape)\nprint('train bbox shape:', train_bbox.shape)\nprint('test shape:', test_df.shape)\nprint('ss shape:', ss.shape)\nprint('')\n\n# Show first few entries\ntrain_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T15:36:05.645447Z","iopub.execute_input":"2022-08-28T15:36:05.645943Z","iopub.status.idle":"2022-08-28T15:36:05.735567Z","shell.execute_reply.started":"2022-08-28T15:36:05.645906Z","shell.execute_reply":"2022-08-28T15:36:05.734005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Drop bad scans","metadata":{}},{"cell_type":"code","source":"# https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/discussion/344862\nbad_scans = ['1.2.826.0.1.3680043.20574','1.2.826.0.1.3680043.29952']\n\nfor uid in bad_scans:\n    train_df.drop(train_df[train_df['StudyInstanceUID']==uid].index, axis=0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T15:36:06.630096Z","iopub.execute_input":"2022-08-28T15:36:06.630623Z","iopub.status.idle":"2022-08-28T15:36:06.657488Z","shell.execute_reply.started":"2022-08-28T15:36:06.630584Z","shell.execute_reply":"2022-08-28T15:36:06.655878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Debug","metadata":{}},{"cell_type":"code","source":"debug = False\nif len(ss)==3:\n    debug = True\n    \n    # Fix mismatch with test_images folder\n    test_df = pd.DataFrame(columns = ['row_id','StudyInstanceUID','prediction_type'])\n    for i in ['1.2.826.0.1.3680043.22327','1.2.826.0.1.3680043.25399','1.2.826.0.1.3680043.5876']:\n        for j in ['C1','C2','C3','C4','C5','C6','C7','patient_overall']:\n            test_df = test_df.append({'row_id':i+'_'+j,'StudyInstanceUID':i,'prediction_type':j},ignore_index=True)\n    \n    # Sample submission\n    ss = pd.DataFrame(test_df['row_id'])\n    ss['fractured'] = 0.5\n    \n    display(test_df.head(3))","metadata":{"execution":{"iopub.status.busy":"2022-08-28T15:36:07.730474Z","iopub.execute_input":"2022-08-28T15:36:07.730957Z","iopub.status.idle":"2022-08-28T15:36:07.824878Z","shell.execute_reply.started":"2022-08-28T15:36:07.730921Z","shell.execute_reply":"2022-08-28T15:36:07.823119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentations","metadata":{}},{"cell_type":"markdown","source":"Ideas: rotation/flip in x-y plane. Small deformations. Some scans are contained in circle, others aren't. Maybe apply a mask to be consistent. ","metadata":{}},{"cell_type":"code","source":"# Data augmentations (https://kornia.readthedocs.io/en/latest/augmentation.module.html#geometric)\nif AUGMENTATIONS:\n    augs = transforms.Compose([\n        augmentation.RandomRotation3D((0,0,30), resample='bilinear', p=0.5, same_on_batch=False, keepdim=True),\n        #augmentation.RandomHorizontalFlip3D(same_on_batch=False, p=0.5, keepdim=True),\n        ])\nelse:\n    augs=None","metadata":{"execution":{"iopub.status.busy":"2022-08-28T16:23:02.518172Z","iopub.execute_input":"2022-08-28T16:23:02.519194Z","iopub.status.idle":"2022-08-28T16:23:02.527342Z","shell.execute_reply.started":"2022-08-28T16:23:02.519145Z","shell.execute_reply":"2022-08-28T16:23:02.52621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualise augmentations","metadata":{}},{"cell_type":"code","source":"path = '../input/rsna-3d-train-tensors-second-half/train_volumes/1.2.826.0.1.3680043.10443.pt'\nvol = torch.load(path).to(torch.float32)\nplt.imshow(vol[60,:,:], cmap='bone')\nplt.axis('off')\nplt.title('Original')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T16:21:32.492123Z","iopub.execute_input":"2022-08-28T16:21:32.493728Z","iopub.status.idle":"2022-08-28T16:21:32.720424Z","shell.execute_reply.started":"2022-08-28T16:21:32.493672Z","shell.execute_reply":"2022-08-28T16:21:32.71875Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot images\nfig, axes = plt.subplots(nrows=3, ncols=6, figsize=(24,12))\n\nfor i in range(18):\n    vol2 = augs(vol)\n\n    # Plot the image\n    x = i // 6\n    y = i % 6\n\n    axes[x, y].imshow(vol2[60,:,:], cmap=\"bone\")\n    axes[x, y].axis('off')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T16:21:41.541604Z","iopub.execute_input":"2022-08-28T16:21:41.543395Z","iopub.status.idle":"2022-08-28T16:21:54.95878Z","shell.execute_reply.started":"2022-08-28T16:21:41.543305Z","shell.execute_reply":"2022-08-28T16:21:54.956408Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Torch dataset","metadata":{}},{"cell_type":"code","source":"# Dataset for train/valid sets only\nclass RSNADataset(Dataset):\n    # Initialise\n    def __init__(self, subset='train', df_table = train_df, transform=None):\n        super().__init__()\n        \n        self.subset = subset\n        self.df_table = df_table.reset_index(drop=True)\n        self.transform = transform\n        self.targets = ['C1','C2','C3','C4','C5','C6','C7','patient_overall']\n        \n        # Identify files in each of the two datasets\n        fh_paths = glob(os.path.join('../input/rsna-3d-train-tensors-first-half/train_volumes', \"*.pt\"))\n        sh_paths = glob(os.path.join('../input/rsna-3d-train-tensors-second-half/train_volumes', \"*.pt\"))\n        \n        fh_list = []\n        sh_list = []\n        for i in fh_paths:\n            fh_list.append(i.split('/')[-1][:-3])\n        \n        for i in sh_paths:\n            sh_list.append(i.split('/')[-1][:-3])\n        \n        self.df_table_fh = self.df_table[self.df_table['StudyInstanceUID'].isin(fh_list)]\n        self.df_table_sh = self.df_table[self.df_table['StudyInstanceUID'].isin(sh_list)]\n        \n        # Image paths\n        self.volume_dir1 = '../input/rsna-3d-train-tensors-first-half/train_volumes'  # <=1000 patient\n        self.volume_dir2 = '../input/rsna-3d-train-tensors-second-half/train_volumes' # >1000 patient\n\n        # Populate labels\n        self.labels = self.df_table[self.targets].values\n        \n    # Get item in position given by index\n    def __getitem__(self, index):\n        if index in self.df_table_fh.index:\n            patient = self.df_table_fh[self.df_table_fh.index==index]['StudyInstanceUID'].iloc[0]\n            path = os.path.join(self.volume_dir1, f\"{patient}.pt\")\n            vol = torch.load(path).to(torch.float32)\n        else:\n            patient = self.df_table_sh[self.df_table_sh.index==index]['StudyInstanceUID'].iloc[0]\n            path = os.path.join(self.volume_dir2, f\"{patient}.pt\")\n            vol = torch.load(path).to(torch.float32)\n        \n        # Data augmentations\n        if self.transform:\n            vol = self.transform(vol)\n        \n        return vol.unsqueeze(0), self.labels[index]\n\n    # Length of dataset\n    def __len__(self):\n        return len(self.df_table['StudyInstanceUID'])","metadata":{"execution":{"iopub.status.busy":"2022-08-28T16:23:09.457182Z","iopub.execute_input":"2022-08-28T16:23:09.458399Z","iopub.status.idle":"2022-08-28T16:23:09.47683Z","shell.execute_reply.started":"2022-08-28T16:23:09.458335Z","shell.execute_reply":"2022-08-28T16:23:09.475713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train/valid datasets\nexperimental=EXPERIMENTAL\nif experimental:\n    train_table, valid_table = train_test_split(train_df, train_size=0.1, test_size=0.01, random_state=0)\n    train_dataset = RSNADataset(subset='train', df_table = train_table, transform=augs)\n    valid_dataset = RSNADataset(subset='valid', df_table = valid_table)\nelse:\n    train_table, valid_table = train_test_split(train_df, train_size=0.85, test_size=0.15, random_state=0)\n    train_dataset = RSNADataset(subset='train', df_table = train_table, transform=augs)\n    valid_dataset = RSNADataset(subset='valid', df_table = valid_table)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T16:23:18.653975Z","iopub.execute_input":"2022-08-28T16:23:18.654729Z","iopub.status.idle":"2022-08-28T16:23:18.695228Z","shell.execute_reply.started":"2022-08-28T16:23:18.654683Z","shell.execute_reply":"2022-08-28T16:23:18.693494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Torch dataloaders","metadata":{}},{"cell_type":"code","source":"# Dataloaders\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True)\nvalid_loader = DataLoader(dataset=valid_dataset, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T16:23:20.055011Z","iopub.execute_input":"2022-08-28T16:23:20.055563Z","iopub.status.idle":"2022-08-28T16:23:20.062871Z","shell.execute_reply.started":"2022-08-28T16:23:20.055523Z","shell.execute_reply":"2022-08-28T16:23:20.061143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\n\nconv output size = floor((W-F+2P)/S + 1)","metadata":{}},{"cell_type":"code","source":"# Experiment with architecture\n'''\narr = np.ones((4,1,224,224,224))\nx = torch.tensor(arr, dtype=torch.float32)\n\nconv1 = nn.Conv3d(in_channels=1, out_channels=16, kernel_size=7, stride=1, padding=0)\npool = nn.MaxPool3d(kernel_size=2, stride=2, padding=0)\nnorm1 = nn.BatchNorm3d(num_features=16)\nconv2 = nn.Conv3d(in_channels=16, out_channels=32, kernel_size=3, stride=1, padding=0)\nnorm2 = nn.BatchNorm3d(num_features=32)\nconv3 = nn.Conv3d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=0)\nnorm3 = nn.BatchNorm3d(num_features=64)\navg = nn.AdaptiveAvgPool3d((7, 1, 1))\nflat = nn.Flatten()\nrelu = nn.ReLU()\nlin1 = nn.Linear(in_features=32*13*13*13, out_features=256)\nlin2 = nn.Linear(in_features=256, out_features=8)\n\nout = conv1(x)\nout = relu(out)\nout = pool(out)\nout = norm1(out)\nprint(out.shape)\n\nout = conv2(out)\nout = relu(out)\nout = pool(out)\nout = norm2(out)\nprint(out.shape)\n\nout = conv3(out)\nout = relu(out)\nout = pool(out)\nout = norm3(out)\nprint(out.shape)\n\nout = avg(out)\nprint(out.shape)\nout = flat(out)\nprint(out.shape)\n'''","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-28T12:40:50.1844Z","iopub.execute_input":"2022-08-28T12:40:50.184738Z","iopub.status.idle":"2022-08-28T12:40:50.192605Z","shell.execute_reply.started":"2022-08-28T12:40:50.184707Z","shell.execute_reply":"2022-08-28T12:40:50.191553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3D convolutional neural network\nclass Conv3DNet(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        # Layers\n        self.conv1 = nn.Conv3d(in_channels=1, out_channels=16, kernel_size=7, stride=1, padding=0)\n        self.pool = nn.MaxPool3d(kernel_size=2, stride=2, padding=0)\n        self.norm1 = nn.BatchNorm3d(num_features=16)\n        self.conv2 = nn.Conv3d(in_channels=16, out_channels=32, kernel_size=3, stride=1, padding=0)\n        self.norm2 = nn.BatchNorm3d(num_features=32)\n        self.conv3 = nn.Conv3d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=0)\n        self.norm3 = nn.BatchNorm3d(num_features=64)\n        self.avg = nn.AdaptiveAvgPool3d((7, 1, 1))\n        self.flat = nn.Flatten()\n        self.relu = nn.ReLU()\n        self.lin1 = nn.Linear(in_features=448, out_features=128)\n        self.lin2 = nn.Linear(in_features=128, out_features=8)\n        \n    def forward(self, x):\n        # Conv block 1\n        out = self.conv1(x)\n        out = self.relu(out)\n        out = self.pool(out)\n        out = self.norm1(out)\n        \n        # Conv block 2\n        out = self.conv2(out)\n        out = self.relu(out)\n        out = self.pool(out)\n        out = self.norm2(out)\n        \n        # Conv block 3\n        out = self.conv3(out)\n        out = self.relu(out)\n        out = self.pool(out)\n        out = self.norm3(out)\n        \n        # Average & flatten\n        out = self.avg(out)\n        out = self.flat(out)\n        \n        # Fully connected layer\n        out = self.lin1(out)\n        out = self.relu(out)\n        \n        # Output layer (no sigmoid needed)\n        out = self.lin2(out)\n        \n        return out\n\nmodel = Conv3DNet().to(device)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:40:50.53526Z","iopub.execute_input":"2022-08-28T12:40:50.535673Z","iopub.status.idle":"2022-08-28T12:40:52.158273Z","shell.execute_reply.started":"2022-08-28T12:40:50.535645Z","shell.execute_reply":"2022-08-28T12:40:52.157229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss & optimiser","metadata":{}},{"cell_type":"code","source":"# Replicate competition metric (https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/discussion/341854)\nloss_fn = nn.BCEWithLogitsLoss(reduction='none')\n\ncompetition_weights = {\n    '-' : torch.tensor([1, 1, 1, 1, 1, 1, 1, 7], dtype=torch.float, device=device),\n    '+' : torch.tensor([2, 2, 2, 2, 2, 2, 2, 14], dtype=torch.float, device=device),\n}\n\n# y_hat.shape = (batch_size, num_classes)\n# y.shape = (batch_size, num_classes)\n\n# with row-wise weights normalization (https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/discussion/344565)\ndef competiton_loss_row_norm(y_hat, y):\n    loss = loss_fn(y_hat, y.to(y_hat.dtype))\n    weights = y * competition_weights['+'] + (1 - y) * competition_weights['-']\n    loss = (loss * weights).sum(axis=1)\n    w_sum = weights.sum(axis=1)\n    loss = torch.div(loss, w_sum)\n    return loss.mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:40:52.160442Z","iopub.execute_input":"2022-08-28T12:40:52.160911Z","iopub.status.idle":"2022-08-28T12:40:52.169278Z","shell.execute_reply.started":"2022-08-28T12:40:52.160867Z","shell.execute_reply":"2022-08-28T12:40:52.168321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adam optimiser\noptimiser = optim.AdamW(params=model.parameters(), lr=LEARNING_RATE)\n\n# Learning rate scheduler\nscheduler = lr_scheduler.CosineAnnealingLR(optimiser, T_max=N_EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:40:52.170686Z","iopub.execute_input":"2022-08-28T12:40:52.171303Z","iopub.status.idle":"2022-08-28T12:40:52.182643Z","shell.execute_reply.started":"2022-08-28T12:40:52.171268Z","shell.execute_reply":"2022-08-28T12:40:52.18164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train model","metadata":{}},{"cell_type":"code","source":"loss_hist = []\nval_loss_hist = []\npatience_counter = 0\nbest_val_loss = np.inf\n\n# Loop over epochs\nfor epoch in tqdm(range(N_EPOCHS)):\n    loss_acc = 0\n    val_loss_acc = 0\n    train_count = 0\n    valid_count = 0\n    \n    # Loop over batches\n    for imgs, labels in train_loader:\n        # Send to device\n        imgs = imgs.to(device)\n        labels = labels.to(device)\n\n        # Forward pass\n        preds = model(imgs)\n        L = competiton_loss_row_norm(preds, labels)\n\n        # Backprop\n        L.backward()\n\n        # Update parameters\n        optimiser.step()\n\n        # Zero gradients\n        optimiser.zero_grad()\n        \n        # Track loss\n        loss_acc += L.detach().item()\n        train_count += 1\n    \n    # Update learning rate\n    scheduler.step()\n    \n    # Don't update weights\n    with torch.no_grad():\n        # Validate\n        for val_imgs, val_labels in valid_loader:\n            # Reshape\n            val_imgs = val_imgs.to(device)\n            val_labels = val_labels.to(device)\n\n            # Forward pass\n            val_preds = model(val_imgs)\n            val_L = competiton_loss_row_norm(val_preds,val_labels)\n            \n            # Track loss\n            val_loss_acc += val_L.item()\n            valid_count += 1\n    \n    # Save loss history\n    loss_hist.append(loss_acc/train_count)\n    val_loss_hist.append(val_loss_acc/valid_count)\n    \n    # Print loss\n    if (epoch+1)%1==0:\n        print(f'Epoch {epoch+1}/{N_EPOCHS}, loss {loss_acc/train_count:.5f}, val_loss {val_loss_acc/valid_count:.5f}')\n    \n    # Save model (& early stopping)\n    if (val_loss_acc/valid_count) < best_val_loss:\n        best_val_loss = val_loss_acc/valid_count\n        patience_counter=0\n        print('Valid loss improved --> saving model')\n        torch.save({\n                    'epoch': epoch+1,\n                    'model_state_dict': model.state_dict(),\n                    'optimiser_state_dict': optimiser.state_dict(),\n                    'scheduler_state_dict': scheduler.state_dict(),\n                    'loss': loss_acc/train_count,\n                    'val_loss': val_loss_acc/valid_count,\n                    }, \"Conv3DNet.pt\")\n    else:\n        patience_counter+=1\n        \n        if patience_counter==PATIENCE:\n            break\n        \nprint('')\nprint('Training complete!')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T12:40:52.485714Z","iopub.execute_input":"2022-08-28T12:40:52.486204Z","iopub.status.idle":"2022-08-28T12:46:17.106755Z","shell.execute_reply.started":"2022-08-28T12:40:52.486176Z","shell.execute_reply":"2022-08-28T12:46:17.104835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Learning curves","metadata":{}},{"cell_type":"code","source":"# Plot loss\nplt.figure(figsize=(10,5))\nplt.plot(loss_hist, c='C0', label='loss')\nplt.plot(val_loss_hist, c='C1', label='val_loss')\nplt.title('Competition metric')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}