{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\"\"\"\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\"\"\"\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-13T06:15:13.714279Z","iopub.execute_input":"2022-09-13T06:15:13.714869Z","iopub.status.idle":"2022-09-13T06:15:13.741149Z","shell.execute_reply.started":"2022-09-13T06:15:13.71478Z","shell.execute_reply":"2022-09-13T06:15:13.740207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"max_split_size_mb:3000\"","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:15:13.742799Z","iopub.execute_input":"2022-09-13T06:15:13.743226Z","iopub.status.idle":"2022-09-13T06:15:13.748127Z","shell.execute_reply.started":"2022-09-13T06:15:13.743191Z","shell.execute_reply":"2022-09-13T06:15:13.747048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install monai","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:15:13.749732Z","iopub.execute_input":"2022-09-13T06:15:13.750162Z","iopub.status.idle":"2022-09-13T06:15:27.269325Z","shell.execute_reply.started":"2022-09-13T06:15:13.750117Z","shell.execute_reply":"2022-09-13T06:15:27.268116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install GPUtil\n\nfrom GPUtil import showUtilization as gpu_usage","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:15:27.273885Z","iopub.execute_input":"2022-09-13T06:15:27.274224Z","iopub.status.idle":"2022-09-13T06:15:38.552102Z","shell.execute_reply.started":"2022-09-13T06:15:27.274192Z","shell.execute_reply":"2022-09-13T06:15:38.550947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install pylibjpeg pylibjpeg-libjpeg pydicom\n! pip install -U python-gdcm","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:15:38.554058Z","iopub.execute_input":"2022-09-13T06:15:38.554446Z","iopub.status.idle":"2022-09-13T06:16:01.773958Z","shell.execute_reply.started":"2022-09-13T06:15:38.554404Z","shell.execute_reply":"2022-09-13T06:16:01.772803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pylibjpeg\nimport libjpeg","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:01.776388Z","iopub.execute_input":"2022-09-13T06:16:01.776834Z","iopub.status.idle":"2022-09-13T06:16:01.792949Z","shell.execute_reply.started":"2022-09-13T06:16:01.776779Z","shell.execute_reply":"2022-09-13T06:16:01.791765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom torch.utils.data import Dataset, DataLoader","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:01.795738Z","iopub.execute_input":"2022-09-13T06:16:01.796042Z","iopub.status.idle":"2022-09-13T06:16:03.802686Z","shell.execute_reply.started":"2022-09-13T06:16:01.796013Z","shell.execute_reply":"2022-09-13T06:16:03.801446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import monai.transforms as transforms","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:03.80473Z","iopub.execute_input":"2022-09-13T06:16:03.805505Z","iopub.status.idle":"2022-09-13T06:16:07.89914Z","shell.execute_reply.started":"2022-09-13T06:16:03.805459Z","shell.execute_reply":"2022-09-13T06:16:07.897898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport matplotlib.pyplot as plt\nimport pydicom as dicom\nimport torch\nimport torchvision as vision\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import GroupKFold, KFold\nfrom torch.cuda.amp import GradScaler\nfrom torch.cuda.amp import autocast\nfrom torchvision.models import feature_extraction\nfrom tqdm.notebook import tqdm\nimport pydicom as dicom\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:07.900819Z","iopub.execute_input":"2022-09-13T06:16:07.901235Z","iopub.status.idle":"2022-09-13T06:16:08.092749Z","shell.execute_reply.started":"2022-09-13T06:16:07.901189Z","shell.execute_reply":"2022-09-13T06:16:08.091704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_CSV_PATH = '../input/rsna-2022-cervical-spine-fracture-detection/train.csv'\nTRAIN_DATA_PATH = '../input/cervical-spine-pt/train_volumes'","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.097321Z","iopub.execute_input":"2022-09-13T06:16:08.097618Z","iopub.status.idle":"2022-09-13T06:16:08.102784Z","shell.execute_reply.started":"2022-09-13T06:16:08.097591Z","shell.execute_reply":"2022-09-13T06:16:08.101732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device ='cuda' if torch.cuda.is_available() else 'cpu'\nbatch_size = 8","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.104338Z","iopub.execute_input":"2022-09-13T06:16:08.105502Z","iopub.status.idle":"2022-09-13T06:16:08.175618Z","shell.execute_reply.started":"2022-09-13T06:16:08.105465Z","shell.execute_reply":"2022-09-13T06:16:08.174327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.177349Z","iopub.execute_input":"2022-09-13T06:16:08.177832Z","iopub.status.idle":"2022-09-13T06:16:08.191713Z","shell.execute_reply.started":"2022-09-13T06:16:08.177787Z","shell.execute_reply":"2022-09-13T06:16:08.190465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ndef load_data(path):\n    img = dicom.dcmread(path)\n    data = apply_voi_lut(img.pixel_array, img)\n    data = cv2.resize(data, (img_size, img_size), interpolation=cv2.INTER_NEAREST)\n    norm = (np.min(data), np.max(data))\n    if norm[1] != 0:\n        data = (data-norm[0]) / norm[1]\n    data=(data * 255).astype(np.uint8)\n    return cv2.cvtColor(data, cv2.COLOR_GRAY2RGB)\n\"\"\"","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-09-13T06:16:08.193725Z","iopub.execute_input":"2022-09-13T06:16:08.19406Z","iopub.status.idle":"2022-09-13T06:16:08.203017Z","shell.execute_reply.started":"2022-09-13T06:16:08.194031Z","shell.execute_reply":"2022-09-13T06:16:08.201886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"img = load_data('../input/rsna-2022-cervical-spine-fracture-detection/train_images/1.2.826.0.1.3680043.6200/1.dcm')\nplt.figure()\nplt.imshow(img)\nplt.title('Image 0')\n\nimg = load_data(f'{TRAIN_DATA_PATH}/1.2.826.0.1.3680043.10005/1.dcm')\nplt.figure()\nplt.imshow(img)\nplt.title('Image 1')\"\"\"","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-09-13T06:16:08.204199Z","iopub.execute_input":"2022-09-13T06:16:08.204851Z","iopub.status.idle":"2022-09-13T06:16:08.215539Z","shell.execute_reply.started":"2022-09-13T06:16:08.204721Z","shell.execute_reply":"2022-09-13T06:16:08.214546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(TRAIN_CSV_PATH)\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.217094Z","iopub.execute_input":"2022-09-13T06:16:08.218375Z","iopub.status.idle":"2022-09-13T06:16:08.249153Z","shell.execute_reply.started":"2022-09-13T06:16:08.218337Z","shell.execute_reply":"2022-09-13T06:16:08.248134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_label_string(row):\n    label = str(row['patient_overall'])\n    for i in range(7):\n        label += str(row['C'+str(i+1)])\n    label_int = int(label, 2)\n    return label_int","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.250861Z","iopub.execute_input":"2022-09-13T06:16:08.251767Z","iopub.status.idle":"2022-09-13T06:16:08.258682Z","shell.execute_reply.started":"2022-09-13T06:16:08.251727Z","shell.execute_reply":"2022-09-13T06:16:08.25747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['labels'] = train_df.apply(lambda x: create_label_string(x), axis = 1)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.260321Z","iopub.execute_input":"2022-09-13T06:16:08.261633Z","iopub.status.idle":"2022-09-13T06:16:08.359824Z","shell.execute_reply.started":"2022-09-13T06:16:08.261593Z","shell.execute_reply":"2022-09-13T06:16:08.358856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.361364Z","iopub.execute_input":"2022-09-13T06:16:08.361742Z","iopub.status.idle":"2022-09-13T06:16:08.373624Z","shell.execute_reply.started":"2022-09-13T06:16:08.361705Z","shell.execute_reply":"2022-09-13T06:16:08.372401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FractureDetectDataset(Dataset):\n    def __init__(self, df, path, transform=None):\n        super().__init__()\n        self.df = df\n        self.path = path\n        self.transform = transform\n    \n    def __getitem__(self, i):\n        PATH = os.path.join(self.path, self.df.iloc[i, 0])\n        try:\n            img = torch.load(f'{PATH}.pt')\n            img_tensor = img.permute((2, 0, 1))\n        except Exception as e:\n            print(e.message)\n            pass\n        #print(temp.shape)\n        if self.transform is not None:\n            inputs = self.transform(img_tensor)\n        else:\n            inputs = img_tensor\n        targets = torch.as_tensor(self.df.iloc[i,[1, 2, 3, 4, 5, 6 , 7, 8]].astype('float32').values)\n        return {\"inputs\": inputs.unsqueeze(0), \"target\": targets}\n\n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.375227Z","iopub.execute_input":"2022-09-13T06:16:08.376234Z","iopub.status.idle":"2022-09-13T06:16:08.387851Z","shell.execute_reply.started":"2022-09-13T06:16:08.376196Z","shell.execute_reply":"2022-09-13T06:16:08.386845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"train_transform = transforms.Compose([transforms.Resize((img_size, img_size, stack_size)),\n                                      transforms.ToTensor()\n])\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.389468Z","iopub.execute_input":"2022-09-13T06:16:08.38993Z","iopub.status.idle":"2022-09-13T06:16:08.39986Z","shell.execute_reply.started":"2022-09-13T06:16:08.389894Z","shell.execute_reply":"2022-09-13T06:16:08.398647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH = os.path.join(TRAIN_DATA_PATH, train_df.iloc[0, 0])\ntry:\n    img = torch.load(f'{PATH}.pt')\n    img_tensor = img.permute((2, 0, 1))\nexcept Exception as e:\n    print(e)\n    pass\ntargets = torch.as_tensor(train_df.iloc[0,[1, 2, 3, 4, 5, 6 , 7, 8]].astype('float32').values)\nprint(targets, img_tensor.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.401937Z","iopub.execute_input":"2022-09-13T06:16:08.402715Z","iopub.status.idle":"2022-09-13T06:16:08.430743Z","shell.execute_reply.started":"2022-09-13T06:16:08.402645Z","shell.execute_reply":"2022-09-13T06:16:08.429458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpu_usage()","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.434844Z","iopub.execute_input":"2022-09-13T06:16:08.435215Z","iopub.status.idle":"2022-09-13T06:16:08.472689Z","shell.execute_reply.started":"2022-09-13T06:16:08.435176Z","shell.execute_reply":"2022-09-13T06:16:08.471479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.474453Z","iopub.execute_input":"2022-09-13T06:16:08.475121Z","iopub.status.idle":"2022-09-13T06:16:08.481245Z","shell.execute_reply.started":"2022-09-13T06:16:08.475076Z","shell.execute_reply":"2022-09-13T06:16:08.480076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list1 = list(train_df['labels'].value_counts())\ncount = 0\nfor val in list1:\n    if val<2:\n        count +=1\nprint(count)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.483484Z","iopub.execute_input":"2022-09-13T06:16:08.485127Z","iopub.status.idle":"2022-09-13T06:16:08.498039Z","shell.execute_reply.started":"2022-09-13T06:16:08.485097Z","shell.execute_reply":"2022-09-13T06:16:08.497035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"counts = train_df['labels'].value_counts()\n\nres = train_df[~train_df['labels'].isin(counts[counts < 2].index)]\nprint(res.shape, train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.499607Z","iopub.execute_input":"2022-09-13T06:16:08.499941Z","iopub.status.idle":"2022-09-13T06:16:08.514572Z","shell.execute_reply.started":"2022-09-13T06:16:08.499908Z","shell.execute_reply":"2022-09-13T06:16:08.513495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_edited = res","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.516193Z","iopub.execute_input":"2022-09-13T06:16:08.516886Z","iopub.status.idle":"2022-09-13T06:16:08.521458Z","shell.execute_reply.started":"2022-09-13T06:16:08.51685Z","shell.execute_reply":"2022-09-13T06:16:08.520253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_valid = train_test_split(train_df_edited, test_size=0.2, random_state=43, stratify = train_df_edited.labels)\ndf_train.shape, df_valid.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.523333Z","iopub.execute_input":"2022-09-13T06:16:08.524098Z","iopub.status.idle":"2022-09-13T06:16:08.543175Z","shell.execute_reply.started":"2022-09-13T06:16:08.524044Z","shell.execute_reply":"2022-09-13T06:16:08.542127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = FractureDetectDataset(df = df_train, path = TRAIN_DATA_PATH, transform=None)\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.551756Z","iopub.execute_input":"2022-09-13T06:16:08.552435Z","iopub.status.idle":"2022-09-13T06:16:08.558641Z","shell.execute_reply.started":"2022-09-13T06:16:08.552395Z","shell.execute_reply":"2022-09-13T06:16:08.55736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpu_usage()","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.560909Z","iopub.execute_input":"2022-09-13T06:16:08.561538Z","iopub.status.idle":"2022-09-13T06:16:08.600348Z","shell.execute_reply.started":"2022-09-13T06:16:08.561429Z","shell.execute_reply":"2022-09-13T06:16:08.59885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.602469Z","iopub.execute_input":"2022-09-13T06:16:08.603794Z","iopub.status.idle":"2022-09-13T06:16:08.609432Z","shell.execute_reply.started":"2022-09-13T06:16:08.60374Z","shell.execute_reply":"2022-09-13T06:16:08.608233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()\ndata = next(iter(train_dataloader))\nprint(data['inputs'].shape, data['target'].shape, \"\\n\", time.time()-start)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.611073Z","iopub.execute_input":"2022-09-13T06:16:08.612134Z","iopub.status.idle":"2022-09-13T06:16:08.743343Z","shell.execute_reply.started":"2022-09-13T06:16:08.612095Z","shell.execute_reply":"2022-09-13T06:16:08.742237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['target']","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.744965Z","iopub.execute_input":"2022-09-13T06:16:08.746027Z","iopub.status.idle":"2022-09-13T06:16:08.756353Z","shell.execute_reply.started":"2022-09-13T06:16:08.745956Z","shell.execute_reply":"2022-09-13T06:16:08.754754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.758266Z","iopub.execute_input":"2022-09-13T06:16:08.759273Z","iopub.status.idle":"2022-09-13T06:16:08.764971Z","shell.execute_reply.started":"2022-09-13T06:16:08.759227Z","shell.execute_reply":"2022-09-13T06:16:08.763656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Detector(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv3d(in_channels=1, out_channels=16, kernel_size=3, stride=1, padding=0) #(-1, 1, 40, 100, 100) -> #(-1, 16, 38, 98, 98)\n        self.pooling1 = nn.MaxPool3d(kernel_size=2, stride=2, padding=0) #(8, 16, 38, 98,98) -> (8, 16, 19, 49, 49)\n        self.norm1 = nn.BatchNorm3d(num_features=16) #(8, 16, 19, 49, 49)\n        self.layer1 = nn.Sequential(self.conv1, self.pooling1, self.norm1) #(8, 3, 40, 100, 100) -> (8, 16, 19, 49, 49)\n        \n        self.conv2 = nn.Conv3d(in_channels=16, out_channels=32, kernel_size=3, stride=(1, 2, 2), padding=0) #(8, 16, 19, 49, 49) -> (8, 32, 17, 24, 24)\n        #self.norm2 = nn.BatchNorm3d(num_features = 64)\n    \n        self.layer2 = nn.Sequential(self.conv2)\n        \n        #self.conv4 = nn.Conv3d(in_channels=64, out_channels=256, kernel_size=3, stride=1, padding=0) #(8, 32, 17, 47, 47) -> (8, 256, 15, 45, 45)\n        self.final_pool = nn.AvgPool3d((7, 5, 5), stride = (2, 3, 3)) #(8, 32, 17, 24, 24) -> (8, 32, 6, 7, 7)\n        #self.norm4 = nn.BatchNorm3d(num_features = 256)\n        self.layer3 = nn.Sequential( self.final_pool)\n        \n        self.flat = nn.Flatten()\n        \n        self.relu = nn.ReLU()\n        self.layer4 = nn.Sequential(nn.Linear(in_features=32*6*7*7, out_features=256), self.relu,\n                                    nn.Linear(in_features=256, out_features=32), self.relu,\n                                    nn.Linear(in_features=32, out_features = 8)\n                                    )\n        \n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        #print(x.shape)\n        out = self.layer1(x)\n        #print(out.shape)\n        out = self.layer2(out)\n        #print(out.shape)\n        out = self.layer3(out)\n        #print(out.shape)\n        \n        output = self.flat(out)\n        output = self.layer4(output)\n        #print(output.shape)\n        return output\n    \n    def predict(self, x):\n        preds = self.forward(x)\n        preds = self.sigmoid(preds)\n        return preds","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.76669Z","iopub.execute_input":"2022-09-13T06:16:08.767752Z","iopub.status.idle":"2022-09-13T06:16:08.781508Z","shell.execute_reply.started":"2022-09-13T06:16:08.76771Z","shell.execute_reply":"2022-09-13T06:16:08.780268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.783312Z","iopub.execute_input":"2022-09-13T06:16:08.784297Z","iopub.status.idle":"2022-09-13T06:16:08.799196Z","shell.execute_reply.started":"2022-09-13T06:16:08.784255Z","shell.execute_reply":"2022-09-13T06:16:08.79804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Detector()\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:08.800848Z","iopub.execute_input":"2022-09-13T06:16:08.802063Z","iopub.status.idle":"2022-09-13T06:16:11.906646Z","shell.execute_reply.started":"2022-09-13T06:16:08.802024Z","shell.execute_reply":"2022-09-13T06:16:11.905431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpu_usage()","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:11.908624Z","iopub.execute_input":"2022-09-13T06:16:11.909046Z","iopub.status.idle":"2022-09-13T06:16:11.969632Z","shell.execute_reply.started":"2022-09-13T06:16:11.909008Z","shell.execute_reply":"2022-09-13T06:16:11.96822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()\ntrial_tensor = torch.rand(8, 1, 40, 100, 100).to(device)\nprint(trial_tensor.get_device(), \"\\n\", model.predict(trial_tensor), time.time()-start)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:11.971849Z","iopub.execute_input":"2022-09-13T06:16:11.972297Z","iopub.status.idle":"2022-09-13T06:16:18.157385Z","shell.execute_reply.started":"2022-09-13T06:16:11.972247Z","shell.execute_reply":"2022-09-13T06:16:18.156363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:18.15904Z","iopub.execute_input":"2022-09-13T06:16:18.159426Z","iopub.status.idle":"2022-09-13T06:16:18.399438Z","shell.execute_reply.started":"2022-09-13T06:16:18.159389Z","shell.execute_reply":"2022-09-13T06:16:18.398254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_dataloader))","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:18.400911Z","iopub.execute_input":"2022-09-13T06:16:18.402164Z","iopub.status.idle":"2022-09-13T06:16:18.410547Z","shell.execute_reply.started":"2022-09-13T06:16:18.402122Z","shell.execute_reply":"2022-09-13T06:16:18.409218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:18.41216Z","iopub.execute_input":"2022-09-13T06:16:18.412901Z","iopub.status.idle":"2022-09-13T06:16:18.419766Z","shell.execute_reply.started":"2022-09-13T06:16:18.412862Z","shell.execute_reply":"2022-09-13T06:16:18.418609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(epoch, model, opt, lr_scheduler, criterion, dataloader, device):\n    print(f\"Epoch: {epoch}\")\n    model.train()\n    \n    running_loss = 0\n    total_loss = []\n    lrs = []\n    total_size = 0\n    correct = 0\n    \n    start = time.time()\n    for i, data in enumerate(dataloader):\n        #gpu_usage()\n        gc.collect()\n        inputs = data[\"inputs\"].to(device)\n        targets = data[\"target\"].to(device)\n        #print(i)\n        inputs = inputs.type(torch.cuda.FloatTensor)\n        targets = targets.type(torch.cuda.FloatTensor)\n        #print(ids.shape, \"ids\")\n        batch_size = inputs.size(0)\n        #gpu_usage()\n        output = model.forward(inputs)\n        gc.collect()\n        del inputs\n        loss = criterion(output, targets)\n        #print(output.shape, targets.shape, loss.shape)\n        loss.mean().backward()\n        opt.step()\n        opt.zero_grad()\n        if i%4 == 0 or i+1==len(dataloader):\n            lr_scheduler.step()\n        total_loss.append(torch.sum(loss))\n        gc.collect()\n        del loss\n        #gpu_usage()\n        torch.cuda.empty_cache()\n        output = torch.sigmoid(output)\n        predictions = torch.as_tensor((output - 0.6) > 0, dtype=torch.int32)\n        correct += (predictions == targets).float().sum().item()\n        gc.collect()\n        #gpu_usage()\n        del predictions\n        del targets\n        del output\n        #gpu_usage()\n        torch.cuda.empty_cache()\n        #print(predictions, \"\\n\", targets, \"\\n\", correct)\n        total_size += batch_size\n        accuracy = correct/(total_size*8)\n        #print(correct, total_size)\n        lrs.append(opt.param_groups[0]['lr'])\n        if i%40==0:\n            print(time.time()-start, \":time\")\n            print(\"Training Loss over a batch: {:.4f}; Training Loss over a single value: {:.4f} Accuracy: {:.2f}%\".format(torch.mean(torch.stack(total_loss)), (torch.mean(torch.stack(total_loss)))/(batch_size*8), accuracy*100))\n            start = time.time()\n        break\n    return torch.mean(torch.stack(total_loss)), accuracy, lrs\n\ndef valid_eval(model, dataloader, device):\n    model.eval()\n    \n    total_size = 0\n    total_loss = []\n    correct = 0\n    with torch.no_grad():\n        for data in dataloader:\n            gc.collect()\n            torch.cuda.empty_cache()\n            inputs = data[\"inputs\"].to(device)\n            targets = data[\"target\"].to(device)\n\n            inputs = inputs.type(torch.cuda.FloatTensor)\n            targets = targets.type(torch.cuda.FloatTensor)\n            #print(ids.shape, \"ids\")\n            batch_size = inputs.size(0)\n\n            output = model.forward(inputs)\n            gc.collect()\n            del inputs\n            loss = criterion(output, targets)\n            total_loss.append(torch.sum(loss))\n            gc.collect()\n            del loss\n            torch.cuda.empty_cache()\n            output = torch.sigmoid(output)\n\n            predictions = torch.as_tensor((output - 0.6) > 0, dtype=torch.int32)\n            correct += (predictions == targets).float().sum().item()\n            gc.collect()\n            del predictions\n            del targets\n            del output\n            torch.cuda.empty_cache()\n            total_size += batch_size\n            #gpu_usage()\n        accuracy = correct/(total_size*8)\n    \n    print(\"Validation Loss over a batch: {:.4f}; Training Loss over a single value: {:.4f} Accuracy: {:.2f}%\".format(torch.mean(torch.stack(total_loss)),(torch.mean(torch.stack(total_loss)))/(batch_size*8), accuracy*100))\n    model.train()\n    return torch.mean(torch.stack(total_loss)), accuracy","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:18:57.797193Z","iopub.execute_input":"2022-09-13T06:18:57.797756Z","iopub.status.idle":"2022-09-13T06:18:57.818426Z","shell.execute_reply.started":"2022-09-13T06:18:57.797695Z","shell.execute_reply":"2022-09-13T06:18:57.817239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_epochs(model, opt, lr_scheduler, criterion, device, epochs=25, mode='trial'):\n    min_avg_loss = 1e9\n    max_accuracy = 0\n    history = {\"Train Loss\": [], \"Valid Loss\": [], \"Train Acc\": [], \"Valid Acc\": [], \"LRs\": []}\n    \n    for epoch in range(epochs):\n        train_loss, train_accuracy, lrs = train(epoch, model, opt, lr_scheduler, criterion, dataloader = train_dataloader, device=device)\n        \n        valid_loss, valid_accuracy = valid_eval(model, dataloader=valid_dataloader, device=device)\n        \n        history[\"Train Loss\"].append(train_loss)\n        history[\"Valid Loss\"].append(valid_loss)\n        history[\"Train Acc\"].append(train_accuracy)\n        history[\"Valid Acc\"].append(valid_accuracy)\n        history[\"LRs\"].extend(lrs)\n        \n        if valid_loss < min_avg_loss:\n            print(f\"Validation Loss Improved ({min_avg_loss} ---> {valid_loss})\")\n            min_avg_loss = valid_loss\n            max_accuracy = valid_accuracy\n            best_model_weights = copy.deepcopy(model.state_dict())\n            path = f\"best_model.bin\"\n            torch.save(model.state_dict(), path)\n            print(f\"Model Saved\")\n        \n        print('-'*89)\n        \n    \n    print(\"Minimum Loss: {:.4f} Best Accuracy: {:.2f}\".format(min_avg_loss, max_accuracy))\n    \n    model.load_state_dict(best_model_weights)\n    return model, history","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:19:02.261842Z","iopub.execute_input":"2022-09-13T06:19:02.262542Z","iopub.status.idle":"2022-09-13T06:19:02.271545Z","shell.execute_reply.started":"2022-09-13T06:19:02.262504Z","shell.execute_reply":"2022-09-13T06:19:02.270563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_parameters = filter(lambda p: p.requires_grad, model.parameters())\nparams = sum([np.prod(p.size()) for p in model_parameters])\nparams","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:18.460074Z","iopub.execute_input":"2022-09-13T06:16:18.46082Z","iopub.status.idle":"2022-09-13T06:16:18.474884Z","shell.execute_reply.started":"2022-09-13T06:16:18.460782Z","shell.execute_reply":"2022-09-13T06:16:18.473516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for p in model.parameters():\n    print(p.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:18.476895Z","iopub.execute_input":"2022-09-13T06:16:18.477631Z","iopub.status.idle":"2022-09-13T06:16:18.484793Z","shell.execute_reply.started":"2022-09-13T06:16:18.477574Z","shell.execute_reply":"2022-09-13T06:16:18.483562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.optim import lr_scheduler","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:18.485964Z","iopub.execute_input":"2022-09-13T06:16:18.487196Z","iopub.status.idle":"2022-09-13T06:16:18.493509Z","shell.execute_reply.started":"2022-09-13T06:16:18.487159Z","shell.execute_reply":"2022-09-13T06:16:18.492421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.01\nepochs = 10","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:18.49474Z","iopub.execute_input":"2022-09-13T06:16:18.495727Z","iopub.status.idle":"2022-09-13T06:16:18.502719Z","shell.execute_reply.started":"2022-09-13T06:16:18.495687Z","shell.execute_reply":"2022-09-13T06:16:18.501809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_dataset = FractureDetectDataset(df = df_valid, path = TRAIN_DATA_PATH, transform=None)\nvalid_dataloader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:18.505979Z","iopub.execute_input":"2022-09-13T06:16:18.506584Z","iopub.status.idle":"2022-09-13T06:16:18.514176Z","shell.execute_reply.started":"2022-09-13T06:16:18.50654Z","shell.execute_reply":"2022-09-13T06:16:18.512657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ.get('PYTORCH_CUDA_ALLOC_CONF')","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:18.51557Z","iopub.execute_input":"2022-09-13T06:16:18.516675Z","iopub.status.idle":"2022-09-13T06:16:18.527359Z","shell.execute_reply.started":"2022-09-13T06:16:18.516638Z","shell.execute_reply":"2022-09-13T06:16:18.526374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:16:18.530853Z","iopub.execute_input":"2022-09-13T06:16:18.531814Z","iopub.status.idle":"2022-09-13T06:16:18.536876Z","shell.execute_reply.started":"2022-09-13T06:16:18.531775Z","shell.execute_reply":"2022-09-13T06:16:18.535714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_parameters = filter(lambda parameter: parameter.requires_grad, model.parameters())\noptimizer = torch.optim.AdamW(model_parameters, lr=lr, weight_decay = 0.1)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=500, eta_min=0.001)\ncriterion = nn.BCEWithLogitsLoss(reduction = 'none')\n\n#test 1 epoch\nmodel, history = train_epochs(model, optimizer, scheduler,criterion, device=device, epochs=epochs)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:19:07.368696Z","iopub.execute_input":"2022-09-13T06:19:07.369402Z","iopub.status.idle":"2022-09-13T06:19:07.681655Z","shell.execute_reply.started":"2022-09-13T06:19:07.369364Z","shell.execute_reply":"2022-09-13T06:19:07.680009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:17:06.132434Z","iopub.execute_input":"2022-09-13T06:17:06.13315Z","iopub.status.idle":"2022-09-13T06:17:06.141259Z","shell.execute_reply.started":"2022-09-13T06:17:06.133102Z","shell.execute_reply":"2022-09-13T06:17:06.140042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:17:06.142803Z","iopub.execute_input":"2022-09-13T06:17:06.143865Z","iopub.status.idle":"2022-09-13T06:17:06.153433Z","shell.execute_reply.started":"2022-09-13T06:17:06.143827Z","shell.execute_reply":"2022-09-13T06:17:06.152505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:17:06.155469Z","iopub.execute_input":"2022-09-13T06:17:06.156639Z","iopub.status.idle":"2022-09-13T06:17:06.165994Z","shell.execute_reply.started":"2022-09-13T06:17:06.156575Z","shell.execute_reply":"2022-09-13T06:17:06.16503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:17:06.169478Z","iopub.execute_input":"2022-09-13T06:17:06.170145Z","iopub.status.idle":"2022-09-13T06:17:06.181837Z","shell.execute_reply.started":"2022-09-13T06:17:06.170101Z","shell.execute_reply":"2022-09-13T06:17:06.180873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"import torch\nfrom GPUtil import showUtilization as gpu_usage\nfrom numba import cuda\n\ndef free_gpu_cache():\n    print(\"Initial GPU Usage\")\n    gpu_usage()                             \n\n    torch.cuda.empty_cache()\n\n    cuda.select_device(0)\n    cuda.close()\n    cuda.select_device(0)\n\n    print(\"GPU Usage after emptying the cache\")\n    gpu_usage()\n\nfree_gpu_cache()\"\"\"\n","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:17:06.184619Z","iopub.execute_input":"2022-09-13T06:17:06.185305Z","iopub.status.idle":"2022-09-13T06:17:06.191309Z","shell.execute_reply.started":"2022-09-13T06:17:06.18527Z","shell.execute_reply":"2022-09-13T06:17:06.190186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpu_usage()","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:17:06.193093Z","iopub.execute_input":"2022-09-13T06:17:06.193434Z","iopub.status.idle":"2022-09-13T06:17:15.632611Z","shell.execute_reply.started":"2022-09-13T06:17:06.1934Z","shell.execute_reply":"2022-09-13T06:17:15.631059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()\ndel model\ngpu_usage()","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:17:15.634695Z","iopub.execute_input":"2022-09-13T06:17:15.635481Z","iopub.status.idle":"2022-09-13T06:17:16.041017Z","shell.execute_reply.started":"2022-09-13T06:17:15.635421Z","shell.execute_reply":"2022-09-13T06:17:16.039822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}