{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# ! pip install sqlalchemy\n# ! rm /kaggle/working/sqlite3.db\n!pip install scipy==1.4.1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport scikitplot as skplt\nimport cv2\nimport copy\nfrom PIL import Image\nimport pydicom\nimport scipy.ndimage\nimport imageio\nfrom os import listdir\nfrom sqlalchemy import create_engine\nimport sqlite3\nfrom tqdm import tqdm\nfrom torchvision import transforms\nimport torch\nimport random\nfrom torch.utils import data\nfrom sklearn.metrics import roc_auc_score\nimport torchvision.models as models\nfrom torch.utils.data import Dataset\nimport pydicom as dicom\nimport time\nimport pickle\nfrom sklearn.model_selection import train_test_split\npath = \"../input/rsna-str-pulmonary-embolism-detection/train/\"\ntrain = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/train.csv\")\ntrain=train[[\"StudyInstanceUID\",\"SeriesInstanceUID\",\"SOPInstanceUID\",\"pe_present_on_image\"]]\n# pbar = range(0,train.shape[0])\n\n# pbar = tqdm(pbar, dynamic_ncols=True, smoothing=0.01)\n# bad_rows=[]\n# for i in pbar:\n# #     try:\n#     ds=dicom.dcmread(path+train.iloc[i,0]+'/'+train.iloc[i,1]+'/'+train.iloc[i,2]+'.dcm')\n#     dcm_sample=ds.pixel_array\n# #     except Exception:\n# #         print()\n# #         bad_rows.append(i)\n# train=train.drop(bad_rows)\n        \nprint(train.head())\ntrain=train[:200000]\na = train[\"StudyInstanceUID\"]==\"0038fd5f09f5\"\nprint(len(train[a]))\nprint(train[a][\"pe_present_on_image\"].sum())\ntrain_dataset, test_dataset = train_test_split(train, test_size=0.25, random_state=42)\n\nclass myNet(torch.nn.Module):\n    def __init__(self):\n#         super(myNet, self).__init__()\n        super().__init__()\n\n#         model = torch.hub.load('pytorch/vision:v0.6.0', 'inception_v3', pretrained=False)\n        model=models.inception_v3(pretrained=True,aux_logits=False)\n        self.model=model\n\n#         print(self.inception_layer)\n#         print(list(self.inception_layer.children()))\n\n        self.Linear_layer = torch.nn.Linear(1000, 1)\n        self.logistic_layer=torch.nn.Sigmoid()\n        \n    def forward(self, x):\n#         x = self.inception_layer(x)\n        x=self.model(x)\n#         print(x.logits)\n        x = self.Linear_layer(x)\n#         print(x)\n        x = self.logistic_layer(x)\n        \n        return x\n\n\n\npreprocess = transforms.Compose([\n    transforms.Resize(299),\n    transforms.CenterCrop(299),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\ndef transform_to_hu(image):\n    pixel_array=copy.deepcopy(image.pixel_array)\n    pixel_array = pixel_array.astype(np.int16)\n    pixel_array[pixel_array <= -1000] = 0\n\n    intercept = image.RescaleIntercept\n    slope = image.RescaleSlope\n        \n    if slope != 1:\n        pixel_array = slope * pixel_array.astype(np.float64)\n        pixel_array = pixel_array.astype(np.int16)\n            \n    pixel_array += np.int16(intercept)\n    return pixel_array\n\n\nclass PEDataset(Dataset):\n    def __init__(self, transform,pd):\n        self.transform = transform\n        self.df=pd\n        \n    def __len__(self):\n        return self.df.shape[0]\n    def _load_image_from_index(self, index):\n        ds=dicom.dcmread(path+self.df.iloc[index,0]+'/'+self.df.iloc[index,1]+'/'+self.df.iloc[index,2]+'.dcm')\n        pixel_array=transform_to_hu(ds)\n#         print(ds.RescaleIntercept)\n#         print(pixel_array)\n#         print(pixel_array.max())\n#         print(pixel_array.min())\n#         print(transform_to_hu(ds))\n#         print(transform_to_hu(ds).max())\n#         print(transform_to_hu(ds).min())\n#         plt.figure()\n\n#         plt.subplot(1,2,1)\n#         plt.imshow(ds.pixel_array,cmap=\"Greys_r\")\n#         plt.subplot(1,2,2)\n#         plt.imshow(transform_to_hu(ds),cmap=\"Greys_r\")\n#         plt.show()\n#         pixel_array=transform_to_hu(ds)\n        _max=pixel_array.max()\n        _min=pixel_array.min()\n        temp=Image.fromarray((255 * (pixel_array+0.0 - _min) / (_max+0.0 - _min)).astype(np.uint8)).convert(\"RGB\")\n#         print(temp)\n#         plt.figure()\n#         plt.subplot(1,2,1)\n#         plt.imshow(temp)\n#         plt.subplot(1,2,2)\n#         plt.imshow(transform_to_hu(ds),cmap=\"Greys_r\")\n#         plt.show()\n        img=self.transform(temp)\n\n        return img,self.df.iloc[index,3]\n    def __getitem__(self, index):\n        try:\n            img,label = self._load_image_from_index(index)\n        except:\n            index=random.random()*self.__len__()\n            img,label = self.__getitem__(int(index))\n        \n\n        return img,label\n\ndef data_sampler(dataset, shuffle):\n    if shuffle:\n        return data.RandomSampler(dataset)\n    else:\n        return data.SequentialSampler(dataset)    \ndataloaders={}\ntrain_dataset=PEDataset(preprocess,train_dataset)\ntest_dataset=PEDataset(preprocess,test_dataset)\ntrain_loader= data.DataLoader(train_dataset,batch_size=64,sampler=data_sampler(train_dataset, shuffle=True),drop_last=True)\ntest_loader= data.DataLoader(test_dataset,batch_size=64,sampler=data_sampler(test_dataset, shuffle=True),drop_last=True)\n\ndataloaders[\"train\"]=train_loader\ndataloaders[\"val\"]=test_loader\ndef train_model(model, dataloaders, criterion, optimizer, num_epochs=25, is_inception=False):\n    print(\"begin trainning\")\n    since = time.time()\n\n    val_acc_history = []\n    loss_dur_train = []\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n \n    for epoch in tqdm(range(num_epochs), dynamic_ncols=True, smoothing=0.01):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-' * 10)\n        pred_result = None\n        true_result = None\n        for phase in ['val','train']:\n            cmt =  torch.zeros(2,2, dtype=torch.int32)\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n                \n            running_loss = 0.0\n            running_corrects = 0\n\n            for inputs, labels in tqdm(dataloaders[phase]):\n#                 print(inputs.size())\n                inputs = inputs.cuda()\n                labels = labels.cuda()\n\n                optimizer.zero_grad()\n\n\n                with torch.set_grad_enabled(phase == 'train'):\n\n                    if is_inception and phase == 'train':\n\n                        outputs, aux_outputs = model(inputs)\n                        loss1 = criterion(outputs, labels)\n                        loss2 = criterion(aux_outputs, labels)\n                        loss = loss1 + 0.4*loss2\n                    else:\n                        outputs = model(inputs)\n#                         print(outputs)\n#                         print(labels)\n#                         loss = criterion(outputs, labels)\n#                         print(loss)\n                    labels=labels.to(torch.float32)\n                    \n                    preds = torch.round(outputs).squeeze(1)\n                    if pred_result is None:\n                        pred_result = outputs.squeeze(1)\n                        true_result = labels\n                    else:\n                        pred_result = torch.cat((pred_result,outputs.squeeze(1)))\n                        true_result = torch.cat((true_result,labels))\n                    loss = criterion(outputs.squeeze(1), labels)\n                    loss_dur_train.append(loss.mean().item())\n#                     print(\"qqq\")\n#                     print(preds.size())\n#                     print(labels.size())\n#                     print(torch.sum(preds == labels.data))\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.mean().backward()\n                        optimizer.step()\n                \n                running_loss += loss.mean().item() * inputs.size(0)\n                \n                running_corrects += torch.sum(preds == labels.data)\n#             print(running_corrects.double())\n#             print(len(dataloaders[phase].dataset))\n                stacked = torch.stack((preds,labels.data),dim=1)\n                for p in stacked:\n                    tl, pl = p.tolist()\n                    cmt[int(tl),int(pl)] = cmt[int(tl),int(pl)] + 1\n            if phase == 'train':\n                print('--------------trainning loss---------')\n                print(loss_dur_train)\n            if phase=='val':\n                print(\"---------------roc----------------\")\n                print(pred_result.size())\n                print(true_result.size())\n                print(roc_auc_score(true_result.cpu(), pred_result.cpu()))\n\n                fpr, tpr, threshold = skplt.metrics.roc_curve(true_result.cpu(), pred_result.cpu())\n                roc_auc = skplt.metrics.auc(fpr, tpr)\n                plt.title('Receiver Operating Characteristic')\n                plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n                plt.legend(loc = 'lower right')\n                plt.plot([0, 1], [0, 1],'r--')\n                plt.xlim([0, 1])\n                plt.ylim([0, 1])\n                plt.ylabel('True Positive Rate')\n                plt.xlabel('False Positive Rate')\n                plt.show()\n\n            print(cmt)\n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n            torch.save(\n                    {\n                        \"model\": model.state_dict(),\n                    },\"checkpoint.pt\",\n                )\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n\n \n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n            if phase == 'val':\n                val_acc_history.append(epoch_acc)\n\n        print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model, val_acc_history\nmynet=myNet()\n# ckpt = torch.load(\"/kaggle/working/checkpoint.pt\")\n# mynet.load_state_dict(ckpt[\"model\"])\noptimizer = torch.optim.SGD(mynet.parameters(), lr=0.01)\ncriterion = torch.nn.BCELoss()\nmodel, val = train_model(mynet.cuda(), dataloaders, criterion, optimizer, num_epochs=2, is_inception=False)\nwith open(\"model.pkl\",'ab') as f:\n    pickle.dump(model, f)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# path = \"../input/rsna-str-pulmonary-embolism-detection/\"\n# train = pd.read_csv(path + \"train.csv\")\n# test = pd.read_csv(path + \"test.csv\")\n# train.to_sql('ct_train', engine)\n# test.to_sql('ct_test', engine)\n\n\n# conn = sqlite3.connect('sqlite3.db')\n# cursor = conn.cursor()\n\n# cursor.execute('PRAGMA table_info(ct_train)')\n\n# values = cursor.fetchall()\n# for value in values:\n#     print(value)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# def load_slice(path):\n#     slices = [pydicom.read_file(path + '/' + s) for s in listdir(path)]\n#     slices.sort(key = lambda x: float(x.ImagePositionPatient[2]))\n#     try:\n#         slice_thickness = np.abs(slices[0].ImagePositionPatient[2] - slices[1].ImagePositionPatient[2])\n#     except:\n#         slice_thickness = np.abs(slices[0].SliceLocation - slices[1].SliceLocation)\n        \n#     for s in slices:\n#         s.SliceThickness = slice_thickness\n        \n#     return slices\n\n\n    \n#     return np.array(images, dtype=np.int16)\n# first_patient = load_slice('../input/rsna-str-pulmonary-embolism-detection/train/0003b3d648eb/d2b2960c2bbf')\n# first_patient_pixels = transform_to_hu(first_patient)\n# # print(first_patient_pixels)\n# def sample_stack(stack, rows=7, cols=7, start_with=10, show_every=4):\n#     fig,ax = plt.subplots(rows,cols,figsize=[18,20])\n#     for i in range(rows*cols):\n#         ind = start_with + i*show_every\n#         ax[int(i/rows),int(i % rows)].set_title(f'slice {ind}')\n#         ax[int(i/rows),int(i % rows)].imshow(stack[ind],cmap='bone')\n#         ax[int(i/rows),int(i % rows)].axis('off')\n#     plt.show()\n\n# sample_stack(first_patient_pixels)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(train.loc[train[\"SeriesInstanceUID\"]==\"d2b2960c2bbf\"][\"SOPInstanceUID\"])\n# # print(train.loc[1398587:1398809,\"negative_exam_for_pe\"])\n# print(train.shape)\n# print(train.loc[train[\"negative_exam_for_pe\"]==0].shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# list1 = train[\"SeriesInstanceUID\"].values.tolist()\n# print(np.unique(list1, return_counts=True))\n# dct=dict(zip(*np.unique(list1, return_counts=True)))\n# lst=sorted(list(dct.values()))\n# temp=np.unique(lst, return_counts=True)\n\n# fig = plt.figure() \n  \n# plt.ylabel(\"Count\") \n# plt.xlabel(\"Number of CT images per case\") \n# plt.ylim([0,150])\n# plt.bar(temp[0], temp[1], color ='maroon') \n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}