{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torch.backends.cudnn as cudnn\nimport numpy as np\nimport torchvision\nfrom torchvision import datasets, models, transforms\nimport matplotlib.pyplot as plt\nimport time\nimport math\nimport os\nimport copy\nimport glob\nfrom io import open\nfrom torch.utils.data.dataloader import Dataset, DataLoader\nimport unicodedata\nimport string\nimport torch.nn.functional as F\nimport random\nimport pandas as pd\nfrom pandas import DataFrame\n\n!pip install torchmetrics\nimport torchmetrics\n\n!pip install scikit_learn\nimport sklearn\nfrom sklearn.utils import resample\nimport tqdm\nimport cv2\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nif(device == 'cuda'):\n  torch.cuda.manual_seed_all(777)\nprint(device) ","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:53:58.909578Z","iopub.execute_input":"2023-01-18T19:53:58.910238Z","iopub.status.idle":"2023-01-18T19:54:59.327745Z","shell.execute_reply.started":"2023-01-18T19:53:58.910195Z","shell.execute_reply":"2023-01-18T19:54:59.326375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# dcm파일 변환","metadata":{}},{"cell_type":"code","source":"test_dir = '/kaggle/working/test_image_cv2'\ndf_test = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\ndf_test","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:54:59.330212Z","iopub.execute_input":"2023-01-18T19:54:59.331614Z","iopub.status.idle":"2023-01-18T19:54:59.352511Z","shell.execute_reply.started":"2023-01-18T19:54:59.331564Z","shell.execute_reply":"2023-01-18T19:54:59.351292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install pylibjpeg pylibjpeg-libjpeg pydicom\nimport pydicom \n!mkdir /kaggle/working/test_image_cv2","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:54:59.355155Z","iopub.execute_input":"2023-01-18T19:54:59.355802Z","iopub.status.idle":"2023-01-18T19:55:00.378417Z","shell.execute_reply.started":"2023-01-18T19:54:59.355767Z","shell.execute_reply":"2023-01-18T19:55:00.377086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n\n!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \n\ntry:\n    import pylibjpeg\nexcept:\n   !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{"execution":{"iopub.status.busy":"2023-01-18T20:07:42.323904Z","iopub.execute_input":"2023-01-18T20:07:42.324293Z","iopub.status.idle":"2023-01-18T20:08:42.464779Z","shell.execute_reply.started":"2023-01-18T20:07:42.324249Z","shell.execute_reply":"2023-01-18T20:08:42.463338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dicomsdl as dicoml\nfrom joblib import Parallel, delayed","metadata":{"execution":{"iopub.status.busy":"2023-01-18T20:08:42.467271Z","iopub.execute_input":"2023-01-18T20:08:42.46798Z","iopub.status.idle":"2023-01-18T20:08:42.485736Z","shell.execute_reply.started":"2023-01-18T20:08:42.467935Z","shell.execute_reply":"2023-01-18T20:08:42.484769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(f, size=512, save_folder=None, extension=\"png\"):\n    patient = f.split('/')[-2]\n    image_name = f.split('/')[-1][:-4]\n\n    dicom = dicoml.open(f)\n    img = dicom.pixelData()\n    if img.max() - img.min() != 0:\n        img = (img - img.min()) / (img.max() - img.min())\n\n    if dicom.getPixelDataInfo()['PhotometricInterpretation'] == \"MONOCHROME1\":\n        img = 1 - img\n\n    image = (img * 255).astype(np.uint8)\n\n    \n    img = cv2.resize(image, (size, size))\n\n    file_name = f'{save_folder}' + f\"{patient}_{image_name}.{extension}\"\n\n    cv2.imwrite(file_name, img)","metadata":{"execution":{"iopub.status.busy":"2023-01-18T20:10:44.467519Z","iopub.execute_input":"2023-01-18T20:10:44.467881Z","iopub.status.idle":"2023-01-18T20:10:44.478187Z","shell.execute_reply.started":"2023-01-18T20:10:44.467851Z","shell.execute_reply":"2023-01-18T20:10:44.477327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\nfrom joblib import Parallel, delayed\n\ntrain_images = glob.glob(\"/kaggle/input/rsna-breast-cancer-detection/test_images/*/*.dcm\")\nimage_dir_dicomsdl = '/kaggle/working/test_image_cv2/'\n\nParallel(n_jobs=2)(\n    delayed(process)(f, size = 512, save_folder = image_dir_dicomsdl)\n    for f in tqdm.tqdm(train_images)\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-18T20:11:22.801266Z","iopub.execute_input":"2023-01-18T20:11:22.801663Z","iopub.status.idle":"2023-01-18T20:11:25.015884Z","shell.execute_reply.started":"2023-01-18T20:11:22.801629Z","shell.execute_reply":"2023-01-18T20:11:25.014715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Net 불러오기","metadata":{}},{"cell_type":"code","source":"Net_list = [0,0,0,0,0]\nNet_list[0] = models.efficientnet_b7(pretrained=False)\n\nfor Net in Net_list:\n  if Net!=0:\n    Net.classifier.append(nn.BatchNorm1d(1000))\n    Net.classifier.append(nn.SiLU(inplace=True))\n    Net.classifier.append(nn.Linear(1000,2,bias=True))","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:55:02.227922Z","iopub.execute_input":"2023-01-18T19:55:02.228688Z","iopub.status.idle":"2023-01-18T19:55:03.450512Z","shell.execute_reply.started":"2023-01-18T19:55:02.228644Z","shell.execute_reply":"2023-01-18T19:55:03.449455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Ensemble_Net(nn.Module):\n    def __init__(self, Net_list):\n        super(Ensemble_Net, self).__init__()\n        self.Net1 = Net_list[0]\n        \"\"\"\n        self.Net2 = Net_list[1]\n        self.Net3 = Net_list[2]\n        self.Net4 = Net_list[3]\n        self.Net5 = Net_list[4]\n        \"\"\"\n    \n    def forward(self, input):\n        out1 = self.Net1.forward(input)\n        \"\"\"\n        out2 = self.Net2.forward(input)\n        out3 = self.Net3.forward(input)\n        out4 = self.Net4.forward(input)\n        out5 = self.Net5.forward(input)\n        \n        out1 += out2\n        out1 += out3\n        out1 += out4\n        out1 += out5\n        \"\"\"\n        \n        return out1","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:55:03.452069Z","iopub.execute_input":"2023-01-18T19:55:03.452793Z","iopub.status.idle":"2023-01-18T19:55:03.45986Z","shell.execute_reply.started":"2023-01-18T19:55:03.452743Z","shell.execute_reply":"2023-01-18T19:55:03.458884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Net = Ensemble_Net(Net_list).to(device)\nPATH = '/kaggle/input/first-model-2-epochpth/first_model_2_epoch.pth'\nNet.load_state_dict(torch.load(PATH))","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:55:03.461102Z","iopub.execute_input":"2023-01-18T19:55:03.462082Z","iopub.status.idle":"2023-01-18T19:55:04.219358Z","shell.execute_reply.started":"2023-01-18T19:55:03.462001Z","shell.execute_reply":"2023-01-18T19:55:04.218329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# evaluation","metadata":{}},{"cell_type":"code","source":"class My_dataset(Dataset):\n    def __init__ (self, df_X, transforms=None):\n        self.df_X = df_X\n        self.transforms = transforms\n        \n    def __len__ (self):\n        return self.df_X.shape[0]\n        \n    def __getitem__ (self, i):\n        image_index = str(self.df_X.iloc[i]['prediction_id']) #10008_L\n        image_dir = os.path.join(test_dir, str(self.df_X.iloc[i]['patient_id']) + '_' + str(self.df_X.iloc[i]['image_id']) + '.png')\n        img = cv2.imread(image_dir)\n        img = cv2.resize(img, (512, 1024))\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = np.float32(img)\n        img /= 255\n        \n        if self.transforms != None:\n            img = self.transforms(img) \n        \n        return img, image_index\n    \n    \nmean, std = [0.4914, 0.4822, 0.4465], [0.247, 0.243, 0.261] # 색깔 RGB일떄\nmean1, std1 = 0.4822, 0.243 # 흑백일때, 임의로 정함\n\ntest_transforms = transforms.Compose([transforms.ToTensor(),\n                                      transforms.Normalize(mean, std)])\n\ntest_dataset = My_dataset(df_test, test_transforms)\n\ntest_dataloader = DataLoader(test_dataset, batch_size=2, shuffle=False, num_workers=1)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:55:04.223466Z","iopub.execute_input":"2023-01-18T19:55:04.22376Z","iopub.status.idle":"2023-01-18T19:55:04.233084Z","shell.execute_reply.started":"2023-01-18T19:55:04.223732Z","shell.execute_reply":"2023-01-18T19:55:04.231994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_answer = {}\nimport gc\n\nwith torch.no_grad():\n    for data in test_dataloader:\n        img, img_index = data\n        img = img.to(device)\n        result = Net.forward(img)\n        # result = F.softmax(result,dim=1)\n        # _, result_max = result.max(dim=1)\n        \n        del img\n        result = result.to('cpu')\n        gc.collect()\n        torch.cuda.empty_cache()\n        for i, result_line in enumerate(result):\n            # print(img_index[i], result_line)\n            if total_answer.get(img_index[i]) == None:\n                total_answer[img_index[i]] = result_line\n                \n            else:\n                total_answer[img_index[i]] += result_line\n                \n        del result\n        gc.collect()\n        torch.cuda.empty_cache()\n                \n# print(total_answer)\ntotal_answer_2 = {}\n\nfor key in total_answer:\n    total_answer[key] = total_answer[key].reshape(1,-1)\n    # total_answer[key] = F.softmax(total_answer[key],dim=1)\n    \n    THRESHOLD = 0.3\n    if total_answer[key][0][1] > THRESHOLD:\n        total_answer_2[key]=1\n            \n    else:\n        total_answer_2[key]=0\n        \n    # 값 그대로 출력\n    # total_answer_2[key] = total_answer[key][0][1].item()\n    \n    \"\"\"\n    #0과 1중 출력\n    if total_answer[key][0][1] > total_answer[key][0][0]:\n        total_answer_2[key]=1\n            \n    else:\n        total_answer_2[key]=0\n    \"\"\"\n    \n# print(total_answer_2)","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:55:04.235245Z","iopub.execute_input":"2023-01-18T19:55:04.236144Z","iopub.status.idle":"2023-01-18T19:55:05.433815Z","shell.execute_reply.started":"2023-01-18T19:55:04.236104Z","shell.execute_reply":"2023-01-18T19:55:05.432471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"key_arr = []\nans_arr = []\n\nfor key in total_answer_2:\n    key_arr.append(key)\n    ans_arr.append(int(total_answer_2[key]))\n    \nsubmission_df = DataFrame({'prediction_id': key_arr , 'cancer': ans_arr}, index=None)\nsubmission_df.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:55:05.435439Z","iopub.execute_input":"2023-01-18T19:55:05.435794Z","iopub.status.idle":"2023-01-18T19:55:05.447246Z","shell.execute_reply.started":"2023-01-18T19:55:05.435755Z","shell.execute_reply":"2023-01-18T19:55:05.446322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:55:05.448987Z","iopub.execute_input":"2023-01-18T19:55:05.449963Z","iopub.status.idle":"2023-01-18T19:55:05.461743Z","shell.execute_reply.started":"2023-01-18T19:55:05.449934Z","shell.execute_reply":"2023-01-18T19:55:05.460656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:55:05.463408Z","iopub.execute_input":"2023-01-18T19:55:05.463974Z","iopub.status.idle":"2023-01-18T19:55:05.603818Z","shell.execute_reply.started":"2023-01-18T19:55:05.46394Z","shell.execute_reply":"2023-01-18T19:55:05.602524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2023-01-18T19:55:05.605701Z","iopub.execute_input":"2023-01-18T19:55:05.606526Z","iopub.status.idle":"2023-01-18T19:55:05.613314Z","shell.execute_reply.started":"2023-01-18T19:55:05.606488Z","shell.execute_reply":"2023-01-18T19:55:05.612061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}