{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport cv2\nimport os\nfrom matplotlib import pyplot as plt\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport albumentations as albu\nfrom skimage.color import gray2rgb\nimport functools\nimport torch\nfrom tqdm.auto import tqdm\n\ntrain_csv_path = '../input/rsna-str-pulmonary-embolism-detection/train.csv'\njpeg_dir = '../input/rsna-str-pe-detection-jpeg-256/train-jpegs'\n\ntrain_df = pd.read_csv(train_csv_path)\ntrain_df.head()\n\nrow = train_df.iloc[100]\nimg = cv2.imread(glob.glob(f\"{jpeg_dir}/{row[0]}/{row[1]}/*{row[2]}.jpg\")[0])\nplt.figure(figsize=[12,6])\nplt.subplot(131)\nplt.imshow(img[:,:,0],cmap='gray')\nplt.subplot(132)\nplt.imshow(img[:,:,1],cmap='gray')\nplt.subplot(133)\nplt.imshow(img[:,:,2],cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_augmentation(y=256,x=256):\n    train_transform = [albu.VerticalFlip(p=0.5),\n                           albu.HorizontalFlip(p=0.5),\n                           albu.Downscale(p=1.0,scale_min=0.35,scale_max=0.75,),\n                           albu.Resize(y, x)]\n    return albu.Compose(train_transform)\n\n\nformatted_settings = {\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],}\ndef preprocess_input(\n    x, mean=None, std=None, input_space=\"RGB\", input_range=None, **kwargs\n):\n\n    if input_space == \"BGR\":\n        x = x[..., ::-1].copy()\n\n    if input_range is not None:\n        if x.max() > 1 and input_range[1] == 1:\n            x = x / 255.0\n\n    if mean is not None:\n        mean = np.array(mean)\n        x = x - mean\n\n    if std is not None:\n        std = np.array(std)\n        x = x / std\n\n    return x\n\ndef get_preprocessing(preprocessing_fn):\n    _transform = [\n        albu.Lambda(image=preprocessing_fn),\n        albu.Lambda(image=to_tensor, mask=to_tensor),\n    ]\n    return albu.Compose(_transform)\n\ndef get_validation_augmentation(y=256,x=256):\n    \"\"\"Add paddings to make image shape divisible by 32\"\"\"\n    test_transform = [albu.Resize(y, x)]\n    return albu.Compose(test_transform)\n\ndef to_tensor(x, **kwargs):\n    \"\"\"\n    Convert image or mask.\n    \"\"\"\n    return x.transpose(2, 0, 1).astype('float32')\n\nclass CTDataset2D(Dataset):\n    def __init__(self,df,transforms = albu.Compose([albu.HorizontalFlip()]),preprocessing=None,size=1,mode='val'):\n        self.df_main = df.values\n        if mode=='val':\n            self.df = self.df_main\n        else:\n            self.update_train_df()\n            \n        self.transforms = transforms\n        self.preprocessing = preprocessing\n        self.size=size\n\n\n    def __getitem__(self, idx):\n        row = self.df[idx]\n        img = cv2.imread(glob.glob(f\"{jpeg_dir}/{row[0]}/{row[1]}/*{row[2]}.jpg\")[0])\n        label = row[3:].astype(int) \n        UID_new = row[0:3]                                        #Changed: As we need image UIDs\n        label[2:] = label[2:] if label[0]==1 else 0\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        if self.preprocessing:\n            img = self.preprocessing(image=img)['image']\n        return img,torch.from_numpy(label.reshape(-1)),UID_new\n\n    def __len__(self):\n        return len(self.df)\n    \n    def update_train_df(self):\n        df0 = self.df_main[self.df_main[:,3]==0]\n        df1 = self.df_main[self.df_main[:,3]==1]\n        #np.random.shuffle(df0)\n        self.df = np.concatenate([df0[:len(df1)],df1],axis=0)\n        \n\ndef norm(img):\n    img-=img.min()\n    return img/img.max()\n\nStudyInstanceUID = list(set(train_df['StudyInstanceUID']))\nprint(len(StudyInstanceUID))\nt_df = train_df[train_df['StudyInstanceUID'].isin(StudyInstanceUID[0:6500])]\nv_df = train_df[train_df['StudyInstanceUID'].isin(StudyInstanceUID[6500:])]\n\nclass config:\n    model_name=\"resnet18\"\n    batch_size = 1\n    WORKERS = 4\n    classes =14\n    resume = False\n    epochs = 10\n    MODEL_PATH = 'log/cpt'\n    if not os.path.exists(MODEL_PATH):\n        os.makedirs(MODEL_PATH)\n        \npreprocessing_fn = functools.partial(preprocess_input, **formatted_settings)\ntrain_dataset = CTDataset2D(t_df,\n                            transforms=get_training_augmentation(),\n                            preprocessing=get_preprocessing(preprocessing_fn),mode='train')\nval_dataset = CTDataset2D(v_df,\n                            transforms=get_validation_augmentation(),\n                            preprocessing=get_preprocessing(preprocessing_fn))\n\ntrain = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=False, num_workers=config.WORKERS, pin_memory=True)\nval = DataLoader(val_dataset, batch_size=config.batch_size*2, shuffle=False, num_workers=config.WORKERS, pin_memory=True)\n\nx,y,Uid = train_dataset[-400]\nx.shape,len(y),y,len(train_dataset), len(val_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(np.swapaxes(x,0,2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\"\"\"\nThis script reads training and validation images from folders and save them as csv files\n\n@author: Dipu\n\"\"\"\nimport torch.nn as nn\n\n\nfrom torchvision import datasets, models, transforms\nimport matplotlib.pyplot as plt\nimport time\nimport os\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.ion()   # interactive mode\nmodel_ft = models.resnet50(pretrained=True)\n\n''' NullNet forwards the input(features) to output '''\nclass NullNet(nn.Module): \n    def __init__(self):\n        super(NullNet, self).__init__()\n    def forward(self, x):\n        return x\n    \n    \nmodel_ft.fc = NullNet()\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel_ft = model_ft.to(device)\n\nprint(device)\nimport torchvision.transforms as T\ntrf = T.Compose([T.ToTensor()])\n\n\n#%%\n\n\nimport csv\n\nsince = time.time()\niter1=0\nfile = open('train_features.csv', 'a+', newline ='')\n# writing the data into the file \nwith file:     \n    write = csv.writer(file) \n    \n    for inputs, labels,UIDs_new in train_dataset:\n        inp = np.swapaxes(inputs,0,2)\n        inp=trf(inp).unsqueeze(0)\n        #print('Input Size',inp.size())\n        inputs = inp.to(device)\n        outputs = model_ft(inputs)\n        #print(outputs[1,0])\n        outputs = torch.flatten(outputs)\n\n\n        output_format = np.concatenate((UIDs_new,labels.numpy(),outputs.cpu().data.numpy()),axis=0)\n        write.writerows([output_format]) \n\n        iter1 = iter1 + 1\n\n        if iter1 % 20000 == 500:\n            time_elapsed = time.time() - since\n            print('Time from start {:.0f}m {:.0f}s'.format(\n            time_elapsed // 60, time_elapsed % 60))\n            print('Percentage complete: {:4f}'.format(100*iter1/(len(val_dataset)+len(train_dataset))))\n\n\n\n            #break # delete this break to save features from the entire dataset\n\n\n    for inputs, labels,UIDs_new in val_dataset:\n        inp = np.swapaxes(inputs,0,2)\n        inp=trf(inp).unsqueeze(0)\n        #print('Input Size',inp.size())\n        inputs = inp.to(device)\n        outputs = torch.flatten(model_ft(inputs))\n\n        output_format = np.concatenate((UIDs_new,labels.numpy(),outputs.cpu().data.numpy()),axis=0)\n        write.writerows([output_format]) \n        iter1 = iter1 + 1\n\n        if iter1 % 20000 == 500:\n            time_elapsed = time.time() - since\n            print('Time from start {:.0f}m {:.0f}s'.format(\n            time_elapsed // 60, time_elapsed % 60))\n            print('Percentage complete: {:4f}'.format(100*iter1/(len(val_dataset)+len(train_dataset))))\n\n\n\n            #break # delete this break to save features from the entire dataset","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}