\n
![\"\"](\"https://www.googleapis.com/download/storage/v1/b/kaggle-forum-message-attachments/o/inbox%2F3182053%2Fc3f70391fe09b097d9388fc53b9f4da6%2FScreenshot%202020-10-22%20at%2017.02.29.png?generation=1603378970201159&alt=media\")
EDIT
\nThis is not a PyTorch DataLoader issue. I've moved this to here to avoid confusion.
\nHi Kaggler's, hoping to get your help with this one. Since we're all working with the same data and public kernels maybe one of you met this problem.
\nMy memory usage keeps ramping up during iteration through my data loader until I run into OOM. I've seen this StackOverflow thread talking about situations where num_workers > 0 but for me I can confirm that setting num_workers = 0 doesn't fix the problem.
This occurs just by looping through the data with a pass clause. I normally hit OOM near the end.
Here is my relevant code:
\nclass DicomDatasetRetriever(torch.utils.data.Dataset):\n def __init__(self, df, transforms=[], mix_ratio=1, mode='val'):\n self.df_main = df.copy()\n self.mode = mode\n self.mix_ratio = mix_ratio\n if self.mode == 'val':\n self.df = self.df_main\n else:\n self.update_train_df()\n self.lut = df[['SOPInstanceUID', 'image_path']].set_index('SOPInstanceUID')\n\n if not(len(transforms)):\n self.transforms = None\n else:\n self.transforms = A.Compose(transforms)\n\n self.default_transforms = A.Compose([\n A.Normalize(0.449, 0.226),\n ToTensorV2(),\n ])\n\n def __len__(self):\n return len(self.df)\n\n def __getitem__(self, idx):\n row = self.df.iloc[idx]\n study = row.StudyInstanceUID\n img_id = row.SOPInstanceUID\n\n img = self.load_image(img_id)\n pe_ratio = row.r_pe_present_on_image\n target = row.pe_present_on_image\n\n # transforms\n if self.transforms is not None and self.mode != 'val' and (row[3] == 1 or random.random() < 0.1):\n img = self.transforms(image=img)['image']\n\n # default transformation\n img = self.default_transforms(image=img)['image']\n\n return {\n 'img': img,\n 'img_id': img_id,\n 'study_id': study,\n 'pe_ratio': torch.tensor([pe_ratio]).float(),\n 'target': torch.tensor([target]).float()\n }\n\n def load_image(self, img_id):\n# img = cv2.imread(self.lut.loc[img_id, 'image_path'])\n with open(self.lut.loc[img_id, 'image_path'], 'rb') as f:\n img = JPEG.decode(f.read())\n if img is None:\n print(f\"Warning while trying to load image. No file at {file_path}\")\n img = np.zeros(shape=SHAPE) \n img = img.astype(np.float32)\n img /= 255\n return img\n\n def update_train_df(self):\n df0 = self.df_main[self.df_main.pe_present_on_image==0]\n df1 = self.df_main[self.df_main.pe_present_on_image==1]\n df0 = df0.sample(frac=1)\n upto = min(int(self.mix_ratio * len(df1)), len(df0))\n self.df = pd.concat([df0.iloc[:upto],df1], axis=0)\n self.df = self.df.sample(frac=1)\n\nclass DicomDataLoaders:\n def __init__(self, num_splits, batch_size, num_workers, transforms=[], split_ix=0, mix_ratio=1):\n self.num_splits = num_splits\n self.batch_size = batch_size\n self.num_workers = num_workers\n self.split_ix = split_ix\n\n studies = train_df['StudyInstanceUID'].unique()\n stratify_groups = train_df.drop_duplicates(subset='StudyInstanceUID')['stratify_group']\n\n skf = StratifiedKFold(n_splits=self.num_splits, shuffle=True, random_state=SEED)\n train_ix, val_ix = next(islice(skf.split(\n studies, stratify_groups), self.split_ix, self.split_ix+1));\n\n self.train_studies = studies[train_ix]\n self.train_dataset = DicomDatasetRetriever(\n train_df[train_df.StudyInstanceUID.isin(self.train_studies)],\n transforms=transforms, mode='train', mix_ratio=mix_ratio)\n self.train_loader = torch.utils.data.DataLoader(\n self.train_dataset, batch_size=self.batch_size, shuffle=True,\n num_workers=self.num_workers, pin_memory=True)\n\n self.val_studies = studies[val_ix]\n self.val_dataset = DicomDatasetRetriever(\n train_df[train_df.StudyInstanceUID.isin(self.val_studies)], mode='val')\n self.val_loader = torch.utils.data.DataLoader(\n self.val_dataset, batch_size=self.batch_size, shuffle=False,\n num_workers=self.num_workers, pin_memory=True)\n\ndata_loaders = DicomDataLoaders(num_splits=20,\n batch_size=64,\n num_workers=4,\n split_ix=0,\n transforms=TRANSFORMS,\n mix_ratio=1)\n\nfor data in tqdm(data_loaders.train_loader):\n pass\nFingers crossed someone knows!
\nEDIT
\nI've managed to nail this down to the image loading. It's very weird because if I hardcode a filepath for loading the image (and so it loads the same image every iteration) I don't have the memory issue.
Thanks for the report, see update there: https://www.kaggle.com/product-feedback/104464#1058238
", "rawMarkdown": "Thanks for the report, see update there: https://www.kaggle.com/product-feedback/104464#1058238", "votes": 1 }, { "id": 1057318, "postDate": "2020-10-22T15:01:39.483Z", "content": "You can try to find the best num_workers or pin_memory with or without transform setup with this code snippet:
\n
Thanks, although I'm not sure this solves my OOM issue. Did I miss your point?
", "rawMarkdown": "Thanks, although I'm not sure this solves my OOM issue. Did I miss your point?" } ] }, { "id": 1057145, "postDate": "2020-10-22T12:14:55.540Z", "content": "EDIT
\nThis is not a PyTorch DataLoader issue. I've moved this to here to avoid confusion.
\nHi Kaggler's, hoping to get your help with this one. Since we're all working with the same data and public kernels maybe one of you met this problem.
\nMy memory usage keeps ramping up during iteration through my data loader until I run into OOM. I've seen this StackOverflow thread talking about situations where num_workers > 0 but for me I can confirm that setting num_workers = 0 doesn't fix the problem.
This occurs just by looping through the data with a pass clause. I normally hit OOM near the end.
Here is my relevant code:
\nclass DicomDatasetRetriever(torch.utils.data.Dataset):\n def __init__(self, df, transforms=[], mix_ratio=1, mode='val'):\n self.df_main = df.copy()\n self.mode = mode\n self.mix_ratio = mix_ratio\n if self.mode == 'val':\n self.df = self.df_main\n else:\n self.update_train_df()\n self.lut = df[['SOPInstanceUID', 'image_path']].set_index('SOPInstanceUID')\n\n if not(len(transforms)):\n self.transforms = None\n else:\n self.transforms = A.Compose(transforms)\n\n self.default_transforms = A.Compose([\n A.Normalize(0.449, 0.226),\n ToTensorV2(),\n ])\n\n def __len__(self):\n return len(self.df)\n\n def __getitem__(self, idx):\n row = self.df.iloc[idx]\n study = row.StudyInstanceUID\n img_id = row.SOPInstanceUID\n\n img = self.load_image(img_id)\n pe_ratio = row.r_pe_present_on_image\n target = row.pe_present_on_image\n\n # transforms\n if self.transforms is not None and self.mode != 'val' and (row[3] == 1 or random.random() < 0.1):\n img = self.transforms(image=img)['image']\n\n # default transformation\n img = self.default_transforms(image=img)['image']\n\n return {\n 'img': img,\n 'img_id': img_id,\n 'study_id': study,\n 'pe_ratio': torch.tensor([pe_ratio]).float(),\n 'target': torch.tensor([target]).float()\n }\n\n def load_image(self, img_id):\n# img = cv2.imread(self.lut.loc[img_id, 'image_path'])\n with open(self.lut.loc[img_id, 'image_path'], 'rb') as f:\n img = JPEG.decode(f.read())\n if img is None:\n print(f\"Warning while trying to load image. No file at {file_path}\")\n img = np.zeros(shape=SHAPE) \n img = img.astype(np.float32)\n img /= 255\n return img\n\n def update_train_df(self):\n df0 = self.df_main[self.df_main.pe_present_on_image==0]\n df1 = self.df_main[self.df_main.pe_present_on_image==1]\n df0 = df0.sample(frac=1)\n upto = min(int(self.mix_ratio * len(df1)), len(df0))\n self.df = pd.concat([df0.iloc[:upto],df1], axis=0)\n self.df = self.df.sample(frac=1)\n\nclass DicomDataLoaders:\n def __init__(self, num_splits, batch_size, num_workers, transforms=[], split_ix=0, mix_ratio=1):\n self.num_splits = num_splits\n self.batch_size = batch_size\n self.num_workers = num_workers\n self.split_ix = split_ix\n\n studies = train_df['StudyInstanceUID'].unique()\n stratify_groups = train_df.drop_duplicates(subset='StudyInstanceUID')['stratify_group']\n\n skf = StratifiedKFold(n_splits=self.num_splits, shuffle=True, random_state=SEED)\n train_ix, val_ix = next(islice(skf.split(\n studies, stratify_groups), self.split_ix, self.split_ix+1));\n\n self.train_studies = studies[train_ix]\n self.train_dataset = DicomDatasetRetriever(\n train_df[train_df.StudyInstanceUID.isin(self.train_studies)],\n transforms=transforms, mode='train', mix_ratio=mix_ratio)\n self.train_loader = torch.utils.data.DataLoader(\n self.train_dataset, batch_size=self.batch_size, shuffle=True,\n num_workers=self.num_workers, pin_memory=True)\n\n self.val_studies = studies[val_ix]\n self.val_dataset = DicomDatasetRetriever(\n train_df[train_df.StudyInstanceUID.isin(self.val_studies)], mode='val')\n self.val_loader = torch.utils.data.DataLoader(\n self.val_dataset, batch_size=self.batch_size, shuffle=False,\n num_workers=self.num_workers, pin_memory=True)\n\ndata_loaders = DicomDataLoaders(num_splits=20,\n batch_size=64,\n num_workers=4,\n split_ix=0,\n transforms=TRANSFORMS,\n mix_ratio=1)\n\nfor data in tqdm(data_loaders.train_loader):\n pass\nFingers crossed someone knows!
\nEDIT
\nI've managed to nail this down to the image loading. It's very weird because if I hardcode a filepath for loading the image (and so it loads the same image every iteration) I don't have the memory issue.
Found this https://www.kaggle.com/c/aptos2019-blindness-detection/discussion/106622 which talks about the same issue I'm having
", "rawMarkdown": "Found this https://www.kaggle.com/c/aptos2019-blindness-detection/discussion/106622 which talks about the same issue I'm having" } ], "comments": [ { "id": 1058282, "author_name": "Philippe Modard", "author_url": "", "post_date": "2020-10-23T14:31:15.373000", "content": "Thanks for the report, see update there: https://www.kaggle.com/product-feedback/104464#1058238
", "votes": 1, "replies": [] }, { "id": 1057318, "author_name": "Baran Hashemi", "author_url": "", "post_date": "2020-10-22T15:01:39.483000", "content": "You can try to find the best num_workers or pin_memory with or without transform setup with this code snippet:
\n
Thanks, although I'm not sure this solves my OOM issue. Did I miss your point?
", "votes": 0, "replies": [] } ] }, { "id": 1057605, "author_name": "Alexander Soare", "author_url": "", "post_date": "2020-10-22T19:44:04.863000", "content": "Found this https://www.kaggle.com/c/aptos2019-blindness-detection/discussion/106622 which talks about the same issue I'm having
", "votes": 0, "replies": [] } ], "raw_markdown_by_id": { "1058282": "Thanks for the report, see update there: https://www.kaggle.com/product-feedback/104464#1058238", "1057318": "You can try to find the best num_workers or pin_memory with or without transform setup with this code snippet:\n", "1057145": "**EDIT**\n\nThis is not a PyTorch DataLoader issue. I've moved this to [here](https://www.kaggle.com/c/rsna-str-pulmonary-embolism-detection/discussion/192786) to avoid confusion.\n\n----\n\nHi Kaggler's, hoping to get your help with this one. Since we're all working with the same data and public kernels maybe one of you met this problem.\n\nMy memory usage keeps ramping up during iteration through my data loader until I run into OOM. I've seen [this StackOverflow thread](https://github.com/pytorch/pytorch/issues/13246) talking about situations where `num_workers` > 0 but for me I can confirm that setting `num_workers` = 0 doesn't fix the problem.\n\nThis occurs just by looping through the data with a `pass` clause. I normally hit OOM near the end.\n\nHere is my relevant code:\n\n```\nclass DicomDatasetRetriever(torch.utils.data.Dataset):\n def __init__(self, df, transforms=[], mix_ratio=1, mode='val'):\n self.df_main = df.copy()\n self.mode = mode\n self.mix_ratio = mix_ratio\n if self.mode == 'val':\n self.df = self.df_main\n else:\n self.update_train_df()\n self.lut = df[['SOPInstanceUID', 'image_path']].set_index('SOPInstanceUID')\n \n if not(len(transforms)):\n self.transforms = None\n else:\n self.transforms = A.Compose(transforms)\n \n self.default_transforms = A.Compose([\n A.Normalize(0.449, 0.226),\n ToTensorV2(),\n ])\n\n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, idx):\n row = self.df.iloc[idx]\n study = row.StudyInstanceUID\n img_id = row.SOPInstanceUID\n \n img = self.load_image(img_id)\n pe_ratio = row.r_pe_present_on_image\n target = row.pe_present_on_image\n\n # transforms\n if self.transforms is not None and self.mode != 'val' and (row[3] == 1 or random.random() < 0.1):\n img = self.transforms(image=img)['image']\n \n # default transformation\n img = self.default_transforms(image=img)['image']\n\n return {\n 'img': img,\n 'img_id': img_id,\n 'study_id': study,\n 'pe_ratio': torch.tensor([pe_ratio]).float(),\n 'target': torch.tensor([target]).float()\n }\n\n def load_image(self, img_id):\n# img = cv2.imread(self.lut.loc[img_id, 'image_path'])\n with open(self.lut.loc[img_id, 'image_path'], 'rb') as f:\n img = JPEG.decode(f.read())\n if img is None:\n print(f\"Warning while trying to load image. No file at {file_path}\")\n img = np.zeros(shape=SHAPE) \n img = img.astype(np.float32)\n img /= 255\n return img\n \n def update_train_df(self):\n df0 = self.df_main[self.df_main.pe_present_on_image==0]\n df1 = self.df_main[self.df_main.pe_present_on_image==1]\n df0 = df0.sample(frac=1)\n upto = min(int(self.mix_ratio * len(df1)), len(df0))\n self.df = pd.concat([df0.iloc[:upto],df1], axis=0)\n self.df = self.df.sample(frac=1)\n\nclass DicomDataLoaders:\n def __init__(self, num_splits, batch_size, num_workers, transforms=[], split_ix=0, mix_ratio=1):\n self.num_splits = num_splits\n self.batch_size = batch_size\n self.num_workers = num_workers\n self.split_ix = split_ix\n\n studies = train_df['StudyInstanceUID'].unique()\n stratify_groups = train_df.drop_duplicates(subset='StudyInstanceUID')['stratify_group']\n\n skf = StratifiedKFold(n_splits=self.num_splits, shuffle=True, random_state=SEED)\n train_ix, val_ix = next(islice(skf.split(\n studies, stratify_groups), self.split_ix, self.split_ix+1));\n\n self.train_studies = studies[train_ix]\n self.train_dataset = DicomDatasetRetriever(\n train_df[train_df.StudyInstanceUID.isin(self.train_studies)],\n transforms=transforms, mode='train', mix_ratio=mix_ratio)\n self.train_loader = torch.utils.data.DataLoader(\n self.train_dataset, batch_size=self.batch_size, shuffle=True,\n num_workers=self.num_workers, pin_memory=True)\n\n self.val_studies = studies[val_ix]\n self.val_dataset = DicomDatasetRetriever(\n train_df[train_df.StudyInstanceUID.isin(self.val_studies)], mode='val')\n self.val_loader = torch.utils.data.DataLoader(\n self.val_dataset, batch_size=self.batch_size, shuffle=False,\n num_workers=self.num_workers, pin_memory=True)\n\ndata_loaders = DicomDataLoaders(num_splits=20,\n batch_size=64,\n num_workers=4,\n split_ix=0,\n transforms=TRANSFORMS,\n mix_ratio=1)\n\nfor data in tqdm(data_loaders.train_loader):\n pass\n```\n\nFingers crossed someone knows!\n\n**EDIT**\nI've managed to nail this down to the image loading. It's very weird because if I hardcode a filepath for loading the image (and so it loads the same image every iteration) I don't have the memory issue.", "1057605": "Found this https://www.kaggle.com/c/aptos2019-blindness-detection/discussion/106622 which talks about the same issue I'm having" } }