{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Describing the issue\n\nRunning cells 1,2,3,5 leads to OOM whereas running cells 1,2,4,5 does not, even though they should use about the same amount of RAM\n\nUpdate: This is not the case anymore (on 2023-06-16). Both versions run and use about 29.8 GB RAM by the end. Although RAM usage still displays 5 GB after running cell 3, but this probably gets freed as soon as the large array is created.","metadata":{}},{"cell_type":"code","source":"# Cell 1: Import tools\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-16T09:57:05.020008Z","iopub.execute_input":"2023-06-16T09:57:05.021062Z","iopub.status.idle":"2023-06-16T09:57:05.054551Z","shell.execute_reply.started":"2023-06-16T09:57:05.021018Z","shell.execute_reply":"2023-06-16T09:57:05.053273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cell 2: Gather training and validation data directories\n\nimport os\nroot_dir = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/'\ntrain_dirs = next(os.walk(root_dir + 'train'))[1]\nval_dirs = next(os.walk(root_dir + 'validation'))[1]\n\ndata_dirs = ['train/' + s for s in train_dirs] + ['validation/' + s for s in val_dirs]\n\ndel train_dirs, val_dirs","metadata":{"execution":{"iopub.status.busy":"2023-06-16T09:57:05.056865Z","iopub.execute_input":"2023-06-16T09:57:05.0578Z","iopub.status.idle":"2023-06-16T09:57:14.076006Z","shell.execute_reply.started":"2023-06-16T09:57:05.057752Z","shell.execute_reply":"2023-06-16T09:57:14.074826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cell 3: Sorting training data by size of contrail\n\ndef find_size(data_dir):\n    root_dir = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/'\n    y = np.load(root_dir + data_dir + '/human_pixel_masks.npy')\n    output = np.sum(y)#.copy()\n    return output\n\nc_size = pd.DataFrame(data = {'dir': data_dirs, 'size': map(find_size,data_dirs)})\n\n# I have tried using \"with\" and copy() everywhere, \n# but running this cell still permanently takes over about 5GB RAM.\n# Garbage collection also did not free that memory space.","metadata":{"execution":{"iopub.status.busy":"2023-06-16T09:57:14.077361Z","iopub.execute_input":"2023-06-16T09:57:14.077738Z","iopub.status.idle":"2023-06-16T09:59:38.818774Z","shell.execute_reply.started":"2023-06-16T09:57:14.077703Z","shell.execute_reply":"2023-06-16T09:59:38.81727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Cell 4: Generating c_size just to demonstrate how much RAM cell 3 should use\n\n# c_size = pd.DataFrame(data = {'dir': data_dirs, 'size': 0.0})","metadata":{"execution":{"iopub.status.busy":"2023-06-16T09:59:38.821988Z","iopub.execute_input":"2023-06-16T09:59:38.822488Z","iopub.status.idle":"2023-06-16T09:59:38.827951Z","shell.execute_reply.started":"2023-06-16T09:59:38.822445Z","shell.execute_reply":"2023-06-16T09:59:38.826503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cell 5: Creating a large numpy array\n\nA = np.ones([830,256,256,8,9],dtype=float)\n\n# This leads to OOM when cell 3 is run (i.e. the 5GB RAM is used) but runs fine when only cell 4 is run (total RAM usage is around 29.8GB)","metadata":{"execution":{"iopub.status.busy":"2023-06-16T09:59:38.82895Z","iopub.execute_input":"2023-06-16T09:59:38.829347Z","iopub.status.idle":"2023-06-16T09:59:56.057524Z","shell.execute_reply.started":"2023-06-16T09:59:38.829318Z","shell.execute_reply":"2023-06-16T09:59:56.053324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}