{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Based on [this notebook](https://www.kaggle.com/code/rasoulmojtahedzadeh/test-bug-out-of-memory) by [Rasoul Mojtahedzadeh\n](https://www.kaggle.com/rasoulmojtahedzadeh), which was made as a response to [this discussion](https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/382270).","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \n\ntry:\n    import pylibjpeg\nexcept:\n   !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import psutil\nimport dicomsdl\nimport gc\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:29:44.716154Z","iopub.execute_input":"2023-02-01T20:29:44.716629Z","iopub.status.idle":"2023-02-01T20:29:44.722738Z","shell.execute_reply.started":"2023-02-01T20:29:44.716582Z","shell.execute_reply":"2023-02-01T20:29:44.721226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_memory_usage():\n    # Make DataFrame\n    df = pd.Series(psutil.virtual_memory()._asdict()).to_frame('Usage')\n    # Bytes to MegaBytes\n    df = df / 2**30\n    # Cast to Int\n    df = df.round(1).astype(str) + 'GB'\n    \n    display(df)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:17:21.675978Z","iopub.execute_input":"2023-02-01T20:17:21.676329Z","iopub.status.idle":"2023-02-01T20:17:21.681776Z","shell.execute_reply.started":"2023-02-01T20:17:21.676295Z","shell.execute_reply":"2023-02-01T20:17:21.680705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_memory_usage()","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:17:21.683019Z","iopub.execute_input":"2023-02-01T20:17:21.683362Z","iopub.status.idle":"2023-02-01T20:17:21.721168Z","shell.execute_reply.started":"2023-02-01T20:17:21.683331Z","shell.execute_reply":"2023-02-01T20:17:21.720371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Allocate 29GB of memory\nhuge_29gb_array = np.arange(int(29 * 2**30 / 8), dtype=np.int64)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:17:21.723906Z","iopub.execute_input":"2023-02-01T20:17:21.724233Z","iopub.status.idle":"2023-02-01T20:17:30.897869Z","shell.execute_reply.started":"2023-02-01T20:17:21.724203Z","shell.execute_reply":"2023-02-01T20:17:30.896382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validate memory is indeed allocated, leave ~1GB\ndisplay_memory_usage()","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:17:30.899274Z","iopub.execute_input":"2023-02-01T20:17:30.899608Z","iopub.status.idle":"2023-02-01T20:17:30.913318Z","shell.execute_reply.started":"2023-02-01T20:17:30.899576Z","shell.execute_reply":"2023-02-01T20:17:30.912211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Normally, the following loop will make the memory usgae grow to over 6GB.\n\nWith the huge 29GB array allocated, only ~1GB of memory is available, which should cause a Out-Of-Memory error.\n\nBut somehow, this function does not allocate the memory when it is not available, indicating it does not actually need the memory!?\n\n","metadata":{}},{"cell_type":"code","source":"# Read Train DataFrame\ntrain = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv').head(1000)\n\n# Get File Path to DICOM files\ndef get_file_path(args):\n    patient_id, image_id = args\n    return f'/kaggle/input/rsna-breast-cancer-detection/train_images/{patient_id}/{image_id}.dcm'\n\n# Assign DICOM file path for each training sample\ntrain['file_path'] = train[['patient_id', 'image_id']].apply(get_file_path, axis=1)\n\n# Function which simply reads a DICOM file\ndef read_dicom(fp):\n    dicom = dicomsdl.open(fp)\n    image = dicom.pixelData()\n\n# For 1000 training samples, read the DICOM file\npbar = tqdm(train['file_path'])\nfor fp in pbar:    \n    read_dicom(fp)\n    pbar.set_postfix({'RAM Used (GB)': psutil.virtual_memory()[3]/1000000000})","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:17:30.914535Z","iopub.execute_input":"2023-02-01T20:17:30.914956Z","iopub.status.idle":"2023-02-01T20:29:38.015172Z","shell.execute_reply.started":"2023-02-01T20:17:30.914914Z","shell.execute_reply":"2023-02-01T20:29:38.013854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove the large array\ndel huge_29gb_array\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:29:50.582842Z","iopub.execute_input":"2023-02-01T20:29:50.583308Z","iopub.status.idle":"2023-02-01T20:29:51.524587Z","shell.execute_reply.started":"2023-02-01T20:29:50.583269Z","shell.execute_reply":"2023-02-01T20:29:51.52301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Verify the memory is indeed freed and the 6GB of memory is not allocated\ndisplay_memory_usage()","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:29:55.853978Z","iopub.execute_input":"2023-02-01T20:29:55.854733Z","iopub.status.idle":"2023-02-01T20:29:55.869319Z","shell.execute_reply.started":"2023-02-01T20:29:55.854694Z","shell.execute_reply":"2023-02-01T20:29:55.868208Z"},"trusted":true},"execution_count":null,"outputs":[]}]}