{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Dataset Links :\n\n - [256x256 pngs](https://www.kaggle.com/datasets/theoviel/rsna-breast-cancer-256-pngs)\n - [512x512 pngs](https://www.kaggle.com/datasets/theoviel/rsna-breast-cancer-512-pngs)\n - [768x768 pngs](https://www.kaggle.com/datasets/theoviel/rsna-breast-cancer-768-pngs)\n - [1024x1024 pngs](https://www.kaggle.com/datasets/theoviel/rsna-breast-cancer-1024-pngs)\n\n**Changes :**\n- Invert images with PhotometricInterpretation == \"MONOCHROME1\" ","metadata":{}},{"cell_type":"markdown","source":"## Initialization","metadata":{}},{"cell_type":"code","source":"# !pip install -qU python-gdcm pydicom pylibjpeg","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-08T08:28:19.678587Z","iopub.execute_input":"2023-01-08T08:28:19.679029Z","iopub.status.idle":"2023-01-08T08:28:39.834526Z","shell.execute_reply.started":"2023-01-08T08:28:19.678973Z","shell.execute_reply":"2023-01-08T08:28:39.833163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport glob\n# import gdcm\n# import pydicom\nimport numpy as np\nimport pandas as pd\n# import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# from tqdm.notebook import tqdm\n# from joblib import Parallel, delayed","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:33:51.714656Z","iopub.execute_input":"2023-01-09T19:33:51.715104Z","iopub.status.idle":"2023-01-09T19:33:51.983981Z","shell.execute_reply.started":"2023-01-09T19:33:51.715054Z","shell.execute_reply":"2023-01-09T19:33:51.982779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_images = glob.glob(\"/kaggle/input/rsna-breast-cancer-detection/train_images/*/*.dcm\")\n\n# len(train_images)  # 54706","metadata":{"execution":{"iopub.status.busy":"2022-12-25T07:28:55.561797Z","iopub.execute_input":"2022-12-25T07:28:55.562141Z","iopub.status.idle":"2022-12-25T07:29:28.400907Z","shell.execute_reply.started":"2022-12-25T07:28:55.562112Z","shell.execute_reply":"2022-12-25T07:29:28.399733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save the processed data\n**Images are quite big so resizing them is necessary.**\n  - Use 256 to train your first models, or if you don't have a lot of compute\n  - use 512 to have competitive models\n  - Check if 768/1024 is better, if you have the compute power\n\n**I advise using the `png` format because the jpg compression can be annoying during inference.**","metadata":{}},{"cell_type":"code","source":"\"\"\"\nSAVE_FOLDER = \"output/\"\nSIZE = 256\nEXTENSION = \"png\"\n\nos.makedirs(SAVE_FOLDER, exist_ok=True)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-12-25T07:29:31.997031Z","iopub.execute_input":"2022-12-25T07:29:31.997423Z","iopub.status.idle":"2022-12-25T07:29:32.003206Z","shell.execute_reply.started":"2022-12-25T07:29:31.997391Z","shell.execute_reply":"2022-12-25T07:29:32.001218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ndef process(f, size=512, save_folder=\"\", extension=\"png\"):\n    patient = f.split('/')[-2]\n    image = f.split('/')[-1][:-4]\n\n    dicom = pydicom.dcmread(f)\n    img = dicom.pixel_array\n\n    img = (img - img.min()) / (img.max() - img.min())\n\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        img = 1 - img\n\n    img = cv2.resize(img, (size, size))\n\n    cv2.imwrite(save_folder + f\"{patient}_{image}.{extension}\", (img * 255).astype(np.uint8))\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-12-25T07:29:34.090609Z","iopub.execute_input":"2022-12-25T07:29:34.091012Z","iopub.status.idle":"2022-12-25T07:29:34.100741Z","shell.execute_reply.started":"2022-12-25T07:29:34.090979Z","shell.execute_reply":"2022-12-25T07:29:34.099435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\n_ = Parallel(n_jobs=4)(\n    delayed(process)(uid, size=SIZE, save_folder=SAVE_FOLDER, extension=EXTENSION)\n    for uid in tqdm(train_images[:10])\n)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-12-25T08:49:08.927743Z","iopub.execute_input":"2022-12-25T08:49:08.928348Z","iopub.status.idle":"2022-12-25T09:08:13.416045Z","shell.execute_reply.started":"2022-12-25T08:49:08.928313Z","shell.execute_reply":"2022-12-25T09:08:13.414543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(len([name for name in os.listdir('/kaggle/working/output/')]))","metadata":{"execution":{"iopub.status.busy":"2022-12-26T08:27:59.910637Z","iopub.execute_input":"2022-12-26T08:27:59.911001Z","iopub.status.idle":"2022-12-26T08:27:59.946779Z","shell.execute_reply.started":"2022-12-26T08:27:59.910972Z","shell.execute_reply":"2022-12-26T08:27:59.94554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\n\ndf = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\n\n\n# test = cv2.imread('/kaggle/working/output/10006_462822612.png')\n# test2 = cv2.imread('kaggle/working/output/10006_1459541791.png')\n# X_train = np.array([ cv2.imread(f'/kaggle/working/output/{df[\"patient_id\"][i]}_{df[\"image_id\"][i]}.png') for i in df.index ])\n# y_train = np.array(df['cancer'].to_list())\nXy_balanced = [ (cv2.imread(f'/kaggle/working/output/{df[\"patient_id\"][i]}_{df[\"image_id\"][i]}.png'), df['cancer'][i]) for i in df.index ]\nX_val = np.array([ x[0] for x in Xy_balanced[:2000] ])\ny_val = np.array([ x[1] for x in Xy_balanced[:2000] ])\nXy_balanced = Xy_balanced[2000:]\n\nnCancer = len([ i for i in Xy_balanced if i[1] == 1])\nprint(len(Xy_balanced))\nwhile nCancer/len(Xy_balanced) < 0.4:\n    nL = Xy_balanced.copy()\n    for x in Xy_balanced:\n        if x[1] == 1:\n            nL.append(x)\n    Xy_balanced = nL.copy()\n    nCancer = len([ i for i in Xy_balanced if i[1] == 1])\n\nprint(len(Xy_balanced))\n# plt.imshow(test)\n# plt.imshow(test2)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:33:55.475648Z","iopub.execute_input":"2023-01-09T19:33:55.476039Z","iopub.status.idle":"2023-01-09T19:34:48.013931Z","shell.execute_reply.started":"2023-01-09T19:33:55.476008Z","shell.execute_reply":"2023-01-09T19:34:48.012556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport gc\ndel pd\ndel nL\ngc.collect()\n\nmodel = keras.models.Sequential([\n    keras.layers.Input([256, 256, 3]),\n    keras.layers.Conv2D(50, 3, activation=\"relu\"),\n    keras.layers.MaxPooling2D(pool_size=2),\n    keras.layers.BatchNormalization(),\n    keras.layers.Conv2D(40, 3, activation=\"relu\"),\n    keras.layers.BatchNormalization(),\n    keras.layers.Flatten(),\n    keras.layers.Dense(2, activation=\"softmax\")\n])\n\n\"\"\"\nmodel = keras.models.Sequential([\n    keras.layers.Flatten(input_shape=[256, 256, 3]),\n    keras.layers.Dense(900, activation=\"relu\"),\n    keras.layers.Dense(600, activation=\"relu\"),\n    keras.layers.Dense(400, activation=\"relu\"),\n    keras.layers.Dense(100, activation=\"relu\"),\n    keras.layers.Dense(2, activation=\"softmax\")\n])\n\"\"\"\n\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n              optimizer=keras.optimizers.Adam(),\n              metrics=[\"accuracy\"])\n\nrandom.shuffle(Xy_balanced)\nprint('Test')\nX_train = [ x[0] for x in Xy_balanced[:40000] ]\ny_train = [ x[1] for x in Xy_balanced[:40000] ]\nXy_balanced = Xy_balanced[40000:]\n\ngc.collect()\n\nX_train = np.array(X_train)\ny_train = np.array(y_train)\n\nhistory = model.fit(X_train, y_train, epochs=10, batch_size=16,\n                    validation_data=(X_val, y_val))\n","metadata":{"execution":{"iopub.status.busy":"2023-01-09T19:35:03.90235Z","iopub.execute_input":"2023-01-09T19:35:03.902757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('ModelFinished1')","metadata":{"execution":{"iopub.status.busy":"2023-01-09T14:25:59.375056Z","iopub.execute_input":"2023-01-09T14:25:59.375464Z","iopub.status.idle":"2023-01-09T14:26:12.582328Z","shell.execute_reply.started":"2023-01-09T14:25:59.37543Z","shell.execute_reply":"2023-01-09T14:26:12.580583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = [ x[0] for x in Xy_balanced[:40000] ]\ny_train = [ x[1] for x in Xy_balanced[:40000] ]\n\ngc.collect()\n\nX_train = np.array(X_train)\ny_train = np.array(y_train) \n\nhistory2 = model.fit(X_train, y_train, epochs=10, batch_size=16,\n                    validation_data=(X_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2023-01-09T13:20:35.13804Z","iopub.execute_input":"2023-01-09T13:20:35.138589Z","iopub.status.idle":"2023-01-09T14:25:52.198433Z","shell.execute_reply.started":"2023-01-09T13:20:35.138545Z","shell.execute_reply":"2023-01-09T14:25:52.196637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save('ModelFinished2')","metadata":{},"execution_count":null,"outputs":[]}]}