{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Detección de embolismos pulmonares mediante técnicas de aprendizaje automático\n\n### Trabajo de Fin de Grado\n\n### Curso académico 2022-2023\n\n### Autores:\n- Pablo Palacios López\n- José Antonio Gámez Martín\n- Juan Carlos Alfaro Jiménez","metadata":{}},{"cell_type":"markdown","source":"# 1. Importamos librerias","metadata":{}},{"cell_type":"code","source":"!pip install plot-keras-history\n!pip install split-folders\n!pip install python-gdcm\n!pip install pylibjpeg pylibjpeg-libjpeg pydicom\n!pip install -U imbalanced-learn\n!pip install pydotplus","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:44:19.556192Z","iopub.execute_input":"2023-06-19T15:44:19.556698Z","iopub.status.idle":"2023-06-19T15:45:42.287407Z","shell.execute_reply.started":"2023-06-19T15:44:19.556656Z","shell.execute_reply":"2023-06-19T15:45:42.286221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data reading and manipulation libraries\nimport numpy as np, pandas as pd\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom PIL import Image, ImageEnhance\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow import keras\nimport warnings\n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import applications\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.layers import Dense, Conv2D , MaxPool2D , Flatten , Dropout , BatchNormalization\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\n\nfrom scipy.ndimage.interpolation import zoom\nfrom torch.utils.data import Dataset, DataLoader\n\nimport skimage.io as io\n\nimport os, os.path as osp\nfrom sklearn.model_selection import train_test_split\nimport glob\nimport cv2\nimport random\n\nimport shutil\n\nimport gc\nimport pydicom\nimport pydicom as dcm\n\nimport pathlib\nimport plotly.graph_objects as go\nimport itertools\nimport plotly.express as px\n\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objs as go\nfrom matplotlib import animation, rc\nimport imageio\nfrom IPython import display\nfrom PIL import Image\nimport scipy.ndimage\n\n\nimport splitfolders\n\nfrom collections import defaultdict\n\nfrom plot_keras_history import plot_history","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:45:42.290085Z","iopub.execute_input":"2023-06-19T15:45:42.290758Z","iopub.status.idle":"2023-06-19T15:45:55.658886Z","shell.execute_reply.started":"2023-06-19T15:45:42.290719Z","shell.execute_reply":"2023-06-19T15:45:55.657915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Importamos la base de datos","metadata":{}},{"cell_type":"markdown","source":"Número de imagenes que tenemos en la base de datos","metadata":{}},{"cell_type":"code","source":"# images = glob.glob('/kaggle/input/rsna-str-pulmonary-embolism-detection/train/*/*/*.dcm')\n# print(\"El número de imágenes es: \", len(images))\n# images = pd.Series(images)\n\nprint(\"El número de imágenes es: 1790594\")","metadata":{"execution":{"iopub.status.busy":"2023-06-19T00:13:52.326754Z","iopub.execute_input":"2023-06-19T00:13:52.327114Z","iopub.status.idle":"2023-06-19T00:13:52.335158Z","shell.execute_reply.started":"2023-06-19T00:13:52.327082Z","shell.execute_reply":"2023-06-19T00:13:52.334257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Para mostrar nuestras imagenes debemos utilizar la libreria PyDicom. Un paquete de python que nos permite leer archivos DICOM (Digital Imaging and Communications in Medicine) como por ejemplo en este caso, ya que son imágenes médicas.","metadata":{}},{"cell_type":"code","source":"ds = pydicom.dcmread(\"/kaggle/input/rsna-str-pulmonary-embolism-detection/train/0003b3d648eb/d2b2960c2bbf/11401f58474b.dcm\")\ndcm_sample=ds.pixel_array.astype('float32')\nscaled_image = (np.maximum(dcm_sample, 0) / dcm_sample.max())\nplt.imshow(scaled_image)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:07.970036Z","iopub.execute_input":"2023-06-18T22:01:07.971034Z","iopub.status.idle":"2023-06-18T22:01:08.387437Z","shell.execute_reply.started":"2023-06-18T22:01:07.97099Z","shell.execute_reply":"2023-06-18T22:01:08.386523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. DICOM a JPG","metadata":{}},{"cell_type":"markdown","source":"En este caso, no hemos tenido que realizar la conversión de las imagenes a mano, debido a que hemos encontrado una base de datos que ya nos las proporcionaba en el formato deseado. Este es el caso de la base de datos \"train-jpgs\". Donde tendremos todas las imágenes en un formato JPG 256x256","metadata":{}},{"cell_type":"code","source":"img = io.imread('/kaggle/input/train-jpgs/train-jpegs_partion/0003b3d648eb/d2b2960c2bbf/03d7693b0405.jpg')\nio.imshow(img)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T14:50:05.002476Z","iopub.execute_input":"2023-06-18T14:50:05.003366Z","iopub.status.idle":"2023-06-18T14:50:05.41408Z","shell.execute_reply.started":"2023-06-18T14:50:05.00332Z","shell.execute_reply":"2023-06-18T14:50:05.413192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Analisis exploratorio de datos ","metadata":{}},{"cell_type":"code","source":"PATH = \"../input/rsna-str-pulmonary-embolism-detection/\"\n\ntrain_df = pd.read_csv(PATH + \"train.csv\")\ntest_df = pd.read_csv(PATH + \"test.csv\")\nsub = pd.read_csv(PATH + \"sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-06-18T14:50:07.58413Z","iopub.execute_input":"2023-06-18T14:50:07.584606Z","iopub.status.idle":"2023-06-18T14:50:12.37194Z","shell.execute_reply.started":"2023-06-18T14:50:07.584526Z","shell.execute_reply":"2023-06-18T14:50:12.37097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Vemos que tenemos 16 campos distintos, los campos StudyInstanceUID, SeriesInstanceUID y SOPInstanceUID son strings, mientras que el resto de campos son int64, donde la mayoría tienen valores booleanos 0 o 1, obersevemos una pequeña muestra de nuestros datos. ","metadata":{}},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-17T15:15:17.091171Z","iopub.execute_input":"2023-06-17T15:15:17.091519Z","iopub.status.idle":"2023-06-17T15:15:17.115804Z","shell.execute_reply.started":"2023-06-17T15:15:17.091488Z","shell.execute_reply":"2023-06-17T15:15:17.11487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-06-17T15:15:17.11714Z","iopub.execute_input":"2023-06-17T15:15:17.117564Z","iopub.status.idle":"2023-06-17T15:15:17.140435Z","shell.execute_reply.started":"2023-06-17T15:15:17.117531Z","shell.execute_reply":"2023-06-17T15:15:17.139588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Aquí tenemos una pequeña descripción de cada paramétro.","metadata":{}},{"cell_type":"markdown","source":"| Nombre | Tipo | Descripcion |\n| :--- | :--- | :---- |\n| **StudyInstanceUID** | UID | unique ID for each study (exam) in the data. |\n| **SeriesInstanceUID** | UID | unique ID for each series within the study. |\n| **SOPInstanceUID** | UID | unique ID for each image within the study (and data) |\n| **pe_present_on_image**| image-level | notes whether any form of PE is present on the image.|\n| **negative_exam_for_pe**| exam-level | whether there are any images in the study that have PE present.|\n| **qa_motion** | informational | indicates whether radiologists noted an issue with motion in the study. | \n|**qa_contrast** | informational | indicates whether radiologists noted an issue with contrast in the study.|\n| **flow_artifact** | informational | ---|\n| **rv_lv_ratio_gte_1** | exam-level| indicates whether the RV/LV ratio present in the study is >= 1|\n| **rv_lv_ratio_lt_1** | exam-level| indicates whether the RV/LV ratio present in the study is < 1|\n| **leftsided_pe** | exam-level | indicates that there is PE present on the left side of the images in the study| \n| **chronic_pe** | exam-level | indicates that the PE in the study is chronic|\n| **true_filling_defect_not_pe** | informational | indicates a defect that is NOT PE|\n| **rightsided_pe** | exam-level | indicates that there is PE present on the right side of the images in the study|\n| **acute_and_chronic_pe** | exam-level| indicates that the PE present in the study is both acute AND chronic|\n| **central_pe** | exam-level| indicates that there is PE present in the center of the images in the study|\n| **indeterminate** | exam-level| indicates that while the study is not negative for PE, an ultimate set of exam-level labels could not be created, due to QA issues|","metadata":{}},{"cell_type":"markdown","source":"En este gráfico podemos observar la relación entre los parámetros para evaluar la detección de embolismos pulmonares.\n\nEn el primer caso tenemos que detectar si la imagen que estamos observando es o no negativo con el supuesto PE. En caso de que SI sea negativo (es decir, que no exista ningún tipo de PE) habremos terminado y con esa imagen no hay nada que hacer, en cambio, si el examen NO es negativo (es decir, hay PE aparente en la imagen) pasaremos a Indeterminado.\n\nEn segundo lugar, tenemos que ver si la imagen que estamos observando es o no Indeterminado. Será indeterminado cuando la imagen no se pueda apreciar bien, debido al contraste o el movimiento de la imagen, en cambio, si no es Indeterminado significa que presenta un PE el paciente, ahora tenemos que ver de que se trata el embolismo. \n\nUna vez tenemos ya que nos encontramos ante un PE, tenemos que evaluar los distintos parámetros que nos permitirán detectar la gravedad del embolismo (seleccionar solamente un parametro), la localización del mismo (tenemos que seleccionar al menos un parametro) y el radio del ventrículo derecho hasta el ventrículo izquierdo (seleccionar solamente un parametro).","metadata":{}},{"cell_type":"markdown","source":"![](https://www.googleapis.com/download/storage/v1/b/kaggle-user-content/o/inbox%2F115173%2Fa2a5ee66b5799274141dd547cc3ea466%2FPE%20figure.jpg?generation=1599575183749576&alt=media)\n","metadata":{}},{"cell_type":"markdown","source":"Vamos a observar de que está compuesto nuestro train.csv\n\nhttps://plotly.com/python/subplots/#multiple-subplots","metadata":{}},{"cell_type":"code","source":"columns = [\n 'pe_present_on_image', 'negative_exam_for_pe', 'qa_motion', \n 'qa_contrast', 'flow_artifact', 'rv_lv_ratio_gte_1', \n 'rv_lv_ratio_lt_1', 'leftsided_pe', 'chronic_pe', \n 'true_filling_defect_not_pe', 'rightsided_pe', \n 'acute_and_chronic_pe', 'central_pe', 'indeterminate'\n]\n\nfig = make_subplots(rows=5, cols=3)\n\ntraces = []\nfor col in columns:\n bar = go.Bar(\n x=[0, 1], \n y=[len(train_df[train_df[col]==0]),len(train_df[train_df[col]==1])], name=col,\n text = [\n # obtenemos cada uno de los 0 que hay en la base de datos\n # con sus respectivas columnas\n # por ejemplo, esto nos dará con pe_present_on_image un 0.94\n # y redondeamos a 2 decimas\n str(round(100 * len(train_df[train_df[col]==0]) / len(train_df), 1)) + '%',\n str(round(100 * len(train_df[train_df[col]==1]) / len(train_df), 1)) + '%']\n )\n traces.append(bar)\n\n\n \nfor i in range(len(traces)):\n fig.append_trace(traces[i], (i // 3) + 1, (i % 3) +1)\n #esto es para las row y cols, la posición de cada uno de los gráficos\n #para el de la primera posicion 0, será 0//3 = 0 + 1 = 1\n # 0%3 = 0 + 1 = 1, es decir, que se encontrará en la row=1 col=1\n\nfig.update_layout(\n title_text='Train CSV columnas',\n height=1040,\n width=1000\n)\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-17T15:15:17.142095Z","iopub.execute_input":"2023-06-17T15:15:17.142436Z","iopub.status.idle":"2023-06-17T15:15:22.404715Z","shell.execute_reply.started":"2023-06-17T15:15:17.142404Z","shell.execute_reply":"2023-06-17T15:15:22.403854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Aquí vemos una disposición de las columnas en nuestra base de datos, como vemos, abundan el número de ceros en las columnas indicando en muchos casos que la imagen no presenta ese tipo de parámetro, por ejemplo, en el atributo *pe_present_on_image*, vemos que tenemos un 94.6% de las imágenes que no presentan PE (1,694,054 imágenes que no presentan PE), mientras que el 5.4% restante si presenta embolismos pulmonares (96,540 imágenes que presentan PE).","metadata":{}},{"cell_type":"markdown","source":"**Visualización de las imágenes**","metadata":{}},{"cell_type":"code","source":"TRAIN_PATH = '../input/rsna-str-pulmonary-embolism-detection/train/'","metadata":{"execution":{"iopub.status.busy":"2023-06-14T22:26:36.110649Z","iopub.execute_input":"2023-06-14T22:26:36.111066Z","iopub.status.idle":"2023-06-14T22:26:36.115577Z","shell.execute_reply.started":"2023-06-14T22:26:36.111035Z","shell.execute_reply":"2023-06-14T22:26:36.114395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ahora vamos a seleccionar un examen específico, es decir, las imágenes de un paciente para realizar una visión en formato de video de nuestro paciente. Pero el hecho de solamente printear por pantalla imágenes random de un paciente no nos dice anda, es por ello que tenemos que seguir una secuencia específica","metadata":{}},{"cell_type":"markdown","source":"**Si queremos cambiar el GIF, tenemos que cambiar el número de *selected_exam***","metadata":{}},{"cell_type":"code","source":"selected_exam = 12\nos.listdir(TRAIN_PATH)[selected_exam]","metadata":{"execution":{"iopub.status.busy":"2023-06-14T22:26:36.117203Z","iopub.execute_input":"2023-06-14T22:26:36.117893Z","iopub.status.idle":"2023-06-14T22:26:36.651649Z","shell.execute_reply.started":"2023-06-14T22:26:36.117853Z","shell.execute_reply":"2023-06-14T22:26:36.650508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EXAM_IDs = os.listdir(TRAIN_PATH) #Primer directorio ej: 008c672a586b\n\nSERIES = os.listdir(TRAIN_PATH + '/' + EXAM_IDs[selected_exam]) #Segundo directorio ej:4b35af8968e6\n# print(SERIES)\n\nfiles = os.listdir(TRAIN_PATH + '/' + EXAM_IDs[selected_exam] + '/' + SERIES[0])\n# print(files) \n\nsingle_experiment_files = [TRAIN_PATH + '/' + EXAM_IDs[selected_exam] + '/' + SERIES[0] + '/' + file for file in files]\n\n# print(single_experiment_files)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T22:26:36.653145Z","iopub.execute_input":"2023-06-14T22:26:36.653872Z","iopub.status.idle":"2023-06-14T22:26:36.806432Z","shell.execute_reply.started":"2023-06-14T22:26:36.653823Z","shell.execute_reply":"2023-06-14T22:26:36.805359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Hounsfield Unit (HU) es una medida de radiodensidad ","metadata":{}},{"cell_type":"code","source":"def load_slice(paths):\n slices = [pydicom.read_file(path) for path in paths]\n slices.sort(key = lambda x: int(x.InstanceNumber), reverse = False) #ordenamos las slices \n #No comprendo el InstanceNumber\n return slices\n # Convert to int16 (from sometimes int16), \n # should be possible as values should always be low enough (<32k)\n\n\n\ndef transform_to_hu(slices): \n images = np.stack([file.pixel_array for file in slices])\n images = images.astype(np.int16)\n \n # Set outside-of-scan pixels to 1\n # The intercept is usually -1024, so air is approximately 0\n images[images <= -1000] = 0\n \n # Convert to Hounsfield units (HU)\n for n in range(len(slices)): \n intercept = slices[n].RescaleIntercept\n slope = slices[n].RescaleSlope\n if slope != 1:\n images[n] = slope * images[n].astype(np.float64)\n images[n] = images[n].astype(np.int16) \n images[n] += np.int16(intercept)\n return np.array(images, dtype=np.int16)","metadata":{"execution":{"iopub.status.busy":"2023-06-14T22:26:36.808271Z","iopub.execute_input":"2023-06-14T22:26:36.808704Z","iopub.status.idle":"2023-06-14T22:26:36.820566Z","shell.execute_reply.started":"2023-06-14T22:26:36.808664Z","shell.execute_reply":"2023-06-14T22:26:36.819405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stacked_dicoms = load_slice(single_experiment_files)\nstacked_patient_pixels = transform_to_hu(stacked_dicoms)\n\n\ndef sample_stack(stack, rows=6, cols=6, start_with=10, show_every=3):\n fig,ax = plt.subplots(rows,cols,figsize=[20,22])\n for i in range(rows*cols):\n ind = start_with + i*show_every\n ax[int(i/rows),int(i % rows)].set_title(f'slice {ind}')\n ax[int(i/rows),int(i % rows)].imshow(stack[ind],cmap='gray')\n ax[int(i/rows),int(i % rows)].axis('off')\n plt.show()\n\nprint(f'Total Number of Slices: {len(stacked_patient_pixels)}')\nsample_stack(stacked_patient_pixels, \n show_every = int((len(stacked_patient_pixels)-10)/36))","metadata":{"execution":{"iopub.status.busy":"2023-06-14T22:26:36.822491Z","iopub.execute_input":"2023-06-14T22:26:36.823388Z","iopub.status.idle":"2023-06-14T22:26:44.888766Z","shell.execute_reply.started":"2023-06-14T22:26:36.823207Z","shell.execute_reply":"2023-06-14T22:26:44.88464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"GIF del examen seleccionado","metadata":{}},{"cell_type":"code","source":"imageio.mimsave(f'stacked_{EXAM_IDs[selected_exam]}.gif', stacked_patient_pixels, duration=0.1)\ndisplay.Image(f'stacked_{EXAM_IDs[selected_exam]}.gif', format='png')","metadata":{"execution":{"iopub.status.busy":"2023-06-14T22:26:44.890279Z","iopub.execute_input":"2023-06-14T22:26:44.891282Z","iopub.status.idle":"2023-06-14T22:26:46.510938Z","shell.execute_reply.started":"2023-06-14T22:26:44.891245Z","shell.execute_reply":"2023-06-14T22:26:46.509534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pero en este trabajo, nosotros nos vamos a encargar de realizar un modelo a través de las imágenes JPG que obtuvimos de nuestra nueva Base de Datos.","metadata":{}},{"cell_type":"markdown","source":"Comprobaremos si faltan algunos valores para llevar a cabo limpieza de los datos","metadata":{}},{"cell_type":"code","source":"print('Valores nulos en el train:',train_df.isnull().sum().sum())\nprint('Valores nulos en el test:',test_df.isnull().sum().sum())","metadata":{"execution":{"iopub.status.busy":"2023-06-14T22:26:46.513106Z","iopub.execute_input":"2023-06-14T22:26:46.513968Z","iopub.status.idle":"2023-06-14T22:26:48.404728Z","shell.execute_reply.started":"2023-06-14T22:26:46.513919Z","shell.execute_reply":"2023-06-14T22:26:48.402517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Como vemos no existe ningún tipo de valor nulo, por tanto no tendremos que aplciar ninguna limpieza de datos en este caso.","metadata":{}},{"cell_type":"markdown","source":"Pero tenemos un problema en nuestro gráfico que hemos realizado en el análisis exploratiorio tenemos que para la variable pe_present_on_image tiene un 94.6% de casos que no tienen embolia pulmonar y un 5,6% de datos que si tienen, entonces podemos concluir que los datos están desbalanceados. Para ello llevaremos a cabo un estudio de dichos datos.","metadata":{}},{"cell_type":"markdown","source":"# 6. Clasificación binaria de imágenes y partición 60% + 20% + 20%","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import classification_report\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.decomposition import PCA\nfrom sklearn.tree import DecisionTreeClassifier\n \nfrom pylab import rcParams\n\nfrom imblearn.under_sampling import NearMiss\nfrom imblearn.over_sampling import RandomOverSampler\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom imblearn.combine import SMOTETomek\nfrom imblearn.ensemble import BalancedBaggingClassifier\n \nfrom collections import Counter","metadata":{"execution":{"iopub.status.busy":"2023-06-15T10:27:49.935222Z","iopub.execute_input":"2023-06-15T10:27:49.935567Z","iopub.status.idle":"2023-06-15T10:27:50.714905Z","shell.execute_reply.started":"2023-06-15T10:27:49.935534Z","shell.execute_reply":"2023-06-15T10:27:50.713978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Como hemos comprobado en el apartado anterior, vemos que nuestros datos se encuentran muy desbalanceados en la Base de datos y por tanto no podemos clasificar correctamente nuestras imágenes.\n\nLa base de datos que tenemos llamada *rsna-str-detection-partition-train-test-val-in-jpg* no hemos realizado una división justa del Train, test y validation, ya que no hemos tenido en cuenta este desbalanceo, es por eso por lo que no podremos entrenar nuestro modelo con esta BBDD.\n\nTambién tenemos el problema del tiempo de computo, debido que a la hora de realizar el modelo, tarda mucho (3 horas por cada Epoch). \n\nTendremos que encontrar una solución para estos problemas planteados.\n\nEn primer lugar para solucionar nuestro problema con la divisón de nuestra BBDD, lo que haremos será crear distintos csv y directorios, como por ejemplo: Train Normal (No-Embolia) y Train Not Normal (Embolia), en ambos introduciremos aquellas imágenes que se encuentran en nuestro Train.CSV que coincidan con las especificaciones de ambas carpetas, es decir, para el directorio Train Normal, observaremos en nuestro Train.CSV cuales son aquellas imágenes que NO contienen embolia y las introduciremos.\n\nEste paso lo realizaremos exactamente igual para el Test y Validation.\n","metadata":{}},{"cell_type":"markdown","source":"##### El siguiente código está realizado en Visual Studio Code debido a los tiempos de compilación y el así poder introducir un nuevo dataset en la libreta.","metadata":{}},{"cell_type":"markdown","source":"##### Primero cogeremos nuestra base de datos Train.csv, almacenada en *rsna-str-pulmonary-embolism-detection*\n\nRealizaremos la partición correspondiente que hemos pedido 60% train, 20% test, 20% validation","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/rsna-str-pulmonary-embolism-detection/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:23.030918Z","iopub.execute_input":"2023-06-18T22:01:23.031274Z","iopub.status.idle":"2023-06-18T22:01:26.921872Z","shell.execute_reply.started":"2023-06-18T22:01:23.031245Z","shell.execute_reply":"2023-06-18T22:01:26.920848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Realizamos la división","metadata":{}},{"cell_type":"code","source":"# train, test = train_test_split(df, test_size=0.2)\n\n# train, valid = train_test_split(train, test_size=0.2)\n\n# train.to_csv('train.csv')\n# valid.to_csv('valid.csv')\n# test.to_csv('test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-14T18:43:53.103951Z","iopub.execute_input":"2023-06-14T18:43:53.104301Z","iopub.status.idle":"2023-06-14T18:43:53.109321Z","shell.execute_reply.started":"2023-06-14T18:43:53.10427Z","shell.execute_reply":"2023-06-14T18:43:53.108317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Y almacenamos las divisiones que hemos hecho, en nuestro caso, esas divisiones son las que se encuentran en el directorio *data-binary/data*, con sus respectivos porcentajes","metadata":{}},{"cell_type":"markdown","source":"#### Distribución de nuestro Train.csv","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/data-binary/data/trainDesbalanceado.csv\")\ntrain = train.drop(['Unnamed: 0'], axis=1)\ntrain","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:26.923733Z","iopub.execute_input":"2023-06-18T22:01:26.924084Z","iopub.status.idle":"2023-06-18T22:01:30.072979Z","shell.execute_reply.started":"2023-06-18T22:01:26.924054Z","shell.execute_reply":"2023-06-18T22:01:30.071873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A continuación dividimos nuestra base de datos entre Train1 == Train Not Normal (con embolia) y Train0 == Train Normal (sin embolia)","metadata":{}},{"cell_type":"code","source":"train1 = train.loc[(train[\"pe_present_on_image\"] == 1)].reset_index(drop=True)\nprint(\"Tenemos\", len(train1), \"imágenes\")\ntrain1.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:30.074826Z","iopub.execute_input":"2023-06-18T22:01:30.075257Z","iopub.status.idle":"2023-06-18T22:01:30.128994Z","shell.execute_reply.started":"2023-06-18T22:01:30.075224Z","shell.execute_reply":"2023-06-18T22:01:30.127874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Realizando el estudio, nos hemos percatado de que existen tuplas en las que pe_present_on_image==0 y negative_exam_for_pe == 0 e indetermiante == 0 por tanto resulta un poco contradictorio, con respecto al gráfico del análisis exploratorio, determinar si tiene o no embolia en esos casos, ya que te está diciendo que no es negativo, no es indeterminado, y tampoco tiene embolia, por tanto hemos optado por descartar esos casos para el Train, quedandonos solamente con aquellas imágenes que son negativas directamente y entonces sabemos con certeza que no contienen embolia.","metadata":{}},{"cell_type":"code","source":"train0 = train.loc[(train[\"pe_present_on_image\"] == 0) & (train[\"negative_exam_for_pe\"] == 1)].reset_index(drop=True)\nprint(\"Tenemos\", len(train0), \"imágenes\")\ntrain0.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:30.130989Z","iopub.execute_input":"2023-06-18T22:01:30.131848Z","iopub.status.idle":"2023-06-18T22:01:30.294611Z","shell.execute_reply.started":"2023-06-18T22:01:30.131785Z","shell.execute_reply":"2023-06-18T22:01:30.293457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Unimos ambos datasets para hacer un estudio de los datos","metadata":{}},{"cell_type":"code","source":"trainNuevoDesbalanceado_sinPEeq0_sinNEeq0 = pd.read_csv(\"/kaggle/input/data-binary/data/trainNuevoDesbalanceado_sinPEeq0_sinNEeq0.csv\")\ntrainNuevoDesbalanceado_sinPEeq0_sinNEeq0 = trainNuevoDesbalanceado_sinPEeq0_sinNEeq0.drop(['Unnamed: 0'], axis=1)\ntrainNuevoDesbalanceado_sinPEeq0_sinNEeq0","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:32.82883Z","iopub.execute_input":"2023-06-18T22:01:32.829189Z","iopub.status.idle":"2023-06-18T22:01:35.002747Z","shell.execute_reply.started":"2023-06-18T22:01:32.829153Z","shell.execute_reply":"2023-06-18T22:01:35.001661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=trainNuevoDesbalanceado_sinPEeq0_sinNEeq0['pe_present_on_image']\ny.value_counts()\ny.value_counts().plot.pie(autopct='%.2f')","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:35.004988Z","iopub.execute_input":"2023-06-18T22:01:35.006057Z","iopub.status.idle":"2023-06-18T22:01:35.181976Z","shell.execute_reply.started":"2023-06-18T22:01:35.006023Z","shell.execute_reply":"2023-06-18T22:01:35.180722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Vemos el gran desbalanceo de los datos que existen, por tanto, llevaremos a cabo un RandomUnderSampling para tratarlo","metadata":{}},{"cell_type":"code","source":"X=trainNuevoDesbalanceado_sinPEeq0_sinNEeq0.drop(\"pe_present_on_image\",axis=1)\nX","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:37.334481Z","iopub.execute_input":"2023-06-18T22:01:37.334851Z","iopub.status.idle":"2023-06-18T22:01:37.403447Z","shell.execute_reply.started":"2023-06-18T22:01:37.334815Z","shell.execute_reply":"2023-06-18T22:01:37.402539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"El código que tenemos debajo ha sido realizado en visual studio. En él llevamos a cabo la función de balanceo de datos RandomUnderSampler, el cual nos proporciona un dataset totalmente balanceado, en este caso, eliminamos instancias de la base de datos que tengan pe_present_on_image==1, hasta igualar su número con aquellas instancias que tengan pe_present_on_image==0, dando lugar a lo que ","metadata":{}},{"cell_type":"code","source":"# from imblearn.under_sampling import RandomUnderSampler\n\n# rus = RandomUnderSampler(sampling_strategy=1) # Numerical value\n\n# X_res, y_res = rus.fit_resample(X, y)\n\n# ax = y_res.value_counts().plot.pie(autopct='%.2f')\n\n# _ = ax.set_title(\"Under-sampling\")","metadata":{"execution":{"iopub.status.busy":"2023-06-17T15:15:34.294204Z","iopub.execute_input":"2023-06-17T15:15:34.29459Z","iopub.status.idle":"2023-06-17T15:15:34.299217Z","shell.execute_reply.started":"2023-06-17T15:15:34.29456Z","shell.execute_reply":"2023-06-17T15:15:34.29813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Esta será nuestra base de datos totalmente balanceada","metadata":{}},{"cell_type":"code","source":"trainBalanceado = pd.read_csv(\"/kaggle/input/data-binary/data/trainBalanceado.csv\")\ntrainBalanceado = trainBalanceado.drop(['Unnamed: 0'], axis=1)\ntrainBalanceado","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:41.200719Z","iopub.execute_input":"2023-06-18T22:01:41.201463Z","iopub.status.idle":"2023-06-18T22:01:41.563603Z","shell.execute_reply.started":"2023-06-18T22:01:41.20143Z","shell.execute_reply":"2023-06-18T22:01:41.562711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ahora tenemos nuestra base de datos Train totalmente balanceada, con el mismo número de imágenes pertenecientes a Not Normal y Normal","metadata":{}},{"cell_type":"code","source":"y=trainBalanceado['pe_present_on_image']\ny.value_counts()\ny.value_counts().plot.pie(autopct='%.2f')","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:43.707594Z","iopub.execute_input":"2023-06-18T22:01:43.707972Z","iopub.status.idle":"2023-06-18T22:01:43.831504Z","shell.execute_reply.started":"2023-06-18T22:01:43.707942Z","shell.execute_reply":"2023-06-18T22:01:43.830271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_res = y\ny_res","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:45.92511Z","iopub.execute_input":"2023-06-18T22:01:45.926002Z","iopub.status.idle":"2023-06-18T22:01:45.934841Z","shell.execute_reply.started":"2023-06-18T22:01:45.925959Z","shell.execute_reply":"2023-06-18T22:01:45.933131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_res=trainBalanceado.drop(\"pe_present_on_image\",axis=1)\nX_res","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:48.038616Z","iopub.execute_input":"2023-06-18T22:01:48.039174Z","iopub.status.idle":"2023-06-18T22:01:48.076196Z","shell.execute_reply.started":"2023-06-18T22:01:48.03913Z","shell.execute_reply":"2023-06-18T22:01:48.075235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Vamos a comprobar de que están compuestos nuestra BBDD totalmente balanceada","metadata":{}},{"cell_type":"code","source":"#NOT NORMAL\n\ntrain1 = trainBalanceado.loc[trainBalanceado[\"pe_present_on_image\"] == 1].reset_index(drop=True)\nprint(\"Tenemos\", len(train1), \"imágenes\")\ntrain1.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:50.450614Z","iopub.execute_input":"2023-06-18T22:01:50.450976Z","iopub.status.idle":"2023-06-18T22:01:50.479307Z","shell.execute_reply.started":"2023-06-18T22:01:50.450947Z","shell.execute_reply":"2023-06-18T22:01:50.478341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#NORMAL\n\ntrain0 = trainBalanceado.loc[(trainBalanceado[\"negative_exam_for_pe\"] == 1)].reset_index(drop=True)\nprint(\"Tenemos\", len(train0), \"imágenes\")\ntrain0.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:52.455585Z","iopub.execute_input":"2023-06-18T22:01:52.456301Z","iopub.status.idle":"2023-06-18T22:01:52.484552Z","shell.execute_reply.started":"2023-06-18T22:01:52.456268Z","shell.execute_reply":"2023-06-18T22:01:52.483402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ahora el siguiente paso será introducir en nuestras respectivas carpetas las imágenes.","metadata":{}},{"cell_type":"code","source":"# Train_not_normal\n\n# df = train1\n\n# for i in tqdm(range(len(train1))):\n# imJPG = io.imread(\"D:/Descargas/train-jpegs_partion/\" + df.loc[i,'StudyInstanceUID']+'/'+ df.loc[i,'SeriesInstanceUID']+'/'+ df.loc[i,'SOPInstanceUID']+'.jpg')\n# im = Image.fromarray(imJPG)\n# im.save(\"D:/Descargas/data/train/not_normal/\"+str(i)+\".jpg\")\n# del imJPG, im\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-14T22:26:58.632121Z","iopub.execute_input":"2023-06-14T22:26:58.633203Z","iopub.status.idle":"2023-06-14T22:26:58.638626Z","shell.execute_reply.started":"2023-06-14T22:26:58.633161Z","shell.execute_reply":"2023-06-14T22:26:58.637375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train_not_normal 20000 imagenes\n\n# df = train1\n\n# for i in tqdm(range(20000)):\n# imJPG = io.imread(\"D:/Descargas/train-jpegs_partion/\" + df.loc[i,'StudyInstanceUID']+'/'+ df.loc[i,'SeriesInstanceUID']+'/'+ df.loc[i,'SOPInstanceUID']+'.jpg')\n# im = Image.fromarray(imJPG)\n# im.save(\"D:/Descargas/data/train/20000_not_normal/\"+str(i)+\".jpg\")\n# del imJPG, im\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-14T18:43:59.978145Z","iopub.execute_input":"2023-06-14T18:43:59.978945Z","iopub.status.idle":"2023-06-14T18:43:59.991358Z","shell.execute_reply.started":"2023-06-14T18:43:59.978913Z","shell.execute_reply":"2023-06-14T18:43:59.990576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train normal\n\n# df = train0\n\n# for i in tqdm(range(len(train0))):\n# imJPG = io.imread(\"D:/Descargas/train-jpegs_partion/\" + df.loc[i,'StudyInstanceUID']+'/'+ df.loc[i,'SeriesInstanceUID']+'/'+ df.loc[i,'SOPInstanceUID']+'.jpg')\n# im = Image.fromarray(imJPG)\n# im.save(\"D:/Descargas/data/train/normal/\"+str(i)+\".jpg\")\n# del imJPG, im\n# gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train_normal 20000 imagenes\n\n# df = train0\n\n# for i in tqdm(range(20000)):\n# imJPG = io.imread(\"D:/Descargas/train-jpegs_partion/\" + df.loc[i,'StudyInstanceUID']+'/'+ df.loc[i,'SeriesInstanceUID']+'/'+ df.loc[i,'SOPInstanceUID']+'.jpg')\n# im = Image.fromarray(imJPG)\n# im.save(\"D:/Descargas/data/train/20000_normal/\"+str(i)+\".jpg\")\n# del imJPG, im\n# gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Este código ha sido realizado en Visual Studio Code, en mi ordenador local, debido a los tiempos de comptuación, pero estos directorios se pueden encontrar en ***data-binary/data/train/normal*** y ***data-binary/data/train/not_normal***, ahí están las imágenes y su balanceo. También hemos hecho 2 directorios más ***data-binary/data/train/20000_normal*** y ***data-binary/data/train/20000_not_normal***, cogemos una pequeña meustra para realizar pruebas sobre los modelos hechos, así no nos tardará tanto.","metadata":{}},{"cell_type":"markdown","source":"--------------------------------------------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"--------------------------------------------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"A continuación realizaremos el mismo proceso para el test y el validation, la cosa es que ahora no podemos realizar ningún tipo de balanceo, un error que cometimos anteriormente y estamos solucionando ahora mismo, por tanto el único paso que realizaremos será dividir las imágenes si pe_present_on_image==1 a la carpeta Not Normal y si pe_present_on_image==0 irán a la carpeta Normal","metadata":{}},{"cell_type":"markdown","source":"### Test","metadata":{}},{"cell_type":"markdown","source":"Este es nuestro dataset Test para el problema de clasificación binario","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/data-binary/data/test.csv\")\ntest = test.drop(['Unnamed: 0'], axis=1)\ntest","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:01:57.134952Z","iopub.execute_input":"2023-06-18T22:01:57.135311Z","iopub.status.idle":"2023-06-18T22:01:58.271653Z","shell.execute_reply.started":"2023-06-18T22:01:57.135282Z","shell.execute_reply":"2023-06-18T22:01:58.270716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test1 = test.loc[test[\"pe_present_on_image\"] == 1].reset_index(drop=True)\nprint(\"Tenemos\", len(test1), \"imágenes\")\ntest1.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:02:00.303847Z","iopub.execute_input":"2023-06-18T22:02:00.30421Z","iopub.status.idle":"2023-06-18T22:02:00.33463Z","shell.execute_reply.started":"2023-06-18T22:02:00.304181Z","shell.execute_reply":"2023-06-18T22:02:00.333346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test0 = test.loc[test[\"pe_present_on_image\"] == 0].reset_index(drop=True)\nprint(\"Tenemos\", len(test0), \"imágenes\")\ntest0.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:02:02.082844Z","iopub.execute_input":"2023-06-18T22:02:02.083226Z","iopub.status.idle":"2023-06-18T22:02:02.160215Z","shell.execute_reply.started":"2023-06-18T22:02:02.083196Z","shell.execute_reply":"2023-06-18T22:02:02.159228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Introducimos las imágenes","metadata":{}},{"cell_type":"code","source":"#Not Normal\n\n# df = test1\n\n# for i in tqdm(range(len(test1))):\n# imJPG = io.imread(\"D:/Descargas/train-jpegs_partion/\" + df.loc[i,'StudyInstanceUID']+'/'+df.loc[i,'SeriesInstanceUID']+'/'+df.loc[i,'SOPInstanceUID']+'.jpg')\n# im = Image.fromarray(imJPG)\n# im.save(\"D:/Descargas/data/test/not_normal/\"+str(i)+\".jpg\")\n# del imJPG, im\n# gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Normal\n\n# df = test0\n\n# for i in tqdm(range(len(test0))):\n# imJPG = io.imread(\"D:/Descargas/train-jpegs_partion/\" + df.loc[i,'StudyInstanceUID']+'/'+df.loc[i,'SeriesInstanceUID']+'/'+df.loc[i,'SOPInstanceUID']+'.jpg')\n# im = Image.fromarray(imJPG)\n# im.save(\"D:/Descargas/data/test/normal/\"+str(i)+\".jpg\")\n# del imJPG, im\n# gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Este código ha sido realizado en Visual Studio Code, en mi ordenador local, debido a los tiempos de comptuación, pero estos directorios se pueden encontrar en ***data-binary/data/test/normal*** y ***data-binary/data/test/not_normal***, ahí están las imágenes y su balanceo","metadata":{}},{"cell_type":"markdown","source":"------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"-------------------------------------","metadata":{}},{"cell_type":"code","source":"valid = pd.read_csv(\"/kaggle/input/data-binary/data/valid.csv\")\nvalid = valid.drop(['Unnamed: 0'], axis=1)\nvalid","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:02:14.24875Z","iopub.execute_input":"2023-06-18T22:02:14.249675Z","iopub.status.idle":"2023-06-18T22:02:15.014959Z","shell.execute_reply.started":"2023-06-18T22:02:14.249643Z","shell.execute_reply":"2023-06-18T22:02:15.013767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true=valid['pe_present_on_image']\ny_true","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:02:17.311009Z","iopub.execute_input":"2023-06-18T22:02:17.311367Z","iopub.status.idle":"2023-06-18T22:02:17.321945Z","shell.execute_reply.started":"2023-06-18T22:02:17.311339Z","shell.execute_reply":"2023-06-18T22:02:17.320966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid1 = valid.loc[valid[\"pe_present_on_image\"] == 1].reset_index(drop=True)\nprint(\"Tenemos\", len(valid1), \"imágenes\")\nvalid1.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:02:19.125203Z","iopub.execute_input":"2023-06-18T22:02:19.125565Z","iopub.status.idle":"2023-06-18T22:02:19.167221Z","shell.execute_reply.started":"2023-06-18T22:02:19.125537Z","shell.execute_reply":"2023-06-18T22:02:19.166327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid0 = valid.loc[valid[\"pe_present_on_image\"] == 0].reset_index(drop=True)\nprint(\"Tenemos\", len(valid0), \"imágenes\")\nvalid0.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-06-18T22:02:22.081691Z","iopub.execute_input":"2023-06-18T22:02:22.082693Z","iopub.status.idle":"2023-06-18T22:02:22.146166Z","shell.execute_reply.started":"2023-06-18T22:02:22.082653Z","shell.execute_reply":"2023-06-18T22:02:22.145138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Introducimos las imágenes","metadata":{}},{"cell_type":"code","source":"# Validation not_normal\n\n# df = valid1\n\n# for i in tqdm(range(len(valid1))):\n# imJPG = io.imread(\"D:/Descargas/train-jpegs_partion/\" + df.loc[i,'StudyInstanceUID']+'/'+df.loc[i,'SeriesInstanceUID']+'/'+df.loc[i,'SOPInstanceUID']+'.jpg')\n# im = Image.fromarray(imJPG)\n# im.save(\"D:/Descargas/data/valid/not_normal/\"+str(i)+\".jpg\")\n# del imJPG, im\n# gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validation normal\n\n# df = valid0\n\n# for i in tqdm(range(len(valid0))):\n# imJPG = io.imread(\"D:/Descargas/train-jpegs_partion/\" + df.loc[i,'StudyInstanceUID']+'/'+ df.loc[i,'SeriesInstanceUID']+'/'+ df.loc[i,'SOPInstanceUID']+'.jpg')\n# im = Image.fromarray(imJPG)\n# im.save(\"D:/Descargas/data/valid/normal/\"+str(i)+\".jpg\")\n# del imJPG, im\n# gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Este código ha sido realizado en Visual Studio Code, en mi ordenador local, debido a los tiempos de comptuación, pero estos directorios se pueden encontrar en ***data-binary/data/valid/normal*** y ***data-binary/data/valid/not_normal***, ahí están las imágenes y su balanceo","metadata":{}},{"cell_type":"markdown","source":"El siguiente paso será crear nuestro modelo y ver si clasifica correctamente.","metadata":{}},{"cell_type":"markdown","source":"# 8. Clasificador binario","metadata":{}},{"cell_type":"code","source":"!mkdir /kaggle/models","metadata":{"execution":{"iopub.status.busy":"2023-06-18T14:51:17.096073Z","iopub.execute_input":"2023-06-18T14:51:17.096461Z","iopub.status.idle":"2023-06-18T14:51:18.23862Z","shell.execute_reply.started":"2023-06-18T14:51:17.096431Z","shell.execute_reply":"2023-06-18T14:51:18.237024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"La clase ImageDataGenerator permite al usuario llevar a cabo aumentación de datos mientras entrenamos el modelo.\n\nEsta clase nos proporciona 3 funciones para cargar imágenes, en nuestro caso utilizaremos flow_from_directory, ya que hemos dividido nuestras imágenes en 2 directorios, Not normal y normal.","metadata":{}},{"cell_type":"markdown","source":"Lo que haremos con esta función será generar aumento de datos sobre nuestras imágenes.","metadata":{}},{"cell_type":"code","source":"# valid_datagen = ImageDataGenerator(\n# rescale=1./255)\n\n# valid_generator = valid_datagen.flow_from_directory(\n# '/kaggle/input/data-binary/data/valid',\n# target_size=(256, 256),\n# batch_size=64,\n# class_mode='binary')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_datagen = ImageDataGenerator(\n# rescale=1./255)\n\n# test_generator = valid_datagen.flow_from_directory(\n# '/kaggle/input/data-binary/data/test',\n# target_size=(256, 256),\n# batch_size=64,\n# class_mode='binary')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Definimos la red convolucional\n\n*CONV2D* (https://pyimagesearch.com/2018/12/31/keras-conv2d-and-convolutional-layers/): \n\n* **filters**: El 32 es el número de filtros en la convolución, es decir, la matriz a la que vamos a multiplicar.\n\n* **KernelSize**: Tamaño de la matriz, en este caso será una 3x3, If your input images are greater than 128×128 you may choose to use a kernel size > 3.\n\n* **strides**: The filter takes a 1-pixel step to the right and again the filter is applied to the input volume.\n\n* **padding**: **valid** (the input volume is not zero-padded and the spatial dimensions are allowed to reduce via the natural application of convolution) or **same** (preserve the spatial dimensions of the input volume). if you use SAME, you'll need to use maxpooling to reduce spatial dimensions\n\n* **input_shape**=(256, 256, 3) for 256x256 RGB pictures in data_format=\"channels_last\"\n\n*BatchNormalization* (https://www.baeldung.com/cs/batch-normalization-cnn):\n\n* It is a normalization technique done **between** the layers of a Neural Network. It is applied to the neurons’ output before applying the activation function. It serves to speed up training and use higher learning rates, making learning easier.\n\n*MaxPool2D*: \n\n* Downsamples the input along its spatial dimensions (height and width) by taking the maximum value over an input window (of size defined by pool_size)\n\n*DropOut* (https://machinelearningmastery.com/dropout-regularization-deep-learning-models-keras/):\n\n* It is a technique where randomly selected neurons are ignored during training, This means that their contribution to the activation of downstream neurons is temporally removed on the forward pass, and any weight updates are not applied to the neuron on the backward pass\n\n*Flatten* (https://kevinmlean.medium.com/how-to-use-keras-layers-flatten-c3f29ed1b686):\n\n* This function converts the multi-dimensional arrays into flattened one-dimensional arrays or single-dimensional arrays.\n\n*Dense* :\n* Keras Dense layer is the layer that contains all the neurons that are deeply connected within themselves.\n\n\n\n\n","metadata":{}},{"cell_type":"markdown","source":"### 1er modelo con 20000 imágenes normales en color","metadata":{}},{"cell_type":"code","source":"# train_datagen = ImageDataGenerator(\n# featurewise_center=False, \n# samplewise_center=False, \n# featurewise_std_normalization=False, \n# samplewise_std_normalization=False, \n# rescale=1./255,\n# rotation_range=20,\n# width_shift_range=0.05,\n# height_shift_range=0.05,\n# shear_range=0.05,\n# zoom_range=0.2,\n# horizontal_flip=True,\n# vertical_flip=True,\n# fill_mode='nearest')\n\n# train_generator = train_datagen.flow_from_directory(\n# '/kaggle/input/data-binary/data/train_20000',\n# target_size=(256, 256),\n# batch_size=64,\n# class_mode='binary')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu' , input_shape = ( 256, 256, 3)))\nmodel.add(BatchNormalization())\n\nmodel.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))\nmodel.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))\nmodel.add(Dropout(0.1))\nmodel.add(BatchNormalization())\n\nmodel.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))\nmodel.add(Conv2D(64 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))\nmodel.add(BatchNormalization())\n\nmodel.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))\nmodel.add(Conv2D(128 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(BatchNormalization())\n\nmodel.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))\nmodel.add(Conv2D(256 , (3,3) , strides = 1 , padding = 'same' , activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(BatchNormalization())\n\nmodel.add(MaxPool2D((2,2) , strides = 2 , padding = 'same'))\n\nmodel.add(Flatten())\n\nmodel.add(Dense(units = 128 , activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(units = 1 , activation = 'sigmoid')) #clasificación binaria","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loss = tf.keras.losses.BinaryCrossentropy()\n# model.compile(loss=loss, \n# optimizer='Adam', \n# metrics=['binary_accuracy'])\n\n# learning_rate_reduction = ReduceLROnPlateau(monitor='val_binary_accuracy', patience = 2, verbose=1,factor=0.3, min_lr=0.000001)\n\n# filepath = \"/kaggle/models/saved-model-{epoch:02d}-{val_binary_accuracy:.2f}.hdf5\"\n# checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, \n# save_best_only=False,save_freq='epoch')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = model.fit_generator(\n# train_generator,\n# epochs=25,\n# validation_data=valid_generator,\n# validation_steps=4,\n# callbacks=[checkpoint,learning_rate_reduction],\n# verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.plot(history.history['binary_accuracy'], label='The score of correct predictions on the training set')\n# plt.plot(history.history['val_binary_accuracy'], label='The score of correct predictions on the val set')\n# plt.xlabel('Epoch')\n# plt.ylabel('Score correct answers')\n# plt.legend()\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best_acc = 0\n# best_model = \"\"\n# for i in os.listdir(\"/kaggle/models\"):\n# model.load_weights(\"/kaggle/models/\"+i)\n# loss, acc = model.evaluate_generator(test_generator, steps=3, verbose=0)\n# if acc > best_acc:\n# best_model = i\n# best_acc = acc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.load_weights(\"/kaggle/models/\"+best_model)\n# model.save('/kaggle/working/model1.h5')\n# loss, acc = model.evaluate_generator(test_generator, steps=3, verbose=0)\n# acc = acc *100\n# print(f\"accuracy is: {acc:.2f}%\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_pred = model.predict(valid_generator) # Reemplaza \"valid_generator\" con tus datos de validación\n# y_pred_binary = np.squeeze(np.round(y_pred)).astype(int)\n\n# print(y_pred_binary)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import roc_auc_score\n\n# auc = roc_auc_score(y_true, y_pred_binary) # Reemplaza \"y_true\" y \"y_pred\" con tus etiquetas verdaderas y predicciones\n\n# print(auc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import precision_score, recall_score, f1_score\n\n# precision = precision_score(y_true, y_pred_binary)\n# recall = recall_score(y_true, y_pred_binary)\n# f1 = f1_score(y_true, y_pred_binary)\n\n# print(\"Precision: \",precision)\n# print(\"Recall: \", recall)\n# print(\"f1: \", f1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import confusion_matrix\n\n# cm = confusion_matrix(y_true, y_pred_binary)\n\n# import seaborn as sns\n# import matplotlib.pyplot as plt\n\n# # Muestra la matriz de confusión como un heatmap\n# sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\n# plt.xlabel(\"Predicción\")\n# plt.ylabel(\"Etiqueta verdadera\")\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Con 25 epochs y entrenando las 20000 imagenes:**\n\nAccuracy: 88.54%\n\nAuc: 0.49961171257103115\n\nPrecision: 0.05285942128047391\n\nRecall: 0.15246106328448447\n\nf1: 0.0785017003062243","metadata":{}},{"cell_type":"markdown","source":"### 2º Modelo con 20000 imágenes en color ","metadata":{}},{"cell_type":"code","source":"# model = Sequential()\n# model.add(Conv2D(32, (3, 3), strides=1, padding='same', activation='relu', input_shape=(256, 256, 3)))\n# model.add(BatchNormalization())\n\n# model.add(MaxPool2D((2, 2), strides=2, padding='same'))\n# model.add(Conv2D(64, (3, 3), strides=1, padding='same', activation='relu'))\n# model.add(Dropout(0.1))\n# model.add(BatchNormalization())\n\n# model.add(MaxPool2D((2, 2), strides=2, padding='same'))\n# model.add(Conv2D(128, (3, 3), strides=1, padding='same', activation='relu'))\n# model.add(Dropout(0.2))\n# model.add(BatchNormalization())\n\n# model.add(MaxPool2D((2, 2), strides=2, padding='same'))\n# model.add(Conv2D(256, (3, 3), strides=1, padding='same', activation='relu'))\n# model.add(Dropout(0.2))\n# model.add(BatchNormalization())\n\n# model.add(MaxPool2D((2, 2), strides=2, padding='same'))\n\n# model.add(Flatten())\n\n# model.add(Dense(units=128, activation='relu'))\n# model.add(Dropout(0.2))\n# model.add(Dense(units=1, activation='sigmoid'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loss = tf.keras.losses.BinaryCrossentropy()\n# model.compile(loss=loss, \n# optimizer='Adam', \n# metrics=['binary_accuracy'])\n\n# learning_rate_reduction = ReduceLROnPlateau(monitor='val_binary_accuracy', patience = 2, verbose=1,factor=0.3, min_lr=0.000001)\n\n# filepath = \"/kaggle/models/saved-model-{epoch:02d}-{val_binary_accuracy:.2f}.hdf5\"\n# checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, \n# save_best_only=False,save_freq='epoch')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = model.fit_generator(\n# train_generator,\n# epochs=10,\n# validation_data=valid_generator,\n# validation_steps=4,\n# callbacks=[checkpoint,learning_rate_reduction],\n# verbose=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.plot(history.history['binary_accuracy'], label='The score of correct predictions on the training set')\n# plt.plot(history.history['val_binary_accuracy'], label='The score of correct predictions on the val set')\n# plt.xlabel('Epoch')\n# plt.ylabel('Score correct answers')\n# plt.legend()\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# best_acc = 0\n# best_model = \"\"\n# for i in os.listdir(\"/kaggle/models\"):\n# model.load_weights(\"/kaggle/models/\"+i)\n# loss, acc = model.evaluate_generator(test_generator, steps=3, verbose=0)\n# if acc > best_acc:\n# best_model = i\n# best_acc = acc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.load_weights(\"/kaggle/models/\"+best_model)\n# model.save('/kaggle/working/model1.h5')\n# loss, acc = model.evaluate_generator(test_generator, steps=3, verbose=0)\n# acc = acc *100\n# print(f\"accuracy is: {acc:.2f}%\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_pred = model.predict(valid_generator) # Reemplaza \"valid_generator\" con tus datos de validación\n# y_pred_binary = np.squeeze(np.round(y_pred)).astype(int)\n\n# print(y_pred_binary)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import roc_auc_score\n\n# auc = roc_auc_score(y_true, y_pred_binary) # Reemplaza \"y_true\" y \"y_pred\" con tus etiquetas verdaderas y predicciones\n\n# print(auc)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import precision_score, recall_score, f1_score\n\n# precision = precision_score(y_true, y_pred_binary)\n# recall = recall_score(y_true, y_pred_binary)\n# f1 = f1_score(y_true, y_pred_binary)\n\n# print(\"Precision: \",precision)\n# print(\"Recall: \", recall)\n# print(\"f1: \", f1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.metrics import confusion_matrix\n\n# cm = confusion_matrix(y_true, y_pred_binary)\n\n# import seaborn as sns\n# import matplotlib.pyplot as plt\n\n# # Muestra la matriz de confusión como un heatmap\n# sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\n# plt.xlabel(\"Predicción\")\n# plt.ylabel(\"Etiqueta verdadera\")\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Con 10 epochs y entrenando las 20000 imagenes**:\n\nAccuracy: 88.02%\n\nAuc: 0.4985870047823303\n\nPrecision: 0.05155586175221036\n\nRecall: 0.08851941907077611\n\nf1: 0.06516060371517027","metadata":{}},{"cell_type":"markdown","source":"# 9. NUEVO Clasificador binario - Similar to PE Detection with Keras - Model Creation","metadata":{}},{"cell_type":"code","source":"print('Reading train data...')\ntrain = pd.read_csv(\"/kaggle/input/rsna-str-pulmonary-embolism-detection/train.csv\")\n# train = train.drop(['Unnamed: 0'], axis=1)\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T17:38:33.492698Z","iopub.execute_input":"2023-06-19T17:38:33.49308Z","iopub.status.idle":"2023-06-19T17:38:35.7709Z","shell.execute_reply.started":"2023-06-19T17:38:33.493034Z","shell.execute_reply":"2023-06-19T17:38:35.769693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Reading test data...')\ntest = pd.read_csv(\"/kaggle/input/data-binary/data/test.csv\")\ntest = test.drop(['Unnamed: 0'], axis=1)\ntest = test.drop(['pe_present_on_image','negative_exam_for_pe','qa_motion','qa_contrast','flow_artifact','rv_lv_ratio_gte_1','rv_lv_ratio_lt_1','leftsided_pe','chronic_pe','true_filling_defect_not_pe','rightsided_pe','acute_and_chronic_pe','central_pe','indeterminate'], axis=1)\nprint(test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T17:38:38.18159Z","iopub.execute_input":"2023-06-19T17:38:38.18197Z","iopub.status.idle":"2023-06-19T17:38:38.951146Z","shell.execute_reply.started":"2023-06-19T17:38:38.181942Z","shell.execute_reply":"2023-06-19T17:38:38.950089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Keys","metadata":{}},{"cell_type":"code","source":"print('Reading sample data...')\nss = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/sample_submission.csv\")\nprint(ss.shape)\nss.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T17:38:40.972071Z","iopub.execute_input":"2023-06-19T17:38:40.972421Z","iopub.status.idle":"2023-06-19T17:38:41.095972Z","shell.execute_reply.started":"2023-06-19T17:38:40.972393Z","shell.execute_reply":"2023-06-19T17:38:41.094847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = ss.id\ncounter = [1 for _ in range(10)]\nmapper = []\nfor i in ids:\n n = '_'.join(i.split('_')[1:])\n if n not in mapper:\n mapper.append(n)\n else:\n counter[mapper.index(n)] += 1\nprint(\"List of keys:\")\nprint(mapper, sep='\\n')\nprint()\nprint(\"Count of items per key:\")\nprint(counter)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T17:38:42.817005Z","iopub.execute_input":"2023-06-19T17:38:42.817366Z","iopub.status.idle":"2023-06-19T17:38:43.012753Z","shell.execute_reply.started":"2023-06-19T17:38:42.817337Z","shell.execute_reply":"2023-06-19T17:38:43.011625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After checking the keys, we will now work on the fuction to get the image array from a DICOM image. We will be using a code snippet by eladwar from his notebook here.","metadata":{}},{"cell_type":"markdown","source":"Lee las imágenes en formato DICOM y luego convierte las imágenes DICOM en un formato de imagen compatible con OpenCV (Open Source Computer Vision Library)","metadata":{}},{"cell_type":"code","source":"import vtk\nfrom vtk.util import numpy_support\nimport cv2\n\nreader = vtk.vtkDICOMImageReader()\ndef get_img(path):\n reader.SetFileName(path)\n reader.Update()\n _extent = reader.GetDataExtent()\n ConstPixelDims = [_extent[1]-_extent[0]+1, _extent[3]-_extent[2]+1, _extent[5]-_extent[4]+1]\n\n ConstPixelSpacing = reader.GetPixelSpacing()\n imageData = reader.GetOutput()\n pointData = imageData.GetPointData()\n arrayData = pointData.GetArray(0)\n ArrayDicom = numpy_support.vtk_to_numpy(arrayData)\n ArrayDicom = ArrayDicom.reshape(ConstPixelDims, order='F')\n ArrayDicom = cv2.resize(ArrayDicom,(512,512))\n return ArrayDicom","metadata":{"execution":{"iopub.status.busy":"2023-06-19T17:38:45.579514Z","iopub.execute_input":"2023-06-19T17:38:45.579886Z","iopub.status.idle":"2023-06-19T17:38:45.591254Z","shell.execute_reply.started":"2023-06-19T17:38:45.579857Z","shell.execute_reply":"2023-06-19T17:38:45.589191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"After defining our image reader, we will test it with a sample JPEG image to load.","metadata":{}},{"cell_type":"markdown","source":"Este código lee un archivo DICOM, lo convierte a una representación de 8 bits y muestra dos imágenes en una figura de Matplotlib: una imagen en escala de grises de 8 bits y otra imagen DICOM original de 16 bits. Esto permite comparar la diferencia en la apariencia de las imágenes con diferentes profundidades de bits.","metadata":{}},{"cell_type":"markdown","source":"Se utiliza la función int16_to_uint8 para convertir los datos de la imagen DICOM a escala de grises de 8 bits. Esta función toma cada valor de píxel en la imagen DICOM y realiza una transformación lineal para ajustar el rango de valores de 16 bits a un rango de 8 bits (de 0 a 255).\n\nEn el primer conjunto de ejes (ax[0]), se muestra la imagen convertida a escala de grises de 8 bits utilizando ax[0].imshow(data_row_img, cmap=plt.cm.bone). La opción cmap=plt.cm.bone se utiliza para mostrar la imagen con una paleta de colores específica para imágenes médicas.\n\nEn el segundo conjunto de ejes (ax[1]), se muestra la imagen DICOM original de 16 bits sin convertir utilizando ax[1].imshow(ds, cmap=plt.cm.bone).","metadata":{}},{"cell_type":"code","source":"#test read a dcom file and view it\nfpath = \"../input/rsna-str-pulmonary-embolism-detection/train/0003b3d648eb/d2b2960c2bbf/00ac73cfc372.dcm\"\nds = get_img(fpath)\n\nimport matplotlib.pyplot as plt\n\n#Convert dcom file to 8bit color\nfunc = lambda x: int((2**15 + x)*(255/2**16))\nint16_to_uint8 = np.vectorize(func)\n\ndef show_dicom_images(dcom):\n f, ax = plt.subplots(1,2, figsize=(16,20))\n data_row_img = int16_to_uint8(ds)\n ax[0].imshow(data_row_img, cmap=plt.cm.bone)\n ax[1].imshow(ds, cmap=plt.cm.bone)\n #print(data_row_img)\n ax[0].axis('off')\n ax[0].set_title('8-bit DICOM Image')\n ax[1].axis('off')\n ax[1].set_title('16-bit DICOM Image')\n plt.show()\n \nshow_dicom_images(ds)","metadata":{"execution":{"iopub.status.busy":"2023-06-19T17:38:47.927168Z","iopub.execute_input":"2023-06-19T17:38:47.927554Z","iopub.status.idle":"2023-06-19T17:38:48.521282Z","shell.execute_reply.started":"2023-06-19T17:38:47.927523Z","shell.execute_reply":"2023-06-19T17:38:48.520452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import cv2\n# import albumentations as albu\n# import matplotlib.pyplot as plt\n\n# train_transform = [albu.RandomBrightnessContrast(p=0.3),\n# albu.VerticalFlip(p=0.5),\n# albu.HorizontalFlip(p=0.5),\n# albu.Downscale(p=1.0, scale_min=0.35, scale_max=0.75),\n# albu.Resize(512, 512)]\n# transform = albu.Compose(train_transform)\n\n# def normalize_image(image):\n# min_val = 0\n# max_val = 255\n# image = cv2.normalize(image, None, min_val, max_val, cv2.NORM_MINMAX)\n# return image\n\n# def get_img(path):\n# image = cv2.imread(path, cv2.IMREAD_GRAYSCALE)\n# image = cv2.resize(image, (512, 512))\n# image = normalize_image(image)\n# return image\n\n# def show_images(original_img, transformed_img):\n# fig, axes = plt.subplots(1, 2, figsize=(12, 6))\n \n# axes[0].imshow(original_img, cmap='gray', vmin=0, vmax=255)\n# axes[0].set_title('Original Image')\n# axes[0].axis('off')\n \n# axes[1].imshow(transformed_img, cmap='gray', vmin=0, vmax=255)\n# axes[1].set_title('Transformed Image')\n# axes[1].axis('off')\n \n# for ax in axes:\n# ax.spines['top'].set_visible(False)\n# ax.spines['right'].set_visible(False)\n# ax.spines['bottom'].set_visible(False)\n# ax.spines['left'].set_visible(False)\n \n# plt.tight_layout()\n# plt.show()\n\n\n# image_path = '/kaggle/input/train-jpgs/train-jpegs_partion/0003b3d648eb/d2b2960c2bbf/00ac73cfc372.jpg' # Reemplaza con la ruta de tu imagen JPEG\n\n# # Obtener la imagen original y la imagen transformada\n# original_image = get_img(image_path)\n# transformed_image = transform(image=original_image)['image']\n\n# # Mostrar las imágenes\n# show_images(original_image, transformed_image)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-17T15:44:36.308799Z","iopub.execute_input":"2023-06-17T15:44:36.309181Z","iopub.status.idle":"2023-06-17T15:44:36.894501Z","shell.execute_reply.started":"2023-06-17T15:44:36.309149Z","shell.execute_reply":"2023-06-17T15:44:36.89368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creación del modelo","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Dense, Dropout, Conv2D\nimport tensorflow.keras.backend as K\nimport gc\n\ninputs = Input((512, 512, 3))\n#x = Conv2D(3, (1, 1), activation='relu')(inputs)\nbase_model = keras.applications.Xception(\n include_top=False,\n weights=\"imagenet\"\n)\n\nbase_model.trainable = False\n\noutputs = base_model(inputs, training=False)\noutputs = keras.layers.GlobalAveragePooling2D()(outputs)\noutputs = Dropout(0.25)(outputs)\noutputs = Dense(1024, activation='relu')(outputs)\noutputs = Dense(256, activation='relu')(outputs)\noutputs = Dense(64, activation='relu')(outputs)\nppoi = Dense(1, activation='sigmoid', name='pe_present_on_image')(outputs)\n\nmodel = Model(inputs=inputs, outputs={'pe_present_on_image':ppoi})\n\nopt = keras.optimizers.Adam(lr=0.001)\n\nmodel.compile(optimizer=opt,\n loss='binary_crossentropy',\n metrics=['accuracy'])\n\nmodel.summary()\nmodel.save('pe_detection_model.h5')\ndel model\nK.clear_session()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T17:39:16.373618Z","iopub.execute_input":"2023-06-19T17:39:16.374062Z","iopub.status.idle":"2023-06-19T17:39:19.427583Z","shell.execute_reply.started":"2023-06-19T17:39:16.374027Z","shell.execute_reply":"2023-06-19T17:39:19.426405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import tensorflow as tf\n# from tensorflow.keras.models import Model\n# from tensorflow.keras.layers import Input, Dense, Dropout, Conv2D, MaxPooling2D, Flatten, GlobalAveragePooling2D, Embedding, Dot, Reshape, Multiply\n# from tensorflow.keras.applications import Xception\n# from tensorflow.keras.optimizers import Adam\n# import tensorflow.keras.backend as K\n\n# # Entrada de la imagen\n# inputs = Input((512, 512, 3))\n\n# # Base del modelo preentrenado\n# base_model = Xception(include_top=False, weights=\"imagenet\")\n# base_model.trainable = False\n\n# # Salida de la base del modelo\n# base_outputs = base_model(inputs, training=False)\n# pool_outputs = GlobalAveragePooling2D()(base_outputs)\n# drop_outputs = Dropout(0.25)(pool_outputs)\n# dense_outputs = Dense(64, activation='relu')(drop_outputs)\n\n# # Salida para la variable objetivo pe_present_on_image\n# ppoi = Dense(1, activation='sigmoid', name='pe_present_on_image')(dense_outputs)\n\n# # Modelo completo\n# model = Model(inputs=inputs, outputs=ppoi)\n\n# # Compilación del modelo\n# opt = Adam(lr=0.00001)\n# model.compile(optimizer=opt, loss='binary_crossentropy', metrics=['accuracy'])\n# model.summary()\n\n# model.save('pe_present_on_image_model.h5')\n# del model \n# # elimina la referencia al objeto del modelo de Python, \n# # lo que permite que el recolector de basura de Python \n# # libere la memoria ocupada por ese objeto.\n\n# K.clear_session()\n# # elimina cualquier referencia residual al modelo en el backend de Keras\n# # y libera la memoria asociada con esas referencias.\n\n# gc.collect()\n# # realiza una recolección de basura adicional para \n# # liberar cualquier objeto no utilizado y liberar aún más memoria.","metadata":{"execution":{"iopub.status.busy":"2023-06-17T15:51:18.708414Z","iopub.execute_input":"2023-06-17T15:51:18.708822Z","iopub.status.idle":"2023-06-17T15:51:21.834014Z","shell.execute_reply.started":"2023-06-17T15:51:18.708785Z","shell.execute_reply":"2023-06-17T15:51:21.832907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Entrenamiento del modelo","metadata":{}},{"cell_type":"markdown","source":"Before we can train our model, we would be needing an image generator. This is so that our training code would look much cleaner and nicer.\n\nusing yield, we only iterate through the batch once and don't save it in memory.","metadata":{}},{"cell_type":"code","source":"import cv2\nimport numpy as np\n\ndef convert_to_rgb(array):\n array = array.reshape((512, 512, 1))\n return np.stack([array, array, array], axis=2).reshape((512, 512, 3))\n \ndef custom_dcom_image_generator(batch_size, dataset, test=False, debug=False):\n \n fnames = dataset[['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']]\n \n if not test:\n Y = dataset[['pe_present_on_image']]\n prefix = 'input/rsna-str-pulmonary-embolism-detection/train'\n \n else:\n prefix = 'input/rsna-str-pulmonary-embolism-detection/test'\n \n X = []\n batch = 0\n for st, sr, so in fnames.values:\n if debug:\n print(f\"Current file: ../{prefix}/{st}/{sr}/{so}.dcm\")\n\n dicom = get_img(f\"../{prefix}/{st}/{sr}/{so}.dcm\")\n image = convert_to_rgb(dicom)\n X.append(image)\n \n del st, sr, so\n \n if len(X) == batch_size:\n if test:\n yield np.array(X)\n del X\n else:\n yield np.array(X), Y[batch*batch_size:(batch+1)*batch_size].values\n del X\n \n gc.collect()\n X = []\n batch += 1\n \n if test:\n yield np.array(X)\n else:\n yield np.array(X), Y[batch*batch_size:(batch+1)*batch_size].values\n del Y\n del X\n gc.collect()\n return","metadata":{"execution":{"iopub.status.busy":"2023-06-19T17:39:23.898339Z","iopub.execute_input":"2023-06-19T17:39:23.899244Z","iopub.status.idle":"2023-06-19T17:39:23.912029Z","shell.execute_reply.started":"2023-06-19T17:39:23.899201Z","shell.execute_reply":"2023-06-19T17:39:23.910762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will be training our model with the train data","metadata":{}},{"cell_type":"code","source":"import time\nfrom tensorflow.keras.callbacks import ModelCheckpoint as MC\nfrom tensorflow.keras.models import load_model\nfrom IPython.display import clear_output\nfrom sklearn.metrics import roc_auc_score, precision_score\nimport numpy as np\n\nhistory = {}\nstart = time.time()\ndebug = 0\nbatch_size = 1000\ntrain_size = int(batch_size*0.9)\n\nmax_train_time = 3600 * 4 # horas a segundos de entrenamiento\n\ncheckpoint = MC(filepath='../working/pe_detection_model.h5', monitor='val_loss', save_best_only=True, verbose=1)\n\n# Bucle de entrenamiento\nfor n, (x, y) in enumerate(custom_dcom_image_generator(batch_size, train.sample(frac=1), False, debug)):\n\n if len(x) < 10: # Intenta filtrar datos vacíos o cortos\n break\n\n clear_output(wait=True)\n print(\"Entrenamiento del lote: %i - %i\" % (batch_size*n, batch_size*(n+1)))\n\n model = load_model('../working/pe_detection_model.h5')\n\n class_weights = np.ones_like(y[:, 0]) # Inicialmente, asigna un peso de 1 a todas las muestras\n class_weights[y[:, 0] == 1] = 10 # Asigna un peso más alto a las muestras de la clase con embolia\n\n hist = model.fit(\n x,\n {'pe_present_on_image': y[:, 0]},\n callbacks=checkpoint,\n validation_split=0.2,\n epochs=3,\n batch_size=8,\n verbose=debug,\n sample_weight=class_weights\n )\n\n print(\"Métricas de validación del lote:\")\n model.evaluate(x[train_size:], {'pe_present_on_image': y[train_size:, 0]})\n\n try:\n for key in hist.history.keys():\n history[key] = np.concatenate([history[key], hist.history[key]], axis=0)\n except:\n for key in hist.history.keys():\n history[key] = hist.history[key]\n\n # Para asegurarse de que el modelo no entrene durante demasiado tiempo\n if time.time() - start >= max_train_time:\n print(\"¡Se acabó el tiempo!\")\n break\n\n model.save('pe_detection_model.h5')\n del model, x, y, hist\n K.clear_session()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-19T17:41:00.876699Z","iopub.execute_input":"2023-06-19T17:41:00.877053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import time\n# from tensorflow.keras.callbacks import ModelCheckpoint as MC\n# from tensorflow.keras.models import load_model\n# from IPython.display import clear_output\n# from sklearn.metrics import roc_auc_score, precision_score\n# from tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n\n# history = {}\n# start = time.time()\n# debug = 0\n# batch_size = 1000\n# train_size = int(batch_size*0.9)\n\n# max_train_time = 3600 * 4 #hours to seconds of training\n\n# checkpoint = MC(filepath='../working/pe_detection_model.h5', monitor='val_loss', save_best_only=True, verbose=1)\n# #Train loop\n# for n, (x, y) in enumerate(custom_dcom_image_generator(batch_size, train.sample(frac=1), False, debug)):\n \n# if len(x) < 10: #Tries to filter out empty or short data\n# break\n \n# clear_output(wait=True)\n# print(\"Training batch: %i - %i\" %(batch_size*n, batch_size*(n+1)))\n \n# model = load_model('../working/pe_detection_model.h5')\n# hist = model.fit(\n# x[:train_size], #Y values are in a dict as there's more than one target for training output\n# {'pe_present_on_image':y[:train_size, 0]},\n\n# callbacks = checkpoint,\n\n# validation_split=0.2,\n# epochs=3,\n# batch_size=8,\n# verbose=debug\n# )\n \n# print(\"Metrics for batch validation:\")\n# model.evaluate(x[train_size:],\n# {'pe_present_on_image':y[train_size:, 0]})\n \n# try:\n# for key in hist.history.keys():\n# history[key] = np.concatenate([history[key], hist.history[key]], axis=0)\n# except:\n# for key in hist.history.keys():\n# history[key] = hist.history[key]\n \n# #To make sure that our model don't train overtime\n# if time.time() - start >= max_train_time:\n# print(\"Time's up!\")\n# break\n \n# model.save('pe_detection_model.h5')\n# del model, x, y, hist\n# K.clear_session()\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:02:06.137555Z","iopub.execute_input":"2023-06-19T15:02:06.138247Z","iopub.status.idle":"2023-06-19T15:14:56.911163Z","shell.execute_reply.started":"2023-06-19T15:02:06.138215Z","shell.execute_reply":"2023-06-19T15:14:56.909029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in history.keys():\n if key.startswith('val'):\n continue\n else:\n epoch = range(len(history[key]))\n plt.plot(epoch, history[key]) #X=epoch, Y=value\n plt.plot(epoch, history['val_'+key])\n plt.title(key)\n if 'accuracy' in key:\n plt.axis([0, len(history[key]), -0.1, 1.1]) #Xmin, Xmax, Ymin, Ymax\n plt.legend(['train', 'validation'], loc='upper right')\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:15:06.123255Z","iopub.execute_input":"2023-06-19T15:15:06.123711Z","iopub.status.idle":"2023-06-19T15:15:06.686889Z","shell.execute_reply.started":"2023-06-19T15:15:06.123674Z","shell.execute_reply":"2023-06-19T15:15:06.684862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Reading test data...')\ntest = pd.read_csv(\"/kaggle/input/data-binary/data/test.csv\")\ntest = test.drop(['Unnamed: 0'], axis=1)\ntest = test.drop(['negative_exam_for_pe','qa_motion','qa_contrast','flow_artifact','rv_lv_ratio_gte_1','rv_lv_ratio_lt_1','leftsided_pe','chronic_pe','true_filling_defect_not_pe','rightsided_pe','acute_and_chronic_pe','central_pe','indeterminate'], axis=1)\ntest = test.head(15000)\nprint(test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:15:38.40779Z","iopub.execute_input":"2023-06-19T15:15:38.408173Z","iopub.status.idle":"2023-06-19T15:15:39.323976Z","shell.execute_reply.started":"2023-06-19T15:15:38.408142Z","shell.execute_reply":"2023-06-19T15:15:39.321859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# Supongamos que tienes un DataFrame llamado 'df' y quieres contar los valores en la columna 'variable'\ncounts = test['pe_present_on_image'].value_counts()\n\nprint(counts)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:15:41.898779Z","iopub.execute_input":"2023-06-19T15:15:41.899201Z","iopub.status.idle":"2023-06-19T15:15:41.906219Z","shell.execute_reply.started":"2023-06-19T15:15:41.899169Z","shell.execute_reply":"2023-06-19T15:15:41.905102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nfrom tensorflow.keras.models import load_model\nfrom IPython.display import clear_output\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\nfrom sklearn.metrics import roc_auc_score\nimport numpy as np\n\npredictions = {}\nstopper = 3600 * 4 # Límite de 4 horas para la predicción\npred_start_time = time.time()\n\np, c = time.time(), time.time()\nbatch_size = 500\n\nl = 0\nn = test.shape[0]\n\n\nfor x in custom_dcom_image_generator(batch_size, test, True, False):\n clear_output(wait=True)\n model = load_model(\"../working/pe_detection_model.h5\")\n \n preds = model.predict(x, batch_size=8, verbose=1)\n\n try:\n for key in preds.keys():\n if key not in predictions:\n predictions[key] = []\n predictions[key].extend(preds[key].flatten().tolist())\n except Exception as e:\n print(e)\n break\n\n l = (l + batch_size) % n\n print('Total predicted:', len(predictions['pe_present_on_image']), '/', n)\n p, c = c, time.time()\n print(\"One batch time: %.2f seconds\" % (c - p))\n print(\"ETA: %.2f\" % ((n - l) * (c - p) / batch_size))\n\n if c - pred_start_time >= stopper:\n print(\"Time's up!\")\n break\n \n if len(predictions['pe_present_on_image']) == n:\n # Convertir las predicciones en arrays numpy\n y_true = test['pe_present_on_image'].astype(int)\n y_pred = {key: np.array(value) for key, value in predictions.items()}\n \n # Aplicar umbral a las predicciones\n threshold = 0.5 # Umbral para la conversión\n y_pred = np.where(y_pred['pe_present_on_image'] > threshold, 1, 0)\n \n y_pred = pd.Series(y_pred, name='pe_present_on_image')\n\n # Calcular las métricas\n \n auc = roc_auc_score(y_true, y_pred)\n print(\"AUC:\", auc)\n accuracy = accuracy_score(y_true, y_pred)\n precision = precision_score(y_true, y_pred)\n recall = recall_score(y_true, y_pred)\n f1 = f1_score(y_true, y_pred)\n\n # Imprimir las métricas\n print(\"Accuracy:\", accuracy)\n print(\"Precision:\", precision)\n print(\"Recall:\", recall)\n print(\"F1 Score:\", f1)\n\n break\n\n del model\n K.clear_session()\n\n del x, preds\n gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:15:45.766286Z","iopub.execute_input":"2023-06-19T15:15:45.767317Z","iopub.status.idle":"2023-06-19T15:22:37.392246Z","shell.execute_reply.started":"2023-06-19T15:15:45.767272Z","shell.execute_reply":"2023-06-19T15:22:37.391274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\n\ncm = confusion_matrix(y_true, y_pred)\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Muestra la matriz de confusión como un heatmap\nsns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\nplt.xlabel(\"Predicción\")\nplt.ylabel(\"Etiqueta verdadera\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-19T15:22:51.457853Z","iopub.execute_input":"2023-06-19T15:22:51.458245Z","iopub.status.idle":"2023-06-19T15:22:51.730002Z","shell.execute_reply.started":"2023-06-19T15:22:51.458213Z","shell.execute_reply":"2023-06-19T15:22:51.729069Z"},"trusted":true},"execution_count":null,"outputs":[]}]}