{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"},{"sourceId":4696088,"sourceType":"datasetVersion","datasetId":2687741}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"> **Chargement des données**","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-09T22:44:22.367355Z","iopub.execute_input":"2023-12-09T22:44:22.367792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# Spécification du chemin vers le dossier contenant les fichiers DICOM\ndata_path = '/kaggle/input/rsna-breast-cancer-detection'\n\n# Chargement des  métadonnées\ndf = pd.read_csv(os.path.join(data_path,'/kaggle/input/rsna-breast-cancer-detection/train.csv'))\n\n# Affichage  des premières lignes du métadonnées\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:17:18.249095Z","iopub.execute_input":"2023-12-09T23:17:18.250331Z","iopub.status.idle":"2023-12-09T23:17:18.373877Z","shell.execute_reply.started":"2023-12-09T23:17:18.250288Z","shell.execute_reply":"2023-12-09T23:17:18.372726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **normalisation des valeurs des pixels entre 0 et 1, et affichage des images originales et normalisées côte à côte.**\n\n* Mettre à l'échelle les valeurs des caractéristiques pour les ramener à une échelle commune. ","metadata":{}},{"cell_type":"code","source":"!pip install pydicom[gdcm] pylibjpeg","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:17:36.984087Z","iopub.execute_input":"2023-12-09T23:17:36.984539Z","iopub.status.idle":"2023-12-09T23:17:51.342276Z","shell.execute_reply.started":"2023-12-09T23:17:36.984502Z","shell.execute_reply":"2023-12-09T23:17:51.340648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install pylibjpeg pylibjpeg-libjpeg","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:18:05.701605Z","iopub.execute_input":"2023-12-09T23:18:05.702032Z","iopub.status.idle":"2023-12-09T23:18:19.83238Z","shell.execute_reply.started":"2023-12-09T23:18:05.701994Z","shell.execute_reply":"2023-12-09T23:18:19.830602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nimport os\nimport numpy as np\nimport matplotlib.pyplot as plt\n\ndef normalize_images_for_patient(patient_id, directory_path):\n    patient_dir = os.path.join(directory_path, str(patient_id))\n\n    for filename in os.listdir(patient_dir):\n        file_path = os.path.join(patient_dir, filename)\n        dicom_data = pydicom.dcmread(file_path)\n        pixel_array = dicom_data.pixel_array\n\n        # Normaliser les valeurs des pixels entre 0 et 1\n        pixel_min = pixel_array.min()\n        pixel_max = pixel_array.max()\n        pixel_normalized = (pixel_array - pixel_min) / (pixel_max - pixel_min)\n\n        # Afficher l'image originale et normalisée pour illustration\n        plt.figure(figsize=(8, 4))\n        plt.subplot(1, 2, 1)\n        plt.imshow(pixel_array, cmap='gray')\n        plt.title('Image Originale')\n\n        plt.subplot(1, 2, 2)\n        plt.imshow(pixel_normalized, cmap='gray')\n        plt.title('Image Normalisée')\n\n        plt.show()\n\n# Spécifiez le chemin vers le répertoire contenant les fichiers DICOM (train_images)\ndirectory_path = '/kaggle/input/rsna-breast-cancer-detection/train_images'\n\n# Liste des patients\npatients = ['10006', '10011', '10025', '10038', '10042', '10048', '10049', '10050', '10051', '10086',\n            '10095', '10097', '10102', '10106', '10116', '10119', '10122', '10124', '10126', '10130',\n            '10132', '10136', '1014', '10144', '1015', '10151', '10152', '10153', '10175', '10179']\n\n# Normaliser les images pour chaque patient\nfor patient_id in patients:\n    normalize_images_for_patient(patient_id, directory_path)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:19:34.915983Z","iopub.execute_input":"2023-12-09T23:19:34.916511Z","iopub.status.idle":"2023-12-09T23:28:47.132601Z","shell.execute_reply.started":"2023-12-09T23:19:34.916468Z","shell.execute_reply":"2023-12-09T23:28:47.131315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Nettoyage des données**\n\n* Gérer les valeurs manquantes en les supprimant, en les remplaçant par des valeurs appropriées (moyenne, médiane, etc.), ou en utilisant des techniques plus avancées.\n* Traiter les valeurs aberrantes qui peuvent affecter négativement le modèle.","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport pydicom\nimport os\nimport matplotlib.pyplot as plt\n\n# Spécification du chemin vers le dossier contenant les fichiers DICOM\ndata_path = '/kaggle/input/rsna-breast-cancer-detection'\n\n# Chargement des  métadonnées\ndf = pd.read_csv(os.path.join(data_path,'/kaggle/input/rsna-breast-cancer-detection/train.csv'))\n\n# Affichage  des premières lignes du métadonnées\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:12.846829Z","iopub.execute_input":"2023-12-09T23:41:12.847817Z","iopub.status.idle":"2023-12-09T23:41:12.97255Z","shell.execute_reply.started":"2023-12-09T23:41:12.847639Z","shell.execute_reply":"2023-12-09T23:41:12.971024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum().sum()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:15.164308Z","iopub.execute_input":"2023-12-09T23:41:15.165843Z","iopub.status.idle":"2023-12-09T23:41:15.192128Z","shell.execute_reply.started":"2023-12-09T23:41:15.165792Z","shell.execute_reply":"2023-12-09T23:41:15.19086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:16.714115Z","iopub.execute_input":"2023-12-09T23:41:16.714607Z","iopub.status.idle":"2023-12-09T23:41:16.724124Z","shell.execute_reply.started":"2023-12-09T23:41:16.714567Z","shell.execute_reply":"2023-12-09T23:41:16.722477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:18.42498Z","iopub.execute_input":"2023-12-09T23:41:18.42592Z","iopub.status.idle":"2023-12-09T23:41:18.456113Z","shell.execute_reply.started":"2023-12-09T23:41:18.425875Z","shell.execute_reply":"2023-12-09T23:41:18.454776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:20.964384Z","iopub.execute_input":"2023-12-09T23:41:20.964849Z","iopub.status.idle":"2023-12-09T23:41:20.994022Z","shell.execute_reply.started":"2023-12-09T23:41:20.964815Z","shell.execute_reply":"2023-12-09T23:41:20.99249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Identification des valeurs manquantes dans le métadata\nmissing_values = df.isnull().sum()\n\n# Affichage des colonnes avec des valeurs manquantes et le nombre de valeurs manquantes\nprint(\"Valeurs manquantes par colonne :\")\nprint(missing_values[missing_values > 0])","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:22.344611Z","iopub.execute_input":"2023-12-09T23:41:22.345164Z","iopub.status.idle":"2023-12-09T23:41:22.380656Z","shell.execute_reply.started":"2023-12-09T23:41:22.345123Z","shell.execute_reply":"2023-12-09T23:41:22.378215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_statistics = df.groupby('cancer').describe()\nprint(class_statistics)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:25.264169Z","iopub.execute_input":"2023-12-09T23:41:25.264684Z","iopub.status.idle":"2023-12-09T23:41:25.376189Z","shell.execute_reply.started":"2023-12-09T23:41:25.264648Z","shell.execute_reply":"2023-12-09T23:41:25.374603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#supprimer les lignes et les images dont la valeur de age est Nan\nimport os\nimport pandas as pd\n\n# Supprimer les lignes avec des valeurs NaN dans la colonne \"age\"\ndf = df.dropna(subset=['age'])","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:27.314257Z","iopub.execute_input":"2023-12-09T23:41:27.314753Z","iopub.status.idle":"2023-12-09T23:41:27.330822Z","shell.execute_reply.started":"2023-12-09T23:41:27.314713Z","shell.execute_reply":"2023-12-09T23:41:27.328486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:29.425172Z","iopub.execute_input":"2023-12-09T23:41:29.425878Z","iopub.status.idle":"2023-12-09T23:41:29.457736Z","shell.execute_reply.started":"2023-12-09T23:41:29.425827Z","shell.execute_reply":"2023-12-09T23:41:29.455685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ces données n'ont aucun impact sur le modéle donc il sont a supprimer\ndf=df.drop([\"site_id\" ,\"patient_id\" ,\"machine_id\" ] , axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:31.044704Z","iopub.execute_input":"2023-12-09T23:41:31.045171Z","iopub.status.idle":"2023-12-09T23:41:31.056262Z","shell.execute_reply.started":"2023-12-09T23:41:31.045137Z","shell.execute_reply":"2023-12-09T23:41:31.054618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Encodage des variables catégorielles**\n\n* Convertir les variables catégorielles en variables numériques.\n\n","metadata":{}},{"cell_type":"code","source":" #transformer les données catégorique en numérique\nfrom sklearn import preprocessing \nlabel_encoder = preprocessing.LabelEncoder() \ndf['laterality']= label_encoder.fit_transform(df['laterality']) \ndf['difficult_negative_case']= label_encoder.fit_transform(df['difficult_negative_case']) \ndf['density'] = df['density'].map({'A': 0 , 'B' : 1 ,'C': 2 , 'D' : 3})\ndf['view']= label_encoder.fit_transform(df['view']) \n# L =0 R=1\n# false = 0 true =1\n# A=0 B=1 C=2 D=3  bch tetfasa5 attention\n#At=0 CC=1 lm=2 lmo=3 ml=4 mlo=5","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:34.624724Z","iopub.execute_input":"2023-12-09T23:41:34.626094Z","iopub.status.idle":"2023-12-09T23:41:34.680081Z","shell.execute_reply.started":"2023-12-09T23:41:34.626042Z","shell.execute_reply":"2023-12-09T23:41:34.677975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:38.814804Z","iopub.execute_input":"2023-12-09T23:41:38.815253Z","iopub.status.idle":"2023-12-09T23:41:38.846128Z","shell.execute_reply.started":"2023-12-09T23:41:38.815222Z","shell.execute_reply":"2023-12-09T23:41:38.844277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns \n#utiliser le heatmap pour etudier les correlations\nplt.figure(figsize=(8, 8))\n\n#Définission la plage de valeurs à afficher sur la colormap de -1 à 1, et activez l'annotation pour afficher les valeurs de corrélation sur la heatmap.\nheatmap = sns.heatmap(df.corr(), vmin=-1, vmax=1, annot=True)\n\n#titre à la heatmap. La valeur de pad définit la distance entre le titre et le haut de la heatmap.\nheatmap.set_title('Correlation Heatmap', fontdict={'fontsize':12}, pad=12);","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:41.244351Z","iopub.execute_input":"2023-12-09T23:41:41.244858Z","iopub.status.idle":"2023-12-09T23:41:42.321711Z","shell.execute_reply.started":"2023-12-09T23:41:41.24482Z","shell.execute_reply":"2023-12-09T23:41:42.319886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# selon le heatmap on a remarqué que density et cancer n'on pas une relation trés  forte entre eux alors \n# on a decidé de supprimés cette feature car elle contient plus que la moitié de la dataset des valeurs NaN ce qui influence sur les resultat\ndf.drop([\"density\"], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:45.64024Z","iopub.execute_input":"2023-12-09T23:41:45.641564Z","iopub.status.idle":"2023-12-09T23:41:45.677721Z","shell.execute_reply.started":"2023-12-09T23:41:45.641512Z","shell.execute_reply":"2023-12-09T23:41:45.675744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#on a remarqué que les deux classes ne sont pas repartie d'une facon equitable alors on\n# va utiliser le randomOverSampler pour dupliquer des données de la classe minoritaire.\nfrom imblearn.over_sampling import RandomOverSampler\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nx = df[['image_id','laterality','view','age','cancer','biopsy','invasive','BIRADS','implant','difficult_negative_case']]\ny = df['cancer']\n\nx_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.30, random_state=60)\n\n# Instanciez le suréchantillonneur\noversampler = RandomOverSampler(random_state=42)\n\n# Appliquez le suréchantillonnage seulement sur l'ensemble d'entraînement\nx_train_resampled, y_train_resampled = oversampler.fit_resample(x_train, y_train)\n\n# Vérifiez la distribution des classes après le suréchantillonnage\nprint(pd.Series(y_train_resampled).value_counts())\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:48.504974Z","iopub.execute_input":"2023-12-09T23:41:48.505444Z","iopub.status.idle":"2023-12-09T23:41:48.572722Z","shell.execute_reply.started":"2023-12-09T23:41:48.505408Z","shell.execute_reply":"2023-12-09T23:41:48.571027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> ***prépartion des données pour être utilisées dans un processus d'entraînement de modèle***","metadata":{}},{"cell_type":"code","source":"pip install tensorflow","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:41:51.654674Z","iopub.execute_input":"2023-12-09T23:41:51.655167Z","iopub.status.idle":"2023-12-09T23:42:08.260255Z","shell.execute_reply.started":"2023-12-09T23:41:51.655132Z","shell.execute_reply":"2023-12-09T23:42:08.258645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import ResNet50V2\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout\n\n# Créer un modèle de base\nbase_model = ResNet50V2(weights='imagenet', input_shape=(512, 512, 3), include_top=False)\n\n# Geler les poids du modèle de base\nfor layer in base_model.layers:\n    layer.trainable = False\n\n# Créer un modèle séquentiel\nmodel = Sequential()\n\n# Ajouter le modèle de base au modèle séquentiel\nmodel.add(base_model)\n\n# Ajouter une couche de Global Average Pooling\nmodel.add(GlobalAveragePooling2D())\n\n# Ajouter une couche dense avec 128 neurones et une activation ReLU\nmodel.add(Dense(128, activation='relu'))\n\n# Ajouter une couche de Dropout avec un taux de dropout de 0.2\nmodel.add(Dropout(0.2))\n\n# Ajouter la couche de sortie avec un seul neurone et une activation sigmoïde pour une classification binaire\nmodel.add(Dense(1, activation='sigmoid'))\n\n# Compiler le modèle\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:43:30.555051Z","iopub.execute_input":"2023-12-09T23:43:30.555639Z","iopub.status.idle":"2023-12-09T23:43:33.957344Z","shell.execute_reply.started":"2023-12-09T23:43:30.555596Z","shell.execute_reply":"2023-12-09T23:43:33.955584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:43:37.024088Z","iopub.execute_input":"2023-12-09T23:43:37.024601Z","iopub.status.idle":"2023-12-09T23:43:37.082163Z","shell.execute_reply.started":"2023-12-09T23:43:37.024562Z","shell.execute_reply":"2023-12-09T23:43:37.080904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lire csv data\ndf_train = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:43:39.084264Z","iopub.execute_input":"2023-12-09T23:43:39.084774Z","iopub.status.idle":"2023-12-09T23:43:39.193788Z","shell.execute_reply.started":"2023-12-09T23:43:39.084734Z","shell.execute_reply":"2023-12-09T23:43:39.192395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# nombre total des patients\nlen(df_train)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:43:40.954785Z","iopub.execute_input":"2023-12-09T23:43:40.955918Z","iopub.status.idle":"2023-12-09T23:43:40.963994Z","shell.execute_reply.started":"2023-12-09T23:43:40.955867Z","shell.execute_reply":"2023-12-09T23:43:40.962502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Le nombre de cas positifs et négatifs devrait être le même pour créer \n#un ensemble de données équilibré.\nDF_train = df_train.groupby(['cancer']).apply(lambda x: x.sample(1158, replace = True)\n                                              ).reset_index(drop = True)\nprint('New Data Size:', df_train.shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:43:43.594598Z","iopub.execute_input":"2023-12-09T23:43:43.595095Z","iopub.status.idle":"2023-12-09T23:43:43.621088Z","shell.execute_reply.started":"2023-12-09T23:43:43.595047Z","shell.execute_reply":"2023-12-09T23:43:43.61966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generally, invasive cancer is confirmed by biopsy, not by mammography.\n# Maybe it is also extremely difficult for AI to detect invasive cancer from mammography.\ndata = pd.DataFrame(np.concatenate([['Biopsy but Not Malignant'] * len(DF_train[(DF_train['biopsy'] == 1) & (DF_train['cancer'] == 0)]) , ['Malignant Cancer'] *  len(DF_train[DF_train['cancer'] == 1]), ['Invasive Cancer'] *  len(DF_train[(DF_train['cancer'] == 1) & (DF_train['invasive'] == 1)])]), columns = [\"class\"])\n\nsns.countplot(x = 'class', data = data)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:43:45.524495Z","iopub.execute_input":"2023-12-09T23:43:45.525019Z","iopub.status.idle":"2023-12-09T23:43:45.824918Z","shell.execute_reply.started":"2023-12-09T23:43:45.524979Z","shell.execute_reply":"2023-12-09T23:43:45.823775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Chemin des images\nRSNA_512_path = '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_512/train_images_processed_512'","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:43:48.914044Z","iopub.execute_input":"2023-12-09T23:43:48.914837Z","iopub.status.idle":"2023-12-09T23:43:48.921327Z","shell.execute_reply.started":"2023-12-09T23:43:48.914786Z","shell.execute_reply":"2023-12-09T23:43:48.919789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nRSNA_512_path = '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_512/train_images_processed_512/'\n\n# Creation du chemin pour chaque image\nfor i in range(len(DF_train)):\n    patient_id = str(DF_train.loc[i, 'patient_id'])\n    image_id = str(DF_train.loc[i, 'image_id'])\n    image_path = os.path.join(RSNA_512_path, patient_id, image_id + '.png')\n    DF_train.loc[i, 'path'] = image_path\n\nDF_train.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:43:51.524298Z","iopub.execute_input":"2023-12-09T23:43:51.524801Z","iopub.status.idle":"2023-12-09T23:43:52.023347Z","shell.execute_reply.started":"2023-12-09T23:43:51.524761Z","shell.execute_reply":"2023-12-09T23:43:52.021821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# nombre total des patients\nlen(df_train)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:00.534368Z","iopub.execute_input":"2023-12-09T23:44:00.534842Z","iopub.status.idle":"2023-12-09T23:44:00.544257Z","shell.execute_reply.started":"2023-12-09T23:44:00.534804Z","shell.execute_reply":"2023-12-09T23:44:00.541989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train[df_train['cancer'] == 0])","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:10.804592Z","iopub.execute_input":"2023-12-09T23:44:10.805086Z","iopub.status.idle":"2023-12-09T23:44:10.820432Z","shell.execute_reply.started":"2023-12-09T23:44:10.805049Z","shell.execute_reply":"2023-12-09T23:44:10.818747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The number of positive and negative cases should be the same\n# to create a balanced dataset.\nDF_train = DF_train.groupby(['cancer']).apply(lambda x: x.sample(1158, replace = True)\n                                                      ).reset_index(drop = True)\nprint('New Data Size:', DF_train.shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:15.244618Z","iopub.execute_input":"2023-12-09T23:44:15.245026Z","iopub.status.idle":"2023-12-09T23:44:15.264231Z","shell.execute_reply.started":"2023-12-09T23:44:15.244996Z","shell.execute_reply":"2023-12-09T23:44:15.262563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n\nRSNA_512_path = '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_512/train_images_processed_512/'\n\n# Create diestination pour chaque image\nfor i in range(len(DF_train)):\n    patient_id = str(DF_train.loc[i, 'patient_id'])\n    image_id = str(DF_train.loc[i, 'image_id'])\n    image_path = os.path.join(RSNA_512_path, patient_id, image_id + '.png')\n    DF_train.loc[i, 'path'] = image_path\n\nDF_train.head()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:17.365716Z","iopub.execute_input":"2023-12-09T23:44:17.366676Z","iopub.status.idle":"2023-12-09T23:44:18.000704Z","shell.execute_reply.started":"2023-12-09T23:44:17.366631Z","shell.execute_reply":"2023-12-09T23:44:17.999165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Les images normales et celles liées au cancer doivent être réparties de manière équitable.\n\ntrain_df, val_df = train_test_split(DF_train, \n                                   test_size = 0.30, \n                                   random_state = 2018,\n                                   stratify = DF_train[['cancer']])\n\nprint('train', train_df.shape[0], 'validation', val_df.shape[0])\nprint('train', train_df['cancer'].value_counts())\nprint('validation', val_df['cancer'].value_counts())\ntrain_df.sample(1)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:20.954201Z","iopub.execute_input":"2023-12-09T23:44:20.954714Z","iopub.status.idle":"2023-12-09T23:44:21.0067Z","shell.execute_reply.started":"2023-12-09T23:44:20.954673Z","shell.execute_reply":"2023-12-09T23:44:21.005518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training data\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:23.204585Z","iopub.execute_input":"2023-12-09T23:44:23.205004Z","iopub.status.idle":"2023-12-09T23:44:23.229846Z","shell.execute_reply.started":"2023-12-09T23:44:23.204973Z","shell.execute_reply":"2023-12-09T23:44:23.228542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validation data\nval_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:25.994249Z","iopub.execute_input":"2023-12-09T23:44:25.994696Z","iopub.status.idle":"2023-12-09T23:44:26.015839Z","shell.execute_reply.started":"2023-12-09T23:44:25.994661Z","shell.execute_reply":"2023-12-09T23:44:26.014578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Sélectionnement des images normales à partir des données d'entraînement.\ntrain_df_normal = train_df[train_df['cancer'] == 0].reset_index(drop = True)\nprint(len(train_df_normal))\ntrain_df_normal.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:28.174496Z","iopub.execute_input":"2023-12-09T23:44:28.174965Z","iopub.status.idle":"2023-12-09T23:44:28.199346Z","shell.execute_reply.started":"2023-12-09T23:44:28.174928Z","shell.execute_reply":"2023-12-09T23:44:28.198529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sélectionnement des images liées au cancer à partir des données d'entraînement.\ntrain_df_cancer = train_df[train_df['cancer'] == 1].reset_index(drop = True)\nprint(len(train_df_cancer))\ntrain_df_cancer.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:30.404254Z","iopub.execute_input":"2023-12-09T23:44:30.404834Z","iopub.status.idle":"2023-12-09T23:44:30.431852Z","shell.execute_reply.started":"2023-12-09T23:44:30.404787Z","shell.execute_reply":"2023-12-09T23:44:30.430646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sélectionnement les images normales à partir des données de validation.\nval_df_normal = val_df[val_df['cancer'] == 0].reset_index(drop = True)\nprint(len(val_df_normal))\nval_df_normal.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:32.284098Z","iopub.execute_input":"2023-12-09T23:44:32.284585Z","iopub.status.idle":"2023-12-09T23:44:32.316764Z","shell.execute_reply.started":"2023-12-09T23:44:32.284545Z","shell.execute_reply":"2023-12-09T23:44:32.315519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sélectionnement des images liées au cancer à partir des données de validation.\nval_df_cancer = val_df[val_df['cancer'] == 1].reset_index(drop = True)\nprint(len(val_df_cancer))\nval_df_cancer.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:34.264942Z","iopub.execute_input":"2023-12-09T23:44:34.26599Z","iopub.status.idle":"2023-12-09T23:44:34.287169Z","shell.execute_reply.started":"2023-12-09T23:44:34.265953Z","shell.execute_reply":"2023-12-09T23:44:34.286191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***préparer les images pour un traitement ultérieur, utilisant le chemin spécifié dans la dataframe train_df_normal.***","metadata":{}},{"cell_type":"code","source":"import os\nimport shutil\n\n# Definition des destinations\ndestination_dir = '/kaggle/working/train'\ndestination_dir_sub = '/kaggle/working/train/normal'\n\n# Creation de destination si n'existe pas\nif not os.path.exists(destination_dir):\n    os.makedirs(destination_dir)\n\nif not os.path.exists(destination_dir_sub):\n    os.makedirs(destination_dir_sub)   \n    \n# copier les images normales de l'ensemble d'entraînement \nfor path in train_df_normal['path']:\n    shutil.copy2(path, destination_dir_sub)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:36.584563Z","iopub.execute_input":"2023-12-09T23:44:36.584988Z","iopub.status.idle":"2023-12-09T23:44:41.947516Z","shell.execute_reply.started":"2023-12-09T23:44:36.584955Z","shell.execute_reply":"2023-12-09T23:44:41.946467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndestination_dir = '/kaggle/working/train'\ndestination_dir_sub = '/kaggle/working/train/cancer'\n\nif not os.path.exists(destination_dir):\n    os.makedirs(destination_dir)\n\nif not os.path.exists(destination_dir_sub):\n    os.makedirs(destination_dir_sub)   \n    \nfor path in train_df_cancer['path']:\n    shutil.copy2(path, destination_dir_sub)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:47.154625Z","iopub.execute_input":"2023-12-09T23:44:47.156359Z","iopub.status.idle":"2023-12-09T23:44:49.741279Z","shell.execute_reply.started":"2023-12-09T23:44:47.156312Z","shell.execute_reply":"2023-12-09T23:44:49.739906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"destination_dir = '/kaggle/working/val'\ndestination_dir_sub = '/kaggle/working/val/normal'\n\nif not os.path.exists(destination_dir):\n    os.makedirs(destination_dir)\n\nif not os.path.exists(destination_dir_sub):\n    os.makedirs(destination_dir_sub)   \n    \nfor path in val_df_normal['path']:\n    shutil.copy2(path, destination_dir_sub)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T19:01:49.474911Z","iopub.execute_input":"2023-12-09T19:01:49.475378Z","iopub.status.idle":"2023-12-09T19:01:51.071014Z","shell.execute_reply.started":"2023-12-09T19:01:49.475339Z","shell.execute_reply":"2023-12-09T19:01:51.069895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"destination_dir = '/kaggle/working/val'\ndestination_dir_sub = '/kaggle/working/val/cancer'\n\nif not os.path.exists(destination_dir):\n    os.makedirs(destination_dir)\n\nif not os.path.exists(destination_dir_sub):\n    os.makedirs(destination_dir_sub)   \n    \nfor path in val_df_cancer['path']:\n    shutil.copy2(path, destination_dir_sub)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T19:02:08.100154Z","iopub.execute_input":"2023-12-09T19:02:08.101084Z","iopub.status.idle":"2023-12-09T19:02:09.393285Z","shell.execute_reply.started":"2023-12-09T19:02:08.101041Z","shell.execute_reply":"2023-12-09T19:02:09.391915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Sample Images**","metadata":{}},{"cell_type":"code","source":"import glob\nnormal_train_images = glob.glob('/kaggle/working/train/normal/*.png')\ncancer_train_images = glob.glob('/kaggle/working/train/cancer/*.png')","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:53.834833Z","iopub.execute_input":"2023-12-09T23:44:53.835302Z","iopub.status.idle":"2023-12-09T23:44:53.852283Z","shell.execute_reply.started":"2023-12-09T23:44:53.835265Z","shell.execute_reply":"2023-12-09T23:44:53.851042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:56.013787Z","iopub.execute_input":"2023-12-09T23:44:56.014287Z","iopub.status.idle":"2023-12-09T23:44:56.020022Z","shell.execute_reply.started":"2023-12-09T23:44:56.014243Z","shell.execute_reply":"2023-12-09T23:44:56.018786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Voir les images normales de l'ensemble de données d'entraînement.\nfig, axes = plt.subplots(nrows = 2, ncols = 5, figsize = (15, 10), subplot_kw = {'xticks':[], 'yticks':[]})\nfor i, ax in enumerate(axes.flat):\n    img = cv2.imread(normal_train_images[i])\n    ax.imshow(img)\n    ax.set_title('Normal')\nfig.tight_layout()    \n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:44:58.225369Z","iopub.execute_input":"2023-12-09T23:44:58.225841Z","iopub.status.idle":"2023-12-09T23:45:00.558517Z","shell.execute_reply.started":"2023-12-09T23:44:58.225806Z","shell.execute_reply":"2023-12-09T23:45:00.557403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Voir les images ayant cancer de l'ensemble de données d'entraînement.\nfig, axes = plt.subplots(nrows = 2, ncols = 5, figsize = (15, 10), subplot_kw = {'xticks':[], 'yticks':[]})\nfor i, ax in enumerate(axes.flat):\n    img = cv2.imread(cancer_train_images[i])\n    ax.imshow(img)\n    ax.set_title('Cancer')\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:45:04.134029Z","iopub.execute_input":"2023-12-09T23:45:04.134441Z","iopub.status.idle":"2023-12-09T23:45:05.224637Z","shell.execute_reply.started":"2023-12-09T23:45:04.13441Z","shell.execute_reply":"2023-12-09T23:45:05.223493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Création des générateurs d'images.**\nLe jeu de données a déjà été divisé en ensembles d'entraînement et de validation, et chaque ensemble comprend des fichiers d'images normales et liées au cancer. Ainsi, les générateurs d'images ont été facilement créés.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\ntrain_datagen = ImageDataGenerator(rescale = 1./255.,\n                                   zoom_range = 0.2)\nval_datagen = ImageDataGenerator(rescale = 1./255.,)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T19:05:28.300008Z","iopub.execute_input":"2023-12-09T19:05:28.300552Z","iopub.status.idle":"2023-12-09T19:05:28.309362Z","shell.execute_reply.started":"2023-12-09T19:05:28.300509Z","shell.execute_reply":"2023-12-09T19:05:28.308153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '/kaggle/working/train'\nval_path = '/kaggle/working/val'\n\ntrain_generator = train_datagen.flow_from_directory(\n    train_path,\n    target_size = (512, 512),\n    batch_size = 32,\n    class_mode = 'binary'\n)\nvalidation_generator = val_datagen.flow_from_directory(\n        val_path,\n        target_size = (512, 512),\n        batch_size = 16,\n        class_mode = 'binary'\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T19:05:31.559766Z","iopub.execute_input":"2023-12-09T19:05:31.560183Z","iopub.status.idle":"2023-12-09T19:05:31.637373Z","shell.execute_reply.started":"2023-12-09T19:05:31.56015Z","shell.execute_reply":"2023-12-09T19:05:31.636335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Définition du modèle (transfert d'apprentissage) :** \nDans cette étape, ResNet50V2 a été utilisé comme modèle de base pour le transfert d'apprentissage. Les fonctions d'activation, d'optimisation et de perte finales ont été choisies respectivement comme sigmoid, adam et binary cross entropy.","metadata":{}},{"cell_type":"code","source":"base_model = ResNet50V2(weights = 'imagenet', input_shape = (512, 512, 3), include_top = False)\n\nfor layer in base_model.layers:\n    layer.trainable = False\n    \nmodel = Sequential()\nmodel.add(base_model)\nmodel.add(GlobalAveragePooling2D())\nmodel.add(Dense(128, activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(1, activation = 'sigmoid'))\n\nmodel.compile(optimizer = \"adam\", loss = 'binary_crossentropy', metrics = [\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:45:18.054078Z","iopub.execute_input":"2023-12-09T23:45:18.054592Z","iopub.status.idle":"2023-12-09T23:45:20.969828Z","shell.execute_reply.started":"2023-12-09T23:45:18.05455Z","shell.execute_reply":"2023-12-09T23:45:20.968529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:45:22.554273Z","iopub.execute_input":"2023-12-09T23:45:22.554776Z","iopub.status.idle":"2023-12-09T23:45:22.604832Z","shell.execute_reply.started":"2023-12-09T23:45:22.554738Z","shell.execute_reply":"2023-12-09T23:45:22.60353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Entraîner le modèle : \nLe modèle a été entraîné avec les données d'entraînement et de validation, et un arrêt anticipé a été ajouté pour éviter le surajustement.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\ncallback = tf.keras.callbacks.EarlyStopping(monitor = \"val_loss\", mode = \"min\", patience = 4)\n\nhistory = model.fit(train_generator, validation_data = validation_generator, steps_per_epoch = 20, epochs = 15, callbacks = callback)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T19:09:40.080255Z","iopub.execute_input":"2023-12-09T19:09:40.080757Z","iopub.status.idle":"2023-12-09T21:51:19.421792Z","shell.execute_reply.started":"2023-12-09T19:09:40.08072Z","shell.execute_reply":"2023-12-09T21:51:19.419325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Enregistrement du modèle**","metadata":{}},{"cell_type":"code","source":"model.save('mammography_pred_model.keras')","metadata":{"execution":{"iopub.status.busy":"2023-12-09T21:56:08.524387Z","iopub.execute_input":"2023-12-09T21:56:08.525317Z","iopub.status.idle":"2023-12-09T21:56:09.851039Z","shell.execute_reply.started":"2023-12-09T21:56:08.525275Z","shell.execute_reply":"2023-12-09T21:56:09.850117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Métriques du modèle :** \nL'exactitude (accuracy) et la perte (loss) ont été calculées à la fois pour les données d'entraînement et de validation lors du processus d'entraînement du modèle.","metadata":{}},{"cell_type":"code","source":"accuracy = history.history['accuracy']\nval_accuracy = history.history['val_accuracy']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:45:28.234503Z","iopub.execute_input":"2023-12-09T23:45:28.234934Z","iopub.status.idle":"2023-12-09T23:45:28.241951Z","shell.execute_reply.started":"2023-12-09T23:45:28.234902Z","shell.execute_reply":"2023-12-09T23:45:28.240585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Visualisation de l'exactitude et de la perte (ou précision et coût)**","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (15,10))\n\nplt.subplot(2, 2, 1)\nplt.plot(accuracy, label = \"Training Accuracy\")\nplt.plot(val_accuracy, label = \"Validation Accuracy\")\nplt.ylim(0.4, 1)\nplt.legend(['Train', 'Validation'], loc = 'upper left')\nplt.title(\"Training vs Validation Accuracy\")\nplt.xlabel('epoch')\nplt.ylabel('accuracy')\n\n\nplt.subplot(2, 2, 2)\nplt.plot(loss, label = \"Training Loss\")\nplt.plot(val_loss, label = \"Validation Loss\")\nplt.legend(['Train', 'Validation'], loc = 'upper left')\nplt.title(\"Training vs Validation Loss\")\nplt.xlabel('epoch')\nplt.ylabel('loss')","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:45:32.034363Z","iopub.execute_input":"2023-12-09T23:45:32.034849Z","iopub.status.idle":"2023-12-09T23:45:32.745313Z","shell.execute_reply.started":"2023-12-09T23:45:32.034808Z","shell.execute_reply":"2023-12-09T23:45:32.74382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Prédictions :** \nAfin d'évaluer la qualité du modèle entraîné, les résultats ont été prédits à partir des données de test (validation) et comparés avec les valeurs observées.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import load_model #Cette fonction est utilisée pour charger un modèle préalablement sauvegardé.\nmodel = load_model('/kaggle/working/mammography_pred_model.h5') #Chargement du modèle sauvegardé","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:45:35.755249Z","iopub.execute_input":"2023-12-09T23:45:35.755977Z","iopub.status.idle":"2023-12-09T23:45:38.703961Z","shell.execute_reply.started":"2023-12-09T23:45:35.755915Z","shell.execute_reply":"2023-12-09T23:45:38.702802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Une fois que le modèle est chargé, il peut être utilisé pour effectuer des prédictions sur de nouvelles données sans avoir besoin de re-entraîner le modèle.**","metadata":{}},{"cell_type":"code","source":"pred = model.predict(validation_generator)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:45:41.593946Z","iopub.execute_input":"2023-12-09T23:45:41.59436Z","iopub.status.idle":"2023-12-09T23:50:50.37203Z","shell.execute_reply.started":"2023-12-09T23:45:41.594328Z","shell.execute_reply":"2023-12-09T23:50:50.370347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# prédiction\npred","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:51:13.014501Z","iopub.execute_input":"2023-12-09T23:51:13.014954Z","iopub.status.idle":"2023-12-09T23:51:13.035895Z","shell.execute_reply.started":"2023-12-09T23:51:13.014921Z","shell.execute_reply":"2023-12-09T23:51:13.034857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**création d'une liste y_pred à partir des probabilités de prédictions contenues dans la variable pred, pour avoir au final les prédictions binaires finales du modèle, où chaque élément de la liste représente la classe prédite (0 ou 1).**","metadata":{}},{"cell_type":"code","source":"y_pred = []\nfor prob in pred:\n    if prob >= 0.5:\n        y_pred.append(1)\n    else:\n        y_pred.append(0)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:51:21.284319Z","iopub.execute_input":"2023-12-09T23:51:21.284844Z","iopub.status.idle":"2023-12-09T23:51:21.292928Z","shell.execute_reply.started":"2023-12-09T23:51:21.284807Z","shell.execute_reply":"2023-12-09T23:51:21.291863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:51:23.414518Z","iopub.execute_input":"2023-12-09T23:51:23.415184Z","iopub.status.idle":"2023-12-09T23:51:23.421079Z","shell.execute_reply.started":"2023-12-09T23:51:23.415148Z","shell.execute_reply":"2023-12-09T23:51:23.419879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# indiquer le nombre d'occurrences de chaque classe dans les prédictions\npd.Series(y_pred).value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:51:25.69482Z","iopub.execute_input":"2023-12-09T23:51:25.695279Z","iopub.status.idle":"2023-12-09T23:51:25.707988Z","shell.execute_reply.started":"2023-12-09T23:51:25.695243Z","shell.execute_reply":"2023-12-09T23:51:25.706752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Définition de la variable y_true qui contient les classes réelles associées aux exemples dans l'ensemble de données de validation, et elle est utilisée pour comparer les prédictions du modèle (contenues dans la liste y_pred) avec les vraies classes.**","metadata":{}},{"cell_type":"code","source":"y_true = validation_generator.classes","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:51:28.30411Z","iopub.execute_input":"2023-12-09T23:51:28.305162Z","iopub.status.idle":"2023-12-09T23:51:28.311124Z","shell.execute_reply.started":"2023-12-09T23:51:28.305115Z","shell.execute_reply":"2023-12-09T23:51:28.309768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(y_true)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:51:30.52451Z","iopub.execute_input":"2023-12-09T23:51:30.524962Z","iopub.status.idle":"2023-12-09T23:51:30.535193Z","shell.execute_reply.started":"2023-12-09T23:51:30.524926Z","shell.execute_reply":"2023-12-09T23:51:30.53399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Matrice de confusion :**\nUne matrice de confusion a été créée avec les valeurs prédites et observées. La matrice indique presque des prédictions correctes par le modèle entraîné, sauf qu'il y a eu deux cas observés comme faux positifs. Faux positif signifie qu'un cas est en réalité négatif, mais a été prédit comme positif.","metadata":{}},{"cell_type":"code","source":"len(y_true), len(y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:51:32.964187Z","iopub.execute_input":"2023-12-09T23:51:32.964627Z","iopub.status.idle":"2023-12-09T23:51:32.97324Z","shell.execute_reply.started":"2023-12-09T23:51:32.964593Z","shell.execute_reply":"2023-12-09T23:51:32.971903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Definition des noms de classes\nclass_names = ['Normal', 'Cancer']\n\nconfusion_matrix = pd.crosstab(y_true, y_pred, rownames=['Actual'], colnames=['Predicted'])\n\n# Creatation de heatmap avec les noms de classes\ncm = sns.heatmap(confusion_matrix, annot=True, fmt = '.0f', cmap = \"Blues\",\n           xticklabels = class_names, yticklabels = class_names)\n\ncm.set_xlabel(\"Prédiction\")\ncm.set_ylabel(\"Vérité\")","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:51:34.774635Z","iopub.execute_input":"2023-12-09T23:51:34.77508Z","iopub.status.idle":"2023-12-09T23:51:35.128425Z","shell.execute_reply.started":"2023-12-09T23:51:34.775046Z","shell.execute_reply":"2023-12-09T23:51:35.127522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_true, y_pred))","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:51:49.154255Z","iopub.execute_input":"2023-12-09T23:51:49.154756Z","iopub.status.idle":"2023-12-09T23:51:49.176142Z","shell.execute_reply.started":"2023-12-09T23:51:49.154717Z","shell.execute_reply":"2023-12-09T23:51:49.174782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Analyse des Résultats :** \nIl est crucial dans le domaine médical d'analyser les types de cas qui ont été mal classés par le modèle d'IA, car les erreurs de diagnostic médical doivent être évitées autant que possible. Ainsi, il est nécessaire d'identifier les cas de faux positifs et de faux négatifs. La création d'un tableau de données et d'une table de confusion peut visualiser les résultats. Comme discuté précédemment, il est particulièrement important d'éviter les cas de faux négatifs.","metadata":{}},{"cell_type":"code","source":"confusion = []\n\nfor i, j in zip(y_true, y_pred):\n  if i == 0 and j == 0:\n    confusion.append('TN')\n  elif i == 1 and j == 1:\n    confusion.append('TP')\n  elif i == 0 and j == 1:\n    confusion.append('FP')\n  else:\n    confusion.append('FN')","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:52:00.234176Z","iopub.execute_input":"2023-12-09T23:52:00.234657Z","iopub.status.idle":"2023-12-09T23:52:00.247944Z","shell.execute_reply.started":"2023-12-09T23:52:00.234617Z","shell.execute_reply":"2023-12-09T23:52:00.246589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Si la valeur réelle (i) est 0 et la prédiction (j) est également 0, cela signifie que le modèle a correctement prédit une classe négative (TN, True Negative).\n\n* Si la valeur réelle (i) est 1 et la prédiction (j) est également 1, cela signifie que le modèle a correctement prédit une classe positive (TP, True Positive).\n\n* Si la valeur réelle (i) est 0 mais la prédiction (j) est 1, cela signifie que le modèle a mal prédit une classe positive (FP, False Positive).\n\n* Pour tous les autres cas, où la valeur réelle est 1 mais la prédiction est 0, cela signifie que le modèle a mal prédit une classe négative (FN, False Negative).","metadata":{}},{"cell_type":"code","source":"print(confusion)","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:52:04.004333Z","iopub.execute_input":"2023-12-09T23:52:04.005016Z","iopub.status.idle":"2023-12-09T23:52:04.011229Z","shell.execute_reply.started":"2023-12-09T23:52:04.004978Z","shell.execute_reply":"2023-12-09T23:52:04.00956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_table = pd.DataFrame(data = confusion, columns = [\"Results\"])\nconfusion_table","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:52:07.824147Z","iopub.execute_input":"2023-12-09T23:52:07.824634Z","iopub.status.idle":"2023-12-09T23:52:07.840502Z","shell.execute_reply.started":"2023-12-09T23:52:07.824594Z","shell.execute_reply":"2023-12-09T23:52:07.839088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Après une analyse, nous pouvons examiner le nombre total de chaque type d'erreur (FP, FN) et comprendre que le modèle nécessite des améliorations. En particulier, l'attention doit être accordée aux faux négatifs (FN) dans le contexte médical, car cela pourrait signifier que des cas positifs ont été manqués, ce qui pourrait avoir des conséquences graves.","metadata":{}},{"cell_type":"code","source":"confusion_table = pd.DataFrame({'Predicton':y_pred,\n                                'Truth': y_true,\n                                'Results': confusion})\nconfusion_table","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:55:12.604687Z","iopub.execute_input":"2023-12-09T23:55:12.60519Z","iopub.status.idle":"2023-12-09T23:55:12.62405Z","shell.execute_reply.started":"2023-12-09T23:55:12.605156Z","shell.execute_reply":"2023-12-09T23:55:12.622355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**filtrage des lignes de la colonne \"Results\" du DataFrame confusion_table et afficher uniquement les lignes où la valeur est égale à 'FP' (False Positive)**","metadata":{}},{"cell_type":"code","source":"confusion_table.Results == 'FP'","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:55:37.004097Z","iopub.execute_input":"2023-12-09T23:55:37.00458Z","iopub.status.idle":"2023-12-09T23:55:37.017495Z","shell.execute_reply.started":"2023-12-09T23:55:37.004541Z","shell.execute_reply":"2023-12-09T23:55:37.016044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# liste des images \"false positive\"\nFPs = confusion_table[confusion_table['Results'] == 'FP']\nFPs","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:57:32.474466Z","iopub.execute_input":"2023-12-09T23:57:32.475029Z","iopub.status.idle":"2023-12-09T23:57:32.495855Z","shell.execute_reply.started":"2023-12-09T23:57:32.474985Z","shell.execute_reply":"2023-12-09T23:57:32.494161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FPs.index","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:57:43.604273Z","iopub.execute_input":"2023-12-09T23:57:43.605476Z","iopub.status.idle":"2023-12-09T23:57:43.613532Z","shell.execute_reply.started":"2023-12-09T23:57:43.605401Z","shell.execute_reply":"2023-12-09T23:57:43.612519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# liste des images \"false negative\"\nFNs = confusion_table[confusion_table['Results'] == 'FN']\nFNs","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:58:09.314335Z","iopub.execute_input":"2023-12-09T23:58:09.315644Z","iopub.status.idle":"2023-12-09T23:58:09.331141Z","shell.execute_reply.started":"2023-12-09T23:58:09.315597Z","shell.execute_reply":"2023-12-09T23:58:09.330246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FNs.index","metadata":{"execution":{"iopub.status.busy":"2023-12-09T23:58:19.844471Z","iopub.execute_input":"2023-12-09T23:58:19.844912Z","iopub.status.idle":"2023-12-09T23:58:19.854245Z","shell.execute_reply.started":"2023-12-09T23:58:19.844879Z","shell.execute_reply":"2023-12-09T23:58:19.852812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **Cas de Mauvaise Classification :** \nIl est important de repérer les cas incorrects identifiés par l'IA et d'analyser pourquoi l'IA a fait des jugements erronés pour ces images.","metadata":{}},{"cell_type":"code","source":"import glob\nval_images = glob.glob('/kaggle/working/val/*/*.png')\n#récupérer la liste de tous les fichiers PNG présents dans le répertoire '/kaggle/working/val/","metadata":{"execution":{"iopub.status.busy":"2023-12-10T00:01:00.494232Z","iopub.execute_input":"2023-12-10T00:01:00.495384Z","iopub.status.idle":"2023-12-10T00:01:00.504498Z","shell.execute_reply.started":"2023-12-10T00:01:00.495334Z","shell.execute_reply":"2023-12-10T00:01:00.503206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#afficher un sous-ensemble d'images correspondant aux cas de faux positifs (FP) dans notre ensemble de validation.\nfig, axes = plt.subplots(nrows = 2, ncols = 5, figsize = (15, 10), subplot_kw = {'xticks':[], 'yticks':[]})\nfor i, ax in zip(FPs.index, axes.flat):\n    img = cv2.imread(val_images[i])\n    ax.imshow(img)\n    ax.set_title(\"False Positive Case\")\nfig.tight_layout()    \n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-10T00:01:46.604336Z","iopub.execute_input":"2023-12-10T00:01:46.604855Z","iopub.status.idle":"2023-12-10T00:01:48.032296Z","shell.execute_reply.started":"2023-12-10T00:01:46.604817Z","shell.execute_reply":"2023-12-10T00:01:48.03073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#afficher un sous-ensemble d'images correspondant aux cas de faux négatifs (FN) \nfig, axes = plt.subplots(nrows = 2, ncols = 5, figsize = (15, 10), subplot_kw = {'xticks':[], 'yticks':[]})\nfor i, ax in zip(FNs.index, axes.flat):\n    img = cv2.imread(val_images[i])\n    ax.imshow(img)\n    ax.set_title(\"False Negative Case\")\nfig.tight_layout()    \n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-10T00:02:24.774953Z","iopub.execute_input":"2023-12-10T00:02:24.775418Z","iopub.status.idle":"2023-12-10T00:02:26.188406Z","shell.execute_reply.started":"2023-12-10T00:02:24.775373Z","shell.execute_reply":"2023-12-10T00:02:26.187218Z"},"trusted":true},"execution_count":null,"outputs":[]}]}