{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom sklearn.metrics import classification_report , confusion_matrix , accuracy_score , auc\nfrom sklearn.model_selection import train_test_split\nfrom skimage import io\nfrom skimage.color import rgb2gray\nfrom skimage.transform import resize\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential\nfrom keras.layers import Input, Dense,Conv2D , MaxPooling2D, Flatten,BatchNormalization,Dropout\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.preprocessing import image_dataset_from_directory\nimport cv2\nfrom PIL import Image \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-12T14:33:07.20609Z","iopub.execute_input":"2023-10-12T14:33:07.207165Z","iopub.status.idle":"2023-10-12T14:33:20.835204Z","shell.execute_reply.started":"2023-10-12T14:33:07.207115Z","shell.execute_reply":"2023-10-12T14:33:20.833964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df= pd.read_csv('/kaggle/input/UBC-OCEAN/train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:20.83727Z","iopub.execute_input":"2023-10-12T14:33:20.838177Z","iopub.status.idle":"2023-10-12T14:33:20.880347Z","shell.execute_reply.started":"2023-10-12T14:33:20.838139Z","shell.execute_reply":"2023-10-12T14:33:20.878866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:20.882019Z","iopub.execute_input":"2023-10-12T14:33:20.88309Z","iopub.status.idle":"2023-10-12T14:33:20.924484Z","shell.execute_reply.started":"2023-10-12T14:33:20.883051Z","shell.execute_reply":"2023-10-12T14:33:20.923573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.nunique()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:20.926376Z","iopub.execute_input":"2023-10-12T14:33:20.927035Z","iopub.status.idle":"2023-10-12T14:33:20.943803Z","shell.execute_reply.started":"2023-10-12T14:33:20.927003Z","shell.execute_reply":"2023-10-12T14:33:20.942481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:20.945772Z","iopub.execute_input":"2023-10-12T14:33:20.946533Z","iopub.status.idle":"2023-10-12T14:33:20.96051Z","shell.execute_reply.started":"2023-10-12T14:33:20.946488Z","shell.execute_reply":"2023-10-12T14:33:20.958926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:20.963108Z","iopub.execute_input":"2023-10-12T14:33:20.963434Z","iopub.status.idle":"2023-10-12T14:33:20.989234Z","shell.execute_reply.started":"2023-10-12T14:33:20.963407Z","shell.execute_reply":"2023-10-12T14:33:20.98773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"colors_list = ['yellowgreen', 'lightcoral', 'lightskyblue', 'lightgreen', 'pink']\nexplode_list = [0, 0, 0, 0.1, 0.1] \n\ntrain_df['label'].value_counts().plot(kind='pie',autopct=\"%.1f%%\", shadow= True,\n                                     colors= colors_list, explode= explode_list)\nplt.title(\"Ovarian Cancer Types Distributions\")\nplt.legend(loc= 'upper left', fontsize = 7)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:20.992077Z","iopub.execute_input":"2023-10-12T14:33:20.992643Z","iopub.status.idle":"2023-10-12T14:33:21.258977Z","shell.execute_reply.started":"2023-10-12T14:33:20.992558Z","shell.execute_reply":"2023-10-12T14:33:21.257506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Detecting Outliers from the data","metadata":{}},{"cell_type":"code","source":"train_df.plot(kind='box', figsize=(8, 6))\nplt.title('Defining the outliers')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:21.261316Z","iopub.execute_input":"2023-10-12T14:33:21.26306Z","iopub.status.idle":"2023-10-12T14:33:21.553915Z","shell.execute_reply.started":"2023-10-12T14:33:21.262988Z","shell.execute_reply":"2023-10-12T14:33:21.552805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_image_path = \"/kaggle/input/UBC-OCEAN/train_images\"\ntest_image_path = \"/kaggle/input/UBC-OCEAN/test_images\"\ntrain_image = os.listdir(train_image_path)\ntest_image = os.listdir(test_image_path)\n\nprint(len(train_image))\nprint(len(test_image))","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:21.55528Z","iopub.execute_input":"2023-10-12T14:33:21.555604Z","iopub.status.idle":"2023-10-12T14:33:21.624155Z","shell.execute_reply.started":"2023-10-12T14:33:21.555577Z","shell.execute_reply":"2023-10-12T14:33:21.622895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_thumb_path= '/kaggle/input/UBC-OCEAN/train_thumbnails'\ntest_thumb_path= '/kaggle/input/UBC-OCEAN/test_thumbnails'\ntrain_thumb_image= os.listdir(train_thumb_path)\ntest_thumb_image= os.listdir(test_thumb_path)\n\nprint(len(train_thumb_image))\nprint(len(test_thumb_image))","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:21.627579Z","iopub.execute_input":"2023-10-12T14:33:21.627918Z","iopub.status.idle":"2023-10-12T14:33:21.675858Z","shell.execute_reply.started":"2023-10-12T14:33:21.627893Z","shell.execute_reply":"2023-10-12T14:33:21.674663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['is_tma'].value_counts().reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:21.677208Z","iopub.execute_input":"2023-10-12T14:33:21.677585Z","iopub.status.idle":"2023-10-12T14:33:21.689596Z","shell.execute_reply.started":"2023-10-12T14:33:21.677556Z","shell.execute_reply":"2023-10-12T14:33:21.688758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The data for detected Cancer= True & Not detected Cancer= False  ","metadata":{}},{"cell_type":"code","source":"dec_cancer= train_df[train_df['is_tma']== True]\ndec_cancer","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:21.690812Z","iopub.execute_input":"2023-10-12T14:33:21.691851Z","iopub.status.idle":"2023-10-12T14:33:21.711994Z","shell.execute_reply.started":"2023-10-12T14:33:21.691816Z","shell.execute_reply":"2023-10-12T14:33:21.710119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"not_dec_cancer= train_df[train_df['is_tma']== False]\nnot_dec_cancer","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:21.71503Z","iopub.execute_input":"2023-10-12T14:33:21.716096Z","iopub.status.idle":"2023-10-12T14:33:21.739137Z","shell.execute_reply.started":"2023-10-12T14:33:21.71605Z","shell.execute_reply":"2023-10-12T14:33:21.737906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"not_dec_cancer['image_id_path'] = not_dec_cancer['image_id'].apply(lambda x: f\"{x}_thumbnail.png\")\nnot_dec_cancer","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:21.740779Z","iopub.execute_input":"2023-10-12T14:33:21.741484Z","iopub.status.idle":"2023-10-12T14:33:21.768276Z","shell.execute_reply.started":"2023-10-12T14:33:21.741419Z","shell.execute_reply":"2023-10-12T14:33:21.767044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dec_cancer['image_id_path'] = dec_cancer['image_id'].apply(lambda x: f\"{x}_thumbnail.png\")\ndec_cancer\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:21.770338Z","iopub.execute_input":"2023-10-12T14:33:21.771163Z","iopub.status.idle":"2023-10-12T14:33:21.795052Z","shell.execute_reply.started":"2023-10-12T14:33:21.771118Z","shell.execute_reply":"2023-10-12T14:33:21.793601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Data Processing\n- ","metadata":{}},{"cell_type":"markdown","source":"### Thumbnail Data Images","metadata":{}},{"cell_type":"code","source":"train_images_ = train_image\ntrain_images_[:10]","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:21.797006Z","iopub.execute_input":"2023-10-12T14:33:21.797853Z","iopub.status.idle":"2023-10-12T14:33:21.806762Z","shell.execute_reply.started":"2023-10-12T14:33:21.797801Z","shell.execute_reply":"2023-10-12T14:33:21.805705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['HGSC', 'LGSC', 'EC', 'CC', 'MC']\nfor lab in labels:\n    imgs_with_label = dec_cancer[dec_cancer.label==lab]\n    img_ids = list(imgs_with_label[imgs_with_label.is_tma==True].image_id)\n    figure = plt.figure(figsize = (22,6))\n    i = 0\n    for img_id in img_ids:\n        ax = figure.add_subplot(1,5,i+1)\n        try:\n            io.imshow(f'/kaggle/input/UBC-OCEAN/train_images/{img_id}.png')\n            ax.set_title(f'Image Id:{img_id}, label:{lab}', fontsize=14)\n            plt.tick_params(labelbottom=False, labelleft=False, labelright=False, labeltop=False, \n                            bottom=False, left=False, right=False, top=False)\n            i= i+1\n        except:\n            print(f'Thumbnail #{img_id} does not exist.')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T14:33:21.809599Z","iopub.execute_input":"2023-10-12T14:33:21.811009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_thumbnails_ = train_thumb_image\ntrain_thumbnails_[:5]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,20))\nthumb_path = \"/kaggle/input/UBC-OCEAN/train_thumbnails\"\nj=1\nfor img, lb in zip(not_dec_cancer['image_id_path'][:20],not_dec_cancer['label'][:20]):\n    plt.subplot(6,4,j)\n    path = os.path.join(\"/kaggle/input/UBC-OCEAN/train_thumbnails/\",img)\n    image = plt.imread(path)\n    image = plt.imshow(image)\n    plt.title(f\"Label:{lb}\")\n    j+=1   \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.pairplot(train_df,vars=['image_width', 'image_height', 'image_id'], hue='is_tma')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = [int(file.split('/')[-1].split('.')[0]) for file in os.listdir('/kaggle/input/UBC-OCEAN/train_images')]\ntrain_thumbnails = [int(file.split('/')[-1].split('_')[0]) for file in os.listdir('/kaggle/input/UBC-OCEAN/train_thumbnails')]\ntrain_images.sort()\ntrain_thumbnails.sort()\n\nprint('Number of train images:', len(train_images))\nprint('Number of train thumbnails:', len(train_thumbnails))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_thumbnail = cv2.imread('/kaggle/input/UBC-OCEAN/test_thumbnails/41_thumbnail.png')\nplt.imshow(test_thumbnail)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['image_width'].min())\nprint(train_df['image_width'].max())\nprint(train_df['image_width'].mean())\nprint(train_df['image_height'].min())\nprint(train_df['image_height'].max())\nprint(train_df['image_height'].mean())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_csv(\"/kaggle/input/UBC-OCEAN/test.csv\")\ntest_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}