{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"}],"dockerImageVersionId":30615,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Dependencies","metadata":{}},{"cell_type":"code","source":"import os\nimport PIL\nimport pathlib\nimport cv2\nimport pickle\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout\nimport tensorflow_hub as hub\nfrom tensorflow.keras.callbacks import TensorBoard\n\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom keras.preprocessing.image import ImageDataGenerator\n\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\n\nfrom sklearn.metrics import classification_report, confusion_matrix, ConfusionMatrixDisplay\n\nfrom skimage import io\nfrom skimage.color import rgb2gray\nfrom skimage.transform import rescale, resize, downscale_local_mean\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-08T13:23:39.813716Z","iopub.execute_input":"2023-12-08T13:23:39.814605Z","iopub.status.idle":"2023-12-08T13:23:58.54132Z","shell.execute_reply.started":"2023-12-08T13:23:39.81456Z","shell.execute_reply":"2023-12-08T13:23:58.539929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualising the Dataset","metadata":{}},{"cell_type":"code","source":"class Config():\n    is_submission = False\n    train_thumb_path = \"/kaggle/input/UBC-OCEAN/train_thumbnails\"\n    test_thumb_path = \"/kaggle/input/UBC-OCEAN/test_thumbnails\"\n    test_csv = \"/kaggle/input/UBC-OCEAN/test.csv\"\n    train_csv = \"/kaggle/input/UBC-OCEAN/train.csv\"\n    \n\nconfig = Config()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:58.544135Z","iopub.execute_input":"2023-12-08T13:23:58.545284Z","iopub.status.idle":"2023-12-08T13:23:58.553555Z","shell.execute_reply.started":"2023-12-08T13:23:58.545232Z","shell.execute_reply":"2023-12-08T13:23:58.551011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(config.train_csv)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:58.555524Z","iopub.execute_input":"2023-12-08T13:23:58.55639Z","iopub.status.idle":"2023-12-08T13:23:58.612576Z","shell.execute_reply.started":"2023-12-08T13:23:58.556341Z","shell.execute_reply":"2023-12-08T13:23:58.611333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:58.616464Z","iopub.execute_input":"2023-12-08T13:23:58.617478Z","iopub.status.idle":"2023-12-08T13:23:58.64543Z","shell.execute_reply.started":"2023-12-08T13:23:58.617429Z","shell.execute_reply":"2023-12-08T13:23:58.644162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:58.647463Z","iopub.execute_input":"2023-12-08T13:23:58.647953Z","iopub.status.idle":"2023-12-08T13:23:58.655659Z","shell.execute_reply.started":"2023-12-08T13:23:58.647908Z","shell.execute_reply":"2023-12-08T13:23:58.654494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some images do not have images in the train_thumbnail directory.","metadata":{}},{"cell_type":"code","source":"df = df[df[\"is_tma\"] == False]\ndf.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:58.657228Z","iopub.execute_input":"2023-12-08T13:23:58.658304Z","iopub.status.idle":"2023-12-08T13:23:58.675912Z","shell.execute_reply.started":"2023-12-08T13:23:58.658261Z","shell.execute_reply":"2023-12-08T13:23:58.674748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = df.shape[0]\nunique_images = df['image_id'].nunique()\nlabel = df['label'].unique()\n\nprint(\"number of rows : \",rows)\nprint(\"number of images : \",unique_images)\nprint(\"labels: \",label)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:58.679848Z","iopub.execute_input":"2023-12-08T13:23:58.686468Z","iopub.status.idle":"2023-12-08T13:23:58.708144Z","shell.execute_reply.started":"2023-12-08T13:23:58.686408Z","shell.execute_reply":"2023-12-08T13:23:58.706121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:58.709979Z","iopub.execute_input":"2023-12-08T13:23:58.713155Z","iopub.status.idle":"2023-12-08T13:23:58.732584Z","shell.execute_reply.started":"2023-12-08T13:23:58.713108Z","shell.execute_reply":"2023-12-08T13:23:58.73105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.countplot(data=df, x='label', order=df['label'].value_counts().index)\nplt.title('Distribution of Target Classes')\nplt.xlabel('Label')\nplt.ylabel('Count')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:58.734817Z","iopub.execute_input":"2023-12-08T13:23:58.73576Z","iopub.status.idle":"2023-12-08T13:23:59.083997Z","shell.execute_reply.started":"2023-12-08T13:23:58.735711Z","shell.execute_reply":"2023-12-08T13:23:59.082713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HGSC = df[df['label']==\"HGSC\"]\nEC = df[df['label']==\"EC\"]\nCC = df[df['label']==\"CC\"]\nLGSC = df[df['label']==\"LGSC\"]\nMC = df[df['label']==\"MC\"]","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:59.090115Z","iopub.execute_input":"2023-12-08T13:23:59.090894Z","iopub.status.idle":"2023-12-08T13:23:59.102012Z","shell.execute_reply.started":"2023-12-08T13:23:59.090821Z","shell.execute_reply":"2023-12-08T13:23:59.101001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20, 6))\n\nplt.rcParams['font.size'] = 14\n\n# Set the colors\ncolors = ['red', 'lightblue', 'green','magenta', 'yellow']\n\n# Plot the pie chart for the training set\nplt.subplot(1, 1, 1)\nplt.pie([len(HGSC), len(EC), len(CC), len(LGSC), len(MC)], labels=['HGSC', 'EC', 'CC', 'LGSC', 'MC'], autopct='%1.1f%%', colors=colors)\nplt.title('Training Set')\n\n\n# Add a main title to the figure\nplt.suptitle('Distribution of HGSC, EC, CC, LGSC and MC Images in the Training data', fontsize=20, y=1.05)\n\n# Show the plot\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:59.10349Z","iopub.execute_input":"2023-12-08T13:23:59.104171Z","iopub.status.idle":"2023-12-08T13:23:59.369377Z","shell.execute_reply.started":"2023-12-08T13:23:59.104137Z","shell.execute_reply":"2023-12-08T13:23:59.368194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising the images\nio.imshow('/kaggle/input/UBC-OCEAN/train_thumbnails/4_thumbnail.png')","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:23:59.371168Z","iopub.execute_input":"2023-12-08T13:23:59.371984Z","iopub.status.idle":"2023-12-08T13:24:01.316293Z","shell.execute_reply.started":"2023-12-08T13:23:59.371935Z","shell.execute_reply":"2023-12-08T13:24:01.315114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATA PIPELINE","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/UBC-OCEAN/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:24:01.318087Z","iopub.execute_input":"2023-12-08T13:24:01.318794Z","iopub.status.idle":"2023-12-08T13:24:01.329456Z","shell.execute_reply.started":"2023-12-08T13:24:01.318742Z","shell.execute_reply":"2023-12-08T13:24:01.328136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:24:01.331343Z","iopub.execute_input":"2023-12-08T13:24:01.332228Z","iopub.status.idle":"2023-12-08T13:24:01.348336Z","shell.execute_reply.started":"2023-12-08T13:24:01.332175Z","shell.execute_reply":"2023-12-08T13:24:01.346932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label = df[\"label\"].unique().tolist()\nlabel","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:24:01.350174Z","iopub.execute_input":"2023-12-08T13:24:01.351014Z","iopub.status.idle":"2023-12-08T13:24:01.36097Z","shell.execute_reply.started":"2023-12-08T13:24:01.350965Z","shell.execute_reply":"2023-12-08T13:24:01.360165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = []\nlabels = []\nfor (i, row) in tqdm(df.iterrows(), total = 513):\n    if row['is_tma']:\n        continue\n    img = io.imread(os.path.join(config.train_thumb_path, str(row['image_id'])+\"_thumbnail.png\"))\n    img = resize(img, (224,224), anti_aliasing=False)\n    img = img.reshape(224,224,3)\n    images.append(img)\n    labels.append(row['label'])","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:25:19.007372Z","iopub.execute_input":"2023-12-08T13:25:19.007766Z","iopub.status.idle":"2023-12-08T13:28:27.458897Z","shell.execute_reply.started":"2023-12-08T13:25:19.007731Z","shell.execute_reply":"2023-12-08T13:28:27.457698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = np.array(images)\nlabels = np.array(labels)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:28:38.124952Z","iopub.execute_input":"2023-12-08T13:28:38.125522Z","iopub.status.idle":"2023-12-08T13:28:38.351673Z","shell.execute_reply.started":"2023-12-08T13:28:38.125475Z","shell.execute_reply":"2023-12-08T13:28:38.350639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = images.copy()\n\nenc = OneHotEncoder(sparse_output = False)\nlabel_encoding = enc.fit_transform(labels.reshape(-1,1),)\nY = label_encoding","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:28:38.353701Z","iopub.execute_input":"2023-12-08T13:28:38.354122Z","iopub.status.idle":"2023-12-08T13:28:38.668343Z","shell.execute_reply.started":"2023-12-08T13:28:38.354087Z","shell.execute_reply":"2023-12-08T13:28:38.667246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.8)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:28:39.378944Z","iopub.execute_input":"2023-12-08T13:28:39.37939Z","iopub.status.idle":"2023-12-08T13:28:39.617273Z","shell.execute_reply.started":"2023-12-08T13:28:39.379354Z","shell.execute_reply":"2023-12-08T13:28:39.616036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y","metadata":{"execution":{"iopub.status.busy":"2023-12-08T13:28:49.681961Z","iopub.execute_input":"2023-12-08T13:28:49.682497Z","iopub.status.idle":"2023-12-08T13:28:49.691183Z","shell.execute_reply.started":"2023-12-08T13:28:49.68245Z","shell.execute_reply":"2023-12-08T13:28:49.689611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}