{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#  Exploratory Data Analysis (EDA) Notebook \nThis notebook provides a basic framework for evaluating Exploratory Data Analysis (EDA) skills using Python. Students must execute each cell and pass the associated tests to proceed.","metadata":{}},{"cell_type":"code","source":"\n\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nimport plotly.offline as pyo\nfrom IPython.display import HTML\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:18.339926Z","iopub.execute_input":"2025-01-05T18:25:18.340302Z","iopub.status.idle":"2025-01-05T18:25:18.349451Z","shell.execute_reply.started":"2025-01-05T18:25:18.340275Z","shell.execute_reply":"2025-01-05T18:25:18.347456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_csv_path = \"/kaggle/input/UBC-OCEAN/train.csv\"\ntrain_images_path = \"/kaggle/input/UBC-OCEAN/train_images\"\ntrain_thumbnails_path = \"/kaggle/input/UBC-OCEAN/train_thumbnails\"\ntest_csv_path = \"/kaggle/input/UBC-OCEAN/test.csv\"\ntest_images_path = \"/kaggle/input/UBC-OCEAN/test_images\"\ntest_thumbnails_path = \"/kaggle/input/UBC-OCEAN/test_images\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:18.659899Z","iopub.execute_input":"2025-01-05T18:25:18.660269Z","iopub.status.idle":"2025-01-05T18:25:18.666155Z","shell.execute_reply.started":"2025-01-05T18:25:18.660239Z","shell.execute_reply":"2025-01-05T18:25:18.66459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(train_csv_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:19.694805Z","iopub.execute_input":"2025-01-05T18:25:19.6952Z","iopub.status.idle":"2025-01-05T18:25:19.70467Z","shell.execute_reply.started":"2025-01-05T18:25:19.69517Z","shell.execute_reply":"2025-01-05T18:25:19.703532Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndf.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:19.73969Z","iopub.execute_input":"2025-01-05T18:25:19.740082Z","iopub.status.idle":"2025-01-05T18:25:19.751141Z","shell.execute_reply.started":"2025-01-05T18:25:19.740055Z","shell.execute_reply":"2025-01-05T18:25:19.749991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Verify dataset size\nprint(\"Size of the dataset: \", df.shape[0])\nassert df.shape[0] > 0, \"Dataset size must be greater than 0\"","metadata":{"nbgrader":{"grade":true,"grade_id":"dataset_size","locked":true,"points":1,"solution":false},"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:21.594987Z","iopub.execute_input":"2025-01-05T18:25:21.595459Z","iopub.status.idle":"2025-01-05T18:25:21.602427Z","shell.execute_reply.started":"2025-01-05T18:25:21.595422Z","shell.execute_reply":"2025-01-05T18:25:21.60098Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Visual Exploration\n### Random Image Plotting","metadata":{}},{"cell_type":"code","source":"def plot_images(folder_path: str, resize: bool = False):\n    # Get a list of image file names in the folder\n    image_files = [f for f in os.listdir(folder_path) if f.endswith(('.jpg', '.jpeg', '.png', '.gif'))]\n    num_images_to_plot = 6\n    selected_images = random.sample(image_files, num_images_to_plot)\n    fig, axes = plt.subplots(2, 3, figsize=(12, 8))\n    for i, ax in enumerate(axes.flat):\n        if i < num_images_to_plot:\n            image_path = os.path.join(folder_path, selected_images[i])\n            img = Image.open(image_path)\n            if resize:\n                img = img.resize((512,512))\n            img = np.array(img)\n            ax.imshow(img)\n            ax.set_title(selected_images[i])\n            ax.axis('off')\n    plt.tight_layout()\n    plt.show()\n    \n\n# Test the function\nplot_images('/kaggle/input/UBC-OCEAN/train_thumbnails', resize=True)","metadata":{"nbgrader":{"grade":true,"grade_id":"plot_images","locked":true,"points":2,"solution":false},"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:23.179118Z","iopub.execute_input":"2025-01-05T18:25:23.179543Z","iopub.status.idle":"2025-01-05T18:25:26.152482Z","shell.execute_reply.started":"2025-01-05T18:25:23.179508Z","shell.execute_reply":"2025-01-05T18:25:26.151245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Label Distribution","metadata":{}},{"cell_type":"code","source":"label_df = pd.DataFrame(df['label'].value_counts())\nlabel_df.reset_index(inplace=True)\nplt.bar(label_df['label'], label_df['count'], color='skyblue')\nplt.xlabel('Labels')\nplt.ylabel('Count')\nplt.title('Label Distribution')\nplt.show()\nassert len(label_df) > 0, \"Label distribution must be displayed correctly\"","metadata":{"nbgrader":{"grade":true,"grade_id":"label_distribution","locked":true,"points":3,"solution":false},"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:26.153699Z","iopub.execute_input":"2025-01-05T18:25:26.153961Z","iopub.status.idle":"2025-01-05T18:25:26.375668Z","shell.execute_reply.started":"2025-01-05T18:25:26.153939Z","shell.execute_reply":"2025-01-05T18:25:26.374669Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Class-wise Analysis","metadata":{}},{"cell_type":"code","source":"HGSC = df[df['label']==\"HGSC\"]\nEC = df[df['label']==\"EC\"]\nCC = df[df['label']==\"CC\"]\nLGSC = df[df['label']==\"LGSC\"]\nMC = df[df['label']==\"MC\"]\n\nplt.figure(figsize=(20, 6))\nplt.rcParams['font.size'] = 14\ncolors = ['red', 'lightblue', 'green','magenta', 'yellow']\nplt.pie([len(HGSC), len(EC), len(CC), len(LGSC), len(MC)], \n        labels=['HGSC', 'EC', 'CC', 'LGSC', 'MC'], autopct='%1.1f%%', colors=colors)\nplt.title('Training Set')\nplt.show()\nassert sum([len(HGSC), len(EC), len(CC), len(LGSC), len(MC)]) == len(df), \"Pie chart proportions must match dataset size\"","metadata":{"nbgrader":{"grade":true,"grade_id":"class_analysis","locked":true,"points":4,"solution":false},"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:26.377857Z","iopub.execute_input":"2025-01-05T18:25:26.378136Z","iopub.status.idle":"2025-01-05T18:25:26.548541Z","shell.execute_reply.started":"2025-01-05T18:25:26.378112Z","shell.execute_reply":"2025-01-05T18:25:26.546769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nsns.histplot(data=df, x='label', kde=True, color='blue')\nplt.title('Label Distribution with Seaborn')\nplt.show()\nassert not df['label'].isnull().any(), \"Ensure there are no missing values in the label column\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:26.551156Z","iopub.execute_input":"2025-01-05T18:25:26.551768Z","iopub.status.idle":"2025-01-05T18:25:26.844656Z","shell.execute_reply.started":"2025-01-05T18:25:26.551715Z","shell.execute_reply":"2025-01-05T18:25:26.843448Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nHGSC = df[df['label'] == \"HGSC\"]\nEC = df[df['label'] == \"EC\"]\nCC = df[df['label'] == \"CC\"]\nLGSC = df[df['label'] == \"LGSC\"]\nMC = df[df['label'] == \"MC\"]\n\n# Correction: Adjusted figure size for better pie chart display\nplt.figure(figsize=(10, 10))  # Changed from (20, 6) to (10, 10) for a balanced layout\n\n# Set font size for readability\nplt.rcParams['font.size'] = 14\n\n# Create pie chart with appropriate colors and labels\ncolors = ['red', 'lightblue', 'green', 'magenta', 'yellow']\nplt.pie([len(HGSC), len(EC), len(CC), len(LGSC), len(MC)], \n        labels=['HGSC', 'EC', 'CC', 'LGSC', 'MC'], autopct='%1.1f%%', colors=colors)\nplt.title('Training Set')  # Set chart title\nplt.show()\n\n# Assertion to ensure the pie chart proportions match the dataset size\nassert sum([len(HGSC), len(EC), len(CC), len(LGSC), len(MC)]) == len(df), \"Pie chart proportions must match dataset size\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:26.845865Z","iopub.execute_input":"2025-01-05T18:25:26.846182Z","iopub.status.idle":"2025-01-05T18:25:27.061591Z","shell.execute_reply.started":"2025-01-05T18:25:26.846155Z","shell.execute_reply":"2025-01-05T18:25:27.060548Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Seaborn Visualization\n### Distribution Plot","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Check for missing values in the 'label' column\nif df['label'].isnull().any():\n    raise ValueError(\"Ensure there are no missing values in the 'label' column\")\n\n# Create the histogram plot\nsns.histplot(x='label', data=df, kde=True, color='green')\nplt.title('Label Distribution with Seaborn')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:27.06243Z","iopub.execute_input":"2025-01-05T18:25:27.062681Z","iopub.status.idle":"2025-01-05T18:25:27.343955Z","shell.execute_reply.started":"2025-01-05T18:25:27.062659Z","shell.execute_reply":"2025-01-05T18:25:27.342868Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Correlation Matrix","metadata":{}},{"cell_type":"code","source":"numeric_df = df.select_dtypes(include=['number'])\ncorrelation_matrix = numeric_df.corr()\nsns.heatmap(correlation_matrix, annot=True, cmap='Reds_r')\nplt.title('Correlation Matrix')\nplt.show()\nassert correlation_matrix.shape[0] > 0, \"Correlation matrix must be generated\"\n","metadata":{"nbgrader":{"grade":true,"grade_id":"correlation_matrix","locked":true,"points":3,"solution":false},"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:27.345044Z","iopub.execute_input":"2025-01-05T18:25:27.34538Z","iopub.status.idle":"2025-01-05T18:25:27.583421Z","shell.execute_reply.started":"2025-01-05T18:25:27.345343Z","shell.execute_reply":"2025-01-05T18:25:27.582359Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Scatter Plot","metadata":{}},{"cell_type":"code","source":"sns.scatterplot(data=df, x='image_id', y='is_tma', hue='label', palette='Blues')\nplt.title('Scatter Plot of image_id vs is_tma')\nplt.show()\nassert 'image_id' in df.columns and 'is_tma' in df.columns, \"Ensure 'image_id' and 'is_tma' columns exist in the dataset\"","metadata":{"nbgrader":{"grade":true,"grade_id":"scatter_plot","locked":true,"points":3,"solution":false},"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:27.586213Z","iopub.execute_input":"2025-01-05T18:25:27.586599Z","iopub.status.idle":"2025-01-05T18:25:28.016604Z","shell.execute_reply.started":"2025-01-05T18:25:27.586566Z","shell.execute_reply":"2025-01-05T18:25:28.015173Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Pair Plot","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Make sure the columns 'image_width' and 'image_height' exist\nassert 'image_width' in df.columns and 'image_height' in df.columns, \"Ensure 'image_width' and 'image_height' columns exist in the dataset\"\n\n# Create a scatter plot\nsns.scatterplot(data=df, x='image_width', y='image_height', hue='label', palette='Set2')\n\n# Add a title and display the plot\nplt.title('Scatter Plot of Image Width vs Image Height')\nplt.show()","metadata":{"nbgrader":{"grade":true,"grade_id":"pair_plot","locked":true,"points":3,"solution":false},"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:28.01791Z","iopub.execute_input":"2025-01-05T18:25:28.018548Z","iopub.status.idle":"2025-01-05T18:25:28.376562Z","shell.execute_reply.started":"2025-01-05T18:25:28.018509Z","shell.execute_reply":"2025-01-05T18:25:28.375502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Scatter plot of 'image_width' vs 'image_height'\nsns.scatterplot(x='image_width', y='image_height', hue='label', data=df, palette='Set2')\nplt.title('Scatter Plot of Image Width vs Image Height')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:28.3777Z","iopub.execute_input":"2025-01-05T18:25:28.37802Z","iopub.status.idle":"2025-01-05T18:25:28.791533Z","shell.execute_reply.started":"2025-01-05T18:25:28.377994Z","shell.execute_reply":"2025-01-05T18:25:28.790416Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Box Plot","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Plot the boxplot directly\nsns.boxplot(x='label', y='image_height', data=df, palette='cool')\nplt.title('Box Plot of Image Height by Label')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:28.792561Z","iopub.execute_input":"2025-01-05T18:25:28.792924Z","iopub.status.idle":"2025-01-05T18:25:29.2773Z","shell.execute_reply.started":"2025-01-05T18:25:28.792888Z","shell.execute_reply":"2025-01-05T18:25:29.276198Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Count Plot","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\n\n# Load the dataset\ndf = pd.read_csv(\"/kaggle/input/UBC-OCEAN/train.csv\")\n\n# Plot the count of 'label'\nsns.countplot(x='label', data=df, palette='pastel')\nplt.title('Count Plot of Categorical Feature')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T18:25:29.278267Z","iopub.execute_input":"2025-01-05T18:25:29.278556Z","iopub.status.idle":"2025-01-05T18:25:29.539598Z","shell.execute_reply.started":"2025-01-05T18:25:29.278531Z","shell.execute_reply":"2025-01-05T18:25:29.537889Z"}},"outputs":[],"execution_count":null}]}