{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":37333,"databundleVersionId":3949526,"sourceType":"competition"},{"sourceId":9843237,"sourceType":"datasetVersion","datasetId":6038853}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:09:09.577099Z","iopub.execute_input":"2024-11-14T10:09:09.577657Z","iopub.status.idle":"2024-11-14T10:09:09.616082Z","shell.execute_reply.started":"2024-11-14T10:09:09.577611Z","shell.execute_reply":"2024-11-14T10:09:09.614727Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install pandas\n!pip install numpy\n!pip install matplotlib\n!pip install seaborn\n!pip install opencv-python\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:09:09.618811Z","iopub.execute_input":"2024-11-14T10:09:09.619327Z","iopub.status.idle":"2024-11-14T10:10:20.132965Z","shell.execute_reply.started":"2024-11-14T10:09:09.619273Z","shell.execute_reply":"2024-11-14T10:10:20.131009Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport sys\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nfrom glob import glob\nfrom pprint import pprint\nfrom collections import defaultdict\nimport gc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.135465Z","iopub.execute_input":"2024-11-14T10:10:20.136103Z","iopub.status.idle":"2024-11-14T10:10:20.145241Z","shell.execute_reply.started":"2024-11-14T10:10:20.136028Z","shell.execute_reply":"2024-11-14T10:10:20.143782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/mayo-clinic/train.csv')\ntest_df = pd.read_csv('/kaggle/input/mayo-clinic/test.csv')\nother_df = pd.read_csv('/kaggle/input/mayo-clinic/other.csv')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.148577Z","iopub.execute_input":"2024-11-14T10:10:20.149007Z","iopub.status.idle":"2024-11-14T10:10:20.178587Z","shell.execute_reply.started":"2024-11-14T10:10:20.148965Z","shell.execute_reply":"2024-11-14T10:10:20.177391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.180054Z","iopub.execute_input":"2024-11-14T10:10:20.1805Z","iopub.status.idle":"2024-11-14T10:10:20.198421Z","shell.execute_reply.started":"2024-11-14T10:10:20.180457Z","shell.execute_reply":"2024-11-14T10:10:20.196854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.20083Z","iopub.execute_input":"2024-11-14T10:10:20.201439Z","iopub.status.idle":"2024-11-14T10:10:20.216406Z","shell.execute_reply.started":"2024-11-14T10:10:20.201375Z","shell.execute_reply":"2024-11-14T10:10:20.214949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"other_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.218286Z","iopub.execute_input":"2024-11-14T10:10:20.218771Z","iopub.status.idle":"2024-11-14T10:10:20.237145Z","shell.execute_reply.started":"2024-11-14T10:10:20.218726Z","shell.execute_reply":"2024-11-14T10:10:20.235208Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"patients_train = train_df['patient_id'].nunique()\npatients_test = test_df['patient_id'].nunique()\npatients_other = other_df['patient_id'].nunique()\n\nprint(f\"Number of unique patients in train set: {patients_train}\")\nprint(f\"Number of unique patients in test set: {patients_test}\")\nprint(f\"Number of unique patients in the 'other' set: {patients_other}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.238752Z","iopub.execute_input":"2024-11-14T10:10:20.239282Z","iopub.status.idle":"2024-11-14T10:10:20.251188Z","shell.execute_reply.started":"2024-11-14T10:10:20.23922Z","shell.execute_reply":"2024-11-14T10:10:20.249884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.style.use('Solarize_Light2')\nlabels = train_df.groupby('label')['label'].count().div(len(train_df)).mul(100)\ncenters = train_df.groupby(\"center_id\")['center_id'].count().div(len(train_df)).mul(100)\n\nfig, ax = plt.subplots(1,2, figsize=(16,5))\nsns.barplot(x=labels.index, y=labels.values, ax=ax[0])\nax[0].set_title(\"Distribution of a target variable\"), ax[0].set_ylabel(\"%\")\nsns.barplot(x=centers.index, y=centers.values, ax=ax[1])\nax[1].set_title(\"Images per clinic center\"), ax[1].set_ylabel(\"%\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.253448Z","iopub.execute_input":"2024-11-14T10:10:20.253926Z","iopub.status.idle":"2024-11-14T10:10:20.894822Z","shell.execute_reply.started":"2024-11-14T10:10:20.253871Z","shell.execute_reply":"2024-11-14T10:10:20.893295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Other types of blood clots not a part of this competition:\")\nprint(list(other_df['other_specified'].unique()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.89923Z","iopub.execute_input":"2024-11-14T10:10:20.899702Z","iopub.status.idle":"2024-11-14T10:10:20.907536Z","shell.execute_reply.started":"2024-11-14T10:10:20.899658Z","shell.execute_reply":"2024-11-14T10:10:20.905928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_images = glob(\"/kaggle/input/mayo-clinic-strip-ai/train/*\")\ntest_images = glob(\"/kaggle/input/mayo-clinic-strip-ai/test/*\")\nother_images = glob(\"/kaggle/input/mayo-clinic-strip-ai/other/*\")\nprint(f\"Number of images in a training set: {len(train_images)}\")\nprint(f\"Number of images in a training set: {len(test_images)}\")\nprint(f\"Number of other: {len(other_images)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.909424Z","iopub.execute_input":"2024-11-14T10:10:20.909955Z","iopub.status.idle":"2024-11-14T10:10:20.928452Z","shell.execute_reply.started":"2024-11-14T10:10:20.909897Z","shell.execute_reply":"2024-11-14T10:10:20.927013Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from openslide import OpenSlide","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.930479Z","iopub.execute_input":"2024-11-14T10:10:20.930858Z","iopub.status.idle":"2024-11-14T10:10:20.937292Z","shell.execute_reply.started":"2024-11-14T10:10:20.930817Z","shell.execute_reply":"2024-11-14T10:10:20.935836Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_prop = defaultdict(list)  # 使用defaultdict自动创建列表\nfor i, path in enumerate(train_images):\n    img_path = train_images[i]\n    slide = OpenSlide(img_path)    # 使用OpenSlide读取图像\n    \n    # 收集各种属性\n    img_prop['image_id'].append(img_path[-12:-4])  # 提取图像ID\n    img_prop['width'].append(slide.dimensions[0])   # 图像宽度\n    img_prop['height'].append(slide.dimensions[1])  # 图像高度\n    img_prop['size'].append(round(os.path.getsize(img_path) / 1e6, 2))  # 文件大小(MB)\n    img_prop['path'].append(img_path)  # 图像路径\nimage_data = pd.DataFrame(img_prop)  # 转换为DataFrame\n# 计算宽高比\nimage_data['img_aspect_ratio'] = image_data['width']/image_data['height']\n# 按image_id排序\nimage_data.sort_values(by='image_id', inplace=True)\n# 重置索引\nimage_data.reset_index(inplace=True, drop=True)\n# 将图像属性数据与训练标签数据合并\nimage_data = image_data.merge(train_df, on='image_id')\nimage_data.head()  # 显示前5行数据\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:20.939054Z","iopub.execute_input":"2024-11-14T10:10:20.939541Z","iopub.status.idle":"2024-11-14T10:10:28.230517Z","shell.execute_reply.started":"2024-11-14T10:10:20.939492Z","shell.execute_reply":"2024-11-14T10:10:28.229072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 设置图表样式\nplt.style.use('Solarize_Light2')  # 使用预定义的浅色主题样式\n\n# 创建一个包含2个子图的图表\nfig, ax = plt.subplots(1,2, figsize=(16,5))  # 1行2列，图表大小16x5\n\n# 第一个直方图：显示图像文件大小的分布\nsns.histplot(\n    x='size',           # 使用'size'列的数据\n    data=image_data,    # 数据来源\n    bins=100,           # 将数据分成100个区间\n    ax=ax[0]           # 在第一个子图位置显示\n)\nax[0].set_title(\"Distribution of size\")  # 设置标题\nax[0].set_ylabel(\"%\")                    # 设置y轴标签\n\n# 第二个直方图：显示图像宽高比的分布\nsns.histplot(\n    x='img_aspect_ratio',  # 使用'img_aspect_ratio'列的数据\n    data=image_data,       # 数据来源\n    bins=100,              # 将数据分成100个区间\n    ax=ax[1]              # 在第二个子图位置显示\n)\nax[1].set_title(\"Image aspect ratio\")  # 设置标题\nax[1].set_ylabel(\"%\")                  # 设置y轴标签\n\n# 显示图表\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:28.232498Z","iopub.execute_input":"2024-11-14T10:10:28.233009Z","iopub.status.idle":"2024-11-14T10:10:29.108341Z","shell.execute_reply.started":"2024-11-14T10:10:28.232954Z","shell.execute_reply":"2024-11-14T10:10:29.107049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from PIL import Image\nImage.MAX_IMAGE_PIXELS = None  # you have to set this value to allow displaying high-resolution images","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:29.109793Z","iopub.execute_input":"2024-11-14T10:10:29.110181Z","iopub.status.idle":"2024-11-14T10:10:29.116183Z","shell.execute_reply.started":"2024-11-14T10:10:29.110109Z","shell.execute_reply":"2024-11-14T10:10:29.114624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 选择标签为'CE'的图像路径\nCE_imgs = image_data.loc[image_data['label']=='CE','path']\n# 选择标签为'LAA'的图像路径\nLAA_imgs = image_data.loc[image_data['label']=='LAA','path']\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:29.118616Z","iopub.execute_input":"2024-11-14T10:10:29.119214Z","iopub.status.idle":"2024-11-14T10:10:29.13211Z","shell.execute_reply.started":"2024-11-14T10:10:29.119153Z","shell.execute_reply":"2024-11-14T10:10:29.130544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 设置默认绘图样式\nplt.style.use('default')\n\n# 创建1行5列的子图，总图像大小为16x16\nfig, axes = plt.subplots(1,5, figsize=(16,16))\n\n# 遍历所有子图\nfor ax in axes.reshape(-1):\n    # 从CE_imgs中随机选择一张图片的路径\n    img_path = np.random.choice(CE_imgs)\n    \n    # 使用PIL打开图像\n    img = Image.open(img_path)   \n    \n    # 将图像缩放到300x300大小，使用LANCZOS重采样方法\n    img.thumbnail((300,300), Image.Resampling.LANCZOS)\n    \n    # 在当前子图中显示图像，并设置标题\n    ax.imshow(img)\n    ax.set_title(\"target: CE\")\n\n# 显示整个图表\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:10:29.133672Z","iopub.execute_input":"2024-11-14T10:10:29.134026Z","iopub.status.idle":"2024-11-14T10:12:04.814924Z","shell.execute_reply.started":"2024-11-14T10:10:29.133989Z","shell.execute_reply":"2024-11-14T10:12:04.813108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 创建1行5列的子图，总图像大小为16x16\nfig, axes = plt.subplots(1,5, figsize=(16,16))\n\n# 遍历所有子图\nfor ax in axes.reshape(-1):\n    # 从LAA_imgs中随机选择一张图片的路径\n    img_path = np.random.choice(LAA_imgs)\n    \n    # 使用PIL打开图像\n    img = Image.open(img_path)   \n    \n    # 将图像缩放到300x300大小，使用LANCZOS重采样方法\n    img.thumbnail((300,300), Image.Resampling.LANCZOS)\n    \n    # 在当前子图中显示图像，并设置标题为\"target: LAA\"\n    ax.imshow(img)\n    ax.set_title(\"target: LAA\")\n\n# 显示整个图表\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:12:04.81766Z","iopub.execute_input":"2024-11-14T10:12:04.819051Z","iopub.status.idle":"2024-11-14T10:13:42.237745Z","shell.execute_reply.started":"2024-11-14T10:12:04.818963Z","shell.execute_reply":"2024-11-14T10:13:42.236201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_data():\n    # 读取CSV文件\n    train_df = pd.read_csv('/kaggle/input/mayo-clinic/train.csv')\n    test_df = pd.read_csv('/kaggle/input/mayo-clinic/test.csv')\n    \n    # 查看数据结构\n    print(\"Train DataFrame columns:\", train_df.columns)\n    print(\"\\nFirst few rows of train_df:\")\n    print(train_df.head())\n    \n    print(\"\\nTest DataFrame columns:\", test_df.columns)\n    print(\"\\nFirst few rows of test_df:\")\n    print(test_df.head())\n    \n    return train_df, test_df\n\n# 运行这个函数来查看数据结构\ntrain_df, test_df = process_data()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:13:42.239714Z","iopub.execute_input":"2024-11-14T10:13:42.240239Z","iopub.status.idle":"2024-11-14T10:13:42.270954Z","shell.execute_reply.started":"2024-11-14T10:13:42.240182Z","shell.execute_reply":"2024-11-14T10:13:42.269625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\nimport pandas as pd\nimport numpy as np\nimport cv2\nfrom sklearn.model_selection import train_test_split\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:13:42.272575Z","iopub.execute_input":"2024-11-14T10:13:42.272942Z","iopub.status.idle":"2024-11-14T10:13:42.280229Z","shell.execute_reply.started":"2024-11-14T10:13:42.272903Z","shell.execute_reply":"2024-11-14T10:13:42.278908Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset\nimport cv2\nfrom torchvision import transforms\n\nclass MayoDataset(Dataset):\n    def __init__(self, df, transform=None, is_test=False):\n        self.df = df\n        self.transform = transform\n        self.is_test = is_test\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_path = self.df.iloc[idx]['image_path']\n        image = cv2.imread(img_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        if self.is_test:\n            return image\n        else:\n            label = self.df.iloc[idx]['label']\n            return image, label\n\ndef get_train_transforms():\n    return transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.Resize((224, 224)),\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomRotation(10),\n        transforms.ColorJitter(brightness=0.2, contrast=0.2),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                           std=[0.229, 0.224, 0.225])\n    ])\n\ndef get_valid_transforms():\n    return transforms.Compose([\n        transforms.ToPILImage(),\n        transforms.Resize((224, 224)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                           std=[0.229, 0.224, 0.225])\n    ])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:13:42.28258Z","iopub.execute_input":"2024-11-14T10:13:42.284037Z","iopub.status.idle":"2024-11-14T10:13:42.299685Z","shell.execute_reply.started":"2024-11-14T10:13:42.28396Z","shell.execute_reply":"2024-11-14T10:13:42.298346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_data():\n    # 读取训练集和测试集\n    train_df = pd.read_csv('train.csv')\n    test_df = pd.read_csv('test.csv')\n    \n    # 添加图像路径列\n    train_df['image_path'] = train_df['image_id'].apply(lambda x: f'train/{x}')\n    test_df['image_path'] = test_df['image_id'].apply(lambda x: f'test/{x}')\n    \n    # 将标签转换为数值\n    label_map = {'CE': 0, 'LAA': 1}\n    train_df['label'] = train_df['label'].map(label_map)\n    \n    return train_df, test_df\n\n# 修改 MayoDataset 类\nclass MayoDataset(Dataset):\n    def __init__(self, df, transform=None, is_test=False):\n        self.df = df\n        self.transform = transform\n        self.is_test = is_test\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_path = self.df.iloc[idx]['image_path']\n        # 确保图像路径正确\n        try:\n            image = cv2.imread(img_path)\n            if image is None:\n                raise ValueError(f\"Could not load image at {img_path}\")\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        except Exception as e:\n            print(f\"Error loading image {img_path}: {str(e)}\")\n            raise\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        if self.is_test:\n            return image\n        else:\n            label = self.df.iloc[idx]['label']\n            return image, label\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:13:42.301304Z","iopub.execute_input":"2024-11-14T10:13:42.301687Z","iopub.status.idle":"2024-11-14T10:13:42.320045Z","shell.execute_reply.started":"2024-11-14T10:13:42.30165Z","shell.execute_reply":"2024-11-14T10:13:42.318796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_data():\n    # 使用完整的文件路径\n    DATA_DIR = '/path/to/your/data/directory'  # 替换为你的实际数据目录路径\n    train_df = pd.read_csv(f'{DATA_DIR}/train.csv')\n    test_df = pd.read_csv(f'{DATA_DIR}/test.csv')\n    \n    # 添加图像路径列\n    train_df['image_path'] = train_df['image_id'].apply(lambda x: f'{DATA_DIR}/train/{x}')\n    test_df['image_path'] = test_df['image_id'].apply(lambda x: f'{DATA_DIR}/test/{x}')\n    \n    # 将标签转换为数值\n    label_map = {'CE': 0, 'LAA': 1}\n    train_df['label'] = train_df['label'].map(label_map)\n    \n    return train_df, test_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:19:08.148743Z","iopub.execute_input":"2024-11-14T10:19:08.149289Z","iopub.status.idle":"2024-11-14T10:19:08.158604Z","shell.execute_reply.started":"2024-11-14T10:19:08.149242Z","shell.execute_reply":"2024-11-14T10:19:08.15709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_data():\n    # Kaggle数据目录\n    DATA_DIR = '/kaggle/input/your-dataset-name'  # 替换为你的实际数据集名称\n    \n    # 读取训练集和测试集\n    train_df = pd.read_csv(f'{DATA_DIR}/train.csv')\n    test_df = pd.read_csv(f'{DATA_DIR}/test.csv')\n    \n    # 添加图像路径列\n    train_df['image_path'] = train_df['image_id'].apply(lambda x: f'{DATA_DIR}/train/{x}')\n    test_df['image_path'] = test_df['image_id'].apply(lambda x: f'{DATA_DIR}/test/{x}')\n    \n    # 将标签转换为数值\n    label_map = {'CE': 0, 'LAA': 1}\n    train_df['label'] = train_df['label'].map(label_map)\n    \n    return train_df, test_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:20:29.635991Z","iopub.execute_input":"2024-11-14T10:20:29.636489Z","iopub.status.idle":"2024-11-14T10:20:29.6455Z","shell.execute_reply.started":"2024-11-14T10:20:29.636447Z","shell.execute_reply":"2024-11-14T10:20:29.644027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_data():\n    # 设置数据目录\n    DATA_DIR = '/kaggle/input/mayo-clinic-strip-ai'\n    \n    # 读取训练集和测试集\n    train_df = pd.read_csv(f'{DATA_DIR}/train.csv')\n    test_df = pd.read_csv(f'{DATA_DIR}/test.csv')\n    \n    # 添加图像路径列\n    train_df['image_path'] = train_df['image_id'].apply(lambda x: f'{DATA_DIR}/train/{x}.tif')\n    test_df['image_path'] = test_df['image_id'].apply(lambda x: f'{DATA_DIR}/test/{x}.tif')\n    \n    # 将标签转换为数值\n    label_map = {'CE': 0, 'LAA': 1}\n    train_df['label'] = train_df['label'].map(label_map)\n    \n    return train_df, test_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-14T10:21:51.129174Z","iopub.execute_input":"2024-11-14T10:21:51.130305Z","iopub.status.idle":"2024-11-14T10:21:51.138688Z","shell.execute_reply.started":"2024-11-14T10:21:51.130254Z","shell.execute_reply":"2024-11-14T10:21:51.137395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nimport torch.nn as nn\nimport torch.optim as optim\n\n# 设置 PIL 的限制\nImage.MAX_IMAGE_PIXELS = None  # 取消图片大小限制\n\nclass StripAIDataset(Dataset):\n    def __init__(self, df, image_dir, transform=None):\n        self.df = df\n        self.image_dir = image_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image_path = os.path.join(self.image_dir, f\"{row['image_id']}.tif\")\n        label = 0 if row['label'] == 'CE' else (1 if row['label'] == 'LAA' else 2)\n        \n        try:\n            with Image.open(image_path) as img:\n                img = img.convert('RGB')\n                img = img.resize((64, 64), Image.BILINEAR)  # 使用更快的调整方法\n                \n                if self.transform:\n                    img = self.transform(img)\n                \n                return img, label\n        except Exception as e:\n            print(f\"Error loading {row['image_id']}: {str(e)}\")\n            return torch.zeros((3, 64, 64)), label\n\ndef train_model():\n    # 设置设备\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    print(f\"Using device: {device}\")\n    \n    # 读取少量数据\n    train_df = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/train.csv')\n    train_df = train_df.head(5)  # 只使用5张图片进行测试\n    \n    # 数据转换\n    transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ])\n    \n    # 创建数据集和加载器\n    train_dataset = StripAIDataset(\n        df=train_df,\n        image_dir='/kaggle/input/mayo-clinic-strip-ai/train',\n        transform=transform\n    )\n    \n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=2,\n        shuffle=True,\n        num_workers=0\n    )\n    \n    # 创建模型\n    model = models.mobilenet_v2(weights=None)\n    model.classifier[1] = nn.Linear(model.classifier[1].in_features, 3)\n    model = model.to(device)\n    \n    # 定义损失函数和优化器\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.001)\n    \n    # 训练一个epoch\n    print(\"Starting training...\")\n    model.train()\n    for batch_idx, (inputs, labels) in enumerate(train_loader):\n        inputs, labels = inputs.to(device), labels.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        \n        print(f\"Batch {batch_idx+1}, Loss: {loss.item():.4f}\")\n    \n    print(\"Training completed successfully\")\n    return model\n\ndef main():\n    try:\n        model = train_model()\n        print(\"Model training completed successfully\")\n        \n        # 保存模型\n        torch.save(model.state_dict(), 'model.pth')\n        print(\"Model saved successfully\")\n        \n    except Exception as e:\n        print(f\"An error occurred: {str(e)}\")\n\nif __name__ == '__main__':\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T08:53:51.818578Z","iopub.execute_input":"2024-11-15T08:53:51.819037Z","iopub.status.idle":"2024-11-15T08:55:38.022281Z","shell.execute_reply.started":"2024-11-15T08:53:51.818998Z","shell.execute_reply":"2024-11-15T08:55:38.02104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nfrom PIL import Image\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nimport torch.nn as nn\n\n# 设置 PIL 的限制\nImage.MAX_IMAGE_PIXELS = None\n\ndef predict(model, test_df, image_dir):\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model.to(device)\n    model.eval()  # 设置为评估模式\n    \n    # 数据转换\n    transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ])\n    \n    predictions = []\n    image_ids = []\n    \n    with torch.no_grad():  # 不计算梯度\n        for _, row in test_df.iterrows():\n            image_path = os.path.join(image_dir, f\"{row['image_id']}.tif\")\n            \n            try:\n                # 加载和预处理图片\n                with Image.open(image_path) as img:\n                    img = img.convert('RGB')\n                    img = img.resize((64, 64), Image.BILINEAR)\n                    img = transform(img)\n                    img = img.unsqueeze(0)  # 添加batch维度\n                    img = img.to(device)\n                \n                # 预测\n                outputs = model(img)\n                _, predicted = torch.max(outputs, 1)\n                \n                # 转换预测结果为标签\n                label = 'CE' if predicted.item() == 0 else ('LAA' if predicted.item() == 1 else 'Other')\n                \n                predictions.append(label)\n                image_ids.append(row['image_id'])\n                print(f\"Predicted {row['image_id']}: {label}\")\n                \n            except Exception as e:\n                print(f\"Error processing {row['image_id']}: {str(e)}\")\n                predictions.append('CE')  # 默认预测\n                image_ids.append(row['image_id'])\n    \n    # 创建提交文件\n    submission_df = pd.DataFrame({\n        'image_id': image_ids,\n        'label': predictions\n    })\n    \n    return submission_df\n\ndef main():\n    # 加载测试数据\n    test_df = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/test.csv')\n    test_df = test_df.head(10)  # 先测试10张图片\n    \n    # 加载训练好的模型\n    model = models.mobilenet_v2(weights=None)\n    model.classifier[1] = nn.Linear(model.classifier[1].in_features, 3)\n    model.load_state_dict(torch.load('model.pth'))\n    \n    # 进行预测\n    print(\"Starting predictions...\")\n    submission_df = predict(\n        model=model,\n        test_df=test_df,\n        image_dir='/kaggle/input/mayo-clinic-strip-ai/test'\n    )\n    \n    # 保存预测结果\n    submission_df.to_csv('submission.csv', index=False)\n    print(\"\\nPredictions completed. Results saved to 'submission.csv'\")\n    \n    # 显示部分预测结果\n    print(\"\\nFirst few predictions:\")\n    print(submission_df.head())\n\nif __name__ == '__main__':\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T09:00:52.147711Z","iopub.execute_input":"2024-11-15T09:00:52.148547Z","iopub.status.idle":"2024-11-15T09:02:37.226479Z","shell.execute_reply.started":"2024-11-15T09:00:52.148445Z","shell.execute_reply":"2024-11-15T09:02:37.225325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nfrom PIL import Image\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nimport torch.nn as nn\nimport torch.optim as optim\n\n# 设置 PIL 的限制\nImage.MAX_IMAGE_PIXELS = None  # 取消图片大小限制\n\nclass StripAIDataset(Dataset):\n    def __init__(self, df, image_dir, transform=None):\n        self.df = df\n        self.image_dir = image_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image_path = os.path.join(self.image_dir, f\"{row['image_id']}.tif\")\n        label = 0 if row['label'] == 'CE' else (1 if row['label'] == 'LAA' else 2)\n        \n        try:\n            with Image.open(image_path) as img:\n                img = img.convert('RGB')\n                img = img.resize((64, 64), Image.BILINEAR)  # 使用更快的调整方法\n                \n                if self.transform:\n                    img = self.transform(img)\n                \n                return img, label\n        except Exception as e:\n            print(f\"Error loading {row['image_id']}: {str(e)}\")\n            return torch.zeros((3, 64, 64)), label\n\ndef train_model():\n    # 设置设备\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    print(f\"Using device: {device}\")\n    \n    # 读取少量数据\n    train_df = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/train.csv')\n    train_df = train_df.head(5)  # 只使用5张图片进行测试\n    \n    # 数据转换\n    transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ])\n    \n    # 创建数据集和加载器\n    train_dataset = StripAIDataset(\n        df=train_df,\n        image_dir='/kaggle/input/mayo-clinic-strip-ai/train',\n        transform=transform\n    )\n    \n    train_loader = DataLoader(\n        train_dataset,\n        batch_size=2,\n        shuffle=True,\n        num_workers=0\n    )\n    \n    # 创建模型\n    model = models.mobilenet_v2(weights=None)\n    model.classifier[1] = nn.Linear(model.classifier[1].in_features, 3)\n    model = model.to(device)\n    \n    # 定义损失函数和优化器\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.001)\n    \n    # 训练一个epoch\n    print(\"Starting training...\")\n    model.train()\n    for batch_idx, (inputs, labels) in enumerate(train_loader):\n        inputs, labels = inputs.to(device), labels.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        \n        print(f\"Batch {batch_idx+1}, Loss: {loss.item():.4f}\")\n    \n    print(\"Training completed successfully\")\n    return model\n\ndef main():\n    try:\n        model = train_model()\n        print(\"Model training completed successfully\")\n        \n        # 保存模型\n        torch.save(model.state_dict(), 'model.pth')\n        print(\"Model saved successfully\")\n        \n    except Exception as e:\n        print(f\"An error occurred: {str(e)}\")\n\nif __name__ == '__main__':\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T10:23:46.106883Z","iopub.execute_input":"2024-11-15T10:23:46.107682Z","iopub.status.idle":"2024-11-15T10:25:20.634904Z","shell.execute_reply.started":"2024-11-15T10:23:46.107618Z","shell.execute_reply":"2024-11-15T10:25:20.633228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nfrom PIL import Image\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nimport torch.nn as nn\nimport torch.nn.functional as F\n\ndef predict(model, test_df, image_dir):\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model.to(device)\n    model.eval()\n    \n    transform = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ])\n    \n    results = []\n    \n    with torch.no_grad():\n        for _, row in test_df.iterrows():\n            image_path = os.path.join(image_dir, f\"{row['image_id']}.tif\")\n            \n            try:\n                with Image.open(image_path) as img:\n                    img = img.convert('RGB')\n                    img = img.resize((64, 64), Image.BILINEAR)\n                    img = transform(img)\n                    img = img.unsqueeze(0)\n                    img = img.to(device)\n                \n                outputs = model(img)\n                probabilities = F.softmax(outputs, dim=1)[0]\n                \n                # 获取CE和LAA的概率\n                ce_prob = probabilities[0].item()\n                laa_prob = probabilities[1].item()\n                \n                results.append({\n                    'patient_id': row['patient_id'],\n                    'CE': ce_prob,\n                    'LAA': laa_prob\n                })\n                \n            except Exception as e:\n                print(f\"Error processing {row['image_id']}: {str(e)}\")\n                results.append({\n                    'patient_id': row['patient_id'],\n                    'CE': 0.5,\n                    'LAA': 0.5\n                })\n    \n    # 创建结果DataFrame\n    results_df = pd.DataFrame(results)\n    return results_df\n\ndef main():\n    # 加载测试数据\n    test_df = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/test.csv')\n    test_df = test_df.head(10)  # 先测试10张图片\n    \n    # 加载训练好的模型\n    model = models.mobilenet_v2(weights=None)\n    model.classifier[1] = nn.Linear(model.classifier[1].in_features, 3)\n    model.load_state_dict(torch.load('model.pth'))\n    \n    # 进行预测\n    print(\"Starting predictions...\")\n    results_df = predict(\n        model=model,\n        test_df=test_df,\n        image_dir='/kaggle/input/mayo-clinic-strip-ai/test'\n    )\n    \n    # 设置显示格式\n    pd.set_option('display.float_format', lambda x: '%.6f' % x)\n    \n    # 显示结果\n    print(\"\\nPrediction Results:\")\n    print(results_df)\n    \n    # 保存结果\n    results_df.to_csv('predictions_with_probabilities.csv', index=False)\n    print(\"\\nResults saved to 'predictions_with_probabilities.csv'\")\n\nif __name__ == '__main__':\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-15T10:28:14.030067Z","iopub.execute_input":"2024-11-15T10:28:14.031889Z","iopub.status.idle":"2024-11-15T10:29:51.548136Z","shell.execute_reply.started":"2024-11-15T10:28:14.031793Z","shell.execute_reply":"2024-11-15T10:29:51.546955Z"}},"outputs":[],"execution_count":null}]}