{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":37333,"databundleVersionId":3949526,"sourceType":"competition"},{"sourceId":6774400,"sourceType":"datasetVersion","datasetId":3895136},{"sourceId":7869901,"sourceType":"datasetVersion","datasetId":4617753},{"sourceId":7869997,"sourceType":"datasetVersion","datasetId":4617825},{"sourceId":7870715,"sourceType":"datasetVersion","datasetId":4618317},{"sourceId":7871004,"sourceType":"datasetVersion","datasetId":4618499},{"sourceId":7871459,"sourceType":"datasetVersion","datasetId":4618776},{"sourceId":7875154,"sourceType":"datasetVersion","datasetId":4621378},{"sourceId":8108156,"sourceType":"datasetVersion","datasetId":4789280}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport tensorflow.keras.layers as l\nfrom PIL import Image\nfrom tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-04-17T05:05:37.094673Z","iopub.execute_input":"2024-04-17T05:05:37.095157Z","iopub.status.idle":"2024-04-17T05:05:53.806127Z","shell.execute_reply.started":"2024-04-17T05:05:37.095123Z","shell.execute_reply":"2024-04-17T05:05:53.804984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Disable the decompression bomb limit\nImage.MAX_IMAGE_PIXELS = None","metadata":{"execution":{"iopub.status.busy":"2024-04-17T05:05:53.807786Z","iopub.execute_input":"2024-04-17T05:05:53.808766Z","iopub.status.idle":"2024-04-17T05:05:53.817242Z","shell.execute_reply.started":"2024-04-17T05:05:53.808735Z","shell.execute_reply":"2024-04-17T05:05:53.815825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dftest = pd.read_csv(\"/kaggle/input/mayo-clinic-strip-ai/test.csv\")\ndfother = pd.read_csv(\"/kaggle/input/mayo-clinic-strip-ai/other.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-17T05:05:53.819232Z","iopub.execute_input":"2024-04-17T05:05:53.820246Z","iopub.status.idle":"2024-04-17T05:05:53.855918Z","shell.execute_reply.started":"2024-04-17T05:05:53.820214Z","shell.execute_reply":"2024-04-17T05:05:53.854793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tooBig = []","metadata":{"execution":{"iopub.status.busy":"2024-04-17T05:05:53.860415Z","iopub.execute_input":"2024-04-17T05:05:53.860908Z","iopub.status.idle":"2024-04-17T05:05:53.866322Z","shell.execute_reply.started":"2024-04-17T05:05:53.860868Z","shell.execute_reply":"2024-04-17T05:05:53.865427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tile(img, sz=128, N=16):\n    shape = img.shape\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=255)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    if len(img) < N:\n        img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N] # pick up Top N dark tiles\n    img = img[idxs]\n    return img\n\ndef save_dataset(\n    df: pd.DataFrame, \n    N=16,\n    max_size=20000, \n    crop_size=1024, \n    image_dir='../input/mayo-clinic-strip-ai/train', \n    out_dir='./train_data',\n):\n    format_to_dtype = {\n       'uchar': np.uint8,\n       'char': np.int8,\n       'ushort': np.uint16,\n       'short': np.int16,\n       'uint': np.uint32,\n       'int': np.int32,\n       'float': np.float32,\n       'double': np.float64,\n       'complex': np.complex64,\n       'dpcomplex': np.complex128,\n    }\n    \n    if not os.path.isdir(out_dir):\n        os.makedirs(out_dir)\n        \n    tk0 = tqdm(enumerate(df[\"image_id\"].values), total=len(df))\n    for i, image_id in tk0:\n        if os.path.getsize(f'{image_dir}/{image_id}.tif')/(1000**3) < 2:\n            print(f\"[{i+1}/{len(df)}] image_id: {image_id}\")\n            # open the image\n            image = Image.open(f'{image_dir}/{image_id}.tif')\n            # resize the image\n            image.thumbnail((max_size, max_size), Image.LANCZOS)\n            image = np.array(image)\n            width, height, c = image.shape\n            image = tile(image, sz=crop_size, N=N)\n            for idx, img in enumerate(image):\n                img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)\n                cv2.imwrite(f\"{out_dir}/{image_id}_{idx}.jpg\", img, [cv2.IMWRITE_JPEG_QUALITY, 100])\n\n            del img, image; gc.collect()\n        else:\n            tooBig.append(image_id)\n# def tile(img, sz=128, N=16):\n#     shape = img.shape\n#     pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n#     img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=255)\n#     img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n#     img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n#     if len(img) < N:\n#         img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n#     idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N] # pick up Top N dark tiles\n#     img = img[idxs]\n#     return img\n\n# def save_dataset(\n#     df: pd.DataFrame, \n#     N=16,\n#     max_size=20000, \n#     crop_size=1024, \n#     image_dir='../input/mayo-clinic-strip-ai/test', \n#     out_dir='./train_data',\n# ):\n#     format_to_dtype = {\n#        'uchar': np.uint8,\n#        'char': np.int8,\n#        'ushort': np.uint16,\n#        'short': np.int16,\n#        'uint': np.uint32,\n#        'int': np.int32,\n#        'float': np.float32,\n#        'double': np.float64,\n#        'complex': np.complex64,\n#        'dpcomplex': np.complex128,\n#     }\n    \n#     if not os.path.isdir(out_dir):\n#         os.makedirs(out_dir)\n        \n#     tk0 = tqdm(enumerate(df[\"image_id\"].values), total=len(df))\n#     for i, image_id in tk0:\n#         if os.path.getsize(f'{image_dir}/{image_id}.tif')/(1000**3) < 2:\n#             continue\n#         else:\n#             tooBig.append(image_id)","metadata":{"execution":{"iopub.status.busy":"2024-04-17T05:05:53.867602Z","iopub.execute_input":"2024-04-17T05:05:53.868131Z","iopub.status.idle":"2024-04-17T05:05:53.887523Z","shell.execute_reply.started":"2024-04-17T05:05:53.868081Z","shell.execute_reply":"2024-04-17T05:05:53.886502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"save_dataset(\n    dftest,\n    N=16, \n    max_size=20000,\n    crop_size=1024, \n    image_dir=\"/kaggle/input/mayo-clinic-strip-ai/test\", \n    out_dir=f'/kaggle/working/test'\n)\n# save_dataset(\n#     dfother,\n#     N=16, \n#     max_size=20000,\n#     crop_size=1024, \n#     image_dir=\"/kaggle/input/mayo-clinic-strip-ai/other\", \n#     out_dir=f'/kaggle/working/test'\n# )","metadata":{"execution":{"iopub.status.busy":"2024-04-17T05:05:53.888688Z","iopub.execute_input":"2024-04-17T05:05:53.889487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport sys\nimport zipfile\nfrom IPython.display import FileLink\nfrom pathlib import Path\nimport albumentations as A\nimport SimpleITK as sitk\nimport csv\nfrom scipy.stats import mannwhitneyu\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, display\nimport matplotlib.cm as cm\nfrom sklearn.metrics import classification_report\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.preprocessing import StandardScaler","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df = pd.read_csv(\"/kaggle/input/6835-mayo-radiomics/pyradiomics_features.csv\")","metadata":{"id":"npZwq_30H-qo","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df = train_data_df.sample(frac=1, random_state=69)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for column in train_data_df.columns:\n#     train_data_df[column] = train_data_df[column].fillna(0)\n    \ntrain_data_df = train_data_df.replace(np.nan, 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_df = train_data_df.reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_df = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _weighted_mc_log_loss(y_true, y_pred, epsilon=1e-15):\n    class_cnt = [sum(int(val == cl) for val in y_true) for cl in range(2)]\n    w = [0.5 for _ in range(2)]\n    return -sum(\n        w[cl] * sum(\n            (y == cl) / class_cnt[cl] * np.log(max(min(y_hat, 1 - epsilon), epsilon))\n            for y, y_hat in zip(y_true, y_pred[:, cl])\n        )\n        for cl in range(2)\n    ) / sum(w[cl] for cl in range(2))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Assuming train_data_df is your DataFrame containing the data\n# Replace 'Label' with the actual column name if it's different\n\n\n# Initial counts\nnum_CE = len(train_data_df[train_data_df['Label'] == 0])\nnum_LAA = len(train_data_df[train_data_df['Label'] == 1])\n\n# Calculate the difference in counts between the two classes\ncount_diff = num_CE - num_LAA\n\n# If there are more records with 'CE' label, randomly sample records with 'CE' to match the count of 'LAA'\nif count_diff > 0:\n    train_data_df = train_data_df.drop(train_data_df[train_data_df['Label'] == 0].sample(count_diff).index)\n# If there are more records with 'LAA' label, randomly sample records with 'LAA' to match the count of 'CE'\nelif count_diff < 0:\n    train_data_df = train_data_df.drop(train_data_df[train_data_df['Label'] == 1].sample(-count_diff).index)\n\n# Write the modified DataFrame to a new CSV file\ntrain_data_df.to_csv(\"modified_train_data.csv\", index=False)\n\nprint(\"Number of records with 'CE' and 'LAA' labels are now equal.\")\nprint(\"New size of train_data_df:\", len(train_data_df))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_df, test_df = train_test_split(result_df, test_size=0.2, random_state=42, stratify = result_df['label'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"# Function to extract prefix\ndef extract_prefix(image_id):\n    return '_'.join(image_id.split('_')[:-1])\n\n\n# Initialize DataFrames to store the results\ntrain_big_df = pd.DataFrame()\ntest_big_df = pd.DataFrame()\n\n# Extract the 'Image_ID' column\nimage_id_column = train_data_df['Image_ID']\n\n# Remove non-numeric data from all columns except 'Image_ID'\nfor col in train_data_df.columns:\n    if col != 'Image_ID':\n        train_data_df[col] = pd.to_numeric(train_data_df[col], errors='coerce')\n\n# Add back the 'Image_ID' column\ntrain_data_df['Image_ID'] = image_id_column\n\n# Split the data into train and test based on the prefix of 'Image_ID'\ntrain_mask = train_data_df['Image_ID'].apply(lambda x: extract_prefix(x) in train_df['image_id'].values)\ntrain_big_chunk_df = train_data_df[train_mask]\ntest_big_chunk_df = train_data_df[~train_mask]\n\n# Concatenate DataFrames for the current chunk with the main DataFrames\ntrain_big_df = pd.concat([train_big_df, train_big_chunk_df], ignore_index=True)\ntest_big_df = pd.concat([test_big_df, test_big_chunk_df], ignore_index=True)\n\n# Optional: Drop duplicate rows from the resulting DataFrames\ntrain_big_df = train_big_df.drop_duplicates()\ntest_big_df = test_big_df.drop_duplicates()\n\n# Optional: Reset the index of the resulting DataFrames\ntrain_big_df.reset_index(drop=True, inplace=True)\ntest_big_df.reset_index(drop=True, inplace=True)\"\"\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Run this in case you choose some features only**","metadata":{"id":"68h1le_zC9CO"}},{"cell_type":"code","source":"# Assuming train_big_df is your DataFrame containing the data\ntrain_big_df = train_data_df\nfeatures = train_big_df.iloc[:, 2:]\n\n# Only consider numeric columns\nnumeric_cols = features.columns[features.apply(lambda x: pd.to_numeric(x, errors='coerce').notnull().all())]\nfeatures = features[numeric_cols]\n\n# Assuming 'Label' is the target column\ntarget_column = 'Label'\nclass_labels = train_big_df[target_column].unique()\n\nu_test_results = {}\n\nfor feature in features.columns:\n    feature_results = {}\n    for i in range(len(class_labels)):\n        for j in range(i+1, len(class_labels)):\n            group1 = train_big_df[train_big_df[target_column] == class_labels[i]][feature]\n            group2 = train_big_df[train_big_df[target_column] == class_labels[j]][feature]\n            stat, _ = mannwhitneyu(group1, group2)  # We don't need p-value here\n            pair_label = f\"{class_labels[i]} vs {class_labels[j]}\"\n            feature_results[pair_label] = {'statistic': stat}\n    u_test_results[feature] = feature_results\n\nu_test_results_df = pd.DataFrame(u_test_results)\n\n# Selecting features directly without considering P-values\nselected_features_list = u_test_results_df.columns.tolist()\n\n# Add the target column to the selected features list\nselected_features_list = ['Label'] + selected_features_list\n\n# Filter the DataFrame with selected features\ntrain_big_df = train_big_df[selected_features_list]\n#test_big_df = test_big_df[selected_features_list]","metadata":{"id":"mi17YpNi5-va","outputId":"95938b2e-57a1-4993-ce80-03ef44416c21","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_p_value(feature, target, data):\n    group1 = data[data[target] == 0][feature]  # Group 1: label 0\n    group2 = data[data[target] == 1][feature]  # Group 2: label 1\n    stat, p_value = mannwhitneyu(group1, group2)\n    return p_value\n\n# Target variable\ntarget_variable = 'Label'\n\n# Threshold for significance\nthreshold = 0.07\n\n# Dictionary to store p-values for each feature\np_values = {}\n\n# Iterate over each feature in the DataFrame\nfor idx, feature in enumerate(train_big_df.columns):\n    if idx >= 2:  # Skip the first two columns\n        if feature != target_variable:\n            # Calculate p-value for the current feature\n            p_value = calculate_p_value(feature, target_variable, train_big_df)\n            p_values[feature] = p_value\n\n# Filter significant features based on the threshold\nsignificant_features = [feature for feature, p_value in p_values.items() if p_value <= threshold]\n\n# Print significant features and their p-values\n# for feature in significant_features:\n#     print(f\"Feature: {feature}, p-value: {p_values[feature]}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_red = train_big_df.drop(columns=[col for col in train_big_df.columns if col not in significant_features and col not in [\"Image_ID\", \"Label\"]])\n\ntrain_red","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test_red = test_big_df.drop(columns=[col for col in test_big_df.columns if col not in significant_features and col not in [\"Image_ID\", \"Label\"]])\n\n#test_red","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train_red['Label']\nX_train = train_red.drop('Label', axis=1)","metadata":{"id":"nOVi3srYxdTA","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_test = test_red['Label']\n#X_test = test_red.drop('Label', axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"XGBOOST","metadata":{"id":"3TNUi6S8rDmk"}},{"cell_type":"code","source":"scaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"XGB = GradientBoostingClassifier(n_estimators=200, loss='log_loss', max_depth=4, ccp_alpha=0.0007).fit(X_train, y_train)\nprint(XGB.score(X_train, y_train))\nprint(classification_report(y_train, XGB.predict(X_train), target_names=['CE', 'LAA']))\n#print(XGB.score(X_test, y_test))\n#print(classification_report(y_test, XGB.predict(X_test),  target_names=['CE', 'LAA']))","metadata":{"id":"pkOlXugJGSH6","outputId":"229c0b83-e271-4822-ad66-51290592ee09","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_weighted_mc_log_loss(y_train, XGB.predict_proba(X_train))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pred","metadata":{}},{"cell_type":"code","source":"!pip install /kaggle/input/pyradd/pyradiomics/*.whl\n!yes | dpkg -i --force-depends /kaggle/input/pyvips-python-and-deb-package/linux_packages/archives/*.deb\n!pip install pyvips -f /kaggle/input/pyvips-python-and-deb-package/python_packages/ --no-index\n!pip list | grep pyvips\n!cp -r /kaggle/input/yyyyyyyyyyyy/pyradiomics-3.0.1 /kaggle/working\n!pip install --no-index --find-links /kaggle/working/pyradiomics-3.0.1/ -r requirements.txt\n%cd /kaggle/working/pyradiomics-3.0.1\n!python setup.py build_ext --inplace\nimport pyvips\nfrom radiomics import featureextractor\nfrom radiomics import imageoperations","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parse_images(folder):\n    # get the full paths of the images\n    imgs = [os.path.join(folder, f) for f in os.listdir(folder)]\n\n    df = pd.DataFrame()\n    df['image_path'] = imgs\n    # remove extension, and have the id as first and last component, eg: 006388_0\n    df['image_id'] = df['image_path'].apply(lambda x: '_'.join(x.split('/')[-1].replace('.jpg', '').split('_')[:2]))\n    # remove extension, and have the instance_id as last component only, eg: 0, 1, ...\n    df['instance_id'] = df['image_path'].apply(lambda x: int(x.split('_')[-1].replace('.jpg', '')))\n\n    df = df.sort_values(['image_id', 'instance_id']).reset_index(drop=True)\n\n    return df\n\ndef merge_image_info(image_df, info_df):\n    return image_df.merge(info_df, on='image_id', how='left').reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/test.csv')\n# test_csv = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/other.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = merge_image_info(parse_images('/kaggle/working/test'), test_csv)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize PyRadiomics feature extractor\nextractor = featureextractor.RadiomicsFeatureExtractor()\n\n# Define your lower and upper threshold values for yellow and brown\nlower_yellow = np.array([20, 100, 100])\nupper_yellow = np.array([30, 255, 255])\nlower_orange = np.array([5, 100, 100]) \nupper_orange = np.array([20, 255, 255])\nlower_brown = np.array([20, 50, 20]) \nupper_brown = np.array([40, 255, 200]) \n\n\n# Define the HSV color range for brown\nlower_brown = np.array([10, 50, 20])  \nupper_brown = np.array([40, 255, 255])  \n\n\nlower_blue = np.array([90, 50, 50])\nlower_purple = np.array([120, 50, 50])\nupper_purple = np.array([150, 255, 255])\nlower_dark_purple = np.array([100, 50, 50])  \nupper_dark_purple = np.array([120, 255, 255]) \n\n# Define lower and upper threshold values for white in HSV\nlower_white = np.array([0, 0, 200])\nupper_white = np.array([179, 30, 255])\n\n# Initialize list to store PyRadiomics data for all tiles of all images\npyradiomics_data = []\n\n\nfor i in range(len(df)):  \n    stained_image = cv2.imread(df[\"image_path\"][i])\n    #stained_image = np.array(tile)\n    hsv_image = cv2.cvtColor(stained_image, cv2.COLOR_BGR2HSV)\n\n    # RBC\n    orange_mask = cv2.inRange(hsv_image, lower_orange, upper_orange)\n    yellow_mask = cv2.inRange(hsv_image, lower_yellow, upper_yellow)\n    brown_mask = cv2.inRange(hsv_image, lower_brown, upper_brown)\n    yellow_brown_mask = cv2.bitwise_or(yellow_mask, brown_mask)\n    yellow_brown_mask = cv2.bitwise_or(yellow_brown_mask, orange_mask)\n\n    ########\n    rbc_mask = yellow_brown_mask.copy()\n    rbc_mask[rbc_mask > 0] = 1\n\n    # WBC\n\n    #lower_dark_purple = np.array([100, 50, 50])  \n    #upper_dark_purple = np.array([120, 255, 255])  \n    dark_purple_mask = cv2.inRange(hsv_image, lower_dark_purple, upper_dark_purple)\n\n    blue_mask   = cv2.inRange(hsv_image, lower_blue, upper_purple)\n    purple_mask = cv2.inRange(hsv_image, lower_purple, upper_purple)\n    blue_purple_mask = cv2.bitwise_or(blue_mask, purple_mask)\n    purple_mask = cv2.bitwise_or(blue_purple_mask, dark_purple_mask)\n    wbc_mask = blue_purple_mask.copy()\n    wbc_mask[wbc_mask > 0] = 1\n\n\n            # Fibrin/Platelets\n    white_mask = cv2.inRange(hsv_image, lower_white, upper_white)\n    fibrin_platelets_mask = cv2.bitwise_and(cv2.bitwise_not(yellow_brown_mask), cv2.bitwise_not(blue_purple_mask))\n    fibrin_platelets_mask = cv2.bitwise_and(fibrin_platelets_mask, cv2.bitwise_not(white_mask))\n    fibrin_platelets_mask[fibrin_platelets_mask > 0] = 1  # Ensure all non-zero values are set to 1\n\n\n    RBC = cv2.bitwise_and(stained_image, stained_image, mask = yellow_brown_mask)\n    WBC = cv2.bitwise_and(stained_image, stained_image, mask = blue_purple_mask)\n    FP  = cv2.bitwise_and(stained_image, stained_image, mask = fibrin_platelets_mask)\n\n    try:\n        # Read original image\n        original_image_sitk = sitk.GetImageFromArray(cv2.cvtColor(stained_image, cv2.COLOR_RGB2GRAY))\n\n        # Convert mask images to SimpleITK format\n        maskRBC_image_sitk = sitk.GetImageFromArray(rbc_mask)\n        maskWBC_image_sitk = sitk.GetImageFromArray(wbc_mask)\n        maskFP_image_sitk = sitk.GetImageFromArray(fibrin_platelets_mask)\n\n        # Extract features using PyRadiomics\n        featuresRBC = extractor.execute(original_image_sitk, maskRBC_image_sitk)\n        featuresWBC = extractor.execute(original_image_sitk, maskWBC_image_sitk)\n        featuresFP = extractor.execute(original_image_sitk, maskFP_image_sitk)\n\n        # Append data to list\n        pyradiomics_data.append({\n            'Image_ID': f\"{df['image_id'][i]}_{df['instance_id'][i]}\",\n            **{f\"RBC_{k}\": v for k, v in featuresRBC.items()},\n            **{f\"WBC_{k}\": v for k, v in featuresWBC.items()},\n            **{f\"FP_{k}\": v for k, v in featuresFP.items()}\n        })\n    except Exception as e:\n        print(f\"{i})Error extracting features for tile {df['instance_id'][i]} of image {df['image_id'][i]}: {e}\")\n        continue\n\n# Convert list of dictionaries to DataFrame\npyradiomics_df1 = pd.DataFrame(pyradiomics_data)\n# pyradiomics_df.to_csv('/kaggle/working/pyrad.csv')\nprint(\"All PyRadiomics features saved successfully.\")\npyradiomics_df1\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\n\n# Define the base directory where the images are stored\nbase_directory = '/kaggle/input/mayo-clinic-strip-ai/test'\n\n# Prepare lists to hold data\nimage_ids = []\nimage_paths = []\n\n# List all the images in the 'tooBig' list\nfor img in tooBig:\n    # Construct the path for each image\n    image_path = os.path.join(base_directory, f\"{img}.png\")  # Adjust file extension if necessary\n\n    # Append image ID and path to lists\n    image_ids.append(img)\n    image_paths.append(image_path)\n\n# Create a DataFrame from the lists\nimage_data_df = pd.DataFrame({\n    'image_id': image_ids,\n    'image_path': image_paths\n})\n\n# Display the DataFrame\nimage_data_df\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport os\n\n# Define the base directory where the images are stored\nbase_directory = '/kaggle/input/mayo-clinic-strip-ai/test'\n\n# Prepare lists to hold data\nimage_ids = []\nimage_paths = []\n\n# List all the images in the 'tooBig' list\nfor img in tooBig:\n    # Construct the path for each image\n    image_path = os.path.join(base_directory, f\"{img}.png\")  # Adjust file extension if necessary\n\n    # Append image ID and path to lists\n    image_ids.append(img)\n    image_paths.append(image_path)\n\n# Create a DataFrame from the lists\nimage_data_df = pd.DataFrame({\n    'image_id': image_ids,\n    'image_path': image_paths\n})\n\n# Display the DataFrame\nimage_data_df\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n# Path to the CSV file\ncsv_path = '/kaggle/input/mayo-clinic-strip-ai/test.csv'\n\n# Load the CSV file\ndata = pd.read_csv(csv_path)\n\n# Create an empty DataFrame to store the results\nbig_photo_dataframe = pd.DataFrame(columns=['image_id', 'label', 'center_id'])\n\n# This should be defined somewhere\n\n# Iterate over the image IDs in the tooBig array\nfor image_id in tooBig:\n    # Filter the data for the current image_id\n    filtered_data = data[data['image_id'] == image_id]\n    # Check if any data was found\n    if not filtered_data.empty:\n        # Extract necessary information and create a new DataFrame for safe operations\n        append_df = filtered_data[['image_id', 'label']].copy()\n        # Add the center_id column with all values set to -1\n        append_df['center_id'] = -1\n        # Concatenate the new data with the existing DataFrame\n        big_photo_dataframe = pd.concat([big_photo_dataframe, append_df], ignore_index=True)\n\n# Display the DataFrame\nprint(big_photo_dataframe.head())\n\n# Correcting the output path\ncsv_output_path = '/kaggle/working/big_photo_dataframe.csv'\nbig_photo_dataframe.to_csv(csv_output_path, index=False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from typing import List, Tuple\nclass DataPreparation:\n    def __init__(self, visualize: bool = False, seed: int = 42):\n        self.visualize = visualize\n        self.seed = seed\n\n        other_metadata = pd.read_csv('/kaggle/working/big_photo_dataframe.csv')\n        other_metadata = list(zip(\n            other_metadata['image_id'].tolist(),\n            other_metadata['label'].tolist(),\n            other_metadata['center_id'].tolist(),\n        ))\n        self.other = other_metadata\n        self.all_center_ids = sorted(list({center_id for _, _, center_id in self.other}))\n\n#         other_metadata = pd.read_csv('/content/drive/MyDrive/preprocessing/other.csv').query('label == \\'Other\\'')\n#         other_metadata = list(zip(\n#             other_metadata['image_id'].tolist(),\n#             ['LAA' for _ in range(other_metadata.shape[0])],\n#             [-1 for _ in range(other_metadata.shape[0])],\n#         ))\n#         self.other = self._filter_bad_images(other_metadata)\n\n#         test_metadata = pd.read_csv('/content/drive/MyDrive/preprocessing/test.csv')\n#         self.test = list(zip(\n#             test_metadata['image_id'].tolist(),\n#             ['Unknown' for _ in range(test_metadata.shape[0])],\n#             test_metadata['center_id'].tolist(),\n#         ))\n\n#     @staticmethod\n#     def _filter_bad_images(data: List[Tuple]) -> List[Tuple]:\n#         return [\n#             (image_id, label, center_id)\n#             for image_id, label, center_id in data\n#             if image_id not in BAD_IMAGE_IDS\n#         ]\n\n    @staticmethod\n    def _add_rect_to_numpy(image: np.ndarray, x: int, y: int, size: int, thickness: int) -> None:\n        image[x:x + size, y:y + thickness] = (0, 0, 0)\n        image[x:x + thickness, y:y + size] = (0, 0, 0)\n        image[x:x + size, y + size:y + size + thickness] = (0, 0, 0)\n        image[x + size:x + size + thickness, y:y + size] = (0, 0, 0)\n\n    @staticmethod\n    def _get_blocks_map(image: np.ndarray) -> np.ndarray:\n        pixels_diff = np.sum((image[:-1, :, :] - image[1:, :, :]) ** 2, axis=2)\n        pixels_diff = np.cumsum(np.cumsum(pixels_diff, axis=0), axis=1)\n        blocks_map = np.zeros((\n            (image.shape[0] + BLOCK_SIZE - 1) // BLOCK_SIZE,\n            (image.shape[1] + BLOCK_SIZE - 1) // BLOCK_SIZE,\n        ))\n        for x in range(0, pixels_diff.shape[0], BLOCK_SIZE):\n            for y in range(0, pixels_diff.shape[1], BLOCK_SIZE):\n                nx = min(x + BLOCK_SIZE, pixels_diff.shape[0])\n                ny = min(y + BLOCK_SIZE, pixels_diff.shape[1])\n                block_sum = int(pixels_diff[nx - 1, ny - 1])\n                if x:\n                    block_sum -= int(pixels_diff[x - 1, ny - 1])\n                if y:\n                    block_sum -= int(pixels_diff[nx - 1, y - 1])\n                if x and y:\n                    block_sum += int(pixels_diff[x - 1, y - 1])\n                blocks_map[x // BLOCK_SIZE][y // BLOCK_SIZE] = \\\n                    (block_sum / BLOCK_SIZE / BLOCK_SIZE) > BLOCK_THR\n        return blocks_map\n\n    def _generate_crops_positions(\n            self,\n            image: np.ndarray,\n            crop_thr: float,\n    ) -> Tuple[List[Tuple[int, int]], np.ndarray, np.ndarray, np.ndarray, np.ndarray]:\n        blocks_map = self._get_blocks_map(image)\n\n        if self.visualize:\n            for i in range(blocks_map.shape[0]):\n                for j in range(blocks_map.shape[1]):\n                    if blocks_map[i][j]:\n                        self._add_rect_to_numpy(\n                            image,\n                            i * BLOCK_SIZE,\n                            j * BLOCK_SIZE,\n                            BLOCK_SIZE,\n                            1,\n                        )\n\n        good_crops_starts = []\n        for x in range(0, image.shape[0] - CROP_SIZE + 1, BLOCK_SIZE):\n            for y in range(0, image.shape[1] - CROP_SIZE + 1, BLOCK_SIZE):\n                _x, _y = x // BLOCK_SIZE, y // BLOCK_SIZE\n                crop_sum = blocks_map[_x:_x + BLOCKS_PER_CROP, _y:_y + BLOCKS_PER_CROP].sum()\n                if crop_sum > BLOCKS_PER_CROP * BLOCKS_PER_CROP * crop_thr:\n                    good_crops_starts.append((x, y))\n\n        if self.visualize:\n            for x, y in good_crops_starts:\n                self._add_rect_to_numpy(image, x, y, CROP_SIZE, 1)\n\n        return good_crops_starts\n\n    @staticmethod\n    def _process_crop(crop: np.ndarray) -> np.ndarray:\n        return crop\n\n    def _create_crops(\n        self,\n        image: np.ndarray,\n        crops_starts: List[Tuple[int]],\n    ) -> List[np.ndarray]:\n        return [\n            Image.fromarray(\n                self._process_crop(\n                    image[x:x + CROP_SIZE, y:y + CROP_SIZE],\n                )\n            )\n            for x, y in crops_starts\n        ]\n\n    @staticmethod\n    def _get_unique_crops(crop_starts: List[Tuple[int, int]], order) -> List[Tuple[int, int]]:\n        def inter_size_1d(a: int, b: int, c: int, d: int) -> int:\n            return max(0, min(b, d) - max(a, c))\n\n        def inter_size_2d(crop_start_1: Tuple[int, int], crop_start_2: Tuple[int, int]) -> int:\n            return inter_size_1d(\n                crop_start_1[0], crop_start_1[0] + CROP_SIZE,\n                crop_start_2[0], crop_start_2[0] + CROP_SIZE,\n            ) * inter_size_1d(\n                crop_start_1[1], crop_start_1[1] + CROP_SIZE,\n                crop_start_2[1], crop_start_2[1] + CROP_SIZE,\n            )\n\n        crop_starts_sorted = sorted(crop_starts, key=order)\n        final_crop_starts = []\n        for crop_start in crop_starts_sorted:\n            if any(\n                    inter_size_2d(crop_start, crop_start_prev) > CROP_SIZE * CROP_SIZE // 2\n                    for crop_start_prev in final_crop_starts\n            ):\n                continue\n            final_crop_starts.append(crop_start)\n        return final_crop_starts\n\n    @staticmethod\n    def _read_and_resize_image(image_id: str, base_image_path: str, once_flag) -> np.ndarray:\n        image_path = os.path.join(base_image_path, f'{image_id}.tif')\n        if once_flag: image_path = base_image_path\n        image = pyvips.Image.new_from_file(image_path, access='sequential')\n        return image.resize(1.0 / SCALE_FACTOR).numpy()\n\n    def prepare_crops(\n            self,\n            image_ids: List[int],\n            base_image_path: str,\n            once_flag,\n    ) -> Tuple[List[List[np.ndarray]], List[List[Tuple[int]]], List[Tuple[np.ndarray, np.ndarray]]]:\n        np.random.seed(self.seed)\n        image_crops = []\n        image_crops_indices = []\n        for image_id in tqdm(image_ids):\n            start_time = time()\n            image = self._read_and_resize_image(image_id, base_image_path, once_flag)\n            gc.collect()\n            found_flag = False\n            for crop_thr in np.arange(CROP_THR, -0.1, -0.1):\n                good_crops_starts = self._generate_crops_positions(image, crop_thr)\n                if len(good_crops_starts) < IMAGES_PER_SAMPLE:\n                    continue\n\n                good_crops_starts_unique = []\n                for order in [\n                    lambda x: (x[0], x[1]),\n                    lambda x: (-x[0], -x[1]),\n                ]:\n                    good_crops_starts_unique.extend(self._get_unique_crops(good_crops_starts, order))\n                good_crops_starts_unique = list(set(good_crops_starts_unique))\n\n                if len(good_crops_starts_unique) < IMAGES_PER_SAMPLE:\n                    continue\n\n                good_crops_starts_sample_ids = np.random.choice(\n                    list(range(len(good_crops_starts_unique))),\n                    min(len(good_crops_starts_unique), MAX_CROPS_PER_IMAGE),\n                    replace=False,\n                )\n                good_crops_starts_sample = np.array(good_crops_starts_unique)[good_crops_starts_sample_ids]\n                image_crops_indices.append(good_crops_starts_sample)\n                image_crops.append(self._create_crops(image, good_crops_starts_sample))\n                found_flag = True\n                break\n            if not found_flag:\n                image_crops_indices.append([])\n                image_crops.append([])\n        gc.collect()\n        return image_crops, image_crops_indices\n\n#     def process_train(\n#             self\n#     ) -> Tuple[List[List[np.ndarray]], List[List[Tuple[int]]]]:\n#         return self.prepare_crops(\n#             [image_id for image_id, _, _ in self.train],\n#             '/content/drive/MyDrive/preprocessing/train.csv',\n#             False,\n#         )\n\n    def process_other(\n            self\n    ) -> Tuple[List[List[np.ndarray]], List[List[Tuple[int]]]]:\n        return self.prepare_crops(\n            [image_id for image_id, _, _ in self.other],\n            '/content/drive/MyDrive/preprocessing/train.csv',\n            False,\n        )\n\n#     def process_test(\n#         self\n#     ) -> Tuple[List[List[np.ndarray]], List[List[Tuple[int]]]]:\n#         return self.prepare_crops(\n#             [image_id for image_id, _, _ in self.test],\n#             '/content/drive/MyDrive/preprocessing/train.csv',\n#             False,\n#         )\n\n    def process_once(self, image_path) -> Tuple[List[List[np.ndarray]], List[List[Tuple[int]]]]:\n        self.test = [(\"img_id\", \"unkown\", -1)]\n        return self.prepare_crops(\"img_id\", image_path, True)\n# data_prep = DataPreparation()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom PIL import Image\nfrom time import time\n\n# Constants (These should be defined based on your specific case)\nBLOCK_SIZE = 28\nBLOCKS_PER_CROP = 8\nCROP_SIZE = BLOCK_SIZE * BLOCKS_PER_CROP\nBLOCK_THR = 90\nCROP_THR = 0.6\nMAX_CROPS_PER_IMAGE = 20\nIMAGES_PER_SAMPLE = 4\nEPOCHS_NUM = 10\nSCALE_FACTOR = 24\nTEST_SAMPLE_DUPL_RATE = 20\nTRAIN_SAMPLE_DUPL_RATE = 4\nCENTER_GROUPS = [(11,), (4,), (7,), (1, 5,), (10, 3), (6, 2, 8, 9,)]\n\n# Assume DataPreparation is already defined and imported\ndata_prep = DataPreparation()\n\n# Assume you have the correct image paths and IDs\nimage_ids = tooBig  # These should be the IDs of the images you want to process\nbase_image_path = '/kaggle/input/mayo-clinic-strip-ai/test'\n\n# Check if there is at least one image ID in the list\nif image_ids:\n    # Select the first image ID from the list\n#     first_image_id = image_ids\n    # Process only the first image\n    crops, crop_indices = data_prep.prepare_crops(image_ids, base_image_path, False)\n    # `crops` will now contain the crops for the first image\n    # `crop_indices` will now contain the indices for the crops of the first image\nelse:\n    print(\"No image IDs provided.\")\n\n# # If you need to work with the crops or display them\n# if crops:\n#     first_crop = crops[0]  # Assuming you want the first crop\n#     image_display = Image.fromarray(first_crop)\n#     image_display.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport cv2\nimport SimpleITK as sitk\nfrom radiomics import featureextractor\nimport pandas as pd\nfrom PIL import Image\n\nif image_ids:\n    # Initialize PyRadiomics feature extractor\n    extractor = featureextractor.RadiomicsFeatureExtractor()\n\n    # Define your color thresholds\n    lower_yellow = np.array([20, 100, 100])\n    upper_yellow = np.array([30, 255, 255])\n    lower_orange = np.array([5, 100, 100])\n    upper_orange = np.array([20, 255, 255])\n    lower_brown = np.array([10, 50, 20])\n    upper_brown = np.array([40, 255, 255])\n    lower_blue = np.array([90, 50, 50])\n    lower_purple = np.array([120, 50, 50])\n    upper_purple = np.array([150, 255, 255])\n    lower_dark_purple = np.array([100, 50, 50])\n    upper_dark_purple = np.array([120, 255, 255])\n    lower_white = np.array([0, 0, 200])\n    upper_white = np.array([179, 30, 255])\n\n    # Initialize list to store PyRadiomics data\n    pyradiomics_data = []\n\n    # Iterate over the array of PIL images\n    for i, pil_image in enumerate(crops):\n        print(i)\n        # Convert PIL Image to OpenCV format\n        open_cv_image = cv2.cvtColor(np.array(pil_image), cv2.COLOR_RGB2BGR)\n        hsv_image = cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2HSV)\n\n        # Process the image to create masks\n        orange_mask = cv2.inRange(hsv_image, lower_orange, upper_orange)\n        yellow_mask = cv2.inRange(hsv_image, lower_yellow, upper_yellow)\n        brown_mask = cv2.inRange(hsv_image, lower_brown, upper_brown)\n        blue_purple_mask = cv2.bitwise_or(cv2.inRange(hsv_image, lower_blue, upper_purple), cv2.inRange(hsv_image, lower_purple, upper_purple))\n        blue_purple_mask = cv2.bitwise_or(blue_purple_mask, cv2.inRange(hsv_image, lower_dark_purple, upper_dark_purple))\n        white_mask = cv2.inRange(hsv_image, lower_white, upper_white)\n\n        # Combine masks for RBC, WBC, and FP\n        yellow_brown_orange_mask = cv2.bitwise_or(cv2.bitwise_or(yellow_mask, brown_mask), orange_mask)\n        fibrin_platelets_mask = cv2.bitwise_and(cv2.bitwise_not(yellow_brown_orange_mask), cv2.bitwise_not(blue_purple_mask))\n        fibrin_platelets_mask = cv2.bitwise_and(fibrin_platelets_mask, cv2.bitwise_not(white_mask))\n\n        # Convert masks to binary for feature extraction\n        rbc_mask = np.where(yellow_brown_orange_mask > 0, 1, 0)\n        wbc_mask = np.where(blue_purple_mask > 0, 1, 0)\n        fp_mask = np.where(fibrin_platelets_mask > 0, 1, 0)\n\n        try:\n            # Convert images to SimpleITK format and extract features\n            original_image_sitk = sitk.GetImageFromArray(cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2GRAY))\n            featuresRBC = extractor.execute(original_image_sitk, sitk.GetImageFromArray(rbc_mask.astype(np.uint8)))\n            featuresWBC = extractor.execute(original_image_sitk, sitk.GetImageFromArray(wbc_mask.astype(np.uint8)))\n            featuresFP = extractor.execute(original_image_sitk, sitk.GetImageFromArray(fp_mask.astype(np.uint8)))\n\n            # Append data to list\n            pyradiomics_data.append({\n                'Image_ID': tooBig[i],\n                **{f\"RBC_{k}\": v for k, v in featuresRBC.items()},\n                **{f\"WBC_{k}\": v for k, v in featuresWBC.items()},\n                **{f\"FP_{k}\": v for k, v in featuresFP.items()}\n            })\n        except Exception as e:\n            print(f\"Error extracting features for image {i}: {e}\")\n\n    # Convert list of dictionaries to DataFrame and save to CSV\n    pyradiomics_df2 = pd.DataFrame(pyradiomics_data)\n    # Concatenate the DataFrames\n    pyradiomics_final = pd.concat([pyradiomics_df2, pyradiomics_df1], ignore_index=True)\n\n    # Check the new DataFrame\n    pyradiomics_df = pyradiomics_final\n    pyradiomics_df.to_csv('/kaggle/working/pyrad.csv')\n    pyradiomics_df\nelse:\n    pyradiomics_final = pyradiomics_df1\n\n    # Check the new DataFrame\n    pyradiomics_df = pyradiomics_final\n    pyradiomics_df.to_csv('/kaggle/working/pyrad.csv')\n    pyradiomics_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pandas as pd\n\n# # Assuming df1 already exists and you have the pyradiomics_df from the previous operations\n\n# # Concatenate the DataFrames\n# pyradiomics_final = pd.concat([pyradiomics_df2, pyradiomics_df1], ignore_index=True)\n\n# # Check the new DataFrame\n# pyradiomics_df = pyradiomics_final\n# pyradiomics_df.to_csv('/kaggle/working/pyrad.csv')\n# pyradiomics_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pyradiomics_df = pd.read_csv('/kaggle/working/pyrad.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pyradiomics_df = pyradiomics_df.drop(columns=[col for col in pyradiomics_df.columns if col not in significant_features and col not in [\"Image_ID\", \"Label\"]])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = pyradiomics_df['Image_ID']\nX_test = pyradiomics_df.drop('Image_ID', axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_prefix(image_id):\n    return '_'.join(image_id.split('_')[:-2])\n\n# for i in range(len(ids)):\n#     ids.iloc[i] = extract_prefix(ids.iloc[i])\n\nids = ids.apply(extract_prefix)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = X_test.replace(np.nan, 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = scaler.transform(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.array(XGB.predict_proba(X_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids = ids.to_numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = pd.DataFrame(np.concatenate((ids.reshape(-1,1), preds), axis=1), columns = ['patient_id', 'CE', 'LAA'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds.groupby('patient_id').mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mce = preds['CE'].mean()\nmlaa = preds['LAA'].mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for img in tooBig:\n#     preds.loc['_'.join(img.split('_')[:-1])] = [mce, mlaa]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fill NaN values with [0.5, 0.5] for 'CE' and 'LAA' column s\npreds.fillna({'CE': mce, 'LAA': mlaa}, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"invalid_values_mask = (preds['CE'] > 1) | (preds['CE'] < 0) | (preds['LAA'] > 1) | (preds['LAA'] < 0)\npreds.loc[invalid_values_mask, ['CE', 'LAA']] = [mce, mlaa]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folder_path = '/kaggle/input/mayo-clinic-strip-ai/test'\nfiles = os.listdir(folder_path)\nfile_names = [os.path.splitext(file)[0] for file in files]\nmodified_list = ['_'.join(item.split('_')[:-1]) for item in file_names]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_ids = preds.index\nfor patient_id in modified_list:\n    # Check if patient_id is not in the DataFrame index\n    if patient_id  not in pred_ids:\n        preds.loc[patient_id] = [mce, mlaa]\n        #print(\"0\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds.reset_index()\npreds = preds.sort_values(by='patient_id')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds.drop_duplicates(subset='patient_id', keep='first')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = preds.reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.to_csv('/kaggle/working/submission.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}