{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # Test data 가져오기\n# test_data = RSNADataset(test, _, _,is_train=False)\n# test_loader = DataLoader(test_data, batch_size=16, shuffle=False, num_workers=2)\n\n# # Device 설정 및 모델 가져오기\n# DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# output_size = 1\n# model = EfficientNet4(output_size=output_size).to(DEVICE)\n# model.load_state_dict(torch.load(\"/kaggle/input/checkpoint-efficientb4/EfficientB4_extravasation59.pth\", map_location=DEVICE)[\"state_dict\"])\n\n# # Test data로 prediction 값 구하기\n# id_list = []\n# bowel_injury = []\n# bowel_injury2 = []\n# model.eval()\n# with torch.no_grad():\n#     for i in range(len(test_loader)):\n#         image = test_data.getitem(i)\n#         out = model(image).to(DEVICE)\n#         pred = torch.sigmoid(out)\n        \n#         if ID in id_list:\n#             bowel_injury[id_list.index(ID)].append(pred.item())\n#         else:\n#             id_list.append(ID)\n#             bowel_injury.append([pred.item()])\n\n# for i in range(len(cancer_list)):\n#     bowel_injury2.append(sum(bowel_injury[i]) / len(bowel_injury[i]))\n\n# submission = pd.DataFrame({'patient_id': id_list, 'bowel_injury': bowel_injury2})","metadata":{"execution":{"iopub.status.busy":"2023-10-08T11:43:36.133598Z","iopub.execute_input":"2023-10-08T11:43:36.133967Z","iopub.status.idle":"2023-10-08T11:43:36.141781Z","shell.execute_reply.started":"2023-10-08T11:43:36.133942Z","shell.execute_reply":"2023-10-08T11:43:36.140703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q git+https://github.com/keras-team/keras-cv\n!pip install pydicom\n!pip install tqdm","metadata":{"execution":{"iopub.status.busy":"2023-10-08T11:43:36.143892Z","iopub.execute_input":"2023-10-08T11:43:36.145092Z","iopub.status.idle":"2023-10-08T11:44:22.363362Z","shell.execute_reply.started":"2023-10-08T11:43:36.14505Z","shell.execute_reply":"2023-10-08T11:44:22.362059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)\n\nimport keras\nprint(keras.__version__)\n\nimport h5py\nprint(h5py.__version__)\n\nimport numpy as np\nprint(np.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-10-08T11:44:22.365448Z","iopub.execute_input":"2023-10-08T11:44:22.365775Z","iopub.status.idle":"2023-10-08T11:44:22.373501Z","shell.execute_reply.started":"2023-10-08T11:44:22.365747Z","shell.execute_reply":"2023-10-08T11:44:22.372165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip uninstall keras_core -y\n!pip install keras_core==0.1.7\n!pip uninstall tensorflow -y\n!pip install tensorflow==2.13.0\n!pip uninstall keras -y\n!pip install keras==2.13.1","metadata":{"execution":{"iopub.status.busy":"2023-10-08T11:44:22.374943Z","iopub.execute_input":"2023-10-08T11:44:22.375233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ[\"KERAS_BACKEND\"] = \"tensorflow\"\nimport keras_core as keras\nimport keras_cv\nfrom keras.optimizers import Adam\n\nimport gc\nimport cv2\nimport pydicom\nfrom joblib import Parallel, delayed\n\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nfrom glob import glob\n\nfrom tqdm import tqdm\nimport gc\n\nclass Config:\n    IMAGE_SIZE = [256, 256]\n    RESIZE_DIM = 256\n    BATCH_SIZE = 16\n    AUTOTUNE = tf.data.AUTOTUNE\n    TARGET_COLS  = [\"bowel_healthy\", \"bowel_injury\", \"extravasation_healthy\",\n                   \"extravasation_injury\", \"kidney_healthy\", \"kidney_low\",\n                   \"kidney_high\", \"liver_healthy\", \"liver_low\", \"liver_high\",\n                   \"spleen_healthy\", \"spleen_low\", \"spleen_high\"]\n\nconfig = Config()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_PATH = \"/kaggle/input/rsna-2023-abdominal-trauma-detection\"\nmeta_df = pd.read_csv(f\"{BASE_PATH}/test_series_meta.csv\")\n\n# Checking if patients are repeated by finding the number of unique patient IDs\nnum_rows = meta_df.shape[0]\nunique_patients = meta_df[\"patient_id\"].nunique()\n\nprint(f\"{num_rows=}\")\nprint(f\"{unique_patients=}\")\n\nmeta_df[\"image_path\"] = f\"{BASE_PATH}/test_images/\" + meta_df.patient_id.astype(str) + \"/\" + meta_df.series_id.astype(str)\nmeta_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\ndef _decode_image_python(image_path):\n    image_path = image_path.numpy().decode('utf-8')\n\n    # Check if the path exists\n    dcm_files = [f for f in os.listdir(image_path)]\n    dcm_files = sorted(dcm_files, key=lambda x: int(x.split('.')[0]))\n\n    # 중간 디렉토리 가져옴\n    image_path_ = os.path.join(image_path, dcm_files[len(dcm_files)//2])\n    image = pydicom.dcmread(image_path_)\n\n    # 원하는 크기로 resize\n    resized_image = cv2.resize(image.pixel_array, (Config.IMAGE_SIZE[0], Config.IMAGE_SIZE[1]), interpolation=cv2.INTER_AREA).astype(np.uint8)\n\n    # 이미지 합치기\n    images = np.zeros((Config.IMAGE_SIZE[0], Config.IMAGE_SIZE[1], 3), dtype=np.uint8)\n    images[..., 0] = resized_image\n    images[..., 1] = resized_image\n    images[..., 2] = resized_image\n\n    return tf.cast(images, tf.float32) / 255.0\n\ndef decode_image(image_path):\n    return tf.py_function(_decode_image_python, [image_path], Tout=tf.float32)\n\ndef build_dataset(image_paths):\n    ds = (\n        tf.data.Dataset.from_tensor_slices(image_paths)\n        .map(decode_image, num_parallel_calls=config.AUTOTUNE)\n        .shuffle(config.BATCH_SIZE * 10)\n        .batch(config.BATCH_SIZE)\n        .prefetch(config.AUTOTUNE)\n    )\n    return ds\n\npaths = [path for path in meta_df.image_path.tolist() if os.path.exists(path)]\nds = build_dataset(image_paths=paths)\nimages = next(iter(ds))\nprint(images.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def post_proc(pred):\n    proc_pred = np.empty((pred.shape[0], 2*2 + 3*3), dtype=\"float32\")\n\n    # bowel, extravasation\n    proc_pred[:, 0] = pred[:, 0]\n    proc_pred[:, 1] = 1 - pred[:, 0]\n    proc_pred[:, 2] = pred[:, 1]\n    proc_pred[:, 3] = 1 - pred[:, 2]\n\n    # liver, kidney, sneel\n    proc_pred[:, 4:7] = pred[:, 2:5]\n    proc_pred[:, 7:10] = pred[:, 5:8]\n    proc_pred[:, 10:13] = pred[:, 8:11]\n\n    return proc_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom sklearn.metrics import confusion_matrix\n\n# Custom metric to calculate sensitivity\n# @tf.function\n# @tf.keras.utils.register_keras_serializable()\ndef sensitivity(y_true, y_pred):\n    true_positives = tf.reduce_sum(tf.cast(tf.logical_and(tf.equal(y_true, 1), tf.equal(tf.round(y_pred), 1)), dtype=tf.float32))\n    actual_positives = tf.reduce_sum(tf.cast(tf.equal(y_true, 1), dtype=tf.float32))\n    return true_positives / (actual_positives + tf.keras.backend.epsilon())\n\n# Custom metric to calculate specificity\n# @tf.function\n# @tf.keras.utils.register_keras_serializable()\ndef specificity(y_true, y_pred):\n    true_negatives = tf.reduce_sum(tf.cast(tf.logical_and(tf.equal(y_true, 0), tf.equal(tf.round(y_pred), 0)), dtype=tf.float32))\n    actual_negatives = tf.reduce_sum(tf.cast(tf.equal(y_true, 0), dtype=tf.float32))\n    return true_negatives / (actual_negatives + tf.keras.backend.epsilon())\n\ndef custom_adam_from_config(cls, config, custom_objects=None):\n    config.pop('loss_scale_factor', None)  # remove it if not handled\n    return cls(**config)\n\nAdam.from_config = classmethod(custom_adam_from_config)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_ids = meta_df[\"patient_id\"].unique()\nfinal_df = pd.DataFrame({'patient_id':patient_ids})\n\nmodel_names = [\"bowel\", \"extravasation\", \"kidney\", \"liver\", \"spleen\"]\n\nfor i, name in enumerate(model_names):\n    model_filename = f\"EfficinetnetB3_{name}.keras\"\n    model_path = os.path.join(\"/kaggle/input/checkpoint-efficientb4\", model_filename)\n    print(name)\n    if name ==\"bowel\" or name==\"extravasation\":\n        model = tf.keras.models.load_model(model_path, compile=False)\n        model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[sensitivity, specificity])\n#         custom_objects = {\"Adam\": Adam, \"sensitivity\": sensitivity, \"specificity\": specificity}\n#         model = tf.keras.models.load_model(model_path, custom_objects=custom_objects)\n    else:\n        model = tf.keras.models.load_model(model_path)\n        model.summary()\n        print(f\"Output shape for model {name}: {model.output_shape}\")\n\n    # Getting unique patient IDs from test dataset\n    patient_ids = meta_df[\"patient_id\"].unique()\n\n    # Initializing array to store predictions\n    output_dim = model.output_shape[-1]  # This should be 3 for the kidney model\n    patient_preds = np.zeros(shape=(len(patient_ids), output_dim), dtype=\"float32\")\n\n    # Iterating over each patient\n    for pidx, patient_id in tqdm(enumerate(patient_ids), total=len(patient_ids), desc=\"Patients\"):\n        print(f\"Patient ID: {patient_id}\")\n\n        # Query the dataframe for a particular patient\n        patient_df = meta_df[meta_df[\"patient_id\"] == patient_id]\n\n        # Getting image paths for a patient\n        patient_paths = [path for path in patient_df.image_path.tolist() if os.path.exists(path)]\n\n        # Building dataset for prediction\n        dtest = build_dataset(patient_paths)\n\n        pred = model.predict(dtest)\n\n        # pred = pred.squeeze()  # Removes singleton dimensions, if any. This will convert shape from (1,1) to (1,) if required.\n        print(f\"Predictions shape for patient {patient_id}: {pred.shape}\")\n\n        # If the prediction shape is (1,), reshape it to (1, 1) for consistency.\n        if len(pred.shape) == 1:\n            pred = pred.reshape(1, 1)\n\n        # Use the correct dimension based on the prediction shape.\n        if len(pred.shape) == 2:\n            dim = pred.shape[1]\n        else:\n            # Handle other shapes or raise an exception if unexpected\n            raise ValueError(f\"Unexpected shape for pred: {pred.shape}\")\n\n        pred = np.mean(pred.reshape(1, len(patient_paths), dim), axis=0)\n        pred = np.max(pred, axis=0, keepdims=True)\n        patient_preds[pidx, :output_dim] = pred.squeeze()\n\n\n        # Deleting variables to free up memory\n        del patient_df, patient_paths, dtest, pred; gc.collect()\n\n    temp_df = pd.DataFrame(patient_preds, columns=[f\"{name}_pred_{j}\" for j in range(output_dim)])\n\n    # Merge the temporary DataFrame with the final DataFrame on the patient IDs.\n    final_df = pd.concat([final_df, temp_df], axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df['bowel_pred_1'] = 1 - final_df['bowel_pred_0']\n\n# Reorder the columns to place 'bowel_pred_1' immediately after 'bowel_pred_0'.\ncolumns_order = final_df.columns.tolist()\nbowel_pred_0_index = columns_order.index('bowel_pred_0')\ncolumns_order.insert(bowel_pred_0_index + 1, columns_order.pop(-1))  # Moving the last column to the desired position\nfinal_df = final_df[columns_order]\n\nfinal_df['extravasation_pred_1'] = 1 - final_df['extravasation_pred_0']\n\ncolumns_order = final_df.columns.tolist()\nextravasation_pred_0_index = columns_order.index('extravasation_pred_0')\ncolumns_order.insert(extravasation_pred_0_index + 1, columns_order.pop(-1))  # Moving the last column to the desired position\nfinal_df = final_df[columns_order]\n\nfinal_df.columns = [['patient_id', 'bowel_healthy', 'bowel_injury', 'extravasation_healthy',\n       'extravasation_injury', 'kidney_healthy', 'kidney_low', 'kidney_high',\n       'liver_healthy', 'liver_low', 'liver_high', 'spleen_healthy',\n       'spleen_low', 'spleen_high']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Store submission\nfinal_df.to_csv(\"/kaggle/working/test.csv \",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}