{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"},{"sourceId":13432735,"sourceType":"datasetVersion","datasetId":8523571}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Chỉ cần cho JPEG Lossless:\n!pip install -q --no-deps pylibjpeg pylibjpeg-libjpeg","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-22T05:23:53.896576Z","iopub.execute_input":"2025-10-22T05:23:53.897351Z","iopub.status.idle":"2025-10-22T05:23:58.328772Z","shell.execute_reply.started":"2025-10-22T05:23:53.897309Z","shell.execute_reply":"2025-10-22T05:23:58.327648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pydicom\nfrom pydicom.pixel_data_handlers import pylibjpeg_handler, gdcm_handler\nprint(\"pylibjpeg available:\", pylibjpeg_handler.is_available())\nprint(\"gdcm available:\", gdcm_handler.is_available())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-22T05:23:58.330765Z","iopub.execute_input":"2025-10-22T05:23:58.331156Z","iopub.status.idle":"2025-10-22T05:23:59.227839Z","shell.execute_reply.started":"2025-10-22T05:23:58.33112Z","shell.execute_reply":"2025-10-22T05:23:59.226998Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============== CELL 1 ===============\n# CONFIG + SPLIT 70/30 BY PATIENT (KEEP IMAGE-LEVEL RATIO)\nimport numpy as np\nimport pandas as pd\n\n# ---- Config ----\nSRC_META   = \"/kaggle/input/datacsv2/train_selected_70_30.csv\"   # file bạn đã tạo ở bước trước\nRSNA_ROOT  = \"/kaggle/input/rsna-breast-cancer-detection\"\nDICOM_ROOT = f\"{RSNA_ROOT}/train_images\"\nYOLO_WEIGHTS = \"/kaggle/input/datacsv2/yolo11n_roi.pt\"    # theo yêu cầu\n\nOUT_ROOT   = \"/kaggle/working/dataset\"\nTRAIN_DIR  = f\"{OUT_ROOT}/train_image\"   # (sẽ dùng sau)\nTEST_DIR   = f\"{OUT_ROOT}/test_image\"\n\nTRAIN_CSV  = f\"{OUT_ROOT}/train.csv\"\nTEST_CSV   = f\"{OUT_ROOT}/test.csv\"\n\nSPLIT_TARGET = 0.70    # 70% train - 30% test\nSEED = 42\nMAX_TRIALS = 2000      # số lần thử để tìm chia tối ưu theo tiêu chí\n\n# ---- Load & sanity check ----\nmeta = pd.read_csv(SRC_META)\nneed_cols = {\"site_id\",\"patient_id\",\"image_id\",\"laterality\",\"view\",\"age\",\"implant\",\"machine_id\",\"cancer\"}\nmiss = need_cols - set(meta.columns)\nif miss:\n    raise KeyError(f\"Thiếu cột {miss} trong {SRC_META}\")\n\n# chuẩn hoá kiểu + loại trùng lặp\nmeta[\"patient_id\"] = meta[\"patient_id\"].astype(int)\nmeta[\"image_id\"]   = meta[\"image_id\"].astype(int)\nmeta = meta.drop_duplicates(subset=[\"patient_id\",\"image_id\"]).reset_index(drop=True)\n\n# thống kê toàn cục theo ẢNH\nglob_counts = meta[\"cancer\"].value_counts().reindex([0,1], fill_value=0)\nglob_total  = int(glob_counts.sum())\nglob_pos    = int(glob_counts[1]); glob_neg = int(glob_counts[0])\nglob_posr   = glob_pos / glob_total if glob_total else 0.0\nprint(f\"[GLOBAL] Images={glob_total:,} | 0={glob_neg:,} 1={glob_pos:,} | Pos%={glob_posr*100:.2f}%\")\n\n# tóm tắt theo bệnh nhân\nby_p = meta.groupby(\"patient_id\").agg(\n    n=(\"image_id\",\"nunique\"),\n    pos=(\"cancer\",\"sum\")\n).reset_index()\nby_p[\"neg\"] = by_p[\"n\"] - by_p[\"pos\"]\n\npatients = by_p[\"patient_id\"].tolist()\nn_pat = len(patients)\ntarget_train_pat  = int(round(n_pat * SPLIT_TARGET))\ntarget_train_imgs = int(round(glob_total * SPLIT_TARGET))\n\n# ---- Heuristic: chọn 70% bệnh nhân sao cho số ảnh và Pos% gần toàn cục ----\nrng = np.random.default_rng(SEED)\nbest_score = (1e12, 1e12)  # (|imgs_train - target_imgs|, |posr_train - posr_global|)\nbest_sel = None\n\nfor _ in range(MAX_TRIALS):\n    perm = rng.permutation(n_pat)\n    chosen = set()\n    cur_n = 0; cur_pos = 0\n    for idx in perm:\n        pid = patients[idx]\n        row = by_p.iloc[idx]\n        if (len(chosen) < target_train_pat) or (cur_n < target_train_imgs):\n            chosen.add(pid)\n            cur_n  += int(row[\"n\"])\n            cur_pos+= int(row[\"pos\"])\n    if cur_n == 0:\n        continue\n    cur_posr = cur_pos / cur_n\n    score = (abs(cur_n - target_train_imgs), abs(cur_posr - glob_posr))\n    if score < best_score:\n        best_score = score\n        best_sel = chosen\n\ntrain_pats = best_sel if best_sel else set(rng.choice(patients, size=target_train_pat, replace=False))\ntest_pats  = set(patients) - train_pats\n\nmeta[\"split\"] = np.where(meta[\"patient_id\"].isin(train_pats), \"train\", \"test\")\ntrain_meta = meta[meta[\"split\"]==\"train\"].copy()\ntest_meta  = meta[meta[\"split\"]==\"test\"].copy()\n\n# báo cáo tỉ lệ theo ảnh sau chia\ndef report_ratio(name, df):\n    c = df[\"cancer\"].value_counts().reindex([0,1], fill_value=0)\n    tot = int(c.sum()); neg=int(c[0]); pos=int(c[1])\n    posr = pos/tot*100 if tot else 0\n    print(f\"[{name}] Patients={df['patient_id'].nunique():,} | Images={tot:,} | 0={neg:,} 1={pos:,} | Pos%={posr:.2f}%\")\n\nreport_ratio(\"TRAIN\", train_meta)\nreport_ratio(\"TEST \", test_meta)\n\n# Lưu tạm hai DataFrame vào session (biến) để các cell sau dùng tiếp\nprint(\"\\nSplit xong. Tiếp tục Cell 2 để xử lý test_images + tạo test.csv.\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-22T05:24:01.619168Z","iopub.execute_input":"2025-10-22T05:24:01.619782Z","iopub.status.idle":"2025-10-22T05:24:58.253896Z","shell.execute_reply.started":"2025-10-22T05:24:01.619732Z","shell.execute_reply":"2025-10-22T05:24:58.252844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip -q install ultralytics","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-22T05:26:48.816641Z","iopub.execute_input":"2025-10-22T05:26:48.817093Z","iopub.status.idle":"2025-10-22T05:26:53.109619Z","shell.execute_reply.started":"2025-10-22T05:26:48.817041Z","shell.execute_reply":"2025-10-22T05:26:53.108156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============== CELL 2 (FULL) ===============\n# EXPORT TEST IMAGES (per-patient folders) + WRITE TEST.CSV (9 cols) + QUICK PREVIEW (vertical 512x1024, no padding)\n\nimport os\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport pydicom\nimport matplotlib.pyplot as plt\nfrom ultralytics import YOLO\n\n# ---- Kiểm tra biến từ Cell 1 ----\nrequired_vars = [\"test_meta\", \"DICOM_ROOT\", \"YOLO_WEIGHTS\", \"TEST_DIR\", \"TEST_CSV\", \"OUT_ROOT\"]\nfor v in required_vars:\n    if v not in globals():\n        raise NameError(f\"Biến '{v}' chưa có. Hãy chạy Cell 1 trước.\")\n\n# ---- Cấu hình kích thước dọc ----\nTARGET_W, TARGET_H = 512, 1024   # Width x Height (dọc)\n\n# ---- Utils xử lý ảnh ----\ndef ensure_dir(p):\n    Path(p).mkdir(parents=True, exist_ok=True)\n\ndef dicom_to_uint8(dcm_path):\n    # Đọc DICOM (cần pylibjpeg hoặc gdcm để giải nén JPEG Lossless nếu gặp)\n    ds = pydicom.dcmread(dcm_path, force=True)\n    arr = ds.pixel_array.astype(np.float32)\n\n    # MONOCHROME1 -> đảo mức xám\n    photometric = getattr(ds, \"PhotometricInterpretation\", \"MONOCHROME2\")\n    if photometric == \"MONOCHROME1\":\n        arr = arr.max() - arr\n\n    # Windowing nếu có, fallback percentile\n    try:\n        wc = ds.WindowCenter\n        ww = ds.WindowWidth\n        wc = float(wc[0]) if hasattr(wc, \"__len__\") else float(wc)\n        ww = float(ww[0]) if hasattr(ww, \"__len__\") else float(ww)\n        lo = wc - ww/2\n        hi = wc + ww/2\n        arr = np.clip((arr - lo) / max(hi - lo, 1e-6), 0, 1)\n    except Exception:\n        vmin, vmax = np.percentile(arr, (0.5, 99.5))\n        arr = np.clip((arr - vmin) / max(vmax - vmin, 1e-6), 0, 1)\n\n    img = (arr * 255.0).astype(np.uint8)\n    if img.ndim == 2:\n        img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)\n    return img\n\ndef crop_with_box(img, box, pad=0.03):\n    H, W = img.shape[:2]\n    x1, y1, x2, y2 = box\n    w = x2 - x1; h = y2 - y1\n    x1 -= w*pad; x2 += w*pad; y1 -= h*pad; y2 += h*pad\n    x1 = max(0, int(x1)); y1 = max(0, int(y1))\n    x2 = min(W, int(x2)); y2 = min(H, int(y2))\n    if x2 <= x1 or y2 <= y1:\n        return img\n    return img[y1:y2, x1:x2].copy()\n\ndef resize_stretch(img, out_w=512, out_h=1024):\n    return cv2.resize(img, (out_w, out_h), interpolation=cv2.INTER_AREA)\n\ndef save_png(img, path):\n    Path(path).parent.mkdir(parents=True, exist_ok=True)\n    cv2.imwrite(path, img)\n\ndef yolo_best_box(model, img_bgr):\n    # Ultralytics expects RGB\n    res = model.predict(source=img_bgr[..., ::-1], verbose=False)[0]\n    if res.boxes is None or len(res.boxes) == 0:\n        return None\n    conf = res.boxes.conf.cpu().numpy()\n    b = res.boxes.xyxy.cpu().numpy()\n    return b[int(conf.argmax())]\n\ndef build_prediction_id(row):\n    if \"prediction_id\" in row and pd.notna(row[\"prediction_id\"]):\n        return str(row[\"prediction_id\"])\n    return f\"{int(row['patient_id'])}_{str(row['laterality'])}\"\n\n# ---- Chuẩn bị thư mục test_image ----\nensure_dir(TEST_DIR)\n\n# ---- Load YOLO model ----\nyolo = YOLO(YOLO_WEIGHTS)\n\n# ---- Xử lý và lưu TEST images (theo folder bệnh nhân) ----\nsaved_map = {}  # index of test_meta -> saved png path\nn_total = len(test_meta)\nn_ok = 0\n\nfor i, row in test_meta.iterrows():\n    dcm = f\"{DICOM_ROOT}/{int(row['patient_id'])}/{int(row['image_id'])}.dcm\"\n    if not os.path.exists(dcm):\n        continue\n    try:\n        img = dicom_to_uint8(dcm)\n    except Exception:\n        # Nếu ảnh lỗi giải nén/đọc thì bỏ qua\n        continue\n\n    box = yolo_best_box(yolo, img)\n    if box is not None:\n        img = crop_with_box(img, box, pad=0.03)\n    img = resize_stretch(img, out_w=TARGET_W, out_h=TARGET_H)\n\n    pid = int(row[\"patient_id\"])\n    iid = int(row[\"image_id\"])\n    outp = f\"{TEST_DIR}/{pid}/{iid}.png\"\n    save_png(img, outp)\n    saved_map[i] = outp\n    n_ok += 1\n\nprint(f\"Đã lưu {n_ok}/{n_total} ảnh test vào thư mục: {TEST_DIR}\")\n\n# ---- Cập nhật test_meta theo các ảnh đã lưu ----\ntest_meta = test_meta[test_meta.index.isin(saved_map.keys())].copy()\ntest_meta[\"png_path\"] = test_meta.index.map(saved_map)\n\n# ---- Tạo TEST.CSV: chỉ 9 cột, đúng thứ tự ----\nneed_test_cols = [\"site_id\",\"patient_id\",\"image_id\",\"laterality\",\"view\",\"age\",\"implant\",\"machine_id\",\"prediction_id\"]\ntest_out = test_meta.copy()\nif \"prediction_id\" not in test_out.columns:\n    test_out[\"prediction_id\"] = test_out.apply(build_prediction_id, axis=1)\ntest_out = test_out[need_test_cols]\n\nPath(OUT_ROOT).mkdir(parents=True, exist_ok=True)\ntest_out.to_csv(TEST_CSV, index=False)\nprint(f\"Đã lưu test.csv (9 cột) tại: {TEST_CSV}\")\n\n# ---- Xem nhanh ảnh của 1 bệnh nhân (tối đa 4 ảnh) ----\nif len(test_meta):\n    demo_pid = int(test_meta[\"patient_id\"].iloc[0])\n    demo_rows = test_meta[test_meta[\"patient_id\"] == demo_pid].head(4)\n    fig, axes = plt.subplots(1, len(demo_rows), figsize=(4*len(demo_rows), 8))\n    if len(demo_rows) == 1:\n        axes = [axes]\n    for ax, (_, r) in zip(axes, demo_rows.iterrows()):\n        bgr = cv2.imread(r[\"png_path\"])\n        if bgr is None:\n            ax.axis('off'); continue\n        rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)\n        ax.imshow(rgb)\n        ax.set_title(f\"PID {demo_pid}\\nIMG {int(r['image_id'])} | {r.get('view','?')}/{r.get('laterality','?')}\")\n        ax.axis('off')\n    plt.tight_layout()\n    plt.show()\nelse:\n    print(\"Không có ảnh test nào được lưu – kiểm tra lại DICOM path/YOLO/decoder.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-22T05:27:00.987779Z","iopub.execute_input":"2025-10-22T05:27:00.988387Z","iopub.status.idle":"2025-10-22T06:08:28.158837Z","shell.execute_reply.started":"2025-10-22T05:27:00.988321Z","shell.execute_reply":"2025-10-22T06:08:28.156526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============== CELL 3 (FULL) ===============\n# EXPORT TRAIN IMAGES (per-patient folders) + WRITE TRAIN.CSV (keep all original columns) + QUICK PREVIEW (512x1024 stretch)\n\nimport os\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport pydicom\nimport matplotlib.pyplot as plt\nfrom ultralytics import YOLO\n\n# ---- Kiểm tra biến từ Cell 1 ----\nrequired_vars = [\"train_meta\", \"DICOM_ROOT\", \"YOLO_WEIGHTS\", \"TRAIN_DIR\", \"TRAIN_CSV\", \"OUT_ROOT\"]\nfor v in required_vars:\n    if v not in globals():\n        raise NameError(f\"Biến '{v}' chưa có. Hãy chạy Cell 1 trước.\")\n\n# ---- Cấu hình kích thước dọc ----\nTARGET_W, TARGET_H = 512, 1024   # Width x Height (dọc)\n\n# ---- Utils xử lý ảnh ----\ndef ensure_dir(p):\n    Path(p).mkdir(parents=True, exist_ok=True)\n\ndef dicom_to_uint8(dcm_path):\n    # Đọc DICOM (cần pylibjpeg hoặc gdcm để giải nén JPEG Lossless nếu gặp)\n    ds = pydicom.dcmread(dcm_path, force=True)\n    arr = ds.pixel_array.astype(np.float32)\n\n    # MONOCHROME1 -> đảo mức xám\n    photometric = getattr(ds, \"PhotometricInterpretation\", \"MONOCHROME2\")\n    if photometric == \"MONOCHROME1\":\n        arr = arr.max() - arr\n\n    # Windowing nếu có, fallback percentile\n    try:\n        wc = ds.WindowCenter\n        ww = ds.WindowWidth\n        wc = float(wc[0]) if hasattr(wc, \"__len__\") else float(wc)\n        ww = float(ww[0]) if hasattr(ww, \"__len__\") else float(ww)\n        lo = wc - ww/2\n        hi = wc + ww/2\n        arr = np.clip((arr - lo) / max(hi - lo, 1e-6), 0, 1)\n    except Exception:\n        vmin, vmax = np.percentile(arr, (0.5, 99.5))\n        arr = np.clip((arr - vmin) / max(vmax - vmin, 1e-6), 0, 1)\n\n    img = (arr * 255.0).astype(np.uint8)\n    if img.ndim == 2:\n        img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)\n    return img\n\ndef crop_with_box(img, box, pad=0.03):\n    H, W = img.shape[:2]\n    x1, y1, x2, y2 = box\n    w = x2 - x1; h = y2 - y1\n    x1 -= w*pad; x2 += w*pad; y1 -= h*pad; y2 += h*pad\n    x1 = max(0, int(x1)); y1 = max(0, int(y1))\n    x2 = min(W, int(x2)); y2 = min(H, int(y2))\n    if x2 <= x1 or y2 <= y1:\n        return img\n    return img[y1:y2, x1:x2].copy()\n\ndef resize_stretch(img, out_w=512, out_h=1024):\n    # Resize trực tiếp (méo nếu tỷ lệ khác) — KHÔNG padding\n    return cv2.resize(img, (out_w, out_h), interpolation=cv2.INTER_AREA)\n\ndef save_png(img, path):\n    Path(path).parent.mkdir(parents=True, exist_ok=True)\n    cv2.imwrite(path, img)\n\ndef yolo_best_box(model, img_bgr):\n    # Ultralytics expects RGB\n    res = model.predict(source=img_bgr[..., ::-1], verbose=False)[0]\n    if res.boxes is None or len(res.boxes) == 0:\n        return None\n    conf = res.boxes.conf.cpu().numpy()\n    b = res.boxes.xyxy.cpu().numpy()\n    return b[int(conf.argmax())]\n\n# ---- Chuẩn bị thư mục train_image ----\nensure_dir(TRAIN_DIR)\n\n# ---- Load YOLO model ----\nyolo = YOLO(YOLO_WEIGHTS)\n\n# ---- Xử lý và lưu TRAIN images (theo folder bệnh nhân) ----\nsaved_map = {}  # index of train_meta -> saved png path\nn_total = len(train_meta)\nn_ok = 0\n\nfor i, row in train_meta.iterrows():\n    dcm = f\"{DICOM_ROOT}/{int(row['patient_id'])}/{int(row['image_id'])}.dcm\"\n    if not os.path.exists(dcm):\n        continue\n    try:\n        img = dicom_to_uint8(dcm)\n    except Exception:\n        # Nếu ảnh lỗi giải nén/đọc thì bỏ qua\n        continue\n\n    box = yolo_best_box(yolo, img)\n    if box is not None:\n        img = crop_with_box(img, box, pad=0.03)\n    img = resize_stretch(img, out_w=TARGET_W, out_h=TARGET_H)\n\n    pid = int(row[\"patient_id\"])\n    iid = int(row[\"image_id\"])\n    outp = f\"{TRAIN_DIR}/{pid}/{iid}.png\"\n    save_png(img, outp)\n    saved_map[i] = outp\n    n_ok += 1\n\nprint(f\"Đã lưu {n_ok}/{n_total} ảnh train vào thư mục: {TRAIN_DIR}\")\n\n# ---- Cập nhật train_meta theo các ảnh đã lưu ----\ntrain_meta_saved = train_meta[train_meta.index.isin(saved_map.keys())].copy()\ntrain_meta_saved[\"png_path\"] = train_meta_saved.index.map(saved_map)\n\n# ---- Ghi TRAIN.CSV: giữ NGUYÊN toàn bộ cột gốc + png_path ----\nPath(OUT_ROOT).mkdir(parents=True, exist_ok=True)\ntrain_out = train_meta_saved.copy()  # giữ nguyên schema ban đầu\ntrain_out.to_csv(TRAIN_CSV, index=False)\nprint(f\"Đã lưu train.csv tại: {TRAIN_CSV}\")\n\n# ---- Xem nhanh ảnh của 1 bệnh nhân (tối đa 4 ảnh) ----\nif len(train_meta_saved):\n    demo_pid = int(train_meta_saved[\"patient_id\"].iloc[0])\n    demo_rows = train_meta_saved[train_meta_saved[\"patient_id\"] == demo_pid].head(4)\n    fig, axes = plt.subplots(1, len(demo_rows), figsize=(4*len(demo_rows), 8))\n    if len(demo_rows) == 1:\n        axes = [axes]\n    for ax, (_, r) in zip(axes, demo_rows.iterrows()):\n        bgr = cv2.imread(r[\"png_path\"])\n        if bgr is None:\n            ax.axis('off'); continue\n        rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)\n        ax.imshow(rgb)\n        ax.set_title(f\"PID {demo_pid}\\nIMG {int(r['image_id'])} | {r.get('view','?')}/{r.get('laterality','?')}\")\n        ax.axis('off')\n    plt.tight_layout()\n    plt.show()\nelse:\n    print(\"Không có ảnh train nào được lưu – kiểm tra lại DICOM path/YOLO/decoder.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-22T06:39:33.101335Z","iopub.execute_input":"2025-10-22T06:39:33.102113Z","iopub.status.idle":"2025-10-22T08:20:16.993188Z","shell.execute_reply.started":"2025-10-22T06:39:33.102002Z","shell.execute_reply":"2025-10-22T08:20:16.988543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============== CELL 4 (AUGMENT & BALANCE) ===============\n# Cân bằng train bằng MixUp/CutMix/TankMix theo luật:\n# - Bucket theo (view, laterality, site_id, age)\n# - MixUp/CutMix:\n#     * cùng nhãn -> giữ nhãn\n#     * khác nhãn -> blur NEG 80% & POS 10%, gán NHÃN 1\n# - TankMix: chỉ cùng nhãn\n# Sinh tới khi #pos == #neg. Lưu ảnh mới vào TRAIN_DIR/{patient_id}/{new_image_id}.png\n# Ghi CSV: OUT_ROOT/train_aug_balanced.csv (giữ meta anchor + cột bổ sung cho audit)\n\nimport os, random\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport cv2\n\n# ------------ YÊU CẦU BIẾN TỪ CELL 3 ------------\nrequired_vars = [\"train_meta_saved\", \"TRAIN_DIR\", \"OUT_ROOT\"]\nfor v in required_vars:\n    if v not in globals():\n        raise NameError(f\"Biến '{v}' chưa có. Hãy chạy cell export TRAIN trước.\")\n\n# ------------ CẤU HÌNH ------------\nTARGET_W, TARGET_H = 512, 1024      # ảnh train đã là 512x1024; vẫn ép lại để an toàn\nOUT_AUG_CSV = f\"{OUT_ROOT}/train_aug_balanced.csv\"\n\n# Tỷ lệ giữa MixUp và CutMix khi tạo ảnh mới từ cặp\nP_MIXUP = 0.5                       # (1-P_MIXUP) là CutMix\n# Mỗi khi sinh được 4 ảnh từ cặp, thêm 1 ảnh TankMix cùng nhãn để đa dạng\nTANKMIX_EVERY = 4\n\nSEED = 1337\nrandom.seed(SEED); np.random.seed(SEED)\n\n# ------------ TIỆN ÍCH ------------\ndef ensure_size_stretch(img, w=TARGET_W, h=TARGET_H):\n    return cv2.resize(img, (w, h), interpolation=cv2.INTER_AREA)\n\ndef read_png(pid, iid):\n    p = f\"{TRAIN_DIR}/{int(pid)}/{int(iid)}.png\"\n    if not os.path.exists(p): return None, p\n    return cv2.imread(p), p\n\ndef save_png(img, pid, iid_new):\n    outp = f\"{TRAIN_DIR}/{int(pid)}/{int(iid_new)}.png\"\n    Path(outp).parent.mkdir(parents=True, exist_ok=True)\n    cv2.imwrite(outp, img)\n    return outp\n\ndef new_image_id(base_iid, k):\n    # sinh id mới tránh đè: base*100000 + k + 1\n    return int(base_iid) * 100000 + int(k) + 1\n\ndef gaussian_blur_ratio(img, ratio=0.8):\n    # ratio ~ cỡ mờ (0.1 nhẹ, 0.8 mạnh)\n    H, W = img.shape[:2]\n    base = int(max(3, (min(H, W) * ratio * 0.12) // 2 * 2 + 1))  # số lẻ\n    return cv2.GaussianBlur(img, (base, base), 0)\n\ndef mixup(imgA, imgB, lam=0.5):\n    return (imgA.astype(np.float32)*lam + imgB.astype(np.float32)*(1-lam)).astype(np.uint8)\n\ndef cutmix(imgA, imgB, area_frac=0.33):\n    H, W = imgA.shape[:2]\n    patch = max(1, int(area_frac*H*W))\n    r = np.random.uniform(0.5, 2.0)\n    ph = int(np.sqrt(patch/r)); pw = int(patch/max(ph,1))\n    ph = max(1, min(ph, H)); pw = max(1, min(pw, W))\n    cx = np.random.randint(pw//2, W - pw//2 + 1)\n    cy = np.random.randint(ph//2, H - ph//2 + 1)\n    x1 = cx - pw//2; x2 = x1 + pw\n    y1 = cy - ph//2; y2 = y1 + ph\n    out = imgA.copy()\n    out[y1:y2, x1:x2] = imgB[y1:y2, x1:x2]\n    return out\n\ndef tankmix(imgs4):\n    H, W = TARGET_H, TARGET_W\n    canvas = np.zeros((2*H, 2*W, 3), dtype=imgs4[0].dtype)\n    canvas[0:H,   0:W  ] = imgs4[0]  # TL\n    canvas[0:H,   W:2*W] = imgs4[1]  # TR\n    canvas[H:2*H, 0:W  ] = imgs4[2]  # BL\n    canvas[H:2*H, W:2*W] = imgs4[3]  # BR\n    y0 = H//2; x0 = W//2\n    return canvas[y0:y0+H, x0:x0+W].copy()\n\ndef append_row(anchor_row, iid_new, out_path, label_final, aug_type, src_desc, rows_aug,\n               y_soft=None, lam=None):\n    r = anchor_row.copy()\n    r[\"image_id\"] = iid_new\n    r[\"cancer\"]   = int(label_final)                 # hard label\n    r[\"is_aug\"]   = 1\n    r[\"aug_type\"] = aug_type\n    r[\"png_path\"] = out_path\n    r[\"cancer_soft\"] = float(y_soft) if y_soft is not None else float(label_final)\n    r[\"mix_lambda\"]  = float(lam) if lam is not None else (1.0 if \"tankmix\" in aug_type else 0.5)\n    r[\"source_ids\"]  = src_desc\n    rows_aug.append(r)\n\n# ------------ CHUẨN BỊ DỮ LIỆU NGUỒN ------------\n# Chỉ giữ hàng có file tồn tại\ndef have_file(df):\n    return df[df.apply(lambda r: os.path.exists(f\"{TRAIN_DIR}/{int(r['patient_id'])}/{int(r['image_id'])}.png\"), axis=1)]\n\ndf = have_file(train_meta_saved.copy())\n\n# Yêu cầu các cột để bucket và nhãn\nneed_cols = [\"view\",\"laterality\",\"site_id\",\"age\",\"cancer\",\"patient_id\",\"image_id\"]\nmiss = [c for c in need_cols if c not in df.columns]\nif miss:\n    raise KeyError(f\"Thiếu cột: {miss}\")\n\n# Ép kiểu\ndf = df[df[\"age\"].notna()].copy()\ndf[\"age\"] = df[\"age\"].astype(int)\ndf[\"cancer\"] = df[\"cancer\"].astype(int)\n\n# Gộp bucket theo (view, laterality, site_id, age)\nKEY_COLS = [\"view\",\"laterality\",\"site_id\",\"age\"]\nbuckets = {}\nfor key, g in df.groupby(KEY_COLS):\n    g = g.reset_index(drop=True)\n    g0 = g[g[\"cancer\"]==0].reset_index(drop=True)\n    g1 = g[g[\"cancer\"]==1].reset_index(drop=True)\n    if len(g0)+len(g1) >= 2:\n        buckets[key] = (g0, g1)\n\ntotal0 = (df[\"cancer\"]==0).sum()\ntotal1 = (df[\"cancer\"]==1).sum()\nprint(f\"[BEFORE] neg=0:{total0} | pos=1:{total1} | total={len(df)}\")\n\nrows_aug = []\ngen_from_pairs = 0\ntank_added = 0\n\n# ------------ HÀM SINH ẢNH TỪ CẶP (theo nhãn mong muốn) ------------\ndef gen_from_pair(target_label, rows_aug):\n    \"\"\"\n    target_label=1: ưu tiên lấy cặp khác nhãn (blur NEG 80%, POS 10%) rồi mix -> label=1.\n                    nếu thiếu, lấy cặp 1-1 để giữ label=1.\n    target_label=0: lấy cặp 0-0 để giữ label=0.\n    trả về True nếu sinh được 1 ảnh, False nếu không.\n    \"\"\"\n    # chọn bucket phù hợp\n    if target_label == 1:\n        # ưu tiên bucket có cả 0 và 1\n        cand = [k for k,(g0,g1) in buckets.items() if len(g0)>=1 and len(g1)>=1]\n        mode = \"cross\"\n        if not cand:\n            # fallback: dùng 1-1\n            cand = [k for k,(g0,g1) in buckets.items() if len(g1)>=2]\n            mode = \"same1\"\n        if not cand:\n            return False\n        key = random.choice(cand)\n        g0, g1 = buckets[key]\n\n        if mode == \"cross\":\n            # anchor POS (blur nhẹ), partner NEG (blur mạnh)\n            ra = g1.sample(1).iloc[0]\n            rb = g0.sample(1).iloc[0]\n            imgA, _ = read_png(ra[\"patient_id\"], ra[\"image_id\"])\n            imgB, _ = read_png(rb[\"patient_id\"], rb[\"image_id\"])\n            if imgA is None or imgB is None:\n                return False\n            imgA = ensure_size_stretch(imgA); imgB = ensure_size_stretch(imgB)\n            imgA = gaussian_blur_ratio(imgA, 0.10)\n            imgB = gaussian_blur_ratio(imgB, 0.80)\n            if np.random.rand() < P_MIXUP:\n                out = mixup(imgA, imgB, lam=0.5)\n                aug_type = \"mixup_blur_10_80\"\n                lam_used = 0.5\n            else:\n                out = cutmix(imgA, imgB, area_frac=np.random.uniform(0.25,0.45))\n                aug_type = \"cutmix_blur_10_80\"\n                lam_used = None\n            iid_new = new_image_id(ra[\"image_id\"], len(rows_aug)+gen_from_pairs)\n            path = save_png(out, ra[\"patient_id\"], iid_new)\n            src = f\"A(pos):{int(ra['patient_id'])}:{int(ra['image_id'])};B(neg):{int(rb['patient_id'])}:{int(rb['image_id'])}\"\n            append_row(ra, iid_new, path, 1, aug_type, src, rows_aug, y_soft=1.0, lam=lam_used)\n            return True\n\n        else:  # same1: 1-1 giữ nhãn 1\n            r = g1.sample(2, replace=False)\n            A, B = r.iloc[0], r.iloc[1]\n            imgA, _ = read_png(A[\"patient_id\"], A[\"image_id\"])\n            imgB, _ = read_png(B[\"patient_id\"], B[\"image_id\"])\n            if imgA is None or imgB is None:\n                return False\n            imgA = ensure_size_stretch(imgA); imgB = ensure_size_stretch(imgB)\n            if np.random.rand() < P_MIXUP:\n                out = mixup(imgA, imgB, lam=0.5); aug_type=\"mixup_11\"; lam_used=0.5\n            else:\n                out = cutmix(imgA, imgB, area_frac=np.random.uniform(0.25,0.45)); aug_type=\"cutmix_11\"; lam_used=None\n            iid_new = new_image_id(A[\"image_id\"], len(rows_aug)+gen_from_pairs)\n            path = save_png(out, A[\"patient_id\"], iid_new)\n            src = f\"1-1:{int(A['patient_id'])}:{int(A['image_id'])};{int(B['patient_id'])}:{int(B['image_id'])}\"\n            append_row(A, iid_new, path, 1, aug_type, src, rows_aug, y_soft=1.0, lam=lam_used)\n            return True\n\n    else:  # target_label == 0 → 0-0 giữ nhãn 0\n        cand = [k for k,(g0,g1) in buckets.items() if len(g0)>=2]\n        if not cand:\n            return False\n        key = random.choice(cand)\n        g0, g1 = buckets[key]\n        r = g0.sample(2, replace=False)\n        A, B = r.iloc[0], r.iloc[1]\n        imgA, _ = read_png(A[\"patient_id\"], A[\"image_id\"])\n        imgB, _ = read_png(B[\"patient_id\"], B[\"image_id\"])\n        if imgA is None or imgB is None:\n            return False\n        imgA = ensure_size_stretch(imgA); imgB = ensure_size_stretch(imgB)\n        if np.random.rand() < P_MIXUP:\n            out = mixup(imgA, imgB, lam=0.5); aug_type=\"mixup_00\"; lam_used=0.5\n        else:\n            out = cutmix(imgA, imgB, area_frac=np.random.uniform(0.25,0.45)); aug_type=\"cutmix_00\"; lam_used=None\n        iid_new = new_image_id(A[\"image_id\"], len(rows_aug)+gen_from_pairs)\n        path = save_png(out, A[\"patient_id\"], iid_new)\n        src = f\"0-0:{int(A['patient_id'])}:{int(A['image_id'])};{int(B['patient_id'])}:{int(B['image_id'])}\"\n        append_row(A, iid_new, path, 0, aug_type, src, rows_aug, y_soft=0.0, lam=lam_used)\n        return True\n\n# ------------ TÍNH SỐ LƯỢNG CẦN SINH ------------\ncnt0, cnt1 = total0, total1\ntarget = max(cnt0, cnt1)  # cân bằng bằng cách kéo lớp ít lên bằng lớp nhiều\nneed_pos = max(0, target - cnt1)\nneed_neg = max(0, target - cnt0)\nprint(f\"Need to add -> pos:+{need_pos}, neg:+{need_neg}\")\n\n# ------------ VÒNG SINH ẢNH ------------\nmade_pos = 0; made_neg = 0\nticker = 0\nmax_tries = (need_pos + need_neg) * 20 + 200\n\nwhile (made_pos < need_pos or made_neg < need_neg) and ticker < max_tries:\n    ticker += 1\n    # xen kẽ tạo pos/neg để tránh kẹt\n    if made_pos < need_pos:\n        ok = gen_from_pair(1, rows_aug)\n        if ok:\n            made_pos += 1\n            gen_from_pairs += 1\n            # thỉnh thoảng thêm 1 tankmix cùng nhãn 1\n            if made_pos % TANKMIX_EVERY == 0:\n                # chọn bucket có >=4 pos\n                cand = [k for k,(g0,g1) in buckets.items() if len(g1)>=4]\n                if cand:\n                    key = random.choice(cand); g0, g1 = buckets[key]\n                    group = g1.sample(4, replace=False)\n                    imgs=[]; ids=[]\n                    for _, rr in group.iterrows():\n                        im,_ = read_png(rr[\"patient_id\"], rr[\"image_id\"])\n                        if im is None: imgs=[]; break\n                        imgs.append(ensure_size_stretch(im))\n                        ids.append(f\"{int(rr['patient_id'])}:{int(rr['image_id'])}\")\n                    if len(imgs)==4:\n                        out = tankmix(imgs)\n                        anchor = group.iloc[0]\n                        iid_new = new_image_id(anchor[\"image_id\"], 900000 + made_pos)\n                        path = save_png(out, anchor[\"patient_id\"], iid_new)\n                        append_row(anchor, iid_new, path, 1, \"tankmix_11\", \";\".join(ids), rows_aug, y_soft=1.0, lam=1.0)\n                        tank_added += 1\n        continue\n\n    if made_neg < need_neg:\n        ok = gen_from_pair(0, rows_aug)\n        if ok:\n            made_neg += 1\n            gen_from_pairs += 1\n            # thỉnh thoảng thêm 1 tankmix cùng nhãn 0\n            if made_neg % TANKMIX_EVERY == 0:\n                cand = [k for k,(g0,g1) in buckets.items) if len(g0)>=4]\n                # (nhỏ bug fixed)\n        continue\n\nprint(f\"[GEN] pos:+{made_pos}, neg:+{made_neg}, tankmix:+{tank_added}, tries={ticker}\")\n\n# ------------ GHI CSV AUG ------------\nif rows_aug:\n    aug_df = pd.DataFrame(rows_aug)\n    base_cols  = [\"site_id\",\"patient_id\",\"image_id\",\"laterality\",\"view\",\"age\",\"implant\",\"machine_id\",\"cancer\"]\n    extra_cols = [\"is_aug\",\"aug_type\",\"cancer_soft\",\"mix_lambda\",\"source_ids\",\"png_path\"]\n    # đảm bảo cột nền tảng tồn tại\n    for c in base_cols:\n        if c not in aug_df.columns:\n            aug_df[c] = np.nan\n    # sắp cột\n    cols = base_cols + extra_cols + [c for c in aug_df.columns if c not in base_cols+extra_cols]\n    aug_df = aug_df[cols]\n    Path(OUT_ROOT).mkdir(parents=True, exist_ok=True)\n    aug_df.to_csv(OUT_AUG_CSV, index=False)\n\n    # báo cáo cuối\n    final0 = total0 + (aug_df[\"cancer\"]==0).sum()\n    final1 = total1 + (aug_df[\"cancer\"]==1).sum()\n    print(f\"[AFTER] neg=0:{final0} | pos=1:{final1} | delta(pos-neg)={final1-final0}\")\n    print(f\"Saved: {OUT_AUG_CSV} | New images: {len(aug_df)}\")\nelse:\n    print(\"Không sinh được ảnh augment nào (thiếu bucket phù hợp).\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============== CELL 4 (AUGMENT & BALANCE, SOFT LABEL + MERGED CSV, NO EXTRA COLS) ===============\n# - Bucket theo (view, laterality, site_id, age)\n# - Khác nhãn: blur NEG 80% & POS 10%, cancer (cứng)=1, cancer_soft=SOFT_POS_ON_CROSS (mặc định 0.7)\n# - Cùng nhãn: giữ cancer cứng (0/1), cancer_soft = cancer\n# - TankMix: cùng nhãn 1\n# - Ảnh mới lưu vào TRAIN_DIR/{patient_id}/{new_image_id}.png\n# - Xuất: OUT_ROOT/train_with_aug_soft.csv (GỘP ẢNH CŨ + MỚI)\n#   -> Giữ các cột gốc, chỉ thêm: is_argu, cancer_soft (+ png_path)\n\nimport os, random\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport cv2\n\n# ------------ YÊU CẦU BIẾN TỪ CELL 3 ------------\nrequired_vars = [\"train_meta_saved\", \"TRAIN_DIR\", \"OUT_ROOT\"]\nfor v in required_vars:\n    if v not in globals():\n        raise NameError(f\"Biến '{v}' chưa có. Hãy chạy cell export TRAIN trước.\")\n\n# ------------ CẤU HÌNH ------------\nTARGET_W, TARGET_H = 512, 1024\nOUT_MERGED_CSV = f\"{OUT_ROOT}/train_with_aug_soft.csv\"   # CSV gộp ảnh cũ + mới\n\nP_MIXUP = 0.5           # tỉ lệ chọn MixUp (ngược lại là CutMix)\nTANKMIX_EVERY = 4       # mỗi 4 ảnh pos sinh từ cặp -> thêm 1 ảnh TankMix pos\nSOFT_POS_ON_CROSS = 0.7 # soft-label cho case khác nhãn (pos ⊕ neg)\n\nSEED = 1337\nrandom.seed(SEED); np.random.seed(SEED)\n\n# ------------ TIỆN ÍCH ------------\ndef ensure_size_stretch(img, w=TARGET_W, h=TARGET_H):\n    return cv2.resize(img, (w, h), interpolation=cv2.INTER_AREA)\n\ndef read_png(pid, iid):\n    p = f\"{TRAIN_DIR}/{int(pid)}/{int(iid)}.png\"\n    if not os.path.exists(p): return None, p\n    return cv2.imread(p), p\n\ndef save_png(img, pid, iid_new):\n    outp = f\"{TRAIN_DIR}/{int(pid)}/{int(iid_new)}.png\"\n    Path(outp).parent.mkdir(parents=True, exist_ok=True)\n    cv2.imwrite(outp, img)\n    return outp\n\ndef new_image_id(base_iid, k):\n    return int(base_iid) * 100000 + int(k) + 1\n\ndef gaussian_blur_ratio(img, ratio=0.8):\n    H, W = img.shape[:2]\n    base = int(max(3, (min(H, W) * ratio * 0.12) // 2 * 2 + 1))  # số lẻ\n    return cv2.GaussianBlur(img, (base, base), 0)\n\ndef mixup(imgA, imgB, lam=0.5):\n    return (imgA.astype(np.float32)*lam + imgB.astype(np.float32)*(1-lam)).astype(np.uint8)\n\ndef cutmix(imgA, imgB, area_frac=0.33):\n    H, W = imgA.shape[:2]\n    patch = max(1, int(area_frac*H*W))\n    r = np.random.uniform(0.5, 2.0)\n    ph = int(np.sqrt(patch/r)); pw = int(patch/max(ph,1))\n    ph = max(1, min(ph, H)); pw = max(1, min(pw, W))\n    cx = np.random.randint(pw//2, W - pw//2 + 1)\n    cy = np.random.randint(ph//2, H - ph//2 + 1)\n    x1 = cx - pw//2; x2 = x1 + pw\n    y1 = cy - ph//2; y2 = y1 + ph\n    out = imgA.copy()\n    out[y1:y2, x1:x2] = imgB[y1:y2, x1:x2]\n    return out\n\ndef tankmix(imgs4):\n    H, W = TARGET_H, TARGET_W\n    canvas = np.zeros((2*H, 2*W, 3), dtype=imgs4[0].dtype)\n    canvas[0:H,   0:W  ] = imgs4[0]\n    canvas[0:H,   W:2*W] = imgs4[1]\n    canvas[H:2*H, 0:W  ] = imgs4[2]\n    canvas[H:2*H, W:2*W] = imgs4[3]\n    y0 = H//2; x0 = W//2\n    return canvas[y0:y0+H, x0:x0+W].copy()\n\n# ------------ CHUẨN BỊ NGUỒN ------------\ndef have_file(df):\n    return df[df.apply(lambda r: os.path.exists(f\"{TRAIN_DIR}/{int(r['patient_id'])}/{int(r['image_id'])}.png\"), axis=1)]\n\ndf = have_file(train_meta_saved.copy())\n\nneed_cols = [\"view\",\"laterality\",\"site_id\",\"age\",\"cancer\",\"patient_id\",\"image_id\"]\nmiss = [c for c in need_cols if c not in df.columns]\nif miss:\n    raise KeyError(f\"Thiếu cột: {miss}\")\n\ndf = df[df[\"age\"].notna()].copy()\ndf[\"age\"]    = df[\"age\"].astype(int)\ndf[\"cancer\"] = df[\"cancer\"].astype(int)\n\nKEY_COLS = [\"view\",\"laterality\",\"site_id\",\"age\"]\nbuckets = {}\nfor key, g in df.groupby(KEY_COLS):\n    g = g.reset_index(drop=True)\n    g0 = g[g[\"cancer\"]==0].reset_index(drop=True)\n    g1 = g[g[\"cancer\"]==1].reset_index(drop=True)\n    if len(g0)+len(g1) >= 2:\n        buckets[key] = (g0, g1)\n\ntotal0 = (df[\"cancer\"]==0).sum()\ntotal1 = (df[\"cancer\"]==1).sum()\nprint(f\"[BEFORE] neg=0:{total0} | pos=1:{total1} | total={len(df)}\")\n\n# ------------ SINH ẢNH ------------\nrows_aug = []\ngen_from_pairs = 0\ntank_added = 0\n\ndef append_aug_row(anchor_row, iid_new, out_path, cancer_hard, cancer_soft):\n    r = anchor_row.copy()\n    r[\"image_id\"] = iid_new\n    r[\"cancer\"]   = int(cancer_hard)     # nhãn CỨNG\n    r[\"cancer_soft\"] = float(cancer_soft)\n    r[\"is_argu\"]  = 1\n    r[\"png_path\"] = out_path\n    rows_aug.append(r)\n\ndef gen_from_pair_pos(rows_aug):\n    # ưu tiên cặp khác nhãn\n    cand = [k for k,(g0,g1) in buckets.items() if len(g0)>=1 and len(g1)>=1]\n    mode = \"cross\"\n    if not cand:\n        cand = [k for k,(g0,g1) in buckets.items() if len(g1)>=2]  # fallback: 1-1\n        mode = \"same1\"\n    if not cand:\n        return False\n    key = random.choice(cand)\n    g0, g1 = buckets[key]\n\n    if mode == \"cross\":\n        ra = g1.sample(1).iloc[0]  # anchor POS (blur nhẹ)\n        rb = g0.sample(1).iloc[0]  # partner NEG (blur mạnh)\n        imgA, _ = read_png(ra[\"patient_id\"], ra[\"image_id\"])\n        imgB, _ = read_png(rb[\"patient_id\"], rb[\"image_id\"])\n        if imgA is None or imgB is None:\n            return False\n        imgA = gaussian_blur_ratio(ensure_size_stretch(imgA), 0.10)\n        imgB = gaussian_blur_ratio(ensure_size_stretch(imgB), 0.80)\n        if np.random.rand() < P_MIXUP:\n            out = mixup(imgA, imgB, lam=0.5)\n        else:\n            out = cutmix(imgA, imgB, area_frac=np.random.uniform(0.25,0.45))\n        iid_new = new_image_id(ra[\"image_id\"], len(rows_aug)+gen_from_pairs)\n        path = save_png(out, ra[\"patient_id\"], iid_new)\n        # cancer cứng = 1, soft = SOFT_POS_ON_CROSS\n        append_aug_row(ra, iid_new, path, cancer_hard=1, cancer_soft=SOFT_POS_ON_CROSS)\n        return True\n    else:\n        r = g1.sample(2, replace=False)\n        A, B = r.iloc[0], r.iloc[1]\n        imgA, _ = read_png(A[\"patient_id\"], A[\"image_id\"])\n        imgB, _ = read_png(B[\"patient_id\"], B[\"image_id\"])\n        if imgA is None or imgB is None:\n            return False\n        imgA = ensure_size_stretch(imgA); imgB = ensure_size_stretch(imgB)\n        if np.random.rand() < P_MIXUP:\n            out = mixup(imgA, imgB, lam=0.5)\n        else:\n            out = cutmix(imgA, imgB, area_frac=np.random.uniform(0.25,0.45))\n        iid_new = new_image_id(A[\"image_id\"], len(rows_aug)+gen_from_pairs)\n        path = save_png(out, A[\"patient_id\"], iid_new)\n        append_aug_row(A, iid_new, path, cancer_hard=1, cancer_soft=1.0)\n        return True\n\ndef gen_from_pair_neg(rows_aug):\n    cand = [k for k,(g0,g1) in buckets.items() if len(g0)>=2]\n    if not cand:\n        return False\n    key = random.choice(cand)\n    g0, g1 = buckets[key]\n    r = g0.sample(2, replace=False)\n    A, B = r.iloc[0], r.iloc[1]\n    imgA, _ = read_png(A[\"patient_id\"], A[\"image_id\"])\n    imgB, _ = read_png(B[\"patient_id\"], B[\"image_id\"])\n    if imgA is None or imgB is None:\n        return False\n    imgA = ensure_size_stretch(imgA); imgB = ensure_size_stretch(imgB)\n    if np.random.rand() < P_MIXUP:\n        out = mixup(imgA, imgB, lam=0.5)\n    else:\n        out = cutmix(imgA, imgB, area_frac=np.random.uniform(0.25,0.45))\n    iid_new = new_image_id(A[\"image_id\"], len(rows_aug)+gen_from_pairs)\n    path = save_png(out, A[\"patient_id\"], iid_new)\n    append_aug_row(A, iid_new, path, cancer_hard=0, cancer_soft=0.0)\n    return True\n\n# Số lượng cần sinh (kéo lớp thiếu lên bằng lớp nhiều)\ncnt0, cnt1 = total0, total1\ntarget   = max(cnt0, cnt1)\nneed_pos = max(0, target - cnt1)\nneed_neg = max(0, target - cnt0)\nprint(f\"Need to add -> pos:+{need_pos}, neg:+{need_neg}\")\n\nmade_pos = 0; made_neg = 0\nticker   = 0\nmax_tries = (need_pos + need_neg) * 20 + 200\n\nwhile (made_pos < need_pos or made_neg < need_neg) and ticker < max_tries:\n    ticker += 1\n    if made_pos < need_pos:\n        ok = gen_from_pair_pos(rows_aug)\n        if ok:\n            made_pos += 1\n            gen_from_pairs += 1\n            # TankMix POS định kỳ\n            if made_pos % TANKMIX_EVERY == 0:\n                cand = [k for k,(g0,g1) in buckets.items() if len(g1)>=4]\n                if cand:\n                    key = random.choice(cand); g0, g1 = buckets[key]\n                    group = g1.sample(4, replace=False)\n                    imgs=[]; anchor=None\n                    for _, rr in group.iterrows():\n                        im,_ = read_png(rr[\"patient_id\"], rr[\"image_id\"])\n                        if im is None: imgs=[]; break\n                        imgs.append(ensure_size_stretch(im))\n                        if anchor is None: anchor = rr\n                    if len(imgs)==4 and anchor is not None:\n                        out = tankmix(imgs)\n                        iid_new = new_image_id(anchor[\"image_id\"], 900000 + made_pos)\n                        path = save_png(out, anchor[\"patient_id\"], iid_new)\n                        append_aug_row(anchor, iid_new, path, cancer_hard=1, cancer_soft=1.0)\n                        tank_added += 1\n        continue\n\n    if made_neg < need_neg:\n        ok = gen_from_pair_neg(rows_aug)\n        if ok:\n            made_neg += 1\n            gen_from_pairs += 1\n        continue\n\nprint(f\"[GEN] pos:+{made_pos}, neg:+{made_neg}, tankmix:+{tank_added}, tries={ticker}\")\n\n# ------------ TẠO CSV GỘP (GIỮ CỘT CŨ + is_argu, cancer_soft + png_path) ------------\nif len(rows_aug) > 0:\n    aug_df = pd.DataFrame(rows_aug)\n\n    # base_df = dữ liệu gốc đã có file\n    base_df = df.copy()\n    # thêm png_path nếu thiếu\n    if \"png_path\" not in base_df.columns:\n        base_df[\"png_path\"] = base_df.apply(\n            lambda r: f\"{TRAIN_DIR}/{int(r['patient_id'])}/{int(r['image_id'])}.png\", axis=1\n        )\n    # cờ & soft cho ảnh gốc\n    base_df[\"is_argu\"]     = 0\n    base_df[\"cancer_soft\"] = base_df[\"cancer\"].astype(float)\n\n    # target_cols = cột gốc + 2 cột mới + png_path\n    # (giữ nguyên thứ tự cột gốc như trong base_df)\n    if \"is_argu\" not in base_df.columns:     base_df[\"is_argu\"] = 0\n    if \"cancer_soft\" not in base_df.columns: base_df[\"cancer_soft\"] = base_df[\"cancer\"].astype(float)\n    if \"png_path\" not in base_df.columns:\n        base_df[\"png_path\"] = base_df.apply(\n            lambda r: f\"{TRAIN_DIR}/{int(r['patient_id'])}/{int(r['image_id'])}.png\", axis=1\n        )\n    target_cols = list(base_df.columns)\n\n    # chuẩn hoá aug_df để khớp schema này\n    aug_df_compat = aug_df.copy()\n    # nếu thiếu các cột trong target_cols -> thêm NaN\n    for c in target_cols:\n        if c not in aug_df_compat.columns:\n            aug_df_compat[c] = np.nan\n    # nếu aug_df có cột thừa (ngoài target_cols) -> bỏ\n    aug_df_compat = aug_df_compat[target_cols]\n\n    merged = pd.concat([base_df[target_cols], aug_df_compat], axis=0, ignore_index=True)\n    Path(OUT_ROOT).mkdir(parents=True, exist_ok=True)\n    merged.to_csv(OUT_MERGED_CSV, index=False)\n\n    # báo cáo cuối\n    final0 = (merged[\"cancer\"]==0).sum()\n    final1 = (merged[\"cancer\"]==1).sum()\n    print(f\"[MERGED] neg=0:{final0} | pos=1:{final1} | total={len(merged)}\")\n    print(f\"[MERGED] Saved: {OUT_MERGED_CSV}\")\nelse:\n    print(\"Không sinh được ảnh augment nào (thiếu bucket phù hợp). Không tạo CSV gộp.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-22T09:45:57.208626Z","iopub.execute_input":"2025-10-22T09:45:57.21645Z","iopub.status.idle":"2025-10-22T09:47:55.235613Z","shell.execute_reply.started":"2025-10-22T09:45:57.216262Z","shell.execute_reply":"2025-10-22T09:47:55.234505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =============== ZIP /kaggle/working/dataset ===============\nimport os\nfrom pathlib import Path\nimport zipfile\n\ndef zip_folder(src_dir, zip_path, exclude_hidden=True, exclude_exts=(\".DS_Store\",)):\n    src_dir = Path(src_dir)\n    zip_path = Path(zip_path)\n    zip_path.parent.mkdir(parents=True, exist_ok=True)\n    count_files = 0\n    total_bytes = 0\n\n    for root, dirs, files in os.walk(src_dir):\n        if exclude_hidden:\n            dirs[:] = [d for d in dirs if not d.startswith(\".\") and d not in [\"__MACOSX\", \".ipynb_checkpoints\"]]\n        for f in files:\n            if exclude_hidden and f.startswith(\".\"): \n                continue\n            if any(f.endswith(ext) for ext in exclude_exts):\n                continue\n            count_files += 1\n            try:\n                total_bytes += (Path(root)/f).stat().st_size\n            except Exception:\n                pass\n\n    print(f\"Nén từ: {src_dir}\")\n    print(f\"Số file: {count_files} | Dung lượng ~ {total_bytes/1024/1024:.2f} MB\")\n    if count_files == 0:\n        print(\"Không có file để nén.\"); raise SystemExit\n\n    with zipfile.ZipFile(zip_path, \"w\", compression=zipfile.ZIP_DEFLATED) as zf:\n        idx = 0\n        for root, dirs, files in os.walk(src_dir):\n            if exclude_hidden:\n                dirs[:] = [d for d in dirs if not d.startswith(\".\") and d not in [\"__MACOSX\", \".ipynb_checkpoints\"]]\n            for f in files:\n                if exclude_hidden and f.startswith(\".\"):\n                    continue\n                if any(f.endswith(ext) for ext in exclude_exts):\n                    continue\n                fpath = Path(root) / f\n                arcname = fpath.relative_to(src_dir)\n                zf.write(fpath, arcname=arcname)\n                idx += 1\n                if idx % 500 == 0:\n                    print(f\"  -> Đã thêm {idx}/{count_files} files...\")\n\n    print(f\"[DONE] Zip saved: {zip_path}\")\n    print(\"Vào tab 'Outputs' của notebook để tải file .zip.\")\n\n# GỌI HÀM:\nzip_folder(\"/kaggle/working/dataset\", \"/kaggle/working/dataset.zip\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-22T10:16:16.671184Z","iopub.execute_input":"2025-10-22T10:16:16.671668Z","iopub.status.idle":"2025-10-22T10:19:35.133061Z","shell.execute_reply.started":"2025-10-22T10:16:16.671628Z","shell.execute_reply":"2025-10-22T10:19:35.131903Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.remove(\"/kaggle/working/dataset.zip\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-22T10:15:03.217202Z","iopub.execute_input":"2025-10-22T10:15:03.217641Z","iopub.status.idle":"2025-10-22T10:15:03.443614Z","shell.execute_reply.started":"2025-10-22T10:15:03.217602Z","shell.execute_reply":"2025-10-22T10:15:03.44275Z"}},"outputs":[],"execution_count":null}]}