{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaL4","dataSources":[{"sourceId":84795,"databundleVersionId":11281725,"sourceType":"competition"},{"sourceId":13091650,"sourceType":"datasetVersion","datasetId":8292379},{"sourceId":13519383,"sourceType":"datasetVersion","datasetId":8583998},{"sourceId":13531816,"sourceType":"datasetVersion","datasetId":8583835},{"sourceId":608800,"sourceType":"modelInstanceVersion","modelInstanceId":457029,"modelId":456721},{"sourceId":608802,"sourceType":"modelInstanceVersion","modelInstanceId":457028,"modelId":456721},{"sourceId":608801,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":457030,"modelId":456721},{"sourceId":622207,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":468025,"modelId":483851}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!uv pip install --system --no-index --find-links='/kaggle/input/vllm-vlm/whls' 'vllm' 'torchvision' 'numpy<2' 'google_re2' 'qwen-vl-utils'\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-29T10:16:18.67225Z","iopub.execute_input":"2025-10-29T10:16:18.672417Z","iopub.status.idle":"2025-10-29T10:16:18.941677Z","shell.execute_reply.started":"2025-10-29T10:16:18.672402Z","shell.execute_reply":"2025-10-29T10:16:18.941001Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile utils.py\nimport torch\nimport torchvision.transforms as T\nfrom torchvision.transforms.functional import InterpolationMode\n\nfrom qwen_vl_utils import process_vision_info\n\nimport io\nimport base64\nfrom PIL import Image\n\n\nIMAGENET_MEAN = (0.485, 0.456, 0.406)\nIMAGENET_STD = (0.229, 0.224, 0.225)\n\ndef build_transform(input_size):\n    MEAN, STD = IMAGENET_MEAN, IMAGENET_STD\n    transform = T.Compose([\n        T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),\n        T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),\n        T.ToTensor(),\n        T.Normalize(mean=MEAN, std=STD)\n    ])\n    return transform\n\ndef find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):\n    best_ratio_diff = float('inf')\n    best_ratio = (1, 1)\n    area = width * height\n    for ratio in target_ratios:\n        target_aspect_ratio = ratio[0] / ratio[1]\n        ratio_diff = abs(aspect_ratio - target_aspect_ratio)\n        if ratio_diff < best_ratio_diff:\n            best_ratio_diff = ratio_diff\n            best_ratio = ratio\n        elif ratio_diff == best_ratio_diff:\n            if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:\n                best_ratio = ratio\n    return best_ratio\n\ndef dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):\n    orig_width, orig_height = image.size\n    aspect_ratio = orig_width / orig_height\n\n    # calculate the existing image aspect ratio\n    target_ratios = set(\n        (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if\n        i * j <= max_num and i * j >= min_num)\n    target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])\n\n    # find the closest aspect ratio to the target\n    target_aspect_ratio = find_closest_aspect_ratio(\n        aspect_ratio, target_ratios, orig_width, orig_height, image_size)\n\n    # calculate the target width and height\n    target_width = image_size * target_aspect_ratio[0]\n    target_height = image_size * target_aspect_ratio[1]\n    blocks = target_aspect_ratio[0] * target_aspect_ratio[1]\n\n    # resize the image\n    resized_img = image.resize((target_width, target_height))\n    processed_images = []\n    for i in range(blocks):\n        box = (\n            (i % (target_width // image_size)) * image_size,\n            (i // (target_width // image_size)) * image_size,\n            ((i % (target_width // image_size)) + 1) * image_size,\n            ((i // (target_width // image_size)) + 1) * image_size\n        )\n        # split the image\n        split_img = resized_img.crop(box)\n        processed_images.append(split_img)\n    assert len(processed_images) == blocks\n    if use_thumbnail and len(processed_images) != 1:\n        thumbnail_img = image.resize((image_size, image_size))\n        processed_images.append(thumbnail_img)\n    return processed_images\n\ndef load_image(image_file, input_size=448, max_num=12):\n    image = Image.open(image_file).convert('RGB')\n    transform = build_transform(input_size=input_size)\n    images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)\n    pixel_values = [transform(image) for image in images]\n    pixel_values = torch.stack(pixel_values)\n    return pixel_values\n\ndef image_to_base64(image_path):\n    with open(image_path, \"rb\") as image_file:\n        image_data = image_file.read()\n        base64_bytes = base64.b64encode(image_data)\n        base64_string = base64_bytes.decode(\"utf-8\")\n        return base64_string\n\ndef construct_messages(image, text, role=\"user\"):\n    content = [\n        {\"type\": \"image\", \"image\": image},\n        {\"type\": \"text\", \"text\": text}\n    ]\n\n    return [\n        {\n            \"role\": role,\n            \"content\": content,\n        }\n    ]\n\ndef prepare_inputs_for_vllm(messages, processor):\n    text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n    # qwen_vl_utils 0.0.14+ reqired\n    image_inputs, video_inputs, video_kwargs = process_vision_info(\n        messages,\n        image_patch_size=processor.image_processor.patch_size,\n        return_video_kwargs=True,\n        return_video_metadata=True\n    )\n    \n    mm_data = {}\n    if image_inputs is not None:\n        mm_data['image'] = image_inputs\n    if video_inputs is not None:\n        mm_data['video'] = video_inputs\n\n    return {\n        'prompt': text,\n        'multi_modal_data': mm_data,\n        'mm_processor_kwargs': video_kwargs\n    }\n\nprint(\"functions defined...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T10:16:19.217575Z","iopub.execute_input":"2025-10-29T10:16:19.218266Z","iopub.status.idle":"2025-10-29T10:16:19.224311Z","shell.execute_reply.started":"2025-10-29T10:16:19.21824Z","shell.execute_reply":"2025-10-29T10:16:19.223782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!python utils.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T10:16:19.225138Z","iopub.execute_input":"2025-10-29T10:16:19.225312Z","iopub.status.idle":"2025-10-29T10:16:22.7979Z","shell.execute_reply.started":"2025-10-29T10:16:19.225298Z","shell.execute_reply":"2025-10-29T10:16:22.7972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile solution.py\n\nimport os\nimport io\nimport re\nimport json\nimport time\nimport pickle\nimport argparse\nfrom itertools import combinations\nfrom typing import List, Tuple\nfrom PIL import Image\n\nimport torch\nfrom vllm import LLM, SamplingParams\nfrom transformers import AutoProcessor\n\nfrom utils import prepare_inputs_for_vllm\n\nCHECKPOINT_PATH = \"/kaggle/input/qwen-3-vl/transformers/bb-instruct-fp8/1\"\nGPU_UTIL = 0.80\nMAX_MODEL_LEN = 10240\nMAX_NEW_TOKENS = 1024 * 2\nTEMPERATURE = 0.0\nTOP_K = -1\nBATCH_SIZE = 8\n\nCUTOFF_SECONDS = 55 * 60\nSTART_TIME = time.time()\n\nos.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\nos.environ[\"VLLM_WORKER_MULTIPROC_METHOD\"] = \"spawn\"\n\nOPTION_RE = re.compile(r\"\\b([A-Z])\\s*[\\.\\):\\-]\")\nANSWER_TAG_RE = re.compile(r\"<\\s*answer\\s*>(.*?)<\\s*/\\s*answer\\s*>\", re.IGNORECASE | re.DOTALL)\n\ndef image_from_blob(blob) -> Image.Image:\n    return Image.open(io.BytesIO(blob)).convert(\"RGB\")\n\ndef extract_allowed_letters(question: str) -> List[str]:\n    seen = set()\n    letters = []\n    for ch in OPTION_RE.findall(question or \"\"):\n        ch = ch.upper()\n        if ch not in seen:\n            seen.add(ch)\n            letters.append(ch)\n    # conservative fallback set to reduce overprediction issues\n    return letters if letters else list(\"ABCD\")\n\ndef build_messages(image_pil: Image.Image, question: str):\n    # System: strict rules; allow brief reasoning but require final tag\n    system_text = (\n        \"You are a precise vision-language grader for STEM diagrams.\\n\"\n        \"\\n\"\n        \"Task:\\n\"\n        \"Determine which answer options are TRUE based only on what is visible in the diagram. \"\n        \"If a statement cannot be visually verified, treat it as FALSE. \"\n        \"Drawings are schematic and not necessarily to scale.\\n\"\n        \"\\n\"\n        \"Output rules (strict):\\n\"\n        \"1) First, think briefly step by step, in plain short lines.\\n\"\n        \"2) Then output a single final line wrapped in <answer>...</answer> containing only CAPITAL letters of the correct options in option order (A,B,C,...). \"\n        \"If multiple are correct, concatenate the letters (for example, if A, B and E are correct, output <answer>ABE</answer>). \"\n        \"If none are correct, output <answer></answer>.\\n\"\n        \"3) Do not include any other text after the </answer> tag.\"\n    )\n\n    # User: problem text + explicit \"Answer:\" cue\n    user_text = f\"{question.strip()}\\n\\nAnswer:\"\n    return [\n        {\"role\": \"system\", \"content\": [{\"type\": \"text\", \"text\": system_text}]},\n        {\"role\": \"user\",   \"content\": [\n            {\"type\": \"image\", \"image\": image_pil},\n            {\"type\": \"text\",  \"text\": user_text},\n        ]}\n    ]\n\ndef load_model_and_processor():\n    processor = AutoProcessor.from_pretrained(CHECKPOINT_PATH, trust_remote_code=True)\n    llm = LLM(\n        model=CHECKPOINT_PATH,\n        trust_remote_code=True,\n        tensor_parallel_size=1,\n        gpu_memory_utilization=GPU_UTIL,\n        max_model_len=MAX_MODEL_LEN,\n        enforce_eager=False,\n        seed=0,\n        max_num_seqs=BATCH_SIZE * 2,\n    )\n    sampling = SamplingParams(\n        temperature=TEMPERATURE,\n        top_k=TOP_K,\n        max_tokens=MAX_NEW_TOKENS,\n        stop=[\"</answer>\"],                 # stop right after the closing tag\n        include_stop_str_in_output=True,    # keep the tag so our regex sees it\n    )\n    return llm, processor, sampling\n\ndef build_candidates(letters: List[str]) -> List[str]:\n    # Singles + all pairs; keep triples out for conservative fallback\n    candidates = list(letters)\n    for a, b in combinations(letters, 2):\n        candidates.append(a + b)\n    return candidates\n\ndef deterministic_fallback_prediction(img: Image.Image, question: str, letters: List[str], rid: int) -> str:\n    # Conservative ABCD-based fallback to avoid frequent overprediction exclusions\n    base_letters = letters if letters else list(\"ABCD\")\n    all_cands = build_candidates(base_letters)\n\n    # Deterministic index based on image size + question length\n    w, h = img.size\n    idx = ((31 * (31 * w + h) + len(question))) % len(all_cands) if all_cands else 0\n    return all_cands[idx] if all_cands else \"\"\n\ndef extract_from_answer_tag(text: str, allowed: List[str]) -> str:\n    m = ANSWER_TAG_RE.search(text or \"\")\n    if not m:\n        return \"\"\n    inside = m.group(1)\n    allowed_set = set(allowed)\n    letters = [c for c in re.findall(r\"[A-Z]\", inside) if c in allowed_set]\n    # dedupe while preserving order\n    out, seen = [], set()\n    for c in letters:\n        if c not in seen:\n            seen.add(c)\n            out.append(c)\n    return \"\".join(out)\n\ndef solve(input_path=\"input.pickle\", output_path=\"output.json\"):\n    with open(input_path, \"rb\") as f:\n        data = pickle.load(f)\n\n    results = []\n    llm, processor, sampling = load_model_and_processor()\n\n    N = len(data)\n    i = 0\n    while i < N:\n        # If time exceeded, fallback for remaining samples\n        if time.time() - START_TIME > CUTOFF_SECONDS:\n            print(\"⚠️ Cutoff reached → fallback for remaining items\")\n            for j in range(i, N):\n                item = data[j]\n                rid = item[\"rid\"]\n                q = item[\"question\"]\n                img = image_from_blob(item[\"image\"])\n                letters = extract_allowed_letters(q)\n                pred = deterministic_fallback_prediction(img, q, letters, rid)\n                results.append({\"rid\": rid, \"answer\": pred})\n                print(f\"[RID {rid}] FALLBACK -> {pred}\")\n            break\n\n        # Prepare batch\n        batch = data[i : i + BATCH_SIZE]\n        inputs = []\n        metas: List[Tuple[int, List[str], Image.Image, str]] = []\n\n        for item in batch:\n            rid = item[\"rid\"]\n            q = item[\"question\"]\n            img = image_from_blob(item[\"image\"])\n            letters = extract_allowed_letters(q)\n            msgs = build_messages(img, q)\n            inputs.append(prepare_inputs_for_vllm(msgs, processor))\n            metas.append((rid, letters, img, q))\n\n        # Generate\n        outputs = llm.generate(inputs, sampling_params=sampling)\n\n        # Post-process\n        for out, (rid, letters, img, q) in zip(outputs, metas):\n            raw = (out.outputs[0].text if out.outputs else \"\").strip()\n\n            # 1) Prefer <answer>...</answer>\n            pred = extract_from_answer_tag(raw, letters)\n\n            # 2) If no tag found, fall back to scraping any allowed letters in raw\n            if pred == \"\":\n                picked = [c for c in re.findall(r\"[A-Z]\", raw) if c in set(letters)]\n                pred = \"\".join(picked)\n\n            # 3) If still empty (e.g., truncated), deterministic fallback\n            if pred == \"\":\n                pred = deterministic_fallback_prediction(img, q, letters, rid)\n                print(f\"[RID {rid}] LLM no usable answer -> FALLBACK -> {pred}\")\n            else:\n                print(f\"[RID {rid}] -> {pred}\\nRAW:\\n{raw}\\n---\")\n\n            results.append({\"rid\": rid, \"answer\": pred})\n\n        i += BATCH_SIZE\n\n    with open(output_path, \"w\", encoding=\"utf-8\") as f:\n        json.dump(results, f, ensure_ascii=False)\n    print(f\"✅ Wrote {len(results)} predictions to {output_path}\")\n\nif __name__ == \"__main__\":\n    cwd = os.getcwd()\n    default_input = (\n        \"inputs_1E_2C_3BD_4ACD_5BC_6AC.pickle\"\n        if cwd.startswith(\"/Users/kawchar85/\")\n        else \"input.pickle\"\n    )\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"--input\", type=str, default=default_input)\n    parser.add_argument(\"--output\", type=str, default=\"output.json\")\n    args = parser.parse_args()\n    solve(args.input, args.output)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T10:16:22.799294Z","iopub.execute_input":"2025-10-29T10:16:22.799767Z","iopub.status.idle":"2025-10-29T10:16:22.80712Z","shell.execute_reply.started":"2025-10-29T10:16:22.799744Z","shell.execute_reply":"2025-10-29T10:16:22.806595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!python solution.py --input /kaggle/input/yandex-vlm-test/inputs_1E_2C_3BD_4ACD_5BC_6AC.pickle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T10:16:22.807708Z","iopub.execute_input":"2025-10-29T10:16:22.807876Z","iopub.status.idle":"2025-10-29T10:18:07.700445Z","shell.execute_reply.started":"2025-10-29T10:16:22.807862Z","shell.execute_reply":"2025-10-29T10:18:07.699764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile score.py\nimport json\nimport re\nfrom collections import Counter\n\n# ---------- CONFIG ----------\nOUTPUT_PATH = \"output.json\"\n\n# Provided ground-truth (by rid)\nGT = {\n    1: \"E\",\n    2: \"C\",\n    3: \"BD\",\n    4: \"ACD\",\n    5: \"BC\",\n    6: \"AC\",\n}\n# ----------------------------\n\ndef norm_letters(s: str):\n    \"\"\"Uppercase, keep only A–Z, preserve order, dedupe.\"\"\"\n    seen = set()\n    out = []\n    for ch in re.findall(r\"[A-Z]\", (s or \"\").upper()):\n        if ch not in seen:\n            seen.add(ch)\n            out.append(ch)\n    return out\n\ndef to_set(s: str):\n    return set(norm_letters(s))\n\ndef load_predictions(path):\n    with open(path, \"r\", encoding=\"utf-8\") as f:\n        data = json.load(f)\n    # Expect list of {\"rid\": int, \"answer\": \"AC\"}\n    preds = {}\n    for row in data:\n        rid = int(row[\"rid\"])\n        preds[rid] = row.get(\"answer\", \"\")\n    return preds\n\ndef micro_f1_kept(gt_map, pred_map):\n    \"\"\"Contest-like metric:\n       - drop items where len(pred) > len(gt)\n       - micro-F1 over remaining items\n       - scale by 1000 * (kept/total)\n    \"\"\"\n    total = len(gt_map)\n    kept = []\n    for rid, gt in gt_map.items():\n        p = pred_map.get(rid, \"\")\n        if len(norm_letters(p)) <= len(norm_letters(gt)):\n            kept.append(rid)\n\n    if not kept:\n        return 0.0, 0, total, (0, 0, 0)\n\n    TP = FP = FN = 0\n    for rid in kept:\n        gt_set = to_set(gt_map[rid])\n        pr_set = to_set(pred_map.get(rid, \"\"))\n\n        TP += len(gt_set & pr_set)\n        FP += len(pr_set - gt_set)\n        FN += len(gt_set - pr_set)\n\n    denom = (2*TP + FP + FN)\n    f1 = (2*TP / denom) if denom > 0 else 0.0\n    score = 1000.0 * f1 * (len(kept) / total)\n    return score, len(kept), total, (TP, FP, FN)\n\ndef main():\n    preds = load_predictions(OUTPUT_PATH)\n\n    # Status tallies\n    exact = partial = wrong = overpred = empty = 0\n    per_item = []\n\n    for rid, gt in GT.items():\n        gt_norm = norm_letters(gt)\n        pr_raw = preds.get(rid, \"\")\n        pr_norm = norm_letters(pr_raw)\n\n        pr_set = set(pr_norm)\n        gt_set = set(gt_norm)\n\n        status = \"\"\n        if len(pr_norm) == 0:\n            empty += 1\n            status = \"EMPTY\"\n        elif len(pr_norm) > len(gt_norm):\n            overpred += 1\n            status = \"OVERPRED\"\n        elif pr_set == gt_set and pr_norm == gt_norm:\n            exact += 1\n            status = \"EXACT\"\n        elif len(pr_set & gt_set) > 0:\n            partial += 1\n            status = f\"PARTIAL (hit={sorted(pr_set & gt_set)}, miss={sorted(gt_set - pr_set)}, extra={sorted(pr_set - gt_set)})\"\n        else:\n            wrong += 1\n            status = f\"WRONG (miss={sorted(gt_set)}, extra={sorted(pr_set)})\"\n\n        per_item.append((rid, \"\".join(gt_norm), \"\".join(pr_norm), status))\n\n    score, kept, total, (TP, FP, FN) = micro_f1_kept(GT, preds)\n    prec = TP / (TP + FP) if (TP + FP) > 0 else 0.0\n    rec  = TP / (TP + FN) if (TP + FN) > 0 else 0.0\n\n    print(\"\\n=== Summary ===\")\n    print(f\"Total items:      {total}\")\n    print(f\"Kept (not overpred): {kept}  (dropped: {total-kept})\")\n    print(f\"EXACT: {exact} | PARTIAL: {partial} | WRONG: {wrong} | OVERPRED: {overpred} | EMPTY: {empty}\")\n    print(f\"Micro-F1 (contest-style): {score:.2f}\")\n    print(f\"(On kept items) TP={TP} FP={FP} FN={FN} | Precision={prec:.3f} Recall={rec:.3f}\")\n\n    print(\"\\n=== Per-item breakdown ===\")\n    for rid, gt_s, pr_s, status in sorted(per_item):\n        print(f\"RID {rid}: GT={gt_s:>4}  PRED={pr_s:>4}  -> {status}\")\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T10:18:07.701804Z","iopub.execute_input":"2025-10-29T10:18:07.702268Z","iopub.status.idle":"2025-10-29T10:18:07.707677Z","shell.execute_reply.started":"2025-10-29T10:18:07.702246Z","shell.execute_reply":"2025-10-29T10:18:07.707196Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!python score.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-29T10:18:07.708246Z","iopub.execute_input":"2025-10-29T10:18:07.708407Z","iopub.status.idle":"2025-10-29T10:18:07.934405Z","shell.execute_reply.started":"2025-10-29T10:18:07.708394Z","shell.execute_reply":"2025-10-29T10:18:07.933772Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}