{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -U transformers accelerate \"bitsandbytes>=0.46.1\" huggingface_hub","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-08-23T12:38:01.140587Z","iopub.execute_input":"2026-08-23T12:38:01.141629Z","iopub.status.idle":"2026-08-23T12:38:22.606693Z","shell.execute_reply.started":"2026-08-23T12:38:01.141588Z","shell.execute_reply":"2026-08-23T12:38:22.605961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# !pip install -U transformers accelerate \"bitsandbytes>=0.46.1\" huggingface_hub\nimport os\nimport json\nimport gc\nimport pandas as pd\nimport torch\nfrom tqdm import tqdm\nfrom transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\nfrom transformers import logging\nfrom kaggle_secrets import UserSecretsClient\nfrom huggingface_hub import login\n\nlogging.set_verbosity_error()\n\n# --- Authenticate with Hugging Face ---\ntry:\n    user_secrets = UserSecretsClient()\n    hf_token = user_secrets.get_secret(\"HF_TOKEN\")\n    login(token=hf_token)\n    print(\"Successfully logged into Hugging Face!\")\nexcept Exception as e:\n    print(\"Authentication failed. Did you attach the HF_TOKEN in Kaggle Secrets?\")\n    raise e\n\n# -------------------------------------------\n# 1. Load the Competition Dataset\nCOMPETITION_DATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\ntrain_df = pd.read_csv(f\"{COMPETITION_DATA_PATH}/train.csv\")\n\ntarget_cols = [\n    'ACL', 'MCL', 'Medial Meniscus', 'Lateral Meniscus', 'Medial OA',\n    'Lateral OA', 'PF OA', 'Effusion', 'Synovitis', \"Baker's\",\n    'Contusion', 'Fracture'\n]\n\n# 2. Separate Labeled vs. Unlabeled Studies\ntrain_df['is_gold'] = train_df[target_cols].notna().all(axis=1)\n\nlabeled_df = train_df[train_df['is_gold']].copy()\nunlabeled_df = train_df[~train_df['is_gold']].copy()\n\nprint(f\"Gold Labeled studies: {len(labeled_df)}\")\nprint(f\"Unlabeled reports to process: {len(unlabeled_df)}\")\n\n# 3. Configure 4-bit Quantization for Kaggle's T4 GPUs\nbnb_config = BitsAndBytesConfig(\n    load_in_4bit=True,\n    bnb_4bit_use_double_quant=True,\n    bnb_4bit_quant_type=\"nf4\",\n    bnb_4bit_compute_dtype=torch.float16\n)\n\nmodel_id = \"meta-llama/Meta-Llama-3-8B-Instruct\"\ntokenizer = AutoTokenizer.from_pretrained(model_id)\n\nif tokenizer.pad_token is None:\n    tokenizer.pad_token = tokenizer.eos_token\ntokenizer.padding_side = \"left\"\n\ngc.collect()\ntorch.cuda.empty_cache()\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    quantization_config=bnb_config,\n    device_map={\"\": 0}\n)\n\nBATCH_SIZE = 2 \n\n# 4. Define the CoT Prompt\ndef build_prompt(report_text):\n    return f\"\"\"You are an expert musculoskeletal radiologist performing a STRICT information-extraction task from a knee MRI report.\n\nExtract the status of exactly these 12 targets:\n1. ACL, 2. MCL, 3. Medial Meniscus, 4. Lateral Meniscus, 5. Medial OA, 6. Lateral OA, 7. PF OA, 8. Effusion, 9. Synovitis, 10. Baker's, 11. Contusion, 12. Fracture\n\n==================================================\nCORE EXTRACTION RULES\n==================================================\n1. MULTILINGUAL: If the report is in another language (e.g., Spanish), translate it to English internally before processing.\n2. ABSENCE FROM SILENCE: If a condition is not explicitly mentioned, assume it is normal and output 0.\n3. NEGATION: Output 0 if the structure is described as \"intact\", \"normal\", \"preserved\", \"unremarkable\", \"not torn\", or if the text says \"no [condition]\". Pay close attention to \"No fracture is seen.\"\n4. MENISCUS & OA: Evaluate Medial and Lateral separately.\n5. SYNOVITIS vs EFFUSION: They are different. Do not infer synovitis just because effusion is present.\n6. EXACT KEYS: You MUST copy these 12 keys EXACTLY, character for character. Do not alter spelling, capitalization, or apostrophes (e.g., write \"ACL\" not \"AoCL\", write \"Baker's\" not \"Bakers\").\n\n==================================================\nOUTPUT FORMAT\n==================================================\nStep 1: Write a brief 2-3 sentence analysis of the findings inside <thinking> ... </thinking> tags.\nStep 2: Return ONLY a raw JSON dictionary using this EXACT key skeleton (fill in the 0/1 values):\n{{\"ACL\": <0 or 1>, \"MCL\": <0 or 1>, \"Medial Meniscus\": <0 or 1>, \"Lateral Meniscus\": <0 or 1>, \"Medial OA\": <0 or 1>, \"Lateral OA\": <0 or 1>, \"PF OA\": <0 or 1>, \"Effusion\": <0 or 1>, \"Synovitis\": <0 or 1>, \"Baker's\": <0 or 1>, \"Contusion\": <0 or 1>, \"Fracture\": <0 or 1>}}\nYou MUST start exactly with {{ and end with }}.\n\nEXAMPLE INPUT:\n\"Findings: ACL is intact. Complex tear of the lateral meniscus. Mild joint effusion. No fracture is seen.\"\nEXAMPLE OUTPUT:\n<thinking>\nThe ACL is intact (0). A complex tear of the lateral meniscus is present (1). There is mild joint effusion (1). The text explicitly states no fracture is seen (0). Medial meniscus, OA, synovitis, baker's cyst, and contusion are not mentioned, so they are assumed absent (0).\n</thinking>\n{{\"ACL\": 0, \"MCL\": 0, \"Medial Meniscus\": 0, \"Lateral Meniscus\": 1, \"Medial OA\": 0, \"Lateral OA\": 0, \"PF OA\": 0, \"Effusion\": 1, \"Synovitis\": 0, \"Baker's\": 0, \"Contusion\": 0, \"Fracture\": 0}}\n\nREPORT TO ANALYZE:\n{report_text}\n\"\"\"\n\ndef build_batch_prompts(report_texts):\n    prompts = []\n    for report_text in report_texts:\n        messages = [\n            {\"role\": \"system\", \"content\": \"You are a precise medical data extraction AI. Follow instructions perfectly.\"},\n            {\"role\": \"user\", \"content\": build_prompt(report_text)}\n        ]\n        prompts.append(tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True))\n    return prompts\n\ndef parse_response(response):\n    try:\n        start_idx = response.find('{')\n        end_idx = response.rfind('}')\n        if start_idx == -1 or end_idx == -1:\n            return {col: pd.NA for col in target_cols}, False, \"no_brackets\"\n\n        parsed = json.loads(response[start_idx:end_idx + 1])\n        result = {}\n        unexpected_keys = [k for k in parsed.keys() if k not in target_cols]\n        for col in target_cols:\n            result[col] = parsed[col] if col in parsed else pd.NA\n        status = f\"unexpected_keys:{unexpected_keys}\" if unexpected_keys else \"ok\"\n        return result, len(unexpected_keys) == 0, status\n\n    except json.JSONDecodeError:\n        return {col: pd.NA for col in target_cols}, False, \"json_decode_error\"\n\ndef generate_safe_batch(valid_reports, terminators):\n    \"\"\"Attempts batched generation, falls back to batch_size=1 if OOM occurs.\"\"\"\n    try:\n        prompts = build_batch_prompts(valid_reports)\n        inputs = tokenizer(prompts, return_tensors=\"pt\", padding=True).to(model.device)\n        \n        with torch.no_grad():\n            outputs = model.generate(\n                **inputs,\n                max_new_tokens=500,\n                eos_token_id=terminators,\n                do_sample=False,\n                pad_token_id=tokenizer.pad_token_id\n            )\n            \n        input_len = inputs['input_ids'].shape[-1]\n        decoded = tokenizer.batch_decode(outputs[:, input_len:], skip_special_tokens=True)\n        \n        del inputs, outputs\n        torch.cuda.empty_cache()\n        return decoded\n        \n    except (torch.cuda.OutOfMemoryError, RuntimeError) as e:\n        if \"memory\" not in str(e).lower() and \"cuda\" not in str(e).lower():\n            raise e # Raise if it's a completely different runtime error\n            \n        print(\"\\n⚠️ CUDA OOM caught! Purging cache and falling back to batch size 1...\")\n        torch.cuda.empty_cache()\n        gc.collect()\n        \n        decoded_fallback = []\n        for report in valid_reports:\n            single_prompt = build_batch_prompts([report])\n            single_input = tokenizer(single_prompt, return_tensors=\"pt\", padding=True).to(model.device)\n            \n            with torch.no_grad():\n                single_output = model.generate(\n                    **single_input,\n                    max_new_tokens=500,\n                    eos_token_id=terminators,\n                    do_sample=False,\n                    pad_token_id=tokenizer.pad_token_id\n                )\n                \n            in_len = single_input['input_ids'].shape[-1]\n            dec = tokenizer.batch_decode(single_output[:, in_len:], skip_special_tokens=True)\n            decoded_fallback.extend(dec)\n            \n            del single_input, single_output\n            torch.cuda.empty_cache()\n            gc.collect()\n            \n        return decoded_fallback\n\ndef run_batched_extraction(df, desc):\n    all_preds = []\n    all_statuses = []\n    reports = df['Report'].tolist()\n    terminators = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids(\"<|eot_id|>\")]\n\n    for i in tqdm(range(0, len(reports), BATCH_SIZE), desc=desc):\n        batch_reports = reports[i:i + BATCH_SIZE]\n\n        valid_idx = [j for j, r in enumerate(batch_reports) if pd.notna(r)]\n        batch_results = [{col: pd.NA for col in target_cols} for _ in batch_reports]\n        batch_status = [\"missing_report\" for _ in batch_reports]\n\n        if not valid_idx:\n            all_preds.extend(batch_results)\n            all_statuses.extend(batch_status)\n            continue\n\n        valid_reports = [batch_reports[j] for j in valid_idx]\n        \n        # Wrapped generation call with built-in retry logic\n        decoded = generate_safe_batch(valid_reports, terminators)\n\n        for j, response in zip(valid_idx, decoded):\n            parsed, clean, status = parse_response(response)\n            batch_results[j] = parsed\n            batch_status[j] = status\n\n        all_preds.extend(batch_results)\n        all_statuses.extend(batch_status)\n\n    return all_preds, all_statuses\n\n# 4.5 Measure pseudo-label accuracy against the 58 gold studies\nprint(\"\\n--- Evaluating Extraction Accuracy on Gold Studies ---\")\ngold_preds, gold_statuses = run_batched_extraction(labeled_df, \"Testing Gold Labels\")\ngold_preds_df = pd.DataFrame(gold_preds, index=labeled_df.index)\n\nn_key_issues = sum(1 for s in gold_statuses if s.startswith(\"unexpected_keys\"))\nn_parse_fail = sum(1 for s in gold_statuses if s in (\"no_brackets\", \"json_decode_error\"))\nprint(f\"\\nTotal parse failures: {n_parse_fail}/{len(gold_statuses)}\")\nprint(f\"Responses with unexpected/misspelled keys: {n_key_issues}/{len(gold_statuses)}\")\nif n_key_issues:\n    for s in gold_statuses:\n        if s.startswith(\"unexpected_keys\"):\n            print(f\"  {s}\")\n\nprint(\"\\nAccuracy per Target (Gold vs LLM):\")\ntarget_accuracy = {}\nfor col in target_cols:\n    valid_mask = gold_preds_df[col].notna() & labeled_df[col].notna()\n    if valid_mask.sum() > 0:\n        acc = (gold_preds_df.loc[valid_mask, col].astype(float) == labeled_df.loc[valid_mask, col].astype(float)).mean()\n        target_accuracy[col] = acc\n        n_missing = (~gold_preds_df[col].notna()).sum()\n        print(f\"{col:<20}: {acc:.2%}   (missing/unparsed: {n_missing}/{len(gold_preds_df)})\")\nprint(\"------------------------------------------------------\\n\")\n\nprint(\"\\n--- Inspecting First 3 Outputs ---\")\nfor i in range(min(3, len(labeled_df))):\n    print(f\"REPORT {i+1}:\\n{labeled_df.iloc[i]['Report'][:300]}... [truncated]\")\n    gold_dict = labeled_df.iloc[i][target_cols].to_dict()\n    print(f\"\\nTRUE GOLD : {gold_dict}\")\n    print(f\"LLM PRED  : {gold_preds[i]}\")\n    print(\"=\" * 80)\n\nACCURACY_THRESHOLD = 0.70\nlow_accuracy_targets = [col for col, acc in target_accuracy.items() if acc < ACCURACY_THRESHOLD]\nprint(f\"\\nTargets below {ACCURACY_THRESHOLD:.0%} accuracy — pseudo-labels will be dropped for these: {low_accuracy_targets}\")\n\n# IMPORTANT: Comment this out when you are ready to process all unlabeled rows!\n#import sys\n#sys.exit(\"Stopping after debug print! Review the accuracy table above before continuing.\")\n\n# -------------------------------------------\n# 5. Extraction Loop (Unlabeled Data) — CHUNKED\n# ---> CHANGE THESE NUMBERS FOR EACH NOTEBOOK PART <---\n# PART 1: .iloc[0:1100]\n# PART 2: .iloc[1100:2200]\n# PART 3: .iloc[2200:3300]\n# PART 4: .iloc[3300:]\n\nchunk_df = unlabeled_df.iloc[3300:].copy() \n\nextracted_labels, extraction_statuses = run_batched_extraction(chunk_df, \"Extracting Labels (Part 1)\")\nextracted_df = pd.DataFrame(extracted_labels, index=chunk_df.index)\n\n# Null out any target that failed the gold-accuracy check, competition-wide\nfor col in low_accuracy_targets:\n    extracted_df[col] = pd.NA\n\nchunk_df[target_cols] = extracted_df[target_cols]\n\nn_parse_fail_full = sum(1 for s in extraction_statuses if s in (\"no_brackets\", \"json_decode_error\"))\nprint(f\"\\nChunk parse failures: {n_parse_fail_full}/{len(extraction_statuses)}\")\n\n# 7. Merge and Save the Final Dataset\n# We save ONLY this chunk plus the gold labels so we can merge them all later\nfinal_train_df = pd.concat([labeled_df, chunk_df]).sort_index()\n\n# ---> CHANGE THE FILENAME FOR EACH PART <---\nfinal_train_df.to_csv(\"train_pseudo_labeled_part4.csv\", index=False)\nprint(\"Successfully generated train_pseudo_labeled_part4.csv!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-23T12:45:50.040262Z","iopub.execute_input":"2026-08-23T12:45:50.040715Z"}},"outputs":[],"execution_count":null}]}