momentry_core/scripts/specific_stamp_search.py

#!/opt/homebrew/bin/python3.11
"""
Search for Specific Stamps in the Image (Avoiding Watermark)
"""

import os
import cv2
import types
from PIL import Image
from transformers import AutoProcessor, AutoModelForCausalLM

UUID = "384b0ff44aaaa1f1"
OUTPUT_DIR = f"output/{UUID}/florence2_results"
INPUT_IMG = os.path.join(OUTPUT_DIR, "raw_6846.jpg")


# Patch for compatibility
def patch_model(model):
    inner_model = model.language_model
    original_prepare = inner_model.prepare_inputs_for_generation

    def patched_prepare(
        self,
        input_ids,
        past_key_values=None,
        attention_mask=None,
        inputs_embeds=None,
        **kwargs,
    ):
        is_valid_cache = False
        if past_key_values is not None:
            if isinstance(past_key_values, (list, tuple)) and len(past_key_values) > 0:
                first_layer = past_key_values[0]
                if first_layer is not None and (
                    not isinstance(first_layer, (list, tuple)) or len(first_layer) > 0
                ):
                    is_valid_cache = True

        if not is_valid_cache:
            return {
                "input_ids": input_ids,
                "attention_mask": attention_mask,
                "past_key_values": None,
                "use_cache": True,
            }
        else:
            return original_prepare(
                input_ids,
                past_key_values=past_key_values,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                **kwargs,
            )

    inner_model.prepare_inputs_for_generation = types.MethodType(
        patched_prepare, inner_model
    )


print(f"📷 Loading image from {INPUT_IMG}...")
if not os.path.exists(INPUT_IMG):
    print("❌ Image not found.")
    exit()

image = Image.open(INPUT_IMG).convert("RGB")
print(f"📐 Image Size: {image.width}x{image.height}")

# Mask the watermark area (Top Right Corner) to prevent false positives
# Based on previous error: X: 1721-1813, Y: 23-173.
# We'll cover a slightly larger area to be safe.
img_cv = cv2.imread(INPUT_IMG)
# Draw a black rectangle over the top-right corner
mask_height = 200
mask_width = 200
h, w, _ = img_cv.shape
cv2.rectangle(img_cv, (w - mask_width, 0), (w, mask_height), (0, 0, 0), -1)

# Save masked image
masked_img_path = os.path.join(OUTPUT_DIR, "masked_input.jpg")
cv2.imwrite(masked_img_path, img_cv)
print(f"🎭 Watermark masked and saved to {masked_img_path}")

# Load masked image for AI
masked_image = Image.open(masked_img_path).convert("RGB")

print("🧠 Loading Florence-2 model...")
try:
    processor = AutoProcessor.from_pretrained(
        "microsoft/Florence-2-base", trust_remote_code=True
    )
    model = AutoModelForCausalLM.from_pretrained(
        "microsoft/Florence-2-base", trust_remote_code=True, attn_implementation="eager"
    )
    patch_model(model)

    prompt = "<OPEN_VOCABULARY_DETECTION>"
    # More specific search terms to find a plot-relevant stamp, not a logo
    search_terms = [
        "postage stamp",
        "collection of stamps",
        "stamp album",
        "holding a stamp",
        "envelope with stamp",
    ]

    all_found = []

    for term in search_terms:
        print(f"🔍 Scanning for '{term}'...")
        inputs = processor(text=prompt, images=masked_image, return_tensors="pt")

        generated_ids = model.generate(
            input_ids=inputs["input_ids"],
            pixel_values=inputs["pixel_values"],
            max_new_tokens=1024,
            num_beams=3,
        )

        generated_text = processor.batch_decode(
            generated_ids, skip_special_tokens=False
        )[0]

        try:
            parsed_answer = processor.post_process_generation(
                generated_text,
                task=prompt,
                image_size=(masked_image.width, masked_image.height),
            )
            results = parsed_answer.get("<OPEN_VOCABULARY_DETECTION>", {})
            bboxes = results.get("bboxes", [])
            labels = results.get("bboxes_labels", [])

            if bboxes:
                print(f"✅ Found {len(bboxes)} '{term}'!")
                for i, (box, label) in enumerate(zip(bboxes, labels)):
                    x1, y1, x2, y2 = map(int, box)
                    # Draw on the original unmasked image
                    cv2.rectangle(img_cv, (x1, y1), (x2, y2), (0, 255, 0), 3)
                    cv2.putText(
                        img_cv,
                        f"{label} ({term})",
                        (x1, y1 - 10),
                        cv2.FONT_HERSHEY_SIMPLEX,
                        0.8,
                        (0, 255, 0),
                        2,
                    )
                    all_found.append(True)
            else:
                print(f"   ❌ No '{term}' found.")
        except Exception as e:
            print(f"   ⚠️ Error processing '{term}': {e}")

    final_out = os.path.join(OUTPUT_DIR, "specific_stamp_result.jpg")
    cv2.imwrite(final_out, img_cv)
    print(f"\n🎨 Result image saved to: {final_out}")
    if not all_found:
        print("⚠️ No specific stamps were found in the scene (excluding the watermark).")

except Exception as e:
    print(f"❌ Error: {e}")
    import traceback

    traceback.print_exc()