momentry_core/scripts/search_envelope.py

#!/opt/homebrew/bin/python3.11
"""
Search for Envelope/Stamp in Keyframes
"""

import os
import cv2
import types
from PIL import Image
from transformers import AutoProcessor, AutoModelForCausalLM

UUID = "384b0ff44aaaa1f1"
BASE_DIR = f"output/{UUID}/florence2_results"

# Frames to check
FRAMES = [
    "scan_6756.jpg",  # 112:36
    "scan_6763.jpg",  # 112:43
    "scan_6790.jpg",  # 113:10
    "scan_6813.jpg",  # 113:33
    "scan_6832.jpg",  # 113:52
]


# Patch for compatibility
def patch_model(model):
    inner_model = model.language_model
    original_prepare = inner_model.prepare_inputs_for_generation

    def patched_prepare(
        self,
        input_ids,
        past_key_values=None,
        attention_mask=None,
        inputs_embeds=None,
        **kwargs,
    ):
        is_valid_cache = False
        if past_key_values is not None:
            if isinstance(past_key_values, (list, tuple)) and len(past_key_values) > 0:
                first_layer = past_key_values[0]
                if first_layer is not None and (
                    not isinstance(first_layer, (list, tuple)) or len(first_layer) > 0
                ):
                    is_valid_cache = True

        if not is_valid_cache:
            return {
                "input_ids": input_ids,
                "attention_mask": attention_mask,
                "past_key_values": None,
                "use_cache": True,
            }
        else:
            return original_prepare(
                input_ids,
                past_key_values=past_key_values,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                **kwargs,
            )

    inner_model.prepare_inputs_for_generation = types.MethodType(
        patched_prepare, inner_model
    )


print("🧠 Loading Florence-2 model...")
try:
    processor = AutoProcessor.from_pretrained(
        "microsoft/Florence-2-base", trust_remote_code=True
    )
    model = AutoModelForCausalLM.from_pretrained(
        "microsoft/Florence-2-base", trust_remote_code=True, attn_implementation="eager"
    )
    patch_model(model)

    search_terms = ["envelope", "letter", "paper", "hand holding paper"]

    for img_name in FRAMES:
        img_path = os.path.join(BASE_DIR, img_name)
        if not os.path.exists(img_path):
            continue

        print(f"\n🔍 Scanning {img_name}...")
        image = Image.open(img_path).convert("RGB")
        img_cv = cv2.imread(img_path)

        # We will detect "envelope" specifically
        prompt = "<OPEN_VOCABULARY_DETECTION>"

        # Try to detect envelope
        inputs = processor(text=prompt, images=image, return_tensors="pt")
        # We need to provide the text label for OVD?
        # Florence-2 OVD usually generates the label.
        # But we can filter the results.

        generated_ids = model.generate(
            input_ids=inputs["input_ids"],
            pixel_values=inputs["pixel_values"],
            max_new_tokens=1024,
            num_beams=3,
        )
        generated_text = processor.batch_decode(
            generated_ids, skip_special_tokens=False
        )[0]

        try:
            parsed_answer = processor.post_process_generation(
                generated_text, task=prompt, image_size=(image.width, image.height)
            )
            results = parsed_answer.get("<OPEN_VOCABULARY_DETECTION>", {})
            bboxes = results.get("bboxes", [])
            labels = results.get("bboxes_labels", [])

            if bboxes:
                print(f"   ✅ Found objects: {labels}")
                for i, (box, label) in enumerate(zip(bboxes, labels)):
                    # Check if label is relevant
                    if any(
                        kw in label.lower()
                        for kw in ["envelope", "letter", "paper", "hand"]
                    ):
                        x1, y1, x2, y2 = map(int, box)
                        print(f"      📍 '{label}' at ({x1},{y1}) -> ({x2},{y2})")

                        # Draw
                        cv2.rectangle(img_cv, (x1, y1), (x2, y2), (0, 255, 0), 3)
                        cv2.putText(
                            img_cv,
                            label,
                            (x1, y1 - 10),
                            cv2.FONT_HERSHEY_SIMPLEX,
                            0.8,
                            (0, 255, 0),
                            2,
                        )

                        # Crop
                        crop = img_cv[y1:y2, x1:x2]
                        crop_path = os.path.join(
                            BASE_DIR,
                            f"crop_{img_name.replace('.jpg', '')}_{label}_{i}.jpg",
                        )
                        cv2.imwrite(crop_path, crop)
            else:
                print("   ❌ No objects found.")
        except Exception as e:
            print(f"   ⚠️ Error: {e}")

        # Save result
        res_path = os.path.join(BASE_DIR, f"result_env_{img_name}")
        cv2.imwrite(res_path, img_cv)

except Exception as e:
    print(f"❌ Error: {e}")