"""Near-duplicate detection for memory merge bullets (Plan B: Mem0-lite dedup)."""
from __future__ import annotations

import re

_TOKEN_RE = re.compile(r"[a-z0-9_]+")


def _tokens(s: str) -> set[str]:
    return {t for t in _TOKEN_RE.findall((s or "").lower()) if t}


def token_jaccard(a: str, b: str) -> float:
    sa, sb = _tokens(a), _tokens(b)
    if not sa and not sb:
        return 1.0
    if not sa or not sb:
        return 0.0
    return len(sa & sb) / len(sa | sb)


def dedupe_bullet_lines(lines: list[str], *, jaccard_threshold: float = 0.82) -> list[str]:
    """Drop later markdown bullets that are near-duplicates of an earlier bullet line.

    Non-bullet lines are kept verbatim. Comparison uses Jaccard similarity on
    token sets of the bullet body (text after the leading ``-``).
    """
    if not lines or jaccard_threshold <= 0:
        return list(lines)
    kept: list[str] = []
    prior_bodies: list[str] = []
    for ln in lines:
        stripped = ln.strip()
        if not stripped.startswith("-"):
            kept.append(ln)
            continue
        body = stripped.lstrip("-").strip()
        dup = False
        for pb in prior_bodies:
            if token_jaccard(body, pb) >= jaccard_threshold:
                dup = True
                break
        if dup:
            continue
        prior_bodies.append(body)
        kept.append(ln)
    return kept
