"""Semantic dedupe helpers for observations, connections, and UM proposals."""

from __future__ import annotations

import hashlib
import re
from typing import Any

# Proposals use a slightly softer threshold than observations so write/display stay aligned.
PROPOSAL_NEAR_DUPLICATE_JACCARD = 0.45
OBSERVATION_NEAR_DUPLICATE_JACCARD = 0.52

_CONNECTION_ARROW_SPLIT_RE = re.compile(
    r"\s*(?:↔|\\leftrightarrow|\$\\leftrightarrow\$|<->)\s*",
    re.IGNORECASE,
)

_TOKEN_STOP = frozenset(
    {
        "the",
        "and",
        "with",
        "that",
        "this",
        "from",
        "into",
        "about",
        "user",
        "your",
        "you",
        "have",
        "has",
        "been",
        "would",
        "like",
        "me",
        "for",
        "are",
        "was",
        "were",
        "not",
        "but",
        "when",
        "what",
        "how",
        "they",
        "their",
        "them",
        "than",
        "then",
        "also",
        "just",
        "more",
        "some",
        "such",
        "very",
        "will",
        "can",
        "could",
        "should",
        "might",
        "may",
        "did",
        "does",
        "doing",
        "done",
        "being",
        "through",
        "during",
        "while",
        "where",
        "which",
        "who",
        "whom",
        "whose",
        "because",
        "since",
        "until",
        "after",
        "before",
        "over",
        "under",
        "between",
        "across",
        "within",
        "without",
        "noticed",
        "something",
        "worth",
        "working",
        "assistant",
        "mugi",
        "consolidation",
        "links",
        "link",
        "related",
        "thread",
        "threads",
        "connection",
        "connections",
        "leftrightarrow",
    }
)

_OBSERVATION_BOILERPLATE_RE = re.compile(
    r"^(?:i noticed(?: something worth working on)?|i've been thinking about something)\s*[—–-]?\s*",
    re.IGNORECASE,
)


def normalize_connection_headline(text: str) -> str:
    """Strip markdown/LaTeX noise from a connection headline for dedupe."""
    s = str(text or "").strip()
    for pattern, repl in (
        (r"\$\\leftrightarrow\$", " ↔ "),
        (r"\\leftrightarrow", "↔"),
        (r"\*\*(.+?)\*\*", r"\1"),
        (r"`([^`]+)`", r"\1"),
    ):
        s = re.sub(pattern, repl, s)
    return re.sub(r"\s+", " ", s).strip()


def connection_topic_pair(text: str) -> tuple[str, str] | None:
    """Extract sorted topic pair from ``A ↔ B: …`` connection lines, if present."""
    raw = normalize_connection_headline(str(text or ""))
    if not raw:
        return None
    headline = raw.split(":", 1)[0].strip()
    if not headline:
        return None
    parts = _CONNECTION_ARROW_SPLIT_RE.split(headline, maxsplit=1)
    if len(parts) != 2:
        return None
    left = parts[0].strip().strip("*").strip().lower()
    right = parts[1].strip().strip("*").strip().lower()
    if len(left) < 3 or len(right) < 3:
        return None
    return tuple(sorted((left, right)))


def connection_pair_signatures_from_texts(texts: list[str]) -> set[tuple[str, str]]:
    sigs: set[tuple[str, str]] = set()
    for text in texts:
        pair = connection_topic_pair(text)
        if pair:
            sigs.add(pair)
    return sigs


def _http_status_codes(text: str) -> set[str]:
    return {m.rstrip("s") for m in re.findall(r"\b(?:403|404|429|500|502|503)s?\b", str(text or "").lower())}


def _shares_http_failure_cluster(a: str, b: str, *, min_codes: int = 3) -> bool:
    """True when both lines describe the same HTTP failure cluster (404/403/429…)."""
    shared = _http_status_codes(a) & _http_status_codes(b)
    if len(shared) < min_codes:
        return False
    combined = f"{a} {b}".lower()
    return any(
        cue in combined
        for cue in ("web", "http", "404", "403", "429", "rate", "failures", "failure", "forbidden")
    )


def _semantic_tokens(text: str, *, max_tokens: int = 36) -> set[str]:
    cleaned = str(text or "").lower()
    cleaned = _OBSERVATION_BOILERPLATE_RE.sub("", cleaned)
    cleaned = re.sub(r"[^\w\s-]+", " ", cleaned)
    codes = [c.rstrip("s") for c in re.findall(r"\b(?:403|404|429|500|502|503)s?\b", cleaned)]
    tokens = re.findall(r"[a-z][a-z0-9_-]{2,}", cleaned)
    out: list[str] = []
    seen: set[str] = set()
    for tok in [*codes, *tokens]:
        if tok in _TOKEN_STOP or tok in seen:
            continue
        seen.add(tok)
        out.append(tok)
        if len(out) >= max_tokens:
            break
    return set(out)


def semantic_fingerprint(text: str, *, extra: str = "") -> str:
    tokens = sorted(_semantic_tokens(f"{text} {extra}".strip()))
    if not tokens:
        return ""
    payload = "|".join(tokens)
    return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]


def semantic_jaccard(a: str, b: str, *, min_tokens: int = 4) -> float:
    ta = _semantic_tokens(a)
    tb = _semantic_tokens(b)
    if len(ta) < min_tokens or len(tb) < min_tokens:
        return 0.0
    inter = len(ta & tb)
    union = len(ta | tb)
    if union <= 0:
        return 0.0
    return inter / union


def is_near_duplicate_text(
    candidate: str,
    existing_texts: list[str],
    *,
    min_jaccard: float = 0.52,
) -> bool:
    """True when *candidate* is semantically similar to any text in *existing_texts*."""
    cand = str(candidate or "").strip()
    if len(cand) < 14:
        return False
    cand_fp = semantic_fingerprint(cand)
    new_pair = connection_topic_pair(cand)
    existing_pairs = connection_pair_signatures_from_texts(existing_texts)
    if new_pair and new_pair in existing_pairs:
        return True
    for existing in existing_texts:
        other = str(existing or "").strip()
        if not other:
            continue
        if _shares_http_failure_cluster(cand, other):
            return True
        if cand.lower() == other.lower():
            return True
        if len(cand) >= 20 and len(other) >= 20:
            shorter, longer = (cand, other) if len(cand) <= len(other) else (other, cand)
            if shorter.lower() in longer.lower():
                return True
        if cand_fp and cand_fp == semantic_fingerprint(other):
            return True
        if semantic_jaccard(cand, other) >= min_jaccard:
            return True
    return False


def is_duplicate_observation_noticed(
    noticed: str,
    proposed_work: str,
    *,
    existing_noticed: list[str],
    existing_work: list[str] | None = None,
    min_jaccard: float = OBSERVATION_NEAR_DUPLICATE_JACCARD,
) -> bool:
    """True when a pending observation already covers the same topic."""
    n = str(noticed or "").strip()
    w = str(proposed_work or "").strip()
    if not n:
        return False
    combined = f"{n} {w}".strip()
    fp = semantic_fingerprint(n, extra=w)
    for idx, prior in enumerate(existing_noticed):
        prior_n = str(prior or "").strip()
        if not prior_n:
            continue
        prior_w = ""
        if existing_work and idx < len(existing_work):
            prior_w = str(existing_work[idx] or "").strip()
        if _shares_http_failure_cluster(n, f"{prior_n} {prior_w}"):
            return True
        if fp and fp == semantic_fingerprint(prior_n, extra=prior_w):
            return True
        if is_near_duplicate_text(combined, [f"{prior_n} {prior_w}".strip()], min_jaccard=min_jaccard):
            return True
        if is_near_duplicate_text(n, [prior_n], min_jaccard=min_jaccard):
            return True
    return False


def collect_pending_observation_notices(entries: list[dict[str, Any]]) -> tuple[list[str], list[str]]:
    """Extract noticed/proposed_work lists from pending observation queue rows."""
    noticed_out: list[str] = []
    work_out: list[str] = []
    for item in entries:
        if not isinstance(item, dict):
            continue
        obs = item.get("observation") if isinstance(item.get("observation"), dict) else {}
        noticed_out.append(str(obs.get("noticed") or item.get("content") or ""))
        work_out.append(str(obs.get("proposed_work") or ""))
    return noticed_out, work_out


def _walk_change_strings(val: Any, out: list[str]) -> None:
    if isinstance(val, str):
        text = val.strip()
        if len(text) >= 14:
            out.append(text)
    elif isinstance(val, dict):
        for v in val.values():
            _walk_change_strings(v, out)
    elif isinstance(val, list):
        for item in val:
            _walk_change_strings(item, out)


def proposal_canonical_text(proposal: dict[str, Any]) -> str:
    """Primary human-readable text for deduping a User Model proposal."""
    if not isinstance(proposal, dict):
        return ""
    changes = proposal.get("changes")
    texts: list[str] = []
    if isinstance(changes, dict):
        for conn in changes.get("connections") or []:
            if isinstance(conn, dict):
                text = str(conn.get("text") or "").strip()
                if text:
                    return text
        _walk_change_strings(changes, texts)
    if texts:
        return max(texts, key=len)
    rationale = str(proposal.get("rationale") or "").strip()
    if rationale:
        return rationale
    return ""


def collect_proposal_texts(proposal: dict[str, Any]) -> list[str]:
    """All substantive strings from a proposal payload."""
    if not isinstance(proposal, dict):
        return []
    texts: list[str] = []
    changes = proposal.get("changes")
    if isinstance(changes, dict):
        _walk_change_strings(changes, texts)
    rationale = str(proposal.get("rationale") or "").strip()
    if len(rationale) >= 14:
        texts.append(rationale)
    return texts


def dedupe_proposals_for_display(
    proposals: list[dict[str, Any]],
    *,
    baseline_texts: list[str] | None = None,
    min_jaccard: float = PROPOSAL_NEAR_DUPLICATE_JACCARD,
) -> list[dict[str, Any]]:
    """Collapse semantically duplicate pending proposals for the You pane."""
    baseline = [str(t or "").strip() for t in (baseline_texts or []) if str(t or "").strip()]
    seen = list(baseline)
    kept: list[dict[str, Any]] = []
    ordered = sorted(
        proposals,
        key=lambda p: str(p.get("created_at") or p.get("id") or ""),
    )
    for prop in ordered:
        if not isinstance(prop, dict):
            continue
        candidates = collect_proposal_texts(prop)
        if not candidates:
            kept.append(prop)
            continue
        if any(is_near_duplicate_text(text, seen, min_jaccard=min_jaccard) for text in candidates):
            continue
        kept.append(prop)
        primary = proposal_canonical_text(prop)
        if primary.strip():
            seen.append(primary)
        else:
            seen.extend(candidates)
    return kept


def dedupe_observation_entries_for_display(entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
    """Collapse semantically duplicate pending observations for inbox display."""
    kept: list[dict[str, Any]] = []
    prior_noticed: list[str] = []
    prior_work: list[str] = []
    for item in entries:
        if not isinstance(item, dict):
            kept.append(item)
            continue
        if str(item.get("kind") or "") != "observation":
            kept.append(item)
            continue
        obs = item.get("observation") if isinstance(item.get("observation"), dict) else {}
        noticed = str(obs.get("noticed") or item.get("content") or "")
        work = str(obs.get("proposed_work") or "")
        if is_duplicate_observation_noticed(
            noticed,
            work,
            existing_noticed=prior_noticed,
            existing_work=prior_work,
        ):
            continue
        kept.append(item)
        prior_noticed.append(noticed)
        prior_work.append(work)
    return kept
