"""Knowledge base tools: digest corpora and retrieve compiled notes."""

from __future__ import annotations

import contextvars
import hashlib
import json
import logging
import mailbox
import os
import re
import socket
import threading
import time
import unicodedata
import urllib.error
import urllib.request
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError, as_completed
from datetime import datetime, timezone
import plistlib
from email import policy
from email.parser import BytesParser
from ipaddress import ip_address
from pathlib import Path
from urllib.parse import urlparse
from typing import Any

from langchain_core.tools import tool

from agent_loop_config import get_llm_context_length
from persistence import PROACTIVE_QUEUE_PATH, secure_atomic_write
from tools._logging import _debug_log, _emit_input_status
from tools.live_view_visuals import emit_live_view_preview
from utils.path_safety import _is_path_under_root, is_dangerous_path, safe_resolve_path
from utils.text_io import read_text_file_with_encoding_fallback

_log = logging.getLogger(__name__)


def _emit_knowledge_digest_source_error(filename: str, reason: str) -> None:
    """Emit Activity error only when an entire digest source could not be processed."""
    name = str(filename or "").strip() or "source"
    detail = str(reason or "").strip()
    summary = f"Knowledge digest failed for {name}"
    if detail:
        summary = f"{summary}: {detail[:200]}"
    try:
        from activity_bus import emit_activity_enriched

        emit_activity_enriched(
            {
                "source": "utility_model",
                "state": "error",
                "summary": summary,
                "operation": "knowledge_digest",
            },
            tags=["background"],
        )
    except Exception:
        pass


_DEFAULT_WORKSPACE = os.path.expanduser("~/.mugi/workspace")
KNOWLEDGE_SCHEMA_VERSION = 2
_KNOWLEDGE_DIR_NAME = "knowledge"
_SUPPORTED_TEXT_EXTENSIONS = {".md", ".txt", ".rst", ".csv"}
_SUPPORTED_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".tiff", ".tif"}
_SUPPORTED_EXTENSIONS = _SUPPORTED_TEXT_EXTENSIONS | _SUPPORTED_IMAGE_EXTENSIONS | {
    ".pdf",
    ".html",
    ".htm",
    ".docx",
    ".xlsx",
    ".pptx",
    ".rtf",
    ".epub",
    ".eml",
    ".mbox",
}
_MAX_DIGEST_FILES = 1000
_DIGEST_CHUNK_SIZE = 4000
_DIGEST_CHUNK_OVERLAP = 200
_DIGEST_STALE_TTL = 1800
_DIGEST_LLM_READ_TIMEOUT_DEFAULT_S = 120.0
_DIGEST_CHUNK_FUTURE_GRACE_S = 30.0
_TOPIC_INDEX_RESCAN_SECONDS = 60
_RELEVANT_SNIPPET_MAX_CORPORA = 3
_TOPIC_LINE_RE = re.compile(r"^\s*TOPIC:\s*(.+?)\s*(?:$|\|)", re.IGNORECASE)
_FALLBACK_TOKEN_RE = re.compile(r"[a-zA-Z0-9_]{3,}")
_SOURCE_SEARCH_CONTEXT = 180
_LARGE_SOURCE_WARN_CHARS = 500_000
_SUMMARY_WORKERS = 3
_FILE_SUMMARY_WORKERS = 3
_MAX_CORPORA = 250
_MAX_CONCURRENT_DIGESTS = 3
_SKIP_DIR_NAMES = {
    "__pycache__",
    ".git",
    "node_modules",
    "vendor",
    "dist",
    "build",
    "venv",
    ".venv",
}
_REPO_EXTRA_SKIP_DIRS = frozenset(
    {
        "target",
        "build",
        "dist",
        "out",
        "coverage",
        ".gradle",
        ".idea",
        ".vscode",
        "Pods",
        "DerivedData",
        "Carthage",
    }
)
_REPO_CODE_EXTENSIONS = frozenset(
    {
        ".py",
        ".pyi",
        ".js",
        ".jsx",
        ".ts",
        ".tsx",
        ".go",
        ".rs",
        ".java",
        ".kt",
        ".kts",
        ".c",
        ".cc",
        ".cpp",
        ".h",
        ".hpp",
        ".m",
        ".mm",
        ".rb",
        ".php",
        ".cs",
        ".swift",
        ".scala",
        ".r",
        ".sql",
        ".sh",
        ".bash",
        ".zsh",
        ".fish",
        ".lua",
        ".vim",
        ".zig",
        ".dart",
        ".gradle",
        ".lock",
    }
)
_INDEX_MIN_CHUNK_CHARS = 80
_MBOX_MAX_MESSAGE_BODY_CHARS = 60_000
_EML_MAX_MESSAGE_BODY_CHARS = 60_000
_URL_DIGEST_TIMEOUT_SECONDS = 10
_URL_DIGEST_BLOCK_PRIVATE_IPS = True
_URL_RATE_LIMIT_PER_HOST = 6
_URL_RATE_LIMIT_WINDOW_SECONDS = 60
_URL_CACHE_TTL_SECONDS = 300
_PARALLEL_FILE_CHAR_BUDGET = 700_000
_PARALLEL_SAFE_EXTENSIONS = set(_SUPPORTED_TEXT_EXTENSIONS) | {".html", ".htm"}
_KNOWLEDGE_GAP_FILE = os.path.join("memory", "knowledge_gaps.jsonl")
_KNOWLEDGE_GAP_MAX_LINES = 400
_SECTION_SPLIT_MIN_CHARS = 50_000
_SECTION_MIN_CHARS = 2_000
_SECTION_MAX_COUNT = 40
_HIERARCHICAL_CHUNK_THRESHOLD = 25
_HIERARCHICAL_GROUP_SIZE = 15
# Broken extraction (PDF column order, headers/footers, web boilerplate, logs, etc.) often
# yields huge runs of near-identical lines; summarizing that makes models parrot garbage.
_DEDUP_MIN_REPEAT_RUN = 4
_DEGENERATE_MIN_NONEMPTY_LINES = 35
_DEGENERATE_UNIQUE_LINE_RATIO = 0.12
_DEGENERATE_TOP_LINE_FRACTION = 0.18
_DEGENERATE_EXCERPT_CHARS = 2800
_SUMMARIZER_VERSION = "9"
# Bumped when summary-regeneration behavior changes; surfaced in GET /knowledge/{slug}/status.
SUMMARY_UPGRADE_CODE_REV = "2026-06-05h"
_SUMMARY_UPGRADE_SYS = (
    "Write 6–12 markdown bullets summarizing what happens in the passage: themes, events, and key facts. "
    "Output ONLY the bullets — no headings, labels, instructions, thinking, or commentary about the task. "
    "Do not quote the prompt, list constraints, repeat verse references, or describe the input file."
)
JOB_KIND_DIGEST = "digest"
JOB_KIND_SUMMARY_UPGRADE = "summary_upgrade"
_SUMMARY_UPGRADE_PAUSE_MSG = "Summary regeneration stopped."

# ---------------------------------------------------------------------------
# Adaptive main-model skip: after _MAIN_MODEL_FAIL_THRESHOLD consecutive
# quality-gate failures the main LLM is skipped for _MAIN_MODEL_COOLDOWN_S
# seconds, avoiding wasted HTTP round-trips during bulk digestion.
# ---------------------------------------------------------------------------
_MAIN_MODEL_FAIL_THRESHOLD = 2
_MAIN_MODEL_COOLDOWN_S = 300.0  # 5 minutes
_main_model_consecutive_failures = 0
_main_model_skip_until: float = 0.0
_summary_upgrade_active: contextvars.ContextVar[bool] = contextvars.ContextVar(
    "knowledge_summary_upgrade_active", default=False
)


def _reset_main_model_adaptive_skip() -> None:
    """Clear adaptive main-model cooldown (bulk summary upgrades need a fresh LLM pass)."""
    global _main_model_consecutive_failures, _main_model_skip_until
    _main_model_consecutive_failures = 0
    _main_model_skip_until = 0.0


def _prefer_utility_only_summaries() -> None:
    """Bulk summary upgrades: MLX utility only (main model + parallel chunks race and skip)."""
    global _main_model_skip_until
    _main_model_skip_until = time.time() + 86400.0


def _summary_upgrade_mode_active() -> bool:
    return bool(_summary_upgrade_active.get())
_TOPIC_FALLBACK_EXTRA_STOPWORDS = frozenset(
    {
        "the",
        "and",
        "for",
        "not",
        "but",
        "his",
        "her",
        "him",
        "she",
        "was",
        "were",
        "been",
        "being",
        "had",
        "has",
        "did",
        "does",
        "doing",
        "done",
        "from",
        "into",
        "onto",
        "upon",
        "over",
        "under",
        "than",
        "then",
        "thus",
        "also",
        "only",
        "just",
        "even",
        "such",
        "some",
        "any",
        "all",
        "each",
        "every",
        "other",
        "same",
        "very",
        "one",
        "two",
        "may",
        "might",
        "must",
        "shall",
        "should",
        "would",
        "could",
        "said",
        "came",
        "went",
    }
)

_digest_lock = threading.Lock()
_topic_index_lock = threading.Lock()
_topic_index_cache: dict[str, list[dict[str, str]]] = {}
_topic_index_last_manifest_mtime = 0.0
_topic_index_last_scan = 0.0

_digest_job_lock = threading.Lock()
_digest_jobs: dict[str, dict[str, Any]] = {}
_url_rate_lock = threading.Lock()
_url_rate_window: dict[str, list[float]] = {}
_url_cache_lock = threading.Lock()
_url_cache: dict[str, tuple[float, str]] = {}
_knowledge_gap_lock = threading.Lock()
_startup_repair_lock = threading.Lock()
_startup_repair_done = False
_knowledge_catalog_cache: str | None = None
_knowledge_catalog_last_scan = 0.0
_broad_topic_backfill_last_scan = 0.0


class _UnsupportedSource(RuntimeError):
    pass


def _workspace_path() -> str:
    try:
        from core import get_workspace_path

        ws = (get_workspace_path() or "").strip()
        if ws:
            return os.path.expanduser(ws)
    except Exception:
        pass
    return _DEFAULT_WORKSPACE


def _app_sandbox_enabled() -> bool:
    try:
        repo_root = Path(__file__).resolve().parents[1]
        entitlements_path = repo_root / "MugiApp" / "Mugi" / "Mugi.entitlements"
        if not entitlements_path.is_file():
            return False
        with entitlements_path.open("rb") as fh:
            data = plistlib.load(fh)
        return bool(data.get("com.apple.security.app-sandbox", False))
    except Exception:
        return False


def _knowledge_root() -> str:
    root = os.path.join(_workspace_path(), _KNOWLEDGE_DIR_NAME)
    os.makedirs(root, exist_ok=True)
    return root


def _iso_now() -> str:
    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")


def _slugify(text: str, max_len: int = 60) -> str:
    normalized = unicodedata.normalize("NFKD", (text or "").strip())
    ascii_text = normalized.encode("ascii", "ignore").decode("ascii")
    slug = re.sub(r"[^a-zA-Z0-9_-]", "_", ascii_text).lower()
    slug = re.sub(r"_+", "_", slug).strip("_")
    if slug:
        return slug[:max_len]
    digest = hashlib.sha256((text or "").encode("utf-8", errors="ignore")).hexdigest()[:8]
    return f"corpus_{digest}"


def _manifest_path(slug: str) -> str:
    return os.path.join(_knowledge_root(), slug, "manifest.json")


def _synthesis_path(slug: str) -> str:
    return os.path.join(_knowledge_root(), slug, "synthesis.md")


def read_synthesis_text(slug: str) -> str:
    """Return ``synthesis.md`` body when present; never raises on a missing path."""
    path = _synthesis_path(slug)
    if not path or not os.path.isfile(path):
        return ""
    try:
        raw = _read_text_file(path)
    except OSError as exc:
        _log.debug("[knowledge] synthesis read failed for %s: %s", slug, exc)
        return ""
    if not raw.strip():
        return ""
    lines = raw.splitlines()
    if lines and lines[0].lstrip().startswith("#"):
        head, body = lines[0], "\n".join(lines[1:]).strip()
        polished = polish_knowledge_summary_body(body)
        return f"{head}\n\n{polished}".strip() if polished else head
    return polish_knowledge_summary_body(raw)


def _index_path(slug: str) -> str:
    return os.path.join(_knowledge_root(), slug, "index.md")


_INDEX_BROAD_TOPICS_RE = re.compile(r"## Broad Topics\n\n.*?(?=\n## |\Z)", re.DOTALL)


def _semantic_index_ready_for_themes(slug: str) -> bool:
    """True when search embeddings are off, or the corpus thread index is complete."""
    try:
        from utils.embedding_model import embedding_enabled

        if not embedding_enabled():
            return True
        import session_search_db as ss

        return bool(
            ss.knowledge_semantic_index_complete_for_thread(f"knowledge:{slug}")
        )
    except Exception:
        # Fail-closed: if readiness cannot be verified, do not wipe preserved themes.
        return False


def _broad_topics_need_semantic_backfill(broad_topics: list[Any] | None) -> bool:
    """True when manifest/index broad topics are empty or only the (none) placeholder."""
    labels = [str(t).strip() for t in (broad_topics or []) if str(t).strip()]
    return not labels or labels == ["(none)"]


def _resolve_semantic_themes_for_finalize(slug: str) -> list[dict[str, Any]]:
    """Discover semantic themes or preserve on-disk rows when disabled / setup fails."""
    try:
        from utils.knowledge_connections_store import (
            _validate_slug,
            preserved_semantic_themes,
        )
        from utils.knowledge_themes import discover_semantic_themes, get_themes_config, themes_enabled

        norm = _validate_slug(slug)
        themes_cfg = get_themes_config()
        if themes_enabled(themes_cfg):
            discovered = discover_semantic_themes(norm, config=themes_cfg)
            if discovered:
                return discovered
            existing = preserved_semantic_themes(norm)
            if existing and not _semantic_index_ready_for_themes(norm):
                return existing
            return []
        return preserved_semantic_themes(norm)
    except ValueError:
        return []
    except Exception as theme_exc:
        _debug_log(
            f"[knowledge] semantic theme discovery failed for {slug}: {theme_exc}",
            "WARN",
        )
        try:
            from utils.knowledge_connections_store import (
                _validate_slug,
                preserved_semantic_themes,
            )

            return preserved_semantic_themes(_validate_slug(slug))
        except (ValueError, Exception):
            return []


def _resolve_entities_for_finalize(slug: str, *, persist_db: bool = True) -> dict[str, Any]:
    """Discover entities/claims or preserve on-disk rows when disabled / setup fails.

    When discovery returns no rows but connections.json still has entities/claims, preserved
    index rows are kept (semantic index ready does not imply a safe wipe). Contradictions are
    re-detected from the preserved claims when possible; otherwise on-disk contradiction rows
    are kept when there are no claims to scan.

    When persist_db is False, callers must write chats.db after connections.json is saved
    (digest finalization) so a failed materialize does not leave DB ahead of the index.
    """
    empty: dict[str, Any] = {
        "entities": [],
        "claims": [],
        "contradictions": [],
        "discovered": False,
    }

    def _with_contradictions(
        norm_slug: str,
        entities: list[dict[str, Any]],
        claims: list[dict[str, Any]],
        discovered: bool,
    ) -> dict[str, Any]:
        from utils.knowledge_contradictions import contradictions_enabled, detect_contradictions
        from utils.knowledge_connections_store import preserved_contradictions

        if not contradictions_enabled():
            contradictions = preserved_contradictions(norm_slug)
        else:
            try:
                contradictions = detect_contradictions(
                    entities, claims, corpus_slug=norm_slug
                )
            except Exception as contra_exc:
                _debug_log(
                    f"[knowledge] contradiction detection failed for {norm_slug}: {contra_exc}",
                    "WARN",
                )
                contradictions = preserved_contradictions(norm_slug)
            if not contradictions and not claims:
                preserved_rows = preserved_contradictions(norm_slug)
                if preserved_rows:
                    contradictions = preserved_rows
        return {
            "entities": entities,
            "claims": claims,
            "contradictions": contradictions,
            "discovered": discovered,
        }
    try:
        from chat_storage import replace_corpus_entities_and_claims
        from utils.knowledge_connections_store import (
            _validate_slug,
            preserved_claims,
            preserved_entities,
        )
        from utils.knowledge_entities import (
            discover_entities_and_claims,
            get_entities_config,
            entities_enabled,
        )

        def _persist_entities_claims(
            norm_slug: str,
            entities: list[dict[str, Any]],
            claims: list[dict[str, Any]],
        ) -> None:
            if persist_db:
                replace_corpus_entities_and_claims(norm_slug, entities, claims)

        norm = _validate_slug(slug)
        entities_cfg = get_entities_config()
        if not entities_enabled(entities_cfg):
            entities = preserved_entities(norm)
            claims = preserved_claims(norm)
            _persist_entities_claims(norm, entities, claims)
            return _with_contradictions(norm, entities, claims, False)
        discovered = discover_entities_and_claims(norm, config=entities_cfg)
        if discovered.get("failed"):
            entities = preserved_entities(norm)
            claims = preserved_claims(norm)
            _persist_entities_claims(norm, entities, claims)
            return _with_contradictions(norm, entities, claims, False)
        entities = list(discovered.get("entities") or [])
        claims = list(discovered.get("claims") or [])
        if entities or claims:
            _persist_entities_claims(norm, entities, claims)
            return _with_contradictions(norm, entities, claims, True)
        existing_entities = preserved_entities(norm)
        existing_claims = preserved_claims(norm)
        if existing_entities or existing_claims:
            _persist_entities_claims(norm, existing_entities, existing_claims)
            return _with_contradictions(norm, existing_entities, existing_claims, False)
        _persist_entities_claims(norm, [], [])
        return _with_contradictions(norm, [], [], True)
    except ValueError:
        return empty
    except Exception as ent_exc:
        _debug_log(
            f"[knowledge] entity/claim discovery failed for {slug}: {ent_exc}",
            "WARN",
        )
        try:
            from utils.knowledge_connections_store import (
                _validate_slug,
                preserved_claims,
                preserved_entities,
            )

            norm = _validate_slug(slug)
            entities = preserved_entities(norm)
            claims = preserved_claims(norm)
            if persist_db:
                try:
                    from chat_storage import replace_corpus_entities_and_claims

                    replace_corpus_entities_and_claims(norm, entities, claims)
                except Exception as persist_exc:
                    _debug_log(
                        f"[knowledge] entity/claim DB sync failed for {slug}: {persist_exc}",
                        "WARN",
                    )
            return _with_contradictions(norm, entities, claims, False)
        except ValueError:
            return empty


def _entities_state_for_status(
    slug: str,
    *,
    entities_count: int,
    claims_count: int,
) -> tuple[str, str]:
    """Return (entities_state, entities_algo_version) for get_knowledge_status."""
    from utils.knowledge_entities import ENTITIES_ALGO_VERSION, entities_enabled, get_entities_config

    if not entities_enabled(get_entities_config()):
        return "disabled", ENTITIES_ALGO_VERSION
    if entities_count > 0 or claims_count > 0:
        return "ready", ENTITIES_ALGO_VERSION
    return "empty", ENTITIES_ALGO_VERSION


def _contradictions_state_for_status(
    slug: str,
    *,
    contradictions_count: int,
) -> tuple[str, str]:
    """Return (contradictions_state, contradictions_algo_version) for get_knowledge_status."""
    from utils.knowledge_contradictions import (
        CONTRADICTIONS_ALGO_VERSION,
        contradictions_enabled,
    )

    if not contradictions_enabled():
        return "disabled", CONTRADICTIONS_ALGO_VERSION
    if contradictions_count > 0:
        return "ready", CONTRADICTIONS_ALGO_VERSION
    return "empty", CONTRADICTIONS_ALGO_VERSION


def _corpus_source_slug_map(slug: str) -> dict[str, dict[str, Any]]:
    from utils.knowledge_connections_store import _slug_to_source_map

    manifest = _load_manifest(slug)
    sources = manifest.get("sources") if isinstance(manifest, dict) else None
    if not isinstance(sources, list):
        sources = []
    return _slug_to_source_map(sources)


def _source_name_for_slug(slug_map: dict[str, dict[str, Any]], source_slug: str, corpus_slug: str) -> str:
    from utils.knowledge_connections_store import _sources_for_slugs

    key = (source_slug or "").strip()
    if not key:
        return ""
    rows = _sources_for_slugs(slug_map, [key], corpus_slug)
    if rows and isinstance(rows[0], dict):
        return str(rows[0].get("name") or rows[0].get("filename") or key)
    return key


def _enrich_contradiction_source_names(
    rows: list[dict[str, Any]],
    *,
    slug_map: dict[str, dict[str, Any]],
    corpus_slug: str,
) -> list[dict[str, Any]]:
    out: list[dict[str, Any]] = []
    for row in rows:
        if not isinstance(row, dict):
            continue
        item = dict(row)
        for side in ("claim_a", "claim_b"):
            block = item.get(side)
            if not isinstance(block, dict):
                continue
            sub = dict(block)
            sub["source_name"] = _source_name_for_slug(
                slug_map, str(sub.get("source_slug") or ""), corpus_slug
            )
            item[side] = sub
        out.append(item)
    return out


def _load_corpus_entity_rows(slug: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]:
    """Load entities/claims/contradictions; connections.json is SSOT when present (matches Connections UI)."""
    from utils.knowledge_connections_store import (
        _validate_slug,
        load_connections_index,
        preserved_claims,
        preserved_contradictions,
        preserved_entities,
    )

    try:
        norm = _validate_slug(slug)
    except ValueError:
        return [], [], []
    conn = load_connections_index(norm)
    if isinstance(conn, dict):
        entities: list[dict[str, Any]] = []
        claims: list[dict[str, Any]] = []
        raw_e = conn.get("entities")
        if isinstance(raw_e, list):
            entities = [dict(r) for r in raw_e if isinstance(r, dict)]
        raw_c = conn.get("claims")
        if isinstance(raw_c, list):
            claims = [dict(r) for r in raw_c if isinstance(r, dict)]
        return entities, claims, preserved_contradictions(norm)
    entities = []
    claims = []
    try:
        from chat_storage import load_corpus_claims, load_corpus_entities

        entities = [dict(r) for r in load_corpus_entities(norm)]
        claims = [dict(r) for r in load_corpus_claims(norm)]
    except Exception as db_exc:
        _debug_log(
            f"[knowledge] entity/claim DB load failed for {norm}: {db_exc}",
            "WARN",
        )
    if not entities and not claims:
        entities = preserved_entities(norm)
        claims = preserved_claims(norm)
    return entities, claims, []


def list_corpus_entities(
    slug: str,
    *,
    q: str | None = None,
    limit: int = 40,
) -> dict[str, Any]:
    from utils.knowledge_entities import entities_enabled, get_entities_config
    from utils.knowledge_connections_store import _topic_matches_filter

    if not entities_enabled(get_entities_config()):
        return {"corpus": slug, "entities": [], "state": "disabled", "count": 0}
    try:
        from utils.knowledge_connections_store import _validate_slug

        norm = _validate_slug(slug)
    except ValueError:
        return {"corpus": slug, "entities": [], "state": "empty", "count": 0}
    limit = max(1, min(40, int(limit or 40)))
    query = (q or "").strip().lower()
    raw_entities, _, _ = _load_corpus_entity_rows(norm)
    if query:
        entities = [
            e
            for e in raw_entities
            if _topic_matches_filter(e, topic_low="", query_low=query)
        ]
    else:
        entities = raw_entities
    entities = entities[:limit]
    if entities:
        state = "ready"
    elif raw_entities and query:
        state = "ready"
    else:
        state = "empty"
    return {"corpus": norm, "entities": entities, "state": state, "count": len(entities)}


def get_corpus_entity_detail(slug: str, entity_key: str) -> dict[str, Any]:
    from utils.knowledge_entities import entities_enabled, get_entities_config
    from utils.knowledge_connections_store import _validate_slug

    if not entities_enabled(get_entities_config()):
        return {"corpus": slug, "entity": None, "claims": [], "contradictions": [], "state": "disabled"}
    key = (entity_key or "").strip().lower()
    if not key:
        return {"corpus": slug, "entity": None, "claims": [], "contradictions": [], "state": "empty"}
    try:
        norm = _validate_slug(slug)
    except ValueError:
        return {"corpus": slug, "entity": None, "claims": [], "contradictions": [], "state": "empty"}
    entities, claims, contradictions = _load_corpus_entity_rows(norm)
    from utils.knowledge_contradictions import (
        contradiction_row_matches_entity,
        visible_contradiction_rows,
    )

    slug_map = _corpus_source_slug_map(norm)
    claims_by_id = {
        str(c.get("id") or "").strip(): c
        for c in claims
        if str(c.get("id") or "").strip()
    }
    entity_row = None
    for row in entities:
        ek = str(row.get("entity_key") or row.get("canonical_key") or "").strip().lower()
        if ek == key:
            entity_row = dict(row)
            break
    if entity_row is None:
        return {"corpus": norm, "entity": None, "claims": [], "contradictions": [], "state": "empty"}
    entity_claims = [
        dict(c)
        for c in claims
        if key in [str(k).strip().lower() for k in (c.get("entity_keys") or []) if str(k).strip()]
    ]
    entity_contradictions = [
        dict(x)
        for x in visible_contradiction_rows(contradictions)
        if contradiction_row_matches_entity(x, key, claims_by_id=claims_by_id)
    ]
    entity_contradictions = _enrich_contradiction_source_names(
        entity_contradictions, slug_map=slug_map, corpus_slug=norm
    )
    for claim in entity_claims:
        ss = str(claim.get("source_slug") or "")
        claim["source_name"] = _source_name_for_slug(slug_map, ss, norm)
    return {
        "corpus": norm,
        "entity": entity_row,
        "claims": entity_claims,
        "contradictions": entity_contradictions,
        "state": "ready",
    }


def list_corpus_contradictions(slug: str, *, limit: int = 20) -> dict[str, Any]:
    from utils.knowledge_contradictions import contradictions_enabled

    if not contradictions_enabled():
        return {"corpus": slug, "contradictions": [], "state": "disabled", "count": 0}
    limit = max(1, min(40, int(limit or 20)))
    try:
        from utils.knowledge_connections_store import _validate_slug

        norm = _validate_slug(slug)
    except ValueError:
        return {"corpus": slug, "contradictions": [], "state": "empty", "count": 0}
    from utils.knowledge_contradictions import visible_contradiction_rows

    _, _, contradictions = _load_corpus_entity_rows(norm)
    visible = visible_contradiction_rows(contradictions)
    slug_map = _corpus_source_slug_map(norm)
    rows = _enrich_contradiction_source_names(
        visible[:limit],
        slug_map=slug_map,
        corpus_slug=norm,
    )
    state = "ready" if visible else "empty"
    return {"corpus": norm, "contradictions": rows, "state": state, "count": len(rows)}


def _rewrite_index_broad_topics(slug: str, broad_topics: list[str]) -> None:
    """Refresh index.md ## Broad Topics after semantic backfill (manifest already updated)."""
    path = _index_path(slug)
    if not os.path.isfile(path):
        return
    try:
        body = _read_text_file(path)
        if "## Broad Topics" not in body:
            return
        lines = [f"- {t}" for t in broad_topics if str(t).strip()] or ["- (none)"]
        section = "## Broad Topics\n\n" + "\n".join(lines) + "\n"
        new_body, replaced = _INDEX_BROAD_TOPICS_RE.subn(section, body, count=1)
        if replaced:
            _write_text(path, new_body)
    except Exception as exc:
        _debug_log(f"[knowledge] index broad topics rewrite failed for {slug}: {exc}", "WARN")


def _source_page_path(slug: str, source_slug: str) -> str:
    return os.path.join(_knowledge_root(), slug, "sources", f"{source_slug}.md")


def _sources_dir(slug: str) -> str:
    return os.path.join(_knowledge_root(), slug, "sources")


_STUDIED_SOURCE_PATH_RE = re.compile(r"\*\*Source path:\*\*\s*`([^`]+)`", re.IGNORECASE)
_STUDIED_TITLE_RE = re.compile(r"^#\s+(.+)$", re.MULTILINE)


def _count_disk_studied_sources(slug: str) -> int:
    sources_dir = _sources_dir(slug)
    if not os.path.isdir(sources_dir):
        return 0
    return sum(1 for name in os.listdir(sources_dir) if name.endswith(".md"))


def _manifest_entry_from_studied_md(
    slug: str,
    source_slug: str,
    md_path: str,
    *,
    parent_extract_cache: dict[str, str] | None = None,
) -> dict[str, Any] | None:
    """Rebuild one manifest source row from an on-disk studied note (resume after empty manifest)."""
    md_text = _read_text_file(md_path) if os.path.isfile(md_path) else ""
    if not md_text.strip():
        return None
    path_m = _STUDIED_SOURCE_PATH_RE.search(md_text)
    if not path_m:
        return None
    source_path = path_m.group(1).strip()
    title_m = _STUDIED_TITLE_RE.search(md_text)
    filename = title_m.group(1).strip() if title_m else f"{source_slug}.md"
    content_hash = ""
    chars_extracted = 0
    if "::section::" in source_path:
        parent, idx_part = source_path.split("::section::", 1)
        try:
            section_idx = int(idx_part.split("::", 1)[0])
        except (TypeError, ValueError):
            section_idx = -1
        cache = parent_extract_cache if parent_extract_cache is not None else {}
        if parent not in cache:
            try:
                cache[parent] = _extract_plaintext(parent) if os.path.isfile(parent) else ""
            except Exception:
                cache[parent] = ""
        extracted = cache.get(parent) or ""
        if extracted.strip() and section_idx >= 0:
            sections = _detect_sections(extracted, filename or os.path.basename(parent))
            if section_idx < len(sections):
                _, start, end = sections[section_idx]
                section_text = extracted[start:end].strip()
                if section_text:
                    content_hash = hashlib.sha256(section_text.encode("utf-8", errors="ignore")).hexdigest()
                    chars_extracted = len(section_text)
    if not content_hash:
        fs_path = _manifest_source_fs_path(source_path)
        if fs_path and os.path.isfile(fs_path):
            try:
                content_hash = _sha256_file(fs_path)
                chars_extracted = os.path.getsize(fs_path)
            except OSError:
                content_hash = ""
    if not content_hash:
        content_hash = hashlib.sha256(md_text.encode("utf-8", errors="ignore")).hexdigest()
    topics: list[str] = []
    topics_m = re.search(r"(?s)## Topics\s*\n\n(.*)$", md_text)
    if topics_m:
        for line in topics_m.group(1).splitlines():
            stripped = line.strip()
            if stripped.startswith("- ") and stripped != "- (none)":
                topic = stripped[2:].strip()
                if topic:
                    topics.append(topic)
    entry: dict[str, Any] = {
        "filename": filename,
        "path": source_path,
        "slug": source_slug,
        "content_hash_sha256": content_hash,
        "status": "digested",
        "topics": topics[:5],
        "chars_extracted": chars_extracted,
        "chunks": 0,
    }
    if "::section::" in source_path:
        parent = source_path.split("::section::", 1)[0]
        if parent and os.path.isfile(_canonical_fs_path(parent) or parent):
            try:
                entry["_parent_content_hash"] = _sha256_file(_canonical_fs_path(parent) or parent)
                entry["_source_file_hash"] = entry["_parent_content_hash"]
            except OSError:
                pass
    _annotate_source_repo_paths(entry, source_path)
    return entry


def _repair_manifest_from_disk_sources(slug: str, *, save: bool = True) -> int:
    """Merge studied notes under ``sources/*.md`` into manifest when rows are missing."""
    manifest = _load_manifest(slug)
    if not isinstance(manifest, dict):
        return 0
    sources_dir = _sources_dir(slug)
    if not os.path.isdir(sources_dir):
        return 0
    by_slug: dict[str, dict[str, Any]] = {}
    for src in manifest.get("sources") or []:
        if not isinstance(src, dict):
            continue
        sid = str(src.get("slug") or "").strip()
        if sid:
            by_slug[sid] = dict(src)
    added = 0
    parent_extract_cache: dict[str, str] = {}
    for name in sorted(os.listdir(sources_dir)):
        if not name.endswith(".md"):
            continue
        source_slug = name[:-3]
        if source_slug in by_slug:
            continue
        md_path = os.path.join(sources_dir, name)
        entry = _manifest_entry_from_studied_md(
            slug,
            source_slug,
            md_path,
            parent_extract_cache=parent_extract_cache,
        )
        if not entry:
            continue
        by_slug[source_slug] = entry
        added += 1
    if added <= 0:
        return 0
    manifest["sources"] = sorted(by_slug.values(), key=lambda e: str(e.get("filename") or ""))
    status = str(manifest.get("digest_status") or "")
    # Only promote failed→partial for orphan recovery. Never demote running: disk .md
    # files land before manifest checkpoints during active ingest.
    if status == "failed" and manifest["sources"]:
        manifest["digest_status"] = "partial"
        manifest["started_at"] = None
    manifest["updated_at"] = _iso_now()
    if save:
        _save_manifest(slug, manifest)
        invalidate_topic_index_cache()
    return added


def _corpus_resumable(manifest: dict[str, Any], slug: str) -> bool:
    status = str(manifest.get("digest_status") or "").lower()
    if status == "running":
        return False
    if status not in ("partial", "failed"):
        return False
    if not str(manifest.get("source_path") or "").strip():
        return False
    return bool(manifest.get("sources"))


def _maybe_repair_corpus_manifest(slug: str, manifest: dict[str, Any]) -> dict[str, Any]:
    """Repair empty/missing manifest rows from on-disk studied notes before status/list."""
    if not isinstance(manifest, dict):
        return manifest
    if str(manifest.get("digest_status") or "") == "running":
        return manifest
    with _digest_job_lock:
        if bool((_digest_jobs.get(slug) or {}).get("running")):
            return manifest
    manifest_sources = manifest.get("sources") or []
    disk_count = _count_disk_studied_sources(slug)
    if not manifest_sources and disk_count > 0:
        _repair_manifest_from_disk_sources(slug)
        return _load_manifest(slug) or manifest
    if manifest_sources and disk_count > len(manifest_sources):
        _repair_manifest_from_disk_sources(slug)
        return _load_manifest(slug) or manifest
    return manifest


def _raw_cache_dir(slug: str) -> str:
    return os.path.join(_knowledge_root(), slug, "_raw_cache")


def _raw_cache_key(real_path: str) -> str:
    try:
        real_path = os.path.realpath(os.path.expanduser(str(real_path or "").strip()))
    except OSError:
        real_path = os.path.abspath(os.path.expanduser(str(real_path or "").strip()))
    return hashlib.sha256(real_path.encode("utf-8", errors="ignore")).hexdigest()[:16]


def _save_raw_text_cache(slug: str, real_path: str, text: str) -> None:
    """Persist extracted text so read_knowledge_source avoids re-extraction."""
    try:
        real_path = _canonical_fs_path(real_path) or os.path.abspath(os.path.expanduser(str(real_path or "").strip()))
        cache_dir = _raw_cache_dir(slug)
        os.makedirs(cache_dir, exist_ok=True)
        key = _raw_cache_key(real_path)
        txt_path = os.path.join(cache_dir, f"{key}.txt")
        meta_path = os.path.join(cache_dir, f"{key}.meta")
        secure_atomic_write(txt_path, text)
        try:
            mtime = os.path.getmtime(real_path)
        except OSError:
            mtime = 0.0
        secure_atomic_write(meta_path, json.dumps({"path": real_path, "mtime": mtime}))
    except Exception:
        pass


def _load_raw_text_cache(slug: str, real_path: str) -> str | None:
    """Return cached extracted text if it exists and source mtime matches."""
    try:
        real_path = _canonical_fs_path(real_path) or os.path.abspath(os.path.expanduser(str(real_path or "").strip()))
        cache_dir = _raw_cache_dir(slug)
        key = _raw_cache_key(real_path)
        txt_path = os.path.join(cache_dir, f"{key}.txt")
        meta_path = os.path.join(cache_dir, f"{key}.meta")
        if not os.path.isfile(txt_path) or not os.path.isfile(meta_path):
            return None
        with open(meta_path, "r", encoding="utf-8") as f:
            meta = json.loads(f.read())
        try:
            current_mtime = os.path.getmtime(real_path)
        except OSError:
            return None
        if abs(current_mtime - float(meta.get("mtime", 0))) > 1.0:
            return None
        return _read_text_file(txt_path)
    except Exception:
        return None


def _raw_text_from_manifest_source(slug: str, source: dict[str, Any]) -> str:
    """Load extracted text for a manifest source (cache, disk, or PDF section slice)."""
    path = str(source.get("path") or "").strip()
    if not path:
        return ""
    filename = str(source.get("filename") or os.path.basename(path))
    if "::section::" in path:
        parent, sect_part = path.split("::section::", 1)
        try:
            section_idx = int(sect_part)
        except ValueError:
            section_idx = None
        real, err = _validate_manifest_source_fs_path(parent)
        if err or not real:
            _log.warning("manifest source blocked for slug=%s: %s", slug, err or "invalid path")
            return ""
        text = _load_raw_text_cache(slug, real) if slug else None
        if not text and os.path.isfile(real):
            try:
                text = _extract_plaintext(real)
                if slug and text and text.strip():
                    _save_raw_text_cache(slug, real, text)
            except Exception:
                text = ""
        if not text:
            return ""
        if section_idx is not None:
            sections = _detect_sections(text, filename)
            if sections and section_idx < len(sections):
                _title, start, end = sections[section_idx]
                return text[start:end]
        return text
    real, err = _validate_manifest_source_fs_path(path)
    if err or not real:
        _log.warning("manifest source blocked for slug=%s: %s", slug, err or "invalid path")
        return ""
    cached = _load_raw_text_cache(slug, real) if slug else None
    if cached:
        return cached
    if os.path.isfile(real):
        try:
            text = _extract_plaintext(real)
            if slug and text and text.strip():
                _save_raw_text_cache(slug, real, text)
            return text or ""
        except Exception:
            return ""
    return ""


def _hydrate_raw_texts_for_indexing(
    slug: str,
    sources: list[dict[str, Any]],
    raw_texts: dict[str, str] | None,
) -> dict[str, str]:
    """Fill missing per-source raw text so full reindex keeps unchanged sources' chunk rows."""
    out = dict(raw_texts or {})
    for src in sources:
        if not isinstance(src, dict):
            continue
        sid = str(src.get("slug") or "")
        if not sid:
            continue
        existing = str(out.get(sid) or "")
        if len(existing.strip()) > 500:
            continue
        loaded = _raw_text_from_manifest_source(slug, src)
        if loaded and (len(loaded.strip()) > 500 or not existing.strip()):
            out[sid] = loaded
    return out


def _read_text_file(path: str) -> str:
    return read_text_file_with_encoding_fallback(path)


def _write_text(path: str, text: str) -> None:
    os.makedirs(os.path.dirname(path), exist_ok=True)
    secure_atomic_write(path, text)


def _sha256_file(path: str) -> str:
    h = hashlib.sha256()
    with open(path, "rb") as f:
        while True:
            chunk = f.read(1024 * 1024)
            if not chunk:
                break
            h.update(chunk)
    return h.hexdigest()


def _load_manifest(slug: str) -> dict[str, Any] | None:
    path = _manifest_path(slug)
    if not os.path.isfile(path):
        return None
    try:
        data = json.loads(_read_text_file(path))
        if not isinstance(data, dict):
            return None
        if int(data.get("schema_version", 1)) > KNOWLEDGE_SCHEMA_VERSION:
            return None
        return data
    except Exception:
        return None


def _manifest_schema_too_new(slug: str) -> bool:
    path = _manifest_path(slug)
    if not os.path.isfile(path):
        return False
    try:
        raw = json.loads(_read_text_file(path))
        if not isinstance(raw, dict):
            return False
        return int(raw.get("schema_version", 1)) > KNOWLEDGE_SCHEMA_VERSION
    except Exception:
        return False


def _save_manifest(slug: str, manifest: dict[str, Any]) -> None:
    _write_text(_manifest_path(slug), json.dumps(manifest, ensure_ascii=False, indent=2) + "\n")


def _tokenizer_refs() -> tuple[re.Pattern[str], set[str]]:
    try:
        import sys

        mod = sys.modules.get("agent_loop")
        token_re = getattr(mod, "_TOPIC_TOKEN_RE", None)
        stop = getattr(mod, "_TOPIC_STOPWORDS", None)
        if isinstance(token_re, re.Pattern) and isinstance(stop, (set, frozenset)):
            return token_re, set(stop)
    except Exception:
        pass
    return _FALLBACK_TOKEN_RE, {
        "about",
        "after",
        "again",
        "also",
        "and",
        "assistant",
        "because",
        "been",
        "before",
        "both",
        "can",
        "could",
        "does",
        "from",
        "have",
        "just",
        "like",
        "make",
        "more",
        "most",
        "need",
        "that",
        "their",
        "them",
        "there",
        "they",
        "this",
        "those",
        "what",
        "when",
        "where",
        "which",
        "while",
        "will",
        "with",
        "would",
        "your",
    }


def _extract_html_text(path: str) -> str:
    raw = _read_text_file(path)
    try:
        from bs4 import BeautifulSoup

        soup = BeautifulSoup(raw, "html.parser")
        for tag in soup(["script", "style"]):
            tag.decompose()
        return soup.get_text(separator="\n", strip=True)
    except Exception:
        return re.sub(r"\s+", " ", re.sub(r"(?s)<[^>]+>", " ", raw)).strip()


def _extract_docx_text(path: str) -> str:
    import docx

    doc = docx.Document(path)
    lines = [p.text.strip() for p in doc.paragraphs if (p.text or "").strip()]
    return "\n".join(lines)


def _extract_xlsx_text(path: str) -> str:
    import openpyxl

    wb = openpyxl.load_workbook(path, data_only=True, read_only=True)
    lines: list[str] = []
    for ws in wb.worksheets:
        lines.append(f"# Sheet: {ws.title}")
        for row in ws.iter_rows(values_only=True):
            cells = [str(v).strip() for v in row if v is not None and str(v).strip()]
            if cells:
                lines.append(" | ".join(cells))
    wb.close()
    return "\n".join(lines)


def _extract_pptx_text(path: str) -> str:
    from pptx import Presentation

    prs = Presentation(path)
    lines: list[str] = []
    for idx, slide in enumerate(prs.slides, 1):
        lines.append(f"# Slide {idx}")
        for shape in slide.shapes:
            text = getattr(shape, "text", None)
            if text and str(text).strip():
                lines.append(str(text).strip())
    return "\n".join(lines)


def _extract_rtf_text(path: str) -> str:
    try:
        from striprtf.striprtf import rtf_to_text

        return rtf_to_text(_read_text_file(path))
    except Exception:
        import subprocess

        proc = subprocess.run(
            ["textutil", "-convert", "txt", "-stdout", path],
            capture_output=True,
            text=True,
            timeout=20,
            check=False,
        )
        if proc.returncode != 0:
            raise RuntimeError("RTF extraction failed")
        return proc.stdout


def _extract_epub_text(path: str) -> str:
    from ebooklib import epub

    book = epub.read_epub(path)
    pieces: list[str] = []
    for item in book.get_items():
        name = (getattr(item, "get_name", lambda: "")() or "").lower()
        if not name.endswith((".xhtml", ".html", ".htm")):
            continue
        try:
            content = item.get_content()
            raw = content.decode("utf-8", errors="replace")
        except Exception:
            continue
        try:
            from bs4 import BeautifulSoup

            soup = BeautifulSoup(raw, "html.parser")
            for tag in soup(["script", "style"]):
                tag.decompose()
            text = soup.get_text(separator="\n", strip=True)
        except Exception:
            text = re.sub(r"\s+", " ", re.sub(r"(?s)<[^>]+>", " ", raw)).strip()
        if text:
            pieces.append(text)
    return "\n\n".join(pieces)


def _email_message_text(message_obj: Any, *, max_chars: int) -> str:
    parts: list[str] = []
    if message_obj.is_multipart():
        for part in message_obj.walk():
            ctype = (part.get_content_type() or "").lower()
            disp = (part.get("Content-Disposition") or "").lower()
            if "attachment" in disp:
                continue
            if ctype not in {"text/plain", "text/html"}:
                continue
            payload = part.get_payload(decode=True) or b""
            charset = part.get_content_charset() or "utf-8"
            text = payload.decode(charset, errors="replace")
            if ctype == "text/html":
                text = re.sub(r"\s+", " ", re.sub(r"(?s)<[^>]+>", " ", text)).strip()
            if text.strip():
                parts.append(text.strip())
            if sum(len(p) for p in parts) >= max_chars:
                break
    else:
        payload = message_obj.get_payload(decode=True) or b""
        charset = message_obj.get_content_charset() or "utf-8"
        text = payload.decode(charset, errors="replace")
        if (message_obj.get_content_type() or "").lower() == "text/html":
            text = re.sub(r"\s+", " ", re.sub(r"(?s)<[^>]+>", " ", text)).strip()
        if text.strip():
            parts.append(text.strip())
    joined = "\n\n".join(parts).strip()
    return joined[:max_chars]


def _extract_eml_text(path: str) -> str:
    with open(path, "rb") as fh:
        msg = BytesParser(policy=policy.default).parse(fh)
    header = f"From: {msg.get('From', '')}\nSubject: {msg.get('Subject', '')}\nDate: {msg.get('Date', '')}\n"
    body = _email_message_text(msg, max_chars=_EML_MAX_MESSAGE_BODY_CHARS)
    return f"{header}\n{body}".strip()


def _extract_mbox_text(path: str) -> str:
    pieces: list[str] = []
    box = mailbox.mbox(path)
    count = 0
    for msg in box:
        count += 1
        header = f"From: {msg.get('From', '')}\nSubject: {msg.get('Subject', '')}\nDate: {msg.get('Date', '')}\n"
        body = _email_message_text(msg, max_chars=_MBOX_MAX_MESSAGE_BODY_CHARS)
        pieces.append(f"## Message {count}\n{header}\n{body}".strip())
        if len("\n\n".join(pieces)) >= 1_000_000:
            break
    return "\n\n".join(pieces)


def _is_private_or_local_host(hostname: str) -> bool:
    if not hostname:
        return True
    host = hostname.lower()
    if host in {"localhost", "127.0.0.1", "::1"}:
        return True
    try:
        ip = ip_address(host)
        return bool(ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast)
    except ValueError:
        pass
    try:
        infos = socket.getaddrinfo(hostname, None, proto=socket.IPPROTO_TCP)
    except Exception:
        # DNS failures/timeouts should not be misclassified as private hosts.
        # Let downstream fetch report the real network error instead.
        return False
    for info in infos:
        addr = info[4][0]
        try:
            ip = ip_address(addr)
            if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast:
                return True
        except ValueError:
            continue
    return False


def _fetch_url_text(url: str) -> str:
    parsed = urlparse(url)
    scheme = (parsed.scheme or "").lower()
    if scheme not in {"http", "https"}:
        raise ValueError("Only http(s) URLs are supported.")
    if _URL_DIGEST_BLOCK_PRIVATE_IPS and _is_private_or_local_host(parsed.hostname or ""):
        raise ValueError("Blocked private or local URL host.")
    host = (parsed.hostname or "").lower()
    now = time.time()
    with _url_cache_lock:
        cached = _url_cache.get(url)
        if cached and (now - cached[0]) < _URL_CACHE_TTL_SECONDS:
            return cached[1]
    with _url_rate_lock:
        entries = _url_rate_window.setdefault(host, [])
        entries[:] = [ts for ts in entries if (now - ts) < _URL_RATE_LIMIT_WINDOW_SECONDS]
        if len(entries) >= _URL_RATE_LIMIT_PER_HOST:
            raise RuntimeError(f"URL digest rate limit exceeded for host: {host}")
        entries.append(now)
    from tools.web_tools import read_webpage

    text = read_webpage.invoke({"url": url})
    out = str(text or "")[:120_000]
    with _url_cache_lock:
        _url_cache[url] = (time.time(), out)
    return out


def _extract_plaintext(path: str) -> str:
    ext = Path(path).suffix.lower()
    if ext in _SUPPORTED_TEXT_EXTENSIONS:
        return _read_text_file(path)
    if ext in {".html", ".htm"}:
        return _extract_html_text(path)
    if ext == ".pdf":
        from tools.pdf_tools import extract_pdf_text_from_path

        text = extract_pdf_text_from_path(path)
        if isinstance(text, str) and (text.startswith("No text extracted") or "image-based" in text.lower()):
            from tools.vision_ocr import ocr_scanned_pdf

            raw = ocr_scanned_pdf.invoke({"file_path": path})
            text = raw if isinstance(raw, str) else str(raw)
        return text if isinstance(text, str) else str(text)
    if ext in _SUPPORTED_IMAGE_EXTENSIONS:
        from tools.vision_ocr import ocr_image_text

        raw = ocr_image_text.invoke({"file_path": path})
        text = raw if isinstance(raw, str) else str(raw)
        return text
    if ext == ".docx":
        return _extract_docx_text(path)
    if ext == ".xlsx":
        return _extract_xlsx_text(path)
    if ext == ".pptx":
        return _extract_pptx_text(path)
    if ext == ".rtf":
        return _extract_rtf_text(path)
    if ext == ".epub":
        return _extract_epub_text(path)
    if ext == ".eml":
        return _extract_eml_text(path)
    if ext == ".mbox":
        return _extract_mbox_text(path)
    raise _UnsupportedSource(f"unsupported format ({ext})")


def _chunk_text(text: str, chunk_size: int = _DIGEST_CHUNK_SIZE, overlap: int = _DIGEST_CHUNK_OVERLAP) -> list[str]:
    """LLM summarization chunks — one chunk when under ``chunk_size``; else structure-aware splits."""
    t = (text or "").strip()
    if not t:
        return []
    if len(t) <= chunk_size:
        return [t]
    return [
        str(c.get("text") or "")
        for c in _chunk_text_anchored(text, chunk_size=chunk_size, overlap=overlap, min_chunk_chars=0)
    ]


def normalize_ingest_profile(profile: str | None = None, *, source_path: str = "") -> str:
    """``docs`` (default) or ``repo`` (skip code extensions under git roots)."""
    p = (profile or "").strip().lower()
    if p in ("docs", "repo"):
        return p
    if _path_looks_like_git_repo(source_path):
        return "repo"
    return "docs"


def ingest_chunk_params(profile: str) -> tuple[int, int]:
    """LLM digest / study summarization chunks (not vector index chunks)."""
    if normalize_ingest_profile(profile) == "repo":
        return 2400, 150
    return 3000, 200


def digest_study_chunk_params(profile: str) -> tuple[int, int]:
    """Alias for :func:`ingest_chunk_params` (digest-only chunking)."""
    return ingest_chunk_params(profile)


def _path_looks_like_git_repo(source_path: str) -> bool:
    raw = (source_path or "").strip()
    if not raw or re.match(r"^https?://", raw, flags=re.IGNORECASE):
        return False
    try:
        src = os.path.abspath(os.path.expanduser(raw))
        if os.path.isdir(os.path.join(src, ".git")):
            return True
        parent = os.path.dirname(src)
        return os.path.isdir(os.path.join(parent, ".git"))
    except Exception:
        return False


def _line_number_at(text: str, offset: int) -> int:
    return (text or "")[: max(0, int(offset))].count("\n") + 1


def _merge_adjacent_spans(spans: list[tuple[int, int]], chunk_size: int) -> list[tuple[int, int]]:
    """Coalesce consecutive boundary spans up to ``chunk_size`` (restores sliding-window density)."""
    if not spans:
        return []
    merged: list[tuple[int, int]] = []
    group_start, group_end = spans[0]
    for start, end in spans[1:]:
        if end - group_start > chunk_size:
            merged.append((group_start, group_end))
            group_start, group_end = start, end
        else:
            group_end = end
    merged.append((group_start, group_end))
    return merged


def _window_span_chunks(start: int, end: int, *, chunk_size: int, overlap: int) -> list[tuple[int, int]]:
    """Split an oversized span with overlap (character offsets in parent text)."""
    if end - start <= chunk_size:
        return [(start, end)]
    step = max(1, chunk_size - overlap)
    out: list[tuple[int, int]] = []
    j = 0
    piece_len = end - start
    while j < piece_len:
        cs = start + j
        ce = min(end, cs + chunk_size)
        out.append((cs, ce))
        if ce >= end:
            break
        j += step
    return out


def _chunk_text_anchored(
    text: str,
    *,
    chunk_size: int = _DIGEST_CHUNK_SIZE,
    overlap: int = _DIGEST_CHUNK_OVERLAP,
    min_chunk_chars: int = _INDEX_MIN_CHUNK_CHARS,
) -> list[dict[str, Any]]:
    """Structure-aware chunks with char/line anchors for search hits."""
    raw = text or ""
    if not raw.strip():
        return []
    boundaries = [0]
    for m in re.finditer(r"(?m)^#{1,6}\s", raw):
        if m.start() not in boundaries:
            boundaries.append(m.start())
    for m in re.finditer(r"\n\n+", raw):
        pos = m.end()
        if 0 < pos < len(raw) and pos not in boundaries:
            boundaries.append(pos)
    boundaries.append(len(raw))
    boundaries = sorted(set(boundaries))
    boundary_spans: list[tuple[int, int]] = []
    for i in range(len(boundaries) - 1):
        start, end = boundaries[i], boundaries[i + 1]
        if raw[start:end].strip():
            boundary_spans.append((start, end))
    merged_spans = _merge_adjacent_spans(boundary_spans, chunk_size)
    spans: list[tuple[int, int]] = []
    for start, end in merged_spans:
        spans.extend(_window_span_chunks(start, end, chunk_size=chunk_size, overlap=overlap))
    out: list[dict[str, Any]] = []
    for cs, ce in spans:
        chunk = raw[cs:ce].strip()
        if min_chunk_chars > 0 and len(chunk) < min_chunk_chars:
            continue
        ls = _line_number_at(raw, cs)
        le = _line_number_at(raw, max(cs, ce - 1))
        out.append(
            {
                "text": chunk,
                "char_start": cs,
                "char_end": ce,
                "line_start": ls,
                "line_end": le,
            }
        )
    # Boundary splitting can drop every piece as < 80 chars; summarization still needs chunks.
    if not out and raw.strip():
        left = len(raw) - len(raw.lstrip())
        right = len(raw.rstrip())
        body = raw[left:right]
        if len(body) <= chunk_size:
            out.append(
                {
                    "text": body,
                    "char_start": left,
                    "char_end": right,
                    "line_start": _line_number_at(raw, left),
                    "line_end": _line_number_at(raw, max(left, right - 1)),
                }
            )
        else:
            step = max(1, chunk_size - overlap)
            j = 0
            while j < len(body):
                cs = left + j
                ce = min(left + len(body), cs + chunk_size)
                piece = raw[cs:ce].strip()
                if piece:
                    out.append(
                        {
                            "text": piece,
                            "char_start": cs,
                            "char_end": ce,
                            "line_start": _line_number_at(raw, cs),
                            "line_end": _line_number_at(raw, max(cs, ce - 1)),
                        }
                    )
                if ce >= left + len(body):
                    break
                j += step
    return out


def _kb_source_created_at(
    filename: str,
    *,
    char_start: int | None = None,
    char_end: int | None = None,
    line_start: int | None = None,
    line_end: int | None = None,
) -> str:
    base = f"src:{filename}"
    if char_start is None or char_end is None:
        return base
    token = f"{base}@{int(char_start)}:{int(char_end)}"
    if line_start is not None and line_end is not None:
        token += f"#L{int(line_start)}-{int(line_end)}"
    return token


def _parse_kb_created_at(created_at: str) -> dict[str, Any]:
    """Parse ``src:{name}[@c0:c1][#Ll0-l1]``; filenames may contain ``@`` or ``#``."""
    empty: dict[str, Any] = {
        "source_name": "",
        "char_start": None,
        "char_end": None,
        "line_start": None,
        "line_end": None,
    }
    ca = (created_at or "").strip()
    if not ca.startswith("src:"):
        return dict(empty)
    rest = ca[4:]
    line_start = line_end = char_start = char_end = None
    m_line = re.search(r"#L(\d+)-(\d+)$", rest)
    if m_line:
        line_start = int(m_line.group(1))
        line_end = int(m_line.group(2))
        rest = rest[: m_line.start()]
    m_char = re.search(r"@(\d+):(\d+)$", rest)
    if m_char:
        char_start = int(m_char.group(1))
        char_end = int(m_char.group(2))
        rest = rest[: m_char.start()]
    return {
        "source_name": rest,
        "char_start": char_start,
        "char_end": char_end,
        "line_start": line_start,
        "line_end": line_end,
    }


def _line_fold_for_dedup(line: str) -> str:
    """Normalize a line for dedup: strip trailing small integer tokens (page/line/enum tails) and whitespace."""
    s = line.strip()
    if not s:
        return ""
    s = re.sub(r"\s+\d{1,4}\s*$", "", s)
    return re.sub(r"\s+", " ", s)


def _collapse_consecutive_duplicate_lines(text: str) -> str:
    """Collapse long runs of near-identical non-empty lines (broken PDFs, scraped pages, repeated headers)."""
    lines = text.splitlines()
    out: list[str] = []
    i = 0
    n = len(lines)
    while i < n:
        line = lines[i]
        if not line.strip():
            out.append(line)
            i += 1
            continue
        key = _line_fold_for_dedup(line)
        if not key:
            out.append(line)
            i += 1
            continue
        j = i + 1
        while j < n and lines[j].strip() and _line_fold_for_dedup(lines[j]) == key:
            j += 1
        run = j - i
        if run >= _DEDUP_MIN_REPEAT_RUN:
            out.append(line)
        else:
            out.extend(lines[i:j])
        i = j
    return "\n".join(out)


def _repair_pdf_extract_artifacts(text: str) -> str:
    """Repair common two-column PDF extraction glitches before summarization."""
    t = (text or "").replace("\r", "\n")
    if not t.strip():
        return t
    # keep -\ning -> keeping; there -\nfore -> therefore
    t = re.sub(r"(\w)\s*-\s*\n\s*(\w)", r"\1\2", t)
    repaired: list[str] = []
    for raw_ln in t.splitlines():
        ln = raw_ln.strip()
        if not ln:
            repaired.append("")
            continue
        # 304 305charge -> charge (running page numbers from reader Bibles)
        ln = re.sub(r"^[\d\s]{4,14}(?=[A-Za-z\"'“(])", "", ln)
        repaired.append(ln)
    return "\n".join(repaired)


def _normalize_extracted_text_for_digest(text: str) -> str:
    t = (text or "").strip()
    if not t:
        return t
    t = _repair_pdf_extract_artifacts(t)
    t = _collapse_consecutive_duplicate_lines(t)
    t = re.sub(r"\n{4,}", "\n\n\n", t)
    return t.strip()


def _is_degenerate_extracted_text(text: str) -> bool:
    """True when line-level diversity is so low that LLM summarization usually parrots noise."""
    lines = [_line_fold_for_dedup(ln) for ln in text.splitlines() if ln.strip()]
    lines = [ln for ln in lines if ln]
    n = len(lines)
    if n < _DEGENERATE_MIN_NONEMPTY_LINES:
        return False
    # Ignore short/noisy lines for degeneracy scoring; they are common in PDFs
    # (verse numbers, running headers, page references) and can create false positives.
    informative = [
        ln
        for ln in lines
        if len(ln) >= 18 and sum(1 for c in ln if c.isalnum()) >= 10
    ]
    sample = informative if len(informative) >= 20 else lines
    sn = len(sample)
    if sn < _DEGENERATE_MIN_NONEMPTY_LINES:
        return False
    unique = len(set(sample))
    if unique / sn <= _DEGENERATE_UNIQUE_LINE_RATIO:
        return True
    top_line, top_count = Counter(sample).most_common(1)[0]
    # A short repeated token/fragment should not classify a document as degenerate.
    if len(top_line) >= 24 and (top_count / sn) >= _DEGENERATE_TOP_LINE_FRACTION:
        return True
    return False


def _alnum_ratio(s: str) -> float:
    letters = sum(1 for c in s if c.isalnum())
    return letters / max(1, len(s))


def _has_dominated_by_repeated_window(text: str) -> bool:
    """True when a single moderate-length window appears many times after whitespace fold (log dumps, boilerplate)."""
    flat = " ".join((text or "").split())
    if len(flat) < 5000:
        return False
    for window in (72, 120):
        if len(flat) < window * 18:
            continue
        offsets = [0, len(flat) // 4, len(flat) // 2, (3 * len(flat)) // 4]
        for off in offsets:
            if off + window > len(flat):
                continue
            chunk = flat[off : off + window]
            if _alnum_ratio(chunk) < 0.25:
                continue
            count = flat.count(chunk)
            if count < 18:
                continue
            # Require substantial document coverage, not just a recurring header/footer.
            if (count * window) / max(1, len(flat)) >= 0.35:
                return True
    return False


_SCRIPTURE_REF_TOKEN_RE = re.compile(
    r"(?:\b\d{1,3}:\d{1,3}\b|\b\d{1,2}\s+[A-Za-z][\w']*\s+\d{1,3}:\d{1,3}\b)",
    re.IGNORECASE,
)


def _summary_line_has_citation_spam(line: str) -> bool:
    """True when a summary line is mostly verse/page/header noise (common in Bible PDFs)."""
    text = re.sub(r"^\s*[-*•]\s*", "", (line or "").strip())
    if not text:
        return True
    refs = _SCRIPTURE_REF_TOKEN_RE.findall(text)
    if len(refs) >= 2 and len({r.lower() for r in refs}) < len(refs):
        return True
    head = text[:100]
    if len(refs) >= 2 and _SCRIPTURE_REF_TOKEN_RE.sub("", head).strip() == "":
        return True
    digits = sum(1 for c in head if c.isdigit())
    if len(head) >= 24 and digits / len(head) > 0.22:
        return True
    if re.match(r"^[\d:\s]+(?:\d+\s+\w+\s+\d+:\d+\s*){2,}", text, re.I):
        return True
    return False


def polish_knowledge_summary_body(text: str) -> str:
    """Clean citation/page noise from stored or fallback summary bullets."""
    raw = (text or "").strip()
    if not raw:
        return ""
    if "**Automatic summary skipped.**" in raw:
        return raw
    lines = raw.splitlines()
    out: list[str] = []
    for ln in lines:
        stripped = ln.strip()
        if not stripped:
            if out and out[-1]:
                out.append("")
            continue
        if stripped.startswith("#"):
            out.append(stripped)
            continue
        bullet = stripped.lstrip("-*•").strip()
        cleaned = _strip_summary_citation_noise(bullet)
        if not cleaned or _summary_line_has_citation_spam(cleaned):
            continue
        if stripped.startswith(("-", "*", "•")):
            out.append(f"- {cleaned}")
        else:
            out.append(cleaned)
    if not out:
        return raw
    return "\n".join(out).strip()


def _strip_summary_citation_noise(line: str) -> str:
    s = (line or "").strip()
    if not s:
        return ""
    s = re.sub(r"(\w)\s+-\s+(\w)", r"\1\2", s)
    s = re.sub(r"(?i)\b[\w.-]+\.(pdf|docx?|md|txt|html?)\b", "", s)
    s = re.sub(r"(?i)\bchunk\s+\d+/\d+\b", "", s)
    for _ in range(8):
        prev = s
        s = re.sub(r"^(?:\s*\d{2,4}\s*)+(?=[A-Za-z])", "", s)
        s = _SCRIPTURE_REF_TOKEN_RE.sub("", s, count=1) if _SCRIPTURE_REF_TOKEN_RE.search(s[:80]) else s
        s = re.sub(r"^\d{1,3}:\d{1,3}\s*", "", s)
        s = re.sub(r"^\s+", "", s)
        if s == prev:
            break
    s = re.sub(r"\s+", " ", s).strip(" -")
    return s


def _is_garbage_summary(summary: str) -> bool:
    """Detect model output that mostly repeats one line (failed summarization)."""
    lines = [ln.strip() for ln in (summary or "").splitlines() if ln.strip()]
    if not lines:
        return True
    bullet_lines = [ln for ln in lines if ln.lstrip().startswith(("-", "*", "•"))]
    if bullet_lines:
        spammy = sum(1 for ln in bullet_lines if _summary_line_has_citation_spam(ln))
        if spammy >= max(2, len(bullet_lines) // 2):
            return True
    lower = "\n".join(lines).lower()
    if (
        "thinking process" in lower
        or "analyze the request" in lower
        or "analyze the input text" in lower
        or "conversation history" in lower
        or "the user wants me" in lower
        or "combine these chunk summaries" in lower
        or "no specific content was provided" in lower
        or "template example" in lower
        or "<|channel>thought" in lower
        or "<think>" in lower
    ):
        return True
    if re.search(r"(?i)\b(i|me|my)\b", lower) and re.search(
        r"(?i)\b(user|prompt|request|chunks?|summary|conversation)\b", lower
    ):
        return True
    ctr = Counter(lines)
    top_line, top_c = ctr.most_common(1)[0]
    line_count = len(lines)
    top_frac = top_c / max(1, line_count)
    if len(top_line) > 24 and top_c >= 5:
        return True
    # Catch shorter but still dominant repeats (e.g., 3-4 long bullets repeated).
    if len(top_line) > 24 and top_c >= 3 and top_frac >= 0.45:
        return True
    if line_count >= 8 and len(ctr) / line_count < 0.25:
        return True
    flat = " ".join(lines)
    # Repeated sentence-level boilerplate can evade line-based checks.
    sentences = [
        s.strip(" -")
        for s in re.split(r"(?<=[.!?])\s+", flat)
        if len(s.strip()) >= 50
    ]
    if len(sentences) >= 3:
        sent_counts = Counter(sentences)
        _, sent_top = sent_counts.most_common(1)[0]
        if sent_top >= 3:
            return True
    if _has_dominated_by_repeated_window(summary):
        return True
    return False


def _degenerate_source_summary(filename: str, source_path: str, normalized_text: str) -> str:
    excerpt = (normalized_text or "").strip()
    if len(excerpt) > _DEGENERATE_EXCERPT_CHARS:
        excerpt = excerpt[:_DEGENERATE_EXCERPT_CHARS].rstrip() + "\n…"
    path_line = f"`{source_path}`" if source_path else "the original file"
    return (
        "**Automatic summary skipped.** Extracted text is extremely repetitive. "
        "That usually means extraction or layout produced the same block many times "
        "(headers/footers, columns, boilerplate, logs, numbered tails), not that the document is one sentence.\n\n"
        "**Do not treat any prior auto-summary of this file as reliable.** "
        "Open the original file for reading, or search with **query_knowledge** using exact wording you care about.\n\n"
        f"**Source:** {filename}  \n**Path:** {path_line}\n\n"
        "Below is a short **deduplicated raw excerpt** for search indexing only (not an interpretation):\n\n"
        f"```text\n{excerpt}\n```\n"
    )


def _digest_llm_read_timeout_s() -> float:
    """Bounded read timeout for digest LLM calls (avoids indefinite MLX hangs)."""
    try:
        from core.config import get_knowledge_config

        raw = (get_knowledge_config() or {}).get("digest_llm_read_timeout_seconds")
        timeout = float(raw if raw is not None else _DIGEST_LLM_READ_TIMEOUT_DEFAULT_S)
    except (TypeError, ValueError):
        timeout = _DIGEST_LLM_READ_TIMEOUT_DEFAULT_S
    return max(30.0, min(timeout, 600.0))


def _knowledge_openai_client(base_url: str, api_key: str):
    """OpenAI client for digest only — shorter read timeout than chat agent loop."""
    from openai import OpenAI

    try:
        import httpx
    except ImportError:
        return OpenAI(base_url=base_url, api_key=api_key, max_retries=1)
    read_s = _digest_llm_read_timeout_s()
    return OpenAI(
        base_url=base_url,
        api_key=api_key,
        timeout=httpx.Timeout(connect=15.0, read=read_s, write=30.0, pool=30.0),
        max_retries=1,
    )


def _main_llm_complete(messages: list[dict[str, str]], max_tokens: int, temperature: float) -> str | None:
    read_timeout = int(_digest_llm_read_timeout_s())

    def _http_chat_complete(base_url: str, model: str) -> str | None:
        endpoint = base_url.rstrip("/") + "/chat/completions"
        payload = {
            "model": model,
            "messages": messages,
            "max_tokens": max_tokens,
            "temperature": temperature,
            "stream": False,
        }
        req = urllib.request.Request(
            endpoint,
            data=json.dumps(payload).encode("utf-8"),
            headers={
                "Content-Type": "application/json",
                "Authorization": "Bearer lm-studio",
            },
            method="POST",
        )
        try:
            with urllib.request.urlopen(req, timeout=read_timeout) as resp:
                body = resp.read().decode("utf-8", errors="replace")
        except Exception:
            return None
        try:
            obj = json.loads(body)
            return str((((obj.get("choices") or [{}])[0].get("message") or {}).get("content") or "")).strip() or None
        except Exception:
            return None

    def _http_list_models(base_url: str) -> list[str]:
        endpoint = base_url.rstrip("/") + "/models"
        req = urllib.request.Request(
            endpoint,
            headers={"Authorization": "Bearer lm-studio"},
            method="GET",
        )
        try:
            with urllib.request.urlopen(req, timeout=12) as resp:
                body = resp.read().decode("utf-8", errors="replace")
            obj = json.loads(body)
        except Exception:
            return []
        out: list[str] = []
        for item in (obj.get("data") or []):
            mid = str((item or {}).get("id") or "").strip()
            if mid and mid not in out:
                out.append(mid)
        return out

    candidate_urls: list[str] = []
    model_pref = ""
    mlx_model_pref = ""
    default_url = "http://127.0.0.1:8000/v1"
    get_client = None
    ensure_local_mlx_server_ready = None
    try:
        from agent_loop import _get_openai_client  # type: ignore
        from agent_loop_config import (  # type: ignore
            DEFAULT_BACKEND_BASE_URL,
            _ensure_local_mlx_server_ready,
            get_backend_base_url,
            get_llmster_model,
            get_mlx_openai_model_id,
        )

        get_client = _get_openai_client
        ensure_local_mlx_server_ready = _ensure_local_mlx_server_ready
        configured_url = str(get_backend_base_url() or "").strip()
        model_pref = str(get_llmster_model() or "").strip()
        # MLX (:8000) registers the loaded weights path as the model id, NOT the LM Studio
        # alias — sending the llmster alias would miss / mis-route on the mlx-vlm server.
        mlx_model_pref = str(get_mlx_openai_model_id() or "").strip()
        if configured_url:
            candidate_urls.append(configured_url)
        cfg_default = str(DEFAULT_BACKEND_BASE_URL or "").strip()
        if cfg_default:
            default_url = cfg_default
    except Exception:
        # Fallback mode: use direct HTTP with default local endpoint.
        pass
    if default_url and default_url not in candidate_urls:
        candidate_urls.append(default_url)

    for url in candidate_urls:
        if ":8000" in url and ensure_local_mlx_server_ready is not None:
            try:
                # Best effort: if MLX is selected but not up, start it before requesting.
                ensure_local_mlx_server_ready(url, on_log=None)
            except Exception:
                pass
        models: list[str] = []
        # On the local MLX server, prefer the resolved MLX model id (weights path) over the
        # LM Studio alias so the request hits the loaded model without a reload.
        if ":8000" in url and mlx_model_pref:
            models.append(mlx_model_pref)
        elif model_pref:
            models.append(model_pref)
        if "local-model" not in models:
            models.append("local-model")
        tried_any = False
        discovered_model_ids: list[str] = []
        for model in models:
            tried_any = True
            if get_client is not None:
                try:
                    client = _knowledge_openai_client(url, "lm-studio")
                    resp = client.chat.completions.create(
                        model=model,
                        messages=messages,
                        max_tokens=max_tokens,
                        temperature=temperature,
                        stream=False,
                    )
                    text = (resp.choices[0].message.content or "").strip()
                    if text:
                        return text
                    continue
                except Exception:
                    pass
            # Fallback to direct HTTP for runtimes without openai/langchain_openai deps.
            text = _http_chat_complete(url, model)
            if text:
                return text
        # If configured aliases failed, discover concrete model IDs from /v1/models.
        if get_client is not None:
            try:
                client = _knowledge_openai_client(url, "lm-studio")
                listed = client.models.list()
                for item in getattr(listed, "data", []) or []:
                    mid = str(getattr(item, "id", "") or "").strip()
                    if mid:
                        discovered_model_ids.append(mid)
            except Exception:
                discovered_model_ids = _http_list_models(url)
        else:
            discovered_model_ids = _http_list_models(url)
        for model in discovered_model_ids:
            if model in models:
                continue
            tried_any = True
            if get_client is not None:
                try:
                    client = _knowledge_openai_client(url, "lm-studio")
                    resp = client.chat.completions.create(
                        model=model,
                        messages=messages,
                        max_tokens=max_tokens,
                        temperature=temperature,
                        stream=False,
                    )
                    text = (resp.choices[0].message.content or "").strip()
                    if text:
                        return text
                    continue
                except Exception:
                    pass
            text = _http_chat_complete(url, model)
            if text:
                return text
        # Last-resort if model list failed: try one plain HTTP attempt.
        if not tried_any:
            text = _http_chat_complete(url, model_pref or "local-model")
            if text:
                return text
    return None


def _extractive_fallback_summary(messages: list[dict[str, str]], *, max_chars: int = 2200) -> str:
    """Best-effort local fallback when no summarization model is available."""
    user_chunks = [str(m.get("content") or "") for m in messages if str(m.get("role") or "") == "user"]
    source = "\n\n".join(user_chunks).strip()
    if not source:
        return "Summary unavailable: no model and no source text."
    # Strip markdown fences and normalize whitespace.
    source = re.sub(r"```.*?```", " ", source, flags=re.S)
    source = re.sub(r"Source:\s+[^\n]+?(?:chunk\s+\d+/\d+)?\s+", " ", source, flags=re.I)
    source = source.replace("\r", "\n")
    # Join hyphenated line breaks from PDF extraction (e.g. "attend-\ning" -> "attending").
    source = re.sub(r"([A-Za-z])-\n([A-Za-z])", r"\1\2", source)
    source = re.sub(r"\n{3,}", "\n\n", source)
    lines = [ln.strip() for ln in source.splitlines() if ln.strip()]
    cleaned_lines: list[str] = []
    for ln in lines:
        if re.match(r"^--\s*\d+\s*of\s*\d+\s*--$", ln):
            continue
        if re.match(r"^[\d\s:.\-–—]+$", ln):
            continue
        if re.search(r"(copyright|permissions|index of chapters|wheaton|esv\.org)", ln, flags=re.I):
            continue
        ln = re.sub(r"\s+", " ", ln)
        ln = re.sub(r"^\W*-\W*", "", ln)
        cleaned_lines.append(ln)
    flat = " ".join(cleaned_lines)
    flat = re.sub(r"\s+", " ", flat).strip()
    if not flat:
        return "Summary unavailable: extracted text is empty."
    # Prefer coherent sentence-like snippets from the cleaned text.
    sentences = re.split(r"(?<=[.!?])\s+", flat)
    candidates: list[str] = []
    for s in sentences:
        s = re.sub(r"\s+", " ", s.strip(" -"))
        # Strip obvious filename/chunk residues that leak from extraction.
        s = re.sub(r"(?i)\b[\w.-]+\.(pdf|docx?|md|txt|html?)\b", "", s)
        s = re.sub(r"(?i)\bchunk\s+\d+/\d+\b", "", s)
        s = re.sub(r"\s+", " ", s).strip(" -")
        if len(s) < 55:
            continue
        alpha = sum(1 for c in s if c.isalpha())
        digit = sum(1 for c in s if c.isdigit())
        if alpha < 28:
            continue
        if digit > max(16, alpha // 2):
            continue
        # Avoid all-caps and table-of-contents style lines.
        words = [w for w in re.split(r"\s+", s) if w]
        if words:
            upper_words = sum(1 for w in words if len(w) >= 3 and w.isupper())
            if upper_words / max(1, len(words)) > 0.45:
                continue
        if re.search(r"(?i)\b(contents|index|copyright)\b", s) and len(s) < 120:
            continue
        candidates.append(s)
    # De-duplicate similar candidates.
    unique: list[str] = []
    seen_norm: set[str] = set()
    for s in candidates:
        norm = re.sub(r"[^a-z0-9]+", " ", s.lower()).strip()
        if not norm or norm in seen_norm:
            continue
        seen_norm.add(norm)
        unique.append(s)
    if not unique:
        return flat[:max_chars].rstrip()
    # Spread picks across the document to avoid only the opening section.
    target = min(6, len(unique))
    if target <= 1:
        picked = unique[:1]
    else:
        idxs = {round(i * (len(unique) - 1) / (target - 1)) for i in range(target)}
        picked = [unique[i] for i in sorted(idxs)]
    # Trim to max_chars budget.
    out: list[str] = []
    total = 0
    for s in picked:
        cost = len(s) + 3
        if out and (total + cost) > max_chars:
            break
        out.append(s)
        total += cost
    if not out:
        out = unique[:1]
    polished = [_strip_summary_citation_noise(s) for s in out]
    polished = [s for s in polished if s and not _summary_line_has_citation_spam(s)]
    if not polished:
        polished = [_strip_summary_citation_noise(s) for s in out if _strip_summary_citation_noise(s)]
    if not polished:
        return flat[:max_chars].rstrip()
    return "\n".join(f"- {s}" for s in polished)


def _complete_with_fallback(
    messages: list[dict[str, str]],
    *,
    max_tokens: int = 512,
    temperature: float = 0.2,
    allow_utility_model: bool = True,
    utility_only: bool = False,
    relax_quality_gates: bool = False,
    activity_operation: str | None = None,
    suppress_utility_activity_errors: bool = False,
    return_method: bool = False,
) -> str | tuple[str, str]:
    global _main_model_consecutive_failures, _main_model_skip_until

    # Adaptive skip: if the main model has failed quality gates repeatedly,
    # don't waste time calling it again until the cooldown expires.
    skip_main = utility_only or time.time() < _main_model_skip_until

    if not skip_main:
        text = _main_llm_complete(messages, max_tokens=max_tokens, temperature=temperature)
        if text and text.strip():
            cleaned = _sanitize_summary_output(text.strip())
            if cleaned and not _is_garbage_summary(cleaned):
                _main_model_consecutive_failures = 0
                _main_model_skip_until = 0.0
                return (cleaned, "main_model") if return_method else cleaned
        # Main model returned nothing usable — count toward adaptive skip.
        _main_model_consecutive_failures += 1
        if _main_model_consecutive_failures >= _MAIN_MODEL_FAIL_THRESHOLD:
            _main_model_skip_until = time.time() + _MAIN_MODEL_COOLDOWN_S
            _debug_log(
                f"[knowledge] Main model failed quality gate {_main_model_consecutive_failures}x "
                f"consecutively; skipping for {_MAIN_MODEL_COOLDOWN_S:.0f}s",
                "INFO",
            )

    if allow_utility_model:
        try:
            from utility_model import utility_complete

            utility_activity_op = activity_operation
            if suppress_utility_activity_errors and not utility_activity_op:
                utility_activity_op = "knowledge_digest"
            text = utility_complete(
                messages,
                max_tokens=max_tokens,
                temperature=temperature,
                activity_operation=utility_activity_op,
                surface_errors=not suppress_utility_activity_errors,
            )
        except Exception:
            text = None
        if text and str(text).strip():
            cleaned = _sanitize_summary_output(str(text).strip())
            if cleaned and (relax_quality_gates or not _is_garbage_summary(cleaned)):
                return (cleaned, "utility_model") if return_method else cleaned
    out = _extractive_fallback_summary(messages)
    return (out, "extractive_fallback") if return_method else out


def _upgrade_llm_complete(
    messages: list[dict[str, str]],
    *,
    max_tokens: int = 512,
    temperature: float = 0.2,
) -> tuple[str, str]:
    """Utility-only LLM for bulk summary regen — never falls back to extractive bullets."""
    raw_len = 0
    try:
        from utility_model import utility_complete

        text = utility_complete(
            messages,
            max_tokens=max_tokens,
            temperature=temperature,
            activity_operation="knowledge_digest",
            surface_errors=False,
        )
        raw_len = len(str(text or ""))
    except Exception as exc:
        _debug_log(f"[knowledge] upgrade utility_complete error: {exc}", "WARN")
        return "", "utility_failed"
    if not text or not str(text).strip():
        return "", "utility_failed"
    cleaned = _sanitize_summary_output(str(text).strip())
    if len(cleaned.strip()) < 20:
        return "", "utility_failed"
    if _is_cot_narration_or_prompt_leak(cleaned):
        return "", "utility_failed"
    return cleaned, "utility_model"


def _complete_with_method(
    messages: list[dict[str, str]],
    *,
    max_tokens: int = 512,
    temperature: float = 0.2,
    allow_utility_model: bool = True,
    utility_only: bool = False,
    relax_quality_gates: bool = False,
    activity_operation: str | None = None,
    suppress_utility_activity_errors: bool = False,
) -> tuple[str, str]:
    out = _complete_with_fallback(
        messages,
        max_tokens=max_tokens,
        temperature=temperature,
        allow_utility_model=allow_utility_model,
        utility_only=utility_only,
        relax_quality_gates=relax_quality_gates,
        activity_operation=activity_operation,
        suppress_utility_activity_errors=suppress_utility_activity_errors,
        return_method=True,
    )
    if isinstance(out, tuple) and len(out) == 2:
        text, method = out
        return str(text or ""), str(method or "unknown")
    return str(out or ""), "unknown"


def _summary_has_repetitive_reference_loop(text: str) -> bool:
    """Detect a single bullet that loops the same scripture ref dozens of times."""
    refs = re.findall(r"(?i)\b\d+\s+\w+\s+\d+:\d+(?:-\d+)?", text or "")
    if len(refs) < 5:
        return False
    _top, top_count = Counter(refs).most_common(1)[0]
    if top_count >= 5:
        return True
    return len(refs) >= 12 and len(set(r.lower() for r in refs)) <= 3


def _summary_echoes_upgrade_prompt(summary: str) -> bool:
    """True when output parrots the upgrade system prompt instead of summarizing."""
    low = (summary or "").lower()
    if re.search(r"(?i)^\s*[-*•]?\s*task\s*:", low):
        return True
    fragments = (
        "write 6",
        "markdown bullets",
        "summarizing the passage",
        "summarizing what happens",
        "output only the bullets",
        "no headings, labels, instructions",
        "commentary about the task",
    )
    hits = sum(1 for frag in fragments if frag in low)
    return hits >= 2


def _summary_line_is_meta_leak(line: str) -> bool:
    """True when a bullet is CoT narration or prompt echo, not a real summary."""
    body = re.sub(r"^\s*[-*•]\s+", "", (line or "").strip())
    if not body:
        return True
    low = body.lower()
    if re.match(r"(?i)^(task|key content|provided text|content)\s*:", body):
        return True
    if re.match(r"(?i)^attempt\s+\d+", body):
        return True
    if re.search(r"(?i)\bmental outline\b", body):
        return True
    if re.match(r'^then\s+["\']', low):
        return True
    if re.search(r"(?i)\bappears at the very beginning\b", body):
        return True
    if re.search(
        r"(?i)^(constraints?|content|specific instruction|input(?:\s+(?:data|text))?|"
        r"observation on input)\s*:",
        body,
    ):
        return True
    if re.search(r"(?i)^source\s*:", body) and re.search(
        r"(?i)\b(note|prompt|typo|the prompt)\b", body
    ):
        return True
    if _summary_has_repetitive_reference_loop(body):
        return True
    if re.match(r"(?i)^constraint\s+\d+\s*:", body):
        return True
    if re.search(r"(?i)\bthe prompt says\b", body):
        return True
    if re.search(r"(?i)\bwait,\s+(?:the last line|i need to check)\b", body):
        return True
    if re.search(r"(?i)\bnever mention the user\b", body):
        return True
    if re.search(r"(?i)\bcover major themes,\s*events\b", body):
        return True
    if re.search(r"(?i)\bdo not include analysis preambles\b", body):
        return True
    if re.search(r"(?i)\bif the excerpt is mostly\b", body):
        return True
    if re.search(r"(?i)\bbad extraction or boilerplate\b", body):
        return True
    if re.search(r"(?i)\bthe provided text\b", body):
        return True
    if re.search(r"(?i)\b(be faithful|do not invent facts)\b", body) and re.search(
        r"(?i)\b(prompt|excerpt|instruction|constraint)\b", body
    ):
        return True
    if re.search(r"(?i)\b(block of text starting|repeated)\b", body) and re.search(
        r"\d+\s+\w+\s+\d+:\d+", body
    ):
        return True
    if re.search(r"(?i)analyze the request", body):
        return True
    if re.search(r"(?i)thinking process", body):
        return True
    if re.search(r"(?i)the user wants", body):
        return True
    if re.search(r"(?i)conversation history", body):
        return True
    if re.search(r"(?i)no specific content", body):
        return True
    if re.search(r"(?i)template example", body):
        return True
    if re.search(r"(?i)summarize the source document", body):
        return True
    if re.search(r"(?i)ocr or copy-paste|copy-?paste", body):
        return True
    if re.search(r"(?i)snippet from", body):
        return True
    if re.search(r"(?i)the text provided|text is a snippet", body):
        return True
    if re.search(r"(?i)in the prompt|messy in the|fragmented in the", body):
        return True
    if re.search(r"(?i)let'?s break down", body):
        return True
    if re.search(r"(?i)appears again|looking at the raw", body):
        return True
    if re.search(r"(?i)the last line (?:provided|is)", body):
        return True
    if re.search(r"(?i)verses provided|bible study or similar", body):
        return True
    if "?" in body and re.search(
        r"(?i)\b(?:then|wait|no,|actually)\b", body
    ):
        return True
    # Model echoing raw verse/OCR fragments as a bullet list.
    if len(re.findall(r'"\d+"', body)) >= 4:
        return True
    return False


def _upgrade_summary_has_substance(summary: str, method: str) -> bool:
    """Upgrade summaries need multiple real bullets, not a one-line task echo."""
    bullets = [
        ln.strip()
        for ln in (summary or "").splitlines()
        if ln.strip().startswith(("-", "*", "•"))
    ]
    min_bullets = 2 if str(method or "") == "main_model" else 3
    if len(bullets) < min_bullets:
        return False
    substantive = [
        b
        for b in bullets
        if len(re.sub(r"^[\-*•]\s+", "", b)) >= 24
        and not _summary_line_is_meta_leak(b)
    ]
    return len(substantive) >= min_bullets


def _summary_is_excerpt_commentary(summary: str) -> bool:
    """Model narrates the excerpt/task instead of summarizing passage content."""
    text = (summary or "").strip()
    if not text:
        return True
    lower = text.lower()
    if any(
        m in lower
        for m in (
            "text provided",
            "in the prompt",
            "copy-paste",
            "copy paste",
            "snippet from",
            "let's break down",
            "looks like a copy",
            "bible study or similar",
            "verses provided",
            "messy in the",
            "fragmented in the",
            "appears again",
            "looking at the raw",
            "the last line provided",
            "the last line is",
        )
    ):
        return True
    bullets = [
        ln for ln in text.splitlines() if ln.strip().startswith(("-", "*", "•"))
    ]
    if not bullets:
        return _summary_line_is_meta_leak(text)
    if any(
        _summary_line_is_meta_leak(re.sub(r"^[\-*•]\s+", "", ln.strip())) for ln in bullets
    ):
        return True
    return False


def _is_cot_narration_or_prompt_leak(summary: str) -> bool:
    """Reject model chain-of-thought narration saved as a summary."""
    lines = [ln.strip() for ln in (summary or "").splitlines() if ln.strip()]
    if not lines:
        return True
    bullet_lines = [ln for ln in lines if ln.lstrip().startswith(("-", "*", "•"))]
    if not bullet_lines:
        return _summary_line_is_meta_leak(lines[0])
    if any(_summary_line_is_meta_leak(ln) for ln in bullet_lines):
        return True
    if _summary_has_repetitive_reference_loop(summary):
        return True
    if _summary_is_excerpt_commentary(summary):
        return True
    return False


def _recover_summary_bullets_from_leaked_cot(text: str) -> str:
    """Recover markdown bullets when the model emitted only a 'Thinking Process' outline."""
    raw = (text or "").replace("\r", "\n")
    bullet_re = re.compile(r"^(\s*[-*]\s+)(.+)$")
    lines = raw.splitlines()
    best_run: list[str] = []
    current_run: list[str] = []
    for line in lines:
        m = bullet_re.match(line)
        if m and not _summary_line_is_meta_leak(m.group(2)):
            body = re.sub(r"\*+", "", m.group(2)).strip()
            if len(body) >= 12:
                current_run.append(f"- {body}")
            else:
                if len(current_run) > len(best_run):
                    best_run = current_run
                current_run = []
        else:
            if len(current_run) > len(best_run):
                best_run = current_run
            current_run = []
    if len(current_run) > len(best_run):
        best_run = current_run
    if len(best_run) >= 2 and not _is_cot_narration_or_prompt_leak("\n".join(best_run)):
        return "\n".join(best_run)
    if len(best_run) == 1 and not _summary_line_is_meta_leak(best_run[0]):
        return best_run[0]
    return ""


def _sanitize_summary_output(summary: str) -> str:
    from utils.llm_output_sanitize import sanitize_summary_text

    text = sanitize_summary_text((summary or "").replace("\r", "\n"))
    if len(text.strip()) < 20:
        recovered = _recover_summary_bullets_from_leaked_cot(summary or "")
        if len(recovered.strip()) >= 20:
            text = recovered
    if not text.strip():
        return ""
    lines: list[str] = []
    for raw in text.splitlines():
        line = raw.rstrip()
        if not line.strip():
            lines.append("")
            continue
        bullet = ""
        body = line.strip()
        m = re.match(r"^([\-*]\s+)(.*)$", body)
        if m:
            bullet = m.group(1)
            body = m.group(2)
        # Remove chunk labels that leak from chunk prompts, e.g.:
        # "Source: foo.pdf chunk 3/27" or "foo.pdf chunk 1/8".
        body = re.sub(r"(?i)^source:\s+[^\n]*?\bchunk\s+\d+/\d+\s*", "", body)
        body = re.sub(r"(?i)^(?:[^\s]+\.(?:pdf|md|txt|docx|html?)\s+)?chunk\s+\d+/\d+\s*", "", body)
        body = re.sub(r"\s+", " ", body).strip(" -")
        # Drop obvious meta/instruction lines from assistant leakage.
        if _summary_line_is_meta_leak(f"{bullet}{body}"):
            continue
        if not body:
            continue
        lines.append(f"{bullet}{body}")
    cleaned = "\n".join(lines)
    cleaned = re.sub(r"\n{3,}", "\n\n", cleaned).strip()
    if cleaned and _is_cot_narration_or_prompt_leak(cleaned):
        return ""
    return cleaned


def _annotate_source_repo_paths(entry: dict[str, Any], src_path: str) -> None:
    """Best-effort rel_path/repo_root for GitNexus pivot (Phase 2b)."""
    raw = str(src_path or "").strip()
    if not raw or re.match(r"^https?://", raw, flags=re.IGNORECASE):
        return
    try:
        resolved = os.path.realpath(raw)
        if not os.path.isfile(resolved):
            return
        cur = Path(resolved).parent
        repo_root: str | None = None
        for _ in range(14):
            if (cur / ".git").is_dir():
                repo_root = str(cur)
                break
            if cur.parent == cur:
                break
            cur = cur.parent
        if repo_root:
            entry["repo_root"] = repo_root
            entry["rel_path"] = os.path.relpath(resolved, repo_root)
    except Exception:
        return


def _render_source_markdown(
    filename: str,
    source_path: str,
    summary: str,
    topics: list[str],
    *,
    summary_method: str = "",
) -> str:
    method_line = f"**Summary method:** `{summary_method or 'unknown'}`\n\n"
    return (
        f"# {filename}\n\n"
        f"**Source path:** `{source_path}`\n\n"
        f"**Summarizer version:** `{_SUMMARIZER_VERSION}`\n\n"
        f"{method_line}"
        "## Summary\n\n"
        f"{summary.strip()}\n\n"
        "## Topics\n\n"
        + ("\n".join(f"- {t}" for t in topics) if topics else "- (none)")
        + "\n"
    )


def _extract_topics(summary: str, limit: int = 8) -> list[str]:
    prompt = [
        {
            "role": "system",
            "content": (
                "Return only topic lines: TOPIC: <keyword>. Up to 8 lines.\n"
                "Return conceptual themes only (1-4 words). Include major themes even when "
                "the text uses synonyms (e.g. monarchy, reign, royalty → King or Kingship).\n"
                "Do not return: file paths, section/chapter titles, page/verse/line citations, "
                "RFC/ISO numbers, or full sentences."
            ),
        },
        {"role": "user", "content": summary[:6000]},
    ]
    topics: list[str] = []
    try:
        raw = _complete_with_fallback(prompt, max_tokens=160, temperature=0.1)
        for line in raw.splitlines():
            m = _TOPIC_LINE_RE.match(line)
            if not m:
                continue
            topic = re.sub(r"\s+", " ", m.group(1).strip())
            if topic and topic.lower() not in {t.lower() for t in topics}:
                topics.append(topic)
            if len(topics) >= limit:
                break
    except Exception:
        topics = []
    if topics:
        return topics
    token_re, stopwords = _tokenizer_refs()
    stop = set(stopwords) | _TOPIC_FALLBACK_EXTRA_STOPWORDS
    tokens = [
        t
        for t in token_re.findall(summary.lower())
        if t not in stop and len(t) >= 4
    ]
    return [t for t, _ in Counter(tokens).most_common(limit)]


def _extract_broad_topics_from_synthesis(synthesis_text: str, limit: int = 5) -> list[str]:
    text = (synthesis_text or "").strip()
    if not text:
        return []
    prompt = [
        {
            "role": "system",
            "content": (
                "Return only broad category lines in this exact format: TOPIC: <label>.\n"
                "Use 3-5 broad labels that describe the overall corpus. Prefer one or two words.\n"
                "Return conceptual themes only. Do not return: file paths, section/chapter titles, "
                "page/verse/line citations, RFC/ISO numbers, or full sentences."
            ),
        },
        {"role": "user", "content": text[:6000]},
    ]
    topics: list[str] = []
    try:
        raw = _complete_with_fallback(
            prompt,
            max_tokens=100,
            temperature=0.1,
            allow_utility_model=False,
        )
        for line in raw.splitlines():
            m = _TOPIC_LINE_RE.match(line)
            if not m:
                continue
            topic = re.sub(r"\s+", " ", m.group(1).strip())
            if topic and topic.lower() not in {t.lower() for t in topics}:
                topics.append(topic)
            if len(topics) >= limit:
                break
    except Exception:
        topics = []
    if topics:
        return topics
    # Fallback: derive broad terms from the synthesis itself.
    token_re, stopwords = _tokenizer_refs()
    stop = set(stopwords) | _TOPIC_FALLBACK_EXTRA_STOPWORDS
    tokens = [
        t
        for t in token_re.findall(text.lower())
        if t not in stop and len(t) >= 5
    ]
    return [t.title() for t, _ in Counter(tokens).most_common(limit)]


def _derive_broad_topics_from_manifest(manifest: dict[str, Any], limit: int = 5) -> list[str]:
    token_re, stopwords = _tokenizer_refs()
    stop = set(stopwords) | _TOPIC_FALLBACK_EXTRA_STOPWORDS
    counts: Counter[str] = Counter()
    for topic in manifest.get("topics") or []:
        for token in token_re.findall(str(topic).lower()):
            if token in stop or len(token) < 4:
                continue
            counts[token] += 1
    return [t.title() for t, _ in counts.most_common(limit)]


_SEARCH_STOPWORDS: frozenset[str] = frozenset(
    "a an the of in on at to for by with from and or but not is are was were "
    "be been being have has had do does did will would shall should may might "
    "can could that this these those it its than so as if then into through "
    "rather very also just about up out".split()
)


def _search_in_text(query: str, content: str, context_chars: int = _SOURCE_SEARCH_CONTEXT) -> list[str]:
    """Find passages in *content* whose words overlap with *query* terms.

    Returns up to 5 context windows ranked by term-overlap ratio.
    """
    q = (query or "").lower().strip()
    if not q:
        return []
    query_tokens = {w for w in re.findall(r"[a-zA-Z0-9_]{3,}", q) if w not in _SEARCH_STOPWORDS}
    if not query_tokens:
        return []

    lower = content.lower()
    paragraphs = re.split(r"\n{2,}", content)
    scored: list[tuple[float, int, str]] = []
    offset = 0
    for para in paragraphs:
        para_stripped = para.strip()
        if len(para_stripped) < 20:
            offset += len(para) + 2
            continue
        para_tokens = set(re.findall(r"[a-zA-Z0-9_]{3,}", para_stripped.lower()))
        overlap = query_tokens & para_tokens
        if not overlap:
            offset += len(para) + 2
            continue
        ratio = len(overlap) / len(query_tokens)
        mid = offset + len(para) // 2
        a = max(0, mid - context_chars // 2)
        b = min(len(content), mid + context_chars // 2)
        scored.append((ratio, -len(overlap), content[a:b].strip()))
        offset += len(para) + 2

    scored.sort(key=lambda t: (t[0], t[1]), reverse=True)
    seen: set[str] = set()
    out: list[str] = []
    for _, _, snippet in scored:
        key = snippet[:80]
        if key in seen:
            continue
        seen.add(key)
        out.append(snippet)
        if len(out) >= 5:
            break
    return out


def _walk_candidate_files(
    source_path: str,
    *,
    ingest_profile: str = "docs",
    slug: str | None = None,
) -> tuple[list[str], bool]:
    src = os.path.abspath(os.path.expanduser(source_path))
    profile = normalize_ingest_profile(ingest_profile, source_path=source_path)
    skip_dirs = set(_SKIP_DIR_NAMES)
    if profile == "repo":
        skip_dirs |= _REPO_EXTRA_SKIP_DIRS
    if os.path.isfile(src):
        if os.path.islink(src):
            return [], False
        ext = Path(src).suffix.lower()
        if profile == "repo" and ext in _REPO_CODE_EXTENSIONS:
            return [], False
        if ext in _SUPPORTED_EXTENSIONS:
            return [src], False
        return [], False
    files: list[str] = []
    capped = False
    for root, dirs, names in os.walk(src, topdown=True):
        if slug:
            _abort_if_digest_pause(slug)
        dirs[:] = [d for d in dirs if not d.startswith(".") and d not in skip_dirs]
        for name in names:
            if name.startswith("."):
                continue
            full_path = os.path.join(root, name)
            if os.path.islink(full_path):
                continue
            ext = Path(name).suffix.lower()
            if profile == "repo" and ext in _REPO_CODE_EXTENSIONS:
                continue
            if ext in _SUPPORTED_EXTENSIONS:
                files.append(full_path)
            if len(files) >= _MAX_DIGEST_FILES:
                capped = True
                return files, capped
    return files, capped


def _choose_corpus_slug(label: str, update: bool) -> tuple[str, bool]:
    base = _slugify(label)
    if update:
        return base, False
    root = _knowledge_root()
    if os.path.exists(os.path.join(root, base)):
        # Recovery path: if an older failed attempt left only a directory or a broken manifest,
        # allow reusing the slug so users are not permanently blocked by an orphan corpus folder.
        if _load_manifest(base) is None:
            if _manifest_schema_too_new(base):
                return base, True
            return base, False
        return base, True
    slug = base
    return slug, False


def _outside_workspace_digest_allowed() -> bool:
    try:
        from tools.system import _load_app_prefs_tools

        prefs = _load_app_prefs_tools() or {}
        return bool(prefs.get("allow_knowledge_digest_outside_workspace", False))
    except Exception:
        return False


def _gap_row_id(row: dict[str, Any]) -> str:
    return str(row.get("id") or "").strip()


def _gap_row_is_resolved(row: dict[str, Any]) -> bool:
    """True only for explicit resolved markers (avoid treating ``\"false\"`` strings as resolved)."""
    val = row.get("resolved")
    if val is True:
        return True
    if isinstance(val, str) and val.strip().lower() in ("true", "1", "yes"):
        return True
    return False


def _record_knowledge_gap(query: str, corpus: str | None = None) -> None:
    """Append an unanswered query miss to the canonical gap log (``knowledge_gaps.jsonl``)."""
    q = (query or "").strip()
    if not q:
        return
    ts = _iso_now()
    corp = (corpus or "").strip()[:120]
    payload: dict[str, Any] = {
        "ts": ts,
        "query": q[:400],
        "corpus": corp,
        "id": _knowledge_gap_row_id(ts, q[:400], corp),
        "resolved": False,
    }
    workspace = _workspace_path()
    gaps_path = _knowledge_gaps_path(workspace)
    os.makedirs(os.path.dirname(gaps_path), exist_ok=True)
    with _knowledge_gap_lock:
        existing_rows = _parse_knowledge_gap_file(gaps_path)
        if _should_skip_gap_duplicate(existing_rows, q[:400], corp):
            return
        with open(gaps_path, "a", encoding="utf-8") as fh:
            fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
        _trim_knowledge_gap_file(gaps_path)


def _knowledge_gaps_path(workspace: str | None = None) -> str:
    ws = workspace or _workspace_path()
    return os.path.join(ws, _KNOWLEDGE_GAP_FILE)


def _knowledge_gap_row_id(ts: str, query: str, corpus: str) -> str:
    import hashlib

    raw = f"{ts}:{query}:{corpus}".encode("utf-8")
    return hashlib.sha256(raw).hexdigest()[:16]


def _parse_knowledge_gap_file(gaps_path: str) -> list[dict[str, Any]]:
    if not os.path.isfile(gaps_path):
        return []
    rows: list[dict[str, Any]] = []
    try:
        with open(gaps_path, encoding="utf-8", errors="replace") as fh:
            for line_idx, line in enumerate(fh):
                line = line.strip()
                if not line:
                    continue
                try:
                    row = json.loads(line)
                except json.JSONDecodeError:
                    continue
                if not isinstance(row, dict):
                    continue
                q = str(row.get("query") or "").strip()
                if not q:
                    continue
                ts = str(row.get("ts") or "")
                corp = str(row.get("corpus") or "").strip()[:120]
                if not _gap_row_id(row):
                    if ts:
                        row["id"] = _knowledge_gap_row_id(ts, q, corp)
                    else:
                        row["id"] = _knowledge_gap_row_id(f"line:{line_idx}", q, corp)
                row.setdefault("resolved", False)
                rows.append(row)
    except OSError:
        return []
    return rows


def _write_knowledge_gap_file(gaps_path: str, rows: list[dict[str, Any]]) -> None:
    os.makedirs(os.path.dirname(gaps_path), exist_ok=True)
    lines = [json.dumps(row, ensure_ascii=False) for row in rows if isinstance(row, dict)]
    _write_text(gaps_path, "\n".join(lines).rstrip() + ("\n" if lines else ""))


def _trim_knowledge_gap_file(gaps_path: str) -> None:
    rows = _parse_knowledge_gap_file(gaps_path)
    if len(rows) <= _KNOWLEDGE_GAP_MAX_LINES:
        return
    unresolved = [r for r in rows if not _gap_row_is_resolved(r)]
    resolved = [r for r in rows if _gap_row_is_resolved(r)]
    if len(unresolved) <= _KNOWLEDGE_GAP_MAX_LINES:
        keep_resolved = max(0, _KNOWLEDGE_GAP_MAX_LINES - len(unresolved))
        # resolved[-0:] is resolved[0:] in Python — guard keep_resolved == 0 explicitly.
        rows = unresolved + (resolved[-keep_resolved:] if keep_resolved else [])
    else:
        rows = unresolved[-_KNOWLEDGE_GAP_MAX_LINES:]
    _write_knowledge_gap_file(gaps_path, rows)


def _should_skip_gap_duplicate(rows: list[dict[str, Any]], query: str, corpus: str) -> bool:
    """Skip append when the same query+corpus was logged unresolved within 24h."""
    from datetime import datetime, timezone

    qnorm = query.strip().lower()
    cnorm = corpus.strip().lower()
    now = datetime.now(timezone.utc)
    for row in reversed(rows[-100:]):
        if _gap_row_is_resolved(row):
            continue
        if str(row.get("query") or "").strip().lower() != qnorm:
            continue
        if str(row.get("corpus") or "").strip().lower() != cnorm:
            continue
        ts_raw = str(row.get("ts") or "").strip()
        if not ts_raw:
            # Legacy rows without timestamps still block same-day duplicates.
            return True
        try:
            ts_parse = ts_raw.replace("Z", "+00:00") if ts_raw.endswith("Z") else ts_raw
            dt = datetime.fromisoformat(ts_parse)
            if dt.tzinfo is None:
                dt = dt.replace(tzinfo=timezone.utc)
            if (now - dt).total_seconds() < 86400:
                return True
        except (TypeError, ValueError):
            continue
    return False


def resolve_knowledge_gap(gap_id: str) -> bool:
    """Mark a gap row resolved by id; returns True when a row was updated."""
    gid = (gap_id or "").strip()
    if not gid:
        return False
    gaps_path = _knowledge_gaps_path()
    with _knowledge_gap_lock:
        rows = _parse_knowledge_gap_file(gaps_path)
        found = False
        for row in rows:
            if _gap_row_id(row) == gid:
                row["resolved"] = True
                found = True
        if found:
            _write_knowledge_gap_file(gaps_path, rows)
        return found


def _iso_timestamp_to_epoch(raw: str) -> float:
    text = str(raw or "").strip()
    if not text:
        return 0.0
    try:
        ts_parse = text.replace("Z", "+00:00") if text.endswith("Z") else text
        dt = datetime.fromisoformat(ts_parse)
        if dt.tzinfo is None:
            dt = dt.replace(tzinfo=timezone.utc)
        return dt.timestamp()
    except (TypeError, ValueError):
        return 0.0


def _digest_last_activity_ts(job: dict[str, Any], manifest: dict[str, Any]) -> float:
    """Last heartbeat for stale detection — progress updates bump ``job.updated_at``."""
    job_updated = float(job.get("updated_at") or 0.0)
    job_started = float(job.get("started_at") or 0.0)
    manifest_started = float(manifest.get("started_at") or 0.0)
    manifest_updated = _iso_timestamp_to_epoch(str(manifest.get("updated_at") or ""))
    return max(job_updated, job_started, manifest_started, manifest_updated)


def _stop_digest_job(
    slug: str,
    *,
    error: str,
    clear_progress: bool = True,
    aborted: bool = False,
) -> None:
    """Clear in-memory digest job state (used when manifest and job disagree)."""
    updates: dict[str, Any] = {
        "running": False,
        "done": True,
        "error": error,
        "summary": "",
        "aborted": aborted,
    }
    if clear_progress:
        updates["progress"] = ""
        updates["current_file"] = ""
    _set_digest_job(slug, **updates)


def _digest_pause_requested(slug: str) -> bool:
    with _digest_job_lock:
        job = _digest_jobs.get(slug) or {}
        return bool(job.get("pause_requested"))


def _set_digest_pause_requested(slug: str) -> None:
    with _digest_job_lock:
        _digest_jobs.setdefault(slug, {})["pause_requested"] = True


def _clear_digest_pause_requested(slug: str) -> None:
    with _digest_job_lock:
        if slug in _digest_jobs:
            _digest_jobs[slug].pop("pause_requested", None)


_DIGEST_PAUSE_MSG = "Ingest paused. Continue ingest to resume."


class _DigestPaused(BaseException):
    """Cooperative operator pause during ``_digest_sync``."""


def _abort_if_digest_pause(slug: str) -> None:
    if _digest_pause_requested(slug):
        raise _DigestPaused()
    with _digest_job_lock:
        if bool((_digest_jobs.get(slug) or {}).get("aborted")):
            raise _DigestPaused()


def _complete_paused_digest(
    slug: str,
    label: str,
    planned: list[dict[str, Any]],
    results_by_index: dict[int, dict[str, Any]],
) -> dict[str, Any]:
    if results_by_index:
        _checkpoint_running_manifest(slug, planned, results_by_index)
    _clear_digest_pause_requested(slug)
    _set_digest_job(
        slug,
        running=False,
        done=True,
        error="",
        summary=_DIGEST_PAUSE_MSG,
        progress="",
        current_file="",
    )
    _interrupt_running_digest_manifest(slug)
    manifest = _load_manifest(slug) or {}
    if str(manifest.get("digest_status") or "") != "partial":
        manifest["digest_status"] = "partial"
        manifest["started_at"] = None
        manifest["updated_at"] = _iso_now()
        _save_manifest(slug, manifest)
        invalidate_topic_index_cache()
    return {
        "ok": True,
        "slug": slug,
        "label": label,
        "paused": True,
        "message": _DIGEST_PAUSE_MSG,
    }


def _peek_digest_job_running(slug: str, manifest: dict[str, Any], job: dict[str, Any]) -> bool:
    """Read-only mirror of ``_reconcile_digest_job_with_manifest``'s running verdict.

    Returns whether the job *would* still be running after reconciliation, WITHOUT any side
    effects (no ``_stop_digest_job`` / manifest writes). Used by the HTTP status read path so
    polling the Knowledge pane never stops an ingest or rewrites a manifest — the actual
    zombie cleanup runs from the heartbeat maintenance path (``reconcile_jobs=True``).
    """
    if not job.get("running"):
        return False
    if _digest_pause_requested(slug):
        return True
    manifest_status = str(manifest.get("digest_status") or "")
    is_summary_upgrade = str(job.get("job_kind") or JOB_KIND_DIGEST) == JOB_KIND_SUMMARY_UPGRADE
    if not is_summary_upgrade and manifest_status in ("failed", "complete"):
        return False
    last_activity = _digest_last_activity_ts(job, manifest)
    stale = last_activity > 0 and (time.time() - last_activity) >= _DIGEST_STALE_TTL
    return not stale


def _reconcile_digest_job_with_manifest(slug: str, manifest: dict[str, Any], job: dict[str, Any]) -> dict[str, Any]:
    """Align zombie jobs (manifest failed, job still running) for status polling."""
    if not job.get("running"):
        return job
    if _digest_pause_requested(slug):
        return job
    job_kind = str(job.get("job_kind") or JOB_KIND_DIGEST)
    last_activity = _digest_last_activity_ts(job, manifest)
    stale = last_activity > 0 and (time.time() - last_activity) >= _DIGEST_STALE_TTL

    if job_kind == JOB_KIND_SUMMARY_UPGRADE:
        if stale:
            err = (
                "Summary regeneration had no progress for 30+ minutes and was stopped. "
                "Run Regenerate summaries again to continue."
            )
            _stop_digest_job(slug, error=err, aborted=True)
            with _digest_job_lock:
                return dict(_digest_jobs.get(slug, {}))
        return job

    manifest_status = str(manifest.get("digest_status") or "")
    if manifest_status != "running":
        if manifest_status in ("failed", "complete"):
            err = str(job.get("error") or "").strip()
            if not err and manifest_status == "failed":
                err = "Digest failed."
            _stop_digest_job(slug, error=err, aborted=False)
            with _digest_job_lock:
                return dict(_digest_jobs.get(slug, {}))
        if not stale:
            # Live worker (planning, section progress, refresh races) — heal manifest, don't kill.
            healed = _load_manifest(slug) or dict(manifest)
            if str(healed.get("digest_status") or "") != "running":
                healed["digest_status"] = "running"
                if not healed.get("started_at"):
                    healed["started_at"] = float(job.get("started_at") or time.time())
                healed["updated_at"] = _iso_now()
                _save_manifest(slug, healed)
            return job

    if stale:
        if manifest_status == "running":
            _interrupt_running_digest_manifest(slug)
        err = str(job.get("error") or "").strip()
        if not err:
            err = (
                "Digest had no progress for 30+ minutes and was stopped. "
                "Continue ingest to resume; completed sources were checkpointed."
            )
        _stop_digest_job(slug, error=err, aborted=True)
        with _digest_job_lock:
            return dict(_digest_jobs.get(slug, {}))
    return job


def _checkpoint_running_manifest(
    slug: str,
    planned: list[dict[str, Any]],
    results_by_index: dict[int, dict[str, Any]],
) -> None:
    """Persist per-source progress while digest_status=running (resume after interrupt)."""
    manifest = _load_manifest(slug)
    if not isinstance(manifest, dict) or str(manifest.get("digest_status") or "") != "running":
        return
    by_slug: dict[str, dict[str, Any]] = {}
    for src in manifest.get("sources") or []:
        if not isinstance(src, dict):
            continue
        sid = str(src.get("slug") or "").strip()
        if sid:
            by_slug[sid] = dict(src)
    for row in planned:
        result = results_by_index.get(int(row.get("i") or 0)) or {}
        entry = result.get("entry")
        if not isinstance(entry, dict):
            continue
        status = str(entry.get("status") or "")
        if status not in ("digested", "failed"):
            continue
        sid = str(entry.get("slug") or row.get("source_slug") or "").strip()
        if sid:
            by_slug[sid] = entry
    if not by_slug:
        return
    ordered: list[dict[str, Any]] = []
    seen: set[str] = set()
    for row in planned:
        sid = str(row.get("source_slug") or "").strip()
        if sid and sid in by_slug and sid not in seen:
            ordered.append(by_slug[sid])
            seen.add(sid)
    for sid, entry in sorted(by_slug.items()):
        if sid not in seen:
            ordered.append(entry)
    manifest["sources"] = ordered
    manifest["updated_at"] = _iso_now()
    _save_manifest(slug, manifest)


def _sweep_stale_running_manifests() -> None:
    root = _knowledge_root()
    touched = False
    stale_msg = (
        "Digest had no progress for 30+ minutes and was stopped. "
        "Continue ingest to resume; completed sources were checkpointed."
    )
    for entry in sorted(os.listdir(root)):
        manifest = _load_manifest(entry)
        if not isinstance(manifest, dict):
            continue
        if str(manifest.get("digest_status") or "") != "running":
            continue
        last_activity = _digest_last_activity_ts({}, manifest)
        if last_activity <= 0:
            manifest["digest_status"] = "complete" if manifest.get("sources") else "failed"
            manifest["started_at"] = None
            manifest["updated_at"] = _iso_now()
            _save_manifest(entry, manifest)
            _stop_digest_job(entry, error=stale_msg, aborted=True)
            touched = True
            continue
        if (time.time() - last_activity) < _DIGEST_STALE_TTL:
            continue
        manifest["digest_status"] = "partial" if manifest.get("sources") else "failed"
        manifest["started_at"] = None
        manifest["updated_at"] = _iso_now()
        _save_manifest(entry, manifest)
        _stop_digest_job(entry, error=stale_msg, aborted=True)
        touched = True
    if touched:
        invalidate_topic_index_cache()


def _ensure_startup_manifest_repair() -> None:
    global _startup_repair_done
    run_embedding_sweep = False
    with _startup_repair_lock:
        if _startup_repair_done:
            return
        _sweep_stale_running_manifests()
        root = _knowledge_root()
        if os.path.isdir(root):
            for entry in sorted(os.listdir(root)):
                if entry.startswith(".") or not os.path.isdir(os.path.join(root, entry)):
                    continue
                manifest = _load_manifest(entry)
                if isinstance(manifest, dict):
                    _maybe_repair_corpus_manifest(entry, manifest)
        _startup_repair_done = True
        run_embedding_sweep = True
    if run_embedding_sweep:
        try:
            from utils.embedding_model import startup_embedding_identity_sweep

            startup_embedding_identity_sweep()
        except Exception:
            pass


def _source_slug(stem: str, seen: set[str]) -> str:
    base = _slugify(stem)
    slug = base
    i = 2
    while slug in seen:
        slug = f"{base}_{i}"
        i += 1
    seen.add(slug)
    return slug


def _line_offsets(text: str) -> list[tuple[int, str]]:
    out: list[tuple[int, str]] = []
    cursor = 0
    for line in (text or "").splitlines(keepends=True):
        out.append((cursor, line.rstrip("\r\n")))
        cursor += len(line)
    if not out:
        out.append((0, ""))
    return out


def _clean_section_title(raw: str) -> str:
    t = re.sub(r"\s+", " ", str(raw or "").strip())
    t = re.sub(r"^[#\-\*\d\.\)\s:]+", "", t).strip()
    return t[:120] if t else ""


def _heading_to_sections(
    text: str, headings: list[tuple[int, str]], *, fallback_prefix: str = "Section"
) -> list[tuple[str, int, int]]:
    if not headings:
        return []
    ordered = sorted(headings, key=lambda h: h[0])
    starts = [max(0, min(len(text), int(pos))) for pos, _ in ordered]
    titles = [str(title or "").strip() for _, title in ordered]
    sections: list[tuple[str, int, int]] = []
    first_start = starts[0]
    if first_start >= 120:
        preface = text[:first_start].strip()
        if preface:
            sections.append(("Front Matter", 0, first_start))
    for i, start in enumerate(starts):
        end = starts[i + 1] if (i + 1) < len(starts) else len(text)
        if end <= start:
            continue
        title = _clean_section_title(titles[i]) or f"{fallback_prefix} {i + 1}"
        sections.append((title, start, end))
    return sections


def _merge_small_sections(
    text: str,
    sections: list[tuple[str, int, int]],
    *,
    min_chars: int = _SECTION_MIN_CHARS,
) -> list[tuple[str, int, int]]:
    if not sections:
        return []
    merged: list[list[Any]] = [[t, int(a), int(b)] for t, a, b in sections if int(b) > int(a)]
    if not merged:
        return []
    i = 0
    while i < len(merged):
        title, start, end = merged[i]
        section_chars = len((text[start:end] or "").strip())
        if section_chars >= min_chars or len(merged) == 1:
            i += 1
            continue
        if i > 0:
            merged[i - 1][2] = end
            merged.pop(i)
            i -= 1
            continue
        if i + 1 < len(merged):
            merged[i][2] = merged[i + 1][2]
            merged[i][0] = f"{title} / {merged[i + 1][0]}".strip(" /")
            merged.pop(i + 1)
            continue
        i += 1
    return [(str(t), int(a), int(b)) for t, a, b in merged if int(b) > int(a)]


def _cap_sections(
    sections: list[tuple[str, int, int]], *, max_count: int = _SECTION_MAX_COUNT
) -> list[tuple[str, int, int]]:
    if len(sections) <= max_count:
        return sections
    merged: list[list[Any]] = [[t, int(a), int(b)] for t, a, b in sections]
    while len(merged) > max_count and len(merged) > 1:
        best_i = 0
        best_len = None
        for i in range(len(merged) - 1):
            comb_len = max(0, int(merged[i + 1][2]) - int(merged[i][1]))
            if best_len is None or comb_len < best_len:
                best_len = comb_len
                best_i = i
        merged[best_i][0] = f"{merged[best_i][0]} / {merged[best_i + 1][0]}".strip(" /")
        merged[best_i][2] = merged[best_i + 1][2]
        merged.pop(best_i + 1)
    return [(str(t), int(a), int(b)) for t, a, b in merged if int(b) > int(a)]


def _is_clean_allcaps_heading(stripped: str) -> bool:
    """Return True if *stripped* looks like a genuine all-caps heading (no OCR
    garbage, no embedded page numbers, reasonable length)."""
    if len(stripped) > 45 or len(stripped) < 4:
        return False
    clean = sum(1 for c in stripped if c.isalpha() or c in " '-&,")
    if clean < len(stripped) * 0.90:
        return False
    if re.match(r"^\d", stripped):
        return False
    if re.search(r"\s\d+\s*$", stripped):
        return False
    return True


def _detect_sections(text: str, filename: str) -> list[tuple[str, int, int]]:
    t = (text or "").replace("\r\n", "\n")
    if len(t.strip()) < _SECTION_MIN_CHARS * 2:
        return []
    lines = _line_offsets(t)
    heading_candidates: list[list[tuple[int, str]]] = [[], [], []]
    chapter_re = re.compile(r"^\s*(chapter|part)\s+[\divxlc]+\b[:.\-\s]*(.*)$", re.IGNORECASE)
    numbered_re = re.compile(r"^\s*\d+(?:\.\d+){0,2}[.)]?\s+[A-Z][^\n]{2,120}$")
    artifact_re = re.compile(
        r"^\s*(page\s+\d+|\d+\s*/\s*\d+|copyright|all rights reserved|table of contents)\s*$",
        re.IGNORECASE,
    )

    blank_count = sum(1 for _, line in lines if not line.strip())
    blank_sparse = blank_count < max(10, len(lines) // 20)

    allcaps_relaxed: list[tuple[int, str]] = []

    for idx, (off, line) in enumerate(lines):
        stripped = line.strip()
        if not stripped:
            continue
        m = chapter_re.match(stripped)
        if m:
            suffix = _clean_section_title(m.group(2))
            _scrubbed_word = re.sub(r'[:\-]+$', '', stripped.split()[1])
            title = f"{m.group(1).title()} {_scrubbed_word}"
            if suffix:
                title = f"{title}: {suffix}"
            heading_candidates[0].append((off, title))
            continue
        if numbered_re.match(stripped):
            heading_candidates[2].append((off, _clean_section_title(stripped)))
            continue
        alpha_count = sum(1 for c in stripped if c.isalpha())
        upper_count = sum(1 for c in stripped if c.isalpha() and c.isupper())
        if (
            4 <= len(stripped) <= 60
            and alpha_count >= 4
            and upper_count >= max(3, int(alpha_count * 0.85))
            and not artifact_re.match(stripped)
        ):
            prev_blank = idx == 0 or not lines[idx - 1][1].strip()
            next_blank = idx == (len(lines) - 1) or not lines[idx + 1][1].strip()
            if prev_blank or next_blank:
                heading_candidates[1].append((off, _clean_section_title(stripped.title())))
            elif blank_sparse and _is_clean_allcaps_heading(stripped):
                allcaps_relaxed.append((off, _clean_section_title(stripped.title())))

    if blank_sparse and len(allcaps_relaxed) >= 3:
        title_freq: dict[str, int] = {}
        for _, title in allcaps_relaxed:
            key = title.lower().strip()
            title_freq[key] = title_freq.get(key, 0) + 1
        allcaps_relaxed = [
            (off, title) for off, title in allcaps_relaxed if title_freq[title.lower().strip()] <= 2
        ]
        if len(allcaps_relaxed) >= 3:
            heading_candidates[1] = heading_candidates[1] + allcaps_relaxed

    chosen: list[tuple[int, str]] = []
    for candidate in heading_candidates:
        if len(candidate) >= 3:
            chosen = candidate
            break
    if not chosen:
        return []

    sections = _heading_to_sections(t, chosen, fallback_prefix=Path(filename).stem or "Section")
    sections = _merge_small_sections(t, sections, min_chars=_SECTION_MIN_CHARS)
    sections = _cap_sections(sections, max_count=_SECTION_MAX_COUNT)
    if not sections:
        return []
    avg_len = int(sum(max(0, b - a) for _, a, b in sections) / max(1, len(sections)))
    if avg_len < _SECTION_MIN_CHARS:
        return []
    return sections


def _extract_summary_body(source_md: str) -> str:
    text = (source_md or "").strip()
    if not text:
        return ""
    m = re.search(r"(?s)\n## Summary\s*\n\n(.*?)(?:\n## Topics\s*\n\n|\Z)", text)
    if not m:
        m2 = re.search(r"(?s)\n## Summary\s*\n\n(.*)\Z", text)
        body = (m2.group(1).strip() if m2 else "")
    else:
        body = m.group(1).strip()
    return polish_knowledge_summary_body(body)


def _reuse_existing_sections_if_current(
    *,
    slug: str,
    src: str,
    filename: str,
    parent_file_hash: str,
    old_by_path: dict[str, dict[str, Any]],
    seen_slugs: set[str],
) -> list[dict[str, Any]] | None:
    """Skip PDF re-extract when cached text + manifest sections still match the parent file hash."""
    if not parent_file_hash:
        return None
    src_key = _canonical_fs_path(src) or os.path.abspath(src)
    candidates: list[tuple[int, str, dict[str, Any]]] = []
    for virtual_path, old in old_by_path.items():
        if not isinstance(old, dict) or "::section::" not in virtual_path:
            continue
        if (_canonical_fs_path(virtual_path) or "") != src_key:
            continue
        stored_parent = str(old.get("_parent_content_hash") or old.get("_source_file_hash") or "")
        if stored_parent and stored_parent != parent_file_hash:
            return None
        try:
            idx = int(virtual_path.split("::section::", 1)[1])
        except (TypeError, ValueError):
            continue
        candidates.append((idx, virtual_path, old))
    if len(candidates) < 2:
        return None
    cached = _load_raw_text_cache(slug, src)
    if not cached or not cached.strip():
        return None
    sections = _detect_sections(cached, filename)
    if len(sections) < 2:
        return None
    rows: list[dict[str, Any]] = []
    for idx, virtual_path, old in sorted(candidates, key=lambda t: t[0]):
        if idx >= len(sections):
            return None
        title, start, end = sections[idx]
        section_text = cached[start:end].strip()
        if not section_text:
            continue
        section_title = str(old.get("filename") or _clean_section_title(title) or f"Section {idx + 1}")
        preserved = str(old.get("slug") or "").strip()
        if preserved:
            source_slug = preserved
            seen_slugs.add(source_slug)
        else:
            source_slug = _source_slug(_slugify(section_title), seen_slugs)
        rows.append(
            {
                "src": virtual_path,
                "filename": section_title,
                "ext": ".section",
                "source_slug": source_slug,
                "old": old,
                "_pre_extracted_text": section_text,
                "_parallel_ok": True,
                "_section_of": filename,
                "_parent_file_hash": parent_file_hash,
            }
        )
    if len(rows) < 2:
        return None
    _set_digest_phase(slug, "planning", detail=f"Reusing sections: {filename}")
    return rows


def _expand_large_sources(
    planned: list[dict[str, Any]],
    slug: str,
    seen_slugs: set[str],
    old_by_path: dict[str, dict[str, Any]],
    is_url: bool,
) -> list[dict[str, Any]]:
    if is_url:
        return planned
    expanded: list[dict[str, Any]] = []
    for row in planned:
        _abort_if_digest_pause(slug)
        src = str(row.get("src") or "")
        filename = str(row.get("filename") or os.path.basename(src) or "source")
        _set_digest_job(
            slug,
            running=True,
            progress="planning",
            current_file=f"Planning: {filename}",
        )
        ext = str(row.get("ext") or "").lower()
        if not src:
            expanded.append(row)
            continue
        should_try = ext in {".pdf", ".epub", ".docx"}
        if not should_try:
            try:
                should_try = os.path.getsize(src) > 200_000
            except OSError:
                should_try = False
        if not should_try:
            expanded.append(row)
            continue
        _abort_if_digest_pause(slug)
        try:
            parent_file_hash = _sha256_file(src)
        except OSError:
            parent_file_hash = ""
        reused = _reuse_existing_sections_if_current(
            slug=slug,
            src=src,
            filename=filename,
            parent_file_hash=parent_file_hash,
            old_by_path=old_by_path,
            seen_slugs=seen_slugs,
        )
        if reused is not None:
            expanded.extend(reused)
            continue
        extracted = _load_raw_text_cache(slug, src)
        if (
            extracted
            and should_try
            and len((extracted or "").strip()) < _SECTION_SPLIT_MIN_CHARS
        ):
            extracted = None
        if not extracted:
            _set_digest_phase(slug, "planning", detail=f"Extracting: {filename}")
            try:
                extracted = _extract_plaintext(src)
            except Exception:
                expanded.append(row)
                continue
        _abort_if_digest_pause(slug)
        if extracted and extracted.strip():
            _save_raw_text_cache(slug, src, extracted)
        if len((extracted or "").strip()) < _SECTION_SPLIT_MIN_CHARS:
            expanded.append(row)
            continue
        if not parent_file_hash:
            try:
                parent_file_hash = _sha256_file(src)
            except OSError:
                parent_file_hash = ""
        sections = _detect_sections(extracted, str(row.get("filename") or os.path.basename(src)))
        if len(sections) >= 2:
            section_of = str(row.get("filename") or os.path.basename(src))
            for idx, (title, start, end) in enumerate(sections):
                _abort_if_digest_pause(slug)
                section_text = extracted[start:end].strip()
                if not section_text:
                    continue
                section_title = _clean_section_title(title) or f"Section {idx + 1}"
                virtual_path = f"{src}::section::{idx}"
                old = old_by_path.get(virtual_path)
                expanded.append(
                    {
                        "src": virtual_path,
                        "filename": section_title,
                        "ext": ".section",
                        "source_slug": _source_slug(_slugify(section_title), seen_slugs),
                        "old": old if isinstance(old, dict) else None,
                        "_pre_extracted_text": section_text,
                        "_parallel_ok": True,
                        "_section_of": section_of,
                        "_parent_file_hash": parent_file_hash,
                    }
                )
            continue
        kept = dict(row)
        kept["_pre_extracted_text"] = extracted
        if parent_file_hash:
            kept["_parent_file_hash"] = parent_file_hash
        expanded.append(kept)
    if expanded:
        _set_digest_job(
            slug,
            running=True,
            progress=f"0/{len(expanded)}",
            current_file="",
        )
    return expanded


def _set_digest_job(slug: str, **updates: Any) -> None:
    with _digest_job_lock:
        prev = dict(_digest_jobs.get(slug, {}))
        prev.update(updates)
        prev.setdefault("slug", slug)
        prev.setdefault("started_at", time.time())
        prev["updated_at"] = time.time()
        if updates.get("running") is False:
            prev.pop("progress", None)
            prev.pop("current_file", None)
        _digest_jobs[slug] = prev


def _set_digest_phase(slug: str, phase: str, *, detail: str = "") -> None:
    """Non-``N/M`` progress for planning / synthesis / indexing (UI shows indeterminate)."""
    label = detail.strip() or phase.replace("_", " ").capitalize()
    _set_digest_job(slug, running=True, progress=phase, current_file=label)


def _mark_digest_wrap_up_if_done(slug: str, done_count: int, total: int) -> None:
    """Switch UI off ``N/N`` before the final manifest checkpoint (can be slow on large corpora)."""
    if total > 0 and done_count >= total:
        _set_digest_phase(slug, "synthesizing", detail="Preparing corpus wrap-up…")


def _complete_with_fallback_timed(
    messages: list[dict[str, str]],
    *,
    max_tokens: int = 512,
    temperature: float = 0.2,
    timeout_s: float | None = None,
) -> str:
    limit = timeout_s if timeout_s is not None else max(120.0, _digest_llm_read_timeout_s() * 2.0)
    with ThreadPoolExecutor(max_workers=1) as pool:
        fut = pool.submit(
            _complete_with_fallback,
            messages,
            max_tokens=max_tokens,
            temperature=temperature,
        )
        try:
            out = fut.result(timeout=limit)
        except FuturesTimeoutError:
            _debug_log(f"[knowledge] LLM call timed out after {limit:.0f}s; using extractive fallback", "WARN")
            return _extractive_fallback_summary(messages)
    return str(out or "")


def _summarize_source_text_with_method(
    filename: str,
    text: str,
    *,
    source_path: str = "",
    for_upgrade: bool = False,
) -> tuple[str, list[str], str]:
    upgrade_mode = for_upgrade or _summary_upgrade_mode_active()
    llm_kw = {
        "utility_only": upgrade_mode,
        "relax_quality_gates": upgrade_mode,
        "activity_operation": "knowledge_summary_upgrade" if upgrade_mode else None,
        "suppress_utility_activity_errors": True,
    }

    def _llm_complete(
        messages: list[dict[str, str]],
        *,
        max_tokens: int,
        temperature: float = 0.2,
    ) -> tuple[str, str]:
        if upgrade_mode:
            return _upgrade_llm_complete(
                messages, max_tokens=max_tokens, temperature=temperature
            )
        return _complete_with_method(
            messages,
            max_tokens=max_tokens,
            temperature=temperature,
            allow_utility_model=True,
            **llm_kw,
        )
    raw = text or ""
    raw_len = len(raw)
    raw_degenerate = _is_degenerate_extracted_text(raw)
    normalized = _normalize_extracted_text_for_digest(raw)
    if raw_len and len(normalized) < raw_len * 0.3:
        _debug_log(
            f"[knowledge] Collapsed repetitive lines in {filename}: {raw_len} -> {len(normalized)} chars",
            "INFO",
        )
    if len(normalized) > _LARGE_SOURCE_WARN_CHARS:
        _debug_log(f"[knowledge] Large source {filename}: {len(normalized)} chars before chunking")
    chunks = _chunk_text(normalized)
    if not chunks:
        raise RuntimeError("No text extracted")
    norm_degenerate = _is_degenerate_extracted_text(normalized)
    norm_window_degenerate = _has_dominated_by_repeated_window(normalized)
    raw_window_degenerate = _has_dominated_by_repeated_window(raw)
    normalized_nonempty_lines = sum(1 for ln in normalized.splitlines() if ln.strip())
    recovered_sufficient_text = normalized_nonempty_lines >= 20 and len(normalized) >= 1200
    if (raw_degenerate or raw_window_degenerate) and not (norm_degenerate or norm_window_degenerate):
        if not recovered_sufficient_text:
            _debug_log(
                f"[knowledge] Repetitive extract for {filename} collapsed to too little usable text; skipping LLM summary",
                "INFO",
            )
            summary = _degenerate_source_summary(filename, source_path, normalized)
            return summary, chunks, "degenerate_skip"
        _debug_log(
            f"[knowledge] Raw extract looked repetitive for {filename}, but normalized text recovered enough diversity; proceeding with summary",
            "INFO",
        )
    if norm_degenerate or (norm_window_degenerate and not recovered_sufficient_text):
        _debug_log(f"[knowledge] Degenerate / repetitive extract for {filename}; skipping LLM summary", "INFO")
        summary = _degenerate_source_summary(filename, source_path, normalized)
        return summary, chunks, "degenerate_skip"
    if upgrade_mode:
        _sum_sys = _SUMMARY_UPGRADE_SYS
        _chunk_sys = (
            "Write 3–5 markdown bullets of key facts from this chunk only. "
            "Output ONLY bullets — no labels, thinking, or task commentary."
        )
    else:
        _sum_sys = (
            "Summarize the source document in 6–12 concise markdown bullet points. "
            "Cover major themes, events, and key facts. Be selective — highlight what matters most. "
            "Never mention the user, prompt, request, conversation, chunks, or your process. "
            "Do not include analysis preambles. Be faithful; do not invent facts. "
            "If the excerpt is mostly the same line or phrase repeated (bad extraction or boilerplate), "
            "say so in 1–2 sentences and do NOT repeat that phrase."
        )
        _chunk_sys = (
            "Summarize this chunk from a larger document in 3–5 concise markdown bullets covering key facts only. "
            "Never mention the user, prompt, request, conversation, chunks, or your process. "
            "If the chunk is mostly repetition of one line, state that once only."
        )
    if len(chunks) == 1:
        summary, summary_method = _llm_complete(
            [
                {"role": "system", "content": _sum_sys},
                {"role": "user", "content": f"Source: {filename}\n\n{chunks[0]}"},
            ],
            max_tokens=800,
            temperature=0.2,
        )
        summary = _sanitize_summary_output(summary)
    else:
        work_items = list(enumerate(chunks, 1))
        ordered: dict[int, str] = {}
        if upgrade_mode:
            for j, chunk in work_items:
                chunk_text, _chunk_method = _llm_complete(
                    [
                        {"role": "system", "content": _chunk_sys},
                        {
                            "role": "user",
                            "content": f"Source: {filename} chunk {j}/{len(chunks)}\n\n{chunk}",
                        },
                    ],
                    max_tokens=400,
                    temperature=0.2,
                )
                if chunk_text.strip():
                    ordered[j] = _sanitize_summary_output(chunk_text)
        else:
            with ThreadPoolExecutor(max_workers=min(_SUMMARY_WORKERS, len(work_items))) as pool:
                futures = {
                    pool.submit(
                        _complete_with_method,
                        [
                            {"role": "system", "content": _chunk_sys},
                            {
                                "role": "user",
                                "content": f"Source: {filename} chunk {j}/{len(chunks)}\n\n{chunk}",
                            },
                        ],
                        max_tokens=400,
                        temperature=0.2,
                        allow_utility_model=True,
                        suppress_utility_activity_errors=True,
                    ): j
                    for j, chunk in work_items
                }
                chunk_timeout = _digest_llm_read_timeout_s() + _DIGEST_CHUNK_FUTURE_GRACE_S
                for fut in as_completed(futures):
                    try:
                        chunk_text, _chunk_method = fut.result(timeout=chunk_timeout)
                    except Exception as exc:
                        _debug_log(f"[knowledge] Chunk summary timed out for {filename}: {exc}", "WARN")
                        continue
                    ordered[futures[fut]] = _sanitize_summary_output(chunk_text)
        if upgrade_mode:
            chunk_summaries = [
                ordered[j] for j, _ in work_items if j in ordered and ordered[j].strip()
            ]
            if not chunk_summaries:
                return "", chunks, "utility_failed"
        else:
            chunk_summaries = [
                ordered[j]
                for j, _ in work_items
                if j in ordered and not _is_garbage_summary(ordered[j])
            ]
        if not chunk_summaries:
            summary = _sanitize_summary_output(
                _extractive_fallback_summary([{"role": "user", "content": f"Source: {filename}\n\n{normalized}"}])
            )
            summary_method = "extractive_fallback"
        else:
            combine_input = list(chunk_summaries)
            if len(chunks) > _HIERARCHICAL_CHUNK_THRESHOLD and len(chunk_summaries) > _HIERARCHICAL_GROUP_SIZE:
                grouped: list[str] = []
                for i in range(0, len(chunk_summaries), _HIERARCHICAL_GROUP_SIZE):
                    group = chunk_summaries[i : i + _HIERARCHICAL_GROUP_SIZE]
                    inter_text, _inter_method = _llm_complete(
                        [
                            {
                                "role": "system",
                                "content": (
                                    "Combine these chunk summaries into 6-10 concise markdown bullets. "
                                    "Capture major facts only. Avoid mentioning prompts, chunks, or process."
                                ),
                            },
                            {"role": "user", "content": "\n\n".join(group)},
                        ],
                        max_tokens=500,
                        temperature=0.2,
                    )
                    cleaned = _sanitize_summary_output(inter_text)
                    if cleaned and not _is_garbage_summary(cleaned):
                        grouped.append(cleaned)
                if grouped:
                    combine_input = grouped
            combo_max_tokens = min(2048, max(1024, len(chunk_summaries) * 60))
            summary, summary_method = _llm_complete(
                [
                    {
                        "role": "system",
                        "content": (
                            "Combine these chunk summaries into one coherent markdown summary of 8–15 bullet points. "
                            "Cover the major themes, events, and facts. Merge overlapping points; omit minor details. "
                            "Never mention user instructions, prompts, conversation context, or your process."
                        ),
                    },
                    {"role": "user", "content": "\n\n".join(combine_input)},
                ],
                max_tokens=combo_max_tokens,
                temperature=0.2,
            )
            summary = _sanitize_summary_output(summary)
            if upgrade_mode and summary_method == "utility_failed":
                return "", chunks, "utility_failed"
    if not upgrade_mode:
        if _summary_contains_reasoning_trace(summary):
            _debug_log(f"[knowledge] Discarding reasoning-trace summary for {filename}; using extractive fallback", "WARN")
            summary = _sanitize_summary_output(
                _extractive_fallback_summary([{"role": "user", "content": f"Source: {filename}\n\n{normalized}"}])
            )
            summary_method = "extractive_fallback"
        if _is_garbage_summary(summary):
            _debug_log(f"[knowledge] Discarding repetitive model summary for {filename}; using extractive fallback", "WARN")
            summary = _sanitize_summary_output(
                _extractive_fallback_summary([{"role": "user", "content": f"Source: {filename}\n\n{normalized}"}])
            )
            summary_method = "extractive_fallback"
            if _is_garbage_summary(summary):
                _debug_log(f"[knowledge] Extractive fallback was still low quality for {filename}; using safe placeholder", "WARN")
                summary = _degenerate_source_summary(filename, source_path, normalized)
                summary_method = "degenerate_skip"
    if not upgrade_mode:
        summary = polish_knowledge_summary_body(summary)
    return summary, chunks, summary_method


def _summarize_source_text(filename: str, text: str, *, source_path: str = "") -> tuple[str, list[str]]:
    summary, chunks, _method = _summarize_source_text_with_method(filename, text, source_path=source_path)
    return summary, chunks


def _summary_was_skipped(summary_text: str) -> bool:
    return bool(re.search(r"\*\*Automatic summary skipped\.\*\*", summary_text or ""))


def _summary_needs_upgrade(summary_text: str) -> bool:
    """True when a stored note was produced by an older summarizer or used extractive fallback."""
    text = str(summary_text or "")
    ver_m = re.search(r"\*\*Summarizer version:\*\*\s*`([^`]+)`", text)
    if ver_m and ver_m.group(1) != _SUMMARIZER_VERSION:
        return True
    method_m = re.search(r"\*\*Summary method:\*\*\s*`([^`]+)`", text)
    if method_m and method_m.group(1) in ("extractive_fallback", "degenerate_skip"):
        return True
    body = _extract_summary_body(text)
    if body and (
        _is_cot_narration_or_prompt_leak(body)
        or _summary_is_excerpt_commentary(body)
        or _summary_echoes_upgrade_prompt(body)
        or not _upgrade_summary_has_substance(body, "utility_model")
    ):
        return True
    return False


def _try_main_model_upgrade_summary(
    filename: str,
    text: str,
) -> tuple[str, list[str], str]:
    """One-shot main/chat model summary for bulk upgrade fallback."""
    normalized = _normalize_extracted_text_for_digest(text or "")
    chunks = _chunk_text(normalized) or [normalized[:8000]]
    chunk = chunks[0]
    raw_main = _main_llm_complete(
        [
            {"role": "system", "content": _SUMMARY_UPGRADE_SYS},
            {"role": "user", "content": f"Source: {filename}\n\n{chunk}"},
        ],
        max_tokens=800,
        temperature=0.2,
    )
    if not raw_main or not str(raw_main).strip():
        return "", chunks, "main_model"
    cleaned_main = _sanitize_summary_output(str(raw_main).strip())
    if _summary_acceptable_for_persist("main_model", cleaned_main, for_upgrade=True):
        return cleaned_main, chunks, "main_model"
    return cleaned_main, chunks, "main_model"


def _summary_acceptable_for_persist(
    method: str, summary: str, *, for_upgrade: bool = False
) -> bool:
    """Only LLM summaries are worth persisting during an explicit upgrade pass."""
    text = (summary or "").strip()
    upgrade = for_upgrade or _summary_upgrade_mode_active()
    if str(method or "") not in ("utility_model", "main_model"):
        return False
    if _summary_was_skipped(summary):
        return False
    if upgrade:
        # Bulk regen already paid for utility-only LLM time — persist real output.
        ok = len(text) >= 20 and any(ln.strip() for ln in text.splitlines())
        if ok and (
            _summary_contains_reasoning_trace(text)
            or _is_cot_narration_or_prompt_leak(text)
            or _summary_is_excerpt_commentary(text)
            or _summary_echoes_upgrade_prompt(text)
            or _is_garbage_summary(text)
        ):
            ok = False
        elif ok and not _upgrade_summary_has_substance(text, method):
            ok = False
        return ok
    if len(text) < 40:
        return False
    if _is_garbage_summary(summary):
        return False
    return True


def _summarize_source_for_upgrade(
    filename: str,
    text: str,
    *,
    source_path: str = "",
    max_attempts: int = 5,
    slug: str = "",
    progress_index: int = 0,
    progress_total: int = 0,
    upgraded: int = 0,
    skipped: int = 0,
    failed: int = 0,
) -> tuple[str, list[str], str]:
    """Retry summarization when the LLM is briefly unavailable during bulk upgrades."""
    last: tuple[str, list[str], str] = ("", [], "unknown")
    utility_attempts = min(max(1, max_attempts), 2)
    for attempt in range(utility_attempts):
        if slug:
            detail = filename
            if utility_attempts > 1:
                detail = f"{detail} (attempt {attempt + 1}/{utility_attempts})"
            _set_digest_job(
                slug,
                running=True,
                progress=f"{progress_index}/{progress_total}" if progress_total > 0 else "",
                current_file=f"Summarizing — {detail}",
                job_kind=JOB_KIND_SUMMARY_UPGRADE,
                upgraded=upgraded,
                skipped=skipped,
                failed=failed,
            )
        if attempt > 0:
            time.sleep(min(5.0 * attempt, 30.0))
        summary, chunks, method = _summarize_source_text_with_method(
            filename, text, source_path=source_path, for_upgrade=True
        )
        last = (summary, chunks, method)
        if _summary_acceptable_for_persist(method, summary, for_upgrade=True):
            return last
        # Utility emitted meta/CoT — try main model immediately (don't burn 5 identical retries).
        if attempt == 0:
            if slug:
                _set_digest_job(
                    slug,
                    running=True,
                    progress=f"{progress_index}/{progress_total}" if progress_total > 0 else "",
                    current_file=f"Main model — {filename}",
                    job_kind=JOB_KIND_SUMMARY_UPGRADE,
                    upgraded=upgraded,
                    skipped=skipped,
                    failed=failed,
                )
            main_summary, main_chunks, main_method = _try_main_model_upgrade_summary(
                filename, text
            )
            last = (main_summary, main_chunks, main_method)
            if _summary_acceptable_for_persist(main_method, main_summary, for_upgrade=True):
                return last
    return last


def _summary_contains_reasoning_trace(summary_text: str) -> bool:
    text = str(summary_text or "")
    return bool(
        re.search(r"(?i)\bthinking process\b", text)
        or re.search(r"(?i)\banalyze the request\b", text)
        or re.search(r"(?i)<think>", text)
        or re.search(r"(?i)<\|channel>thought", text)
        or re.search(r"(?i)\bconversation history\b", text)
        or re.search(r"(?i)\bthe user wants me\b", text)
        or re.search(r"(?i)\bno specific content was provided\b", text)
    )


def _normalize_skipped_source_md(source_md: str) -> str:
    text = source_md or ""
    if "Automatic summary skipped." not in text:
        return text
    # Ensure skipped summaries never keep stale lexical topics from older digests.
    text = re.sub(r"(?s)\n## Topics\s*\n\n.*$", "\n## Topics\n\n- (none)\n", text.rstrip() + "\n")
    return text


def _digest_one_source(
    *,
    slug: str,
    src: str,
    filename: str,
    source_slug: str,
    old: dict[str, Any] | None,
    update: bool,
    is_url_input: bool,
    pre_extracted_text: str | None = None,
    parent_file_hash: str | None = None,
) -> dict[str, Any]:
    try:
        if pre_extracted_text is not None:
            text = pre_extracted_text
            content_hash = hashlib.sha256(text.encode("utf-8", errors="ignore")).hexdigest()
        else:
            text = _fetch_url_text(src) if is_url_input else _extract_plaintext(src)
            content_hash = (
                hashlib.sha256(text.encode("utf-8", errors="ignore")).hexdigest() if is_url_input else _sha256_file(src)
            )
        if update and old and str(old.get("content_hash_sha256")) == content_hash:
            entry = dict(old)
            entry["status"] = "digested"
            if parent_file_hash:
                entry["_parent_content_hash"] = str(parent_file_hash)
                entry["_source_file_hash"] = str(parent_file_hash)
            elif not is_url_input:
                real_src = _canonical_fs_path(src)
                if real_src and os.path.isfile(real_src):
                    try:
                        entry["_source_file_hash"] = _sha256_file(real_src)
                    except OSError:
                        pass
            old_path = _source_page_path(slug, source_slug)
            old_summary = _read_text_file(old_path) if os.path.isfile(old_path) else ""
            if _summary_was_skipped(old_summary) or _summary_contains_reasoning_trace(old_summary) or _summary_needs_upgrade(old_summary):
                # Heuristics can improve over time; unchanged content may become summarizable.
                summary, chunks, summary_method = _summarize_source_text_with_method(
                    filename, text, source_path=src
                )
                if _summary_was_skipped(summary):
                    topics = []
                    source_md = _normalize_skipped_source_md(old_summary) if old_summary else (
                        f"# {filename}\n\n"
                        f"**Source path:** `{src}`\n\n"
                        f"**Summarizer version:** `{_SUMMARIZER_VERSION}`\n\n"
                        f"**Summary method:** `{summary_method}`\n\n"
                        "## Summary\n\n"
                        f"{summary.strip()}\n\n"
                        "## Topics\n\n- (none)\n"
                    )
                else:
                    topics = _extract_topics(summary, limit=8)
                    source_md = _render_source_markdown(
                        filename, src, summary, topics, summary_method=summary_method
                    )
                _write_text(old_path, source_md)
                entry["topics"] = topics
                entry["chunks"] = len(chunks)
                entry["chars_extracted"] = len(text)
                return {"kind": "success", "entry": entry, "source_md": source_md}
            return {"kind": "skipped", "entry": entry, "source_md": old_summary}

        summary, chunks, summary_method = _summarize_source_text_with_method(filename, text, source_path=src)
        if _summary_was_skipped(summary):
            topics = []
        else:
            topics = _extract_topics(summary, limit=8)
        source_md = _render_source_markdown(
            filename, src, summary, topics, summary_method=summary_method
        )
        _write_text(_source_page_path(slug, source_slug), source_md)
        entry: dict[str, Any] = {
            "filename": filename,
            "path": src,
            "slug": source_slug,
            "content_hash_sha256": content_hash,
            "status": "digested",
            "topics": topics,
            "chars_extracted": len(text),
            "chunks": len(chunks),
        }
        _annotate_source_repo_paths(entry, src)
        if parent_file_hash:
            entry["_parent_content_hash"] = str(parent_file_hash)
            entry["_source_file_hash"] = str(parent_file_hash)
        elif not is_url_input:
            real_src = _canonical_fs_path(src)
            if real_src and os.path.isfile(real_src):
                try:
                    entry["_source_file_hash"] = _sha256_file(real_src)
                except OSError:
                    pass
        return {
            "kind": "success",
            "source_md": source_md,
            "entry": entry,
        }
    except Exception as exc:
        return {
            "kind": "failed",
            "entry": {
                "filename": filename,
                "path": src,
                "slug": source_slug,
                "status": "failed",
                "error": str(exc)[:300],
                "topics": [],
                "chars_extracted": 0,
                "chunks": 0,
            },
            "error": str(exc),
        }


def _fail_running_digest_manifest(slug: str) -> None:
    manifest = _load_manifest(slug) or {}
    if str(manifest.get("digest_status") or "") != "running":
        return
    manifest["digest_status"] = "failed"
    manifest["started_at"] = None
    manifest["updated_at"] = _iso_now()
    _save_manifest(slug, manifest)
    invalidate_topic_index_cache()


def _interrupt_running_digest_manifest(slug: str) -> None:
    """Mark a stale/interrupted running digest so status polling matches cleared jobs."""
    manifest = _load_manifest(slug) or {}
    if str(manifest.get("digest_status") or "") != "running":
        return
    manifest["digest_status"] = "partial" if manifest.get("sources") else "failed"
    manifest["started_at"] = None
    manifest["updated_at"] = _iso_now()
    _save_manifest(slug, manifest)
    invalidate_topic_index_cache()


def _digest_sync(
    path: str,
    label: str,
    *,
    update: bool,
    emit_progress: bool = True,
    ingest_profile: str | None = None,
    path_filter: set[str] | None = None,
    force_finalize: bool = False,
) -> dict[str, Any]:
    raw_path = (path or "").strip()
    is_url_input = bool(re.match(r"^https?://", raw_path, flags=re.IGNORECASE))
    source_path = ""
    resolve_err = ""
    if is_url_input:
        source_path = raw_path
    else:
        source_path, resolve_err = safe_resolve_path(
            _workspace_path(),
            raw_path,
            allow_write=False,
            create_parents=False,
        )
        if resolve_err or not source_path:
            return {"ok": False, "error": resolve_err or "Path is required."}
        if is_dangerous_path(source_path):
            return {"ok": False, "error": f"Blocked dangerous path: {source_path}"}
        if not os.path.exists(source_path):
            return {"ok": False, "error": f"Source path does not exist: {source_path}"}

    slug, slug_conflict = _choose_corpus_slug(label, update)
    if slug_conflict and not update:
        return {
            "ok": False,
            "slug": slug,
            "error": f"A corpus with label '{label}' already exists. Use update=True to refresh it.",
        }
    if not update and len(list_knowledge_corpora()) >= _MAX_CORPORA:
        return {"ok": False, "error": f"Corpus limit reached ({_MAX_CORPORA}). Remove an existing corpus first."}
    corpus_dir = os.path.join(_knowledge_root(), slug)
    os.makedirs(os.path.join(corpus_dir, "sources"), exist_ok=True)
    profile = normalize_ingest_profile(
        ingest_profile or (_load_manifest(slug) or {}).get("ingest_profile"),
        source_path=source_path,
    )

    with _digest_lock:
        manifest = _load_manifest(slug) or {}
        if update:
            manifest = _maybe_repair_corpus_manifest(slug, manifest)
        prior_digest_status = str(manifest.get("digest_status") or "")
        running = str(manifest.get("digest_status") or "") == "running"
        started_at = float(manifest.get("started_at") or 0.0)
        if running and started_at > 0 and (time.time() - started_at) < _DIGEST_STALE_TTL:
            return {"ok": False, "slug": slug, "error": "Digest already in progress for this corpus."}

        now_iso = _iso_now()
        manifest = {
            "schema_version": KNOWLEDGE_SCHEMA_VERSION,
            "label": label,
            "source_path": source_path,
            "slug": slug,
            "ingest_profile": profile,
            "summarizer_version": _SUMMARIZER_VERSION,
            "created_at": manifest.get("created_at") or now_iso,
            "updated_at": now_iso,
            "digest_status": "running",
            "started_at": time.time(),
            "topics": list(manifest.get("topics") or []),
            "sources": list(manifest.get("sources") or []),
            "security_scoped_bookmarks_required": _app_sandbox_enabled(),
            "security_scoped_bookmarks": (
                dict(manifest.get("security_scoped_bookmarks") or {})
                if isinstance(manifest.get("security_scoped_bookmarks"), dict)
                else {}
            ),
        }
        _save_manifest(slug, manifest)

    old_sources = [s for s in manifest.get("sources", []) if isinstance(s, dict)]
    old_by_path = {str(s.get("path")): s for s in old_sources}
    planned: list[dict[str, Any]] = []
    results_by_index: dict[int, dict[str, Any]] = {}
    try:
        candidates, capped = (
            ([source_path], False) if is_url_input else _walk_candidate_files(source_path, ingest_profile=profile, slug=slug)
        )
        if path_filter is not None:
            filt = {
                os.path.abspath(os.path.expanduser(str(p).split("::section::", 1)[0]))
                for p in path_filter
                if p
            }
            candidates = [c for c in candidates if os.path.abspath(c) in filt]
            if not candidates:
                if force_finalize and manifest.get("sources"):
                    remaining = _remaining_manifest_source_paths(manifest)
                    if remaining:
                        return _digest_sync(
                            path,
                            label,
                            update=update,
                            emit_progress=emit_progress,
                            ingest_profile=ingest_profile,
                            path_filter=remaining,
                            force_finalize=True,
                        )
                    return _digest_sync(
                        path,
                        label,
                        update=update,
                        emit_progress=emit_progress,
                        ingest_profile=ingest_profile,
                        path_filter=None,
                        force_finalize=True,
                    )
                manifest = _load_manifest(slug) or manifest
                manifest["ingest_profile"] = profile
                manifest["updated_at"] = _iso_now()
                manifest["started_at"] = None
                manifest["digest_status"] = "complete" if manifest.get("sources") else "failed"
                _save_manifest(slug, manifest)
                invalidate_topic_index_cache()
                _set_digest_job(
                    slug,
                    running=False,
                    done=True,
                    error="",
                    summary="No stale sources to refresh.",
                )
                return {
                    "ok": True,
                    "slug": slug,
                    "label": label,
                    "message": "No stale sources to refresh.",
                }
        if capped and emit_progress:
            emit_live_view_preview(
                f"File scan reached cap of {_MAX_DIGEST_FILES}. Some files were not included.",
                tool_name="digest_corpus",
                progress_pct=1,
            )
        if not candidates:
            manifest["digest_status"] = "failed"
            manifest["started_at"] = None
            manifest["updated_at"] = _iso_now()
            _save_manifest(slug, manifest)
            return {"ok": False, "slug": slug, "error": "No supported files found"}

        seen_source_slugs: set[str] = set()
        for i, src in enumerate(candidates, 1):
            _abort_if_digest_pause(slug)
            filename = os.path.basename(src)
            ext = Path(src).suffix.lower()
            old = old_by_path.get(src)
            if old and isinstance(old, dict):
                preserved = str(old.get("slug") or "")
                if preserved:
                    source_slug = preserved
                    seen_source_slugs.add(source_slug)
                else:
                    source_slug = _source_slug(Path(filename).stem, seen_source_slugs)
            else:
                source_slug = _source_slug(Path(filename).stem, seen_source_slugs)
            planned.append(
                {
                    "i": i,
                    "src": src,
                    "filename": filename,
                    "ext": ext,
                    "source_slug": source_slug,
                    "old": old if isinstance(old, dict) else None,
                }
            )
        planned = _expand_large_sources(planned, slug, seen_source_slugs, old_by_path, is_url_input)
        raw_texts_by_slug: dict[str, str] = {}
        for row in planned:
            pet = row.get("_pre_extracted_text")
            s_slug = str(row.get("source_slug") or "")
            if pet and s_slug:
                raw_texts_by_slug[s_slug] = str(pet)
        for idx, row in enumerate(planned, 1):
            row["i"] = idx

        def _finish_paused_digest() -> dict[str, Any]:
            return _complete_paused_digest(slug, label, planned, results_by_index)

        if _digest_pause_requested(slug):
            return _finish_paused_digest()

        success_count = 0
        fail_count = 0
        skipped_count = 0
        failed_names: list[str] = []

        total = len(planned)
        safe_parallel = [
            row
            for row in planned
            if (
                bool(row.get("_parallel_ok"))
                or ((not is_url_input) and str(row.get("ext") or "").lower() in _PARALLEL_SAFE_EXTENSIONS)
            )
        ]
        sequential = [row for row in planned if row not in safe_parallel]

        def _consume_result(i: int, result: dict[str, Any], filename: str) -> None:
            nonlocal success_count, fail_count, skipped_count
            results_by_index[i] = result
            kind = str(result.get("kind") or "")
            if kind == "success":
                success_count += 1
            elif kind == "failed":
                fail_count += 1
                failed_names.append(filename)
                entry = result.get("entry") if isinstance(result.get("entry"), dict) else {}
                err = str(result.get("error") or entry.get("error") or "unknown error")
                _emit_knowledge_digest_source_error(filename, err)
            elif kind == "skipped":
                skipped_count += 1
            _checkpoint_running_manifest(slug, planned, results_by_index)

        per_source_timeout_s = min(
            900.0,
            (_digest_llm_read_timeout_s() + _DIGEST_CHUNK_FUTURE_GRACE_S) * 24.0,
        )

        approx_parallel_chars = 0
        for row in safe_parallel:
            if row.get("_pre_extracted_text") is not None:
                continue
            src_path = str(row.get("src") or "")
            try:
                approx_parallel_chars += max(0, int(os.path.getsize(src_path)))
            except OSError:
                approx_parallel_chars += 0
        if approx_parallel_chars > _PARALLEL_FILE_CHAR_BUDGET:
            sequential = planned
            safe_parallel = []

        done_count = 0
        paused = False
        if _digest_pause_requested(slug):
            return _finish_paused_digest()

        if safe_parallel:
            if emit_progress:
                emit_live_view_preview(
                    f"Digesting {len(safe_parallel)} safe-text sources in parallel...",
                    tool_name="digest_corpus",
                    progress_pct=1,
                )
            pool = ThreadPoolExecutor(max_workers=min(_FILE_SUMMARY_WORKERS, len(safe_parallel)))
            futures = {
                pool.submit(
                    _digest_one_source,
                    slug=slug,
                    src=str(row["src"]),
                    filename=str(row["filename"]),
                    source_slug=str(row["source_slug"]),
                    old=row.get("old") if isinstance(row.get("old"), dict) else None,
                    update=bool(update),
                    is_url_input=bool(is_url_input),
                    pre_extracted_text=(
                        str(row.get("_pre_extracted_text")) if row.get("_pre_extracted_text") is not None else None
                    ),
                    parent_file_hash=(
                        str(row.get("_parent_file_hash")) if row.get("_parent_file_hash") else None
                    ),
                ): row
                for row in safe_parallel
            }
            try:
                for fut in as_completed(futures):
                    row = futures[fut]
                    i = int(row["i"])
                    filename = str(row["filename"])
                    try:
                        result = fut.result(timeout=per_source_timeout_s)
                    except Exception as exc:
                        result = {
                            "kind": "failed",
                            "entry": {
                                "filename": filename,
                                "path": str(row.get("src") or ""),
                                "slug": str(row.get("source_slug") or ""),
                                "status": "failed",
                                "error": f"digest timed out: {exc}"[:300],
                                "topics": [],
                                "chars_extracted": 0,
                                "chunks": 0,
                            },
                            "error": str(exc),
                        }
                    done_count += 1
                    section_of = str(row.get("_section_of") or "").strip()
                    display = f"{section_of} -- {filename}" if section_of else filename
                    if emit_progress:
                        emit_live_view_preview(
                            (
                                f"Digesting section {done_count}/{total} of {section_of}..."
                                if section_of
                                else f"Digesting source {done_count}/{total}..."
                            ),
                            tool_name="digest_corpus",
                            progress_pct=int(100 * done_count / max(1, total)),
                        )
                        _emit_input_status(f"Studying: {display}")
                    _set_digest_job(slug, running=True, progress=f"{done_count}/{total}", current_file=display, total=total)
                    _mark_digest_wrap_up_if_done(slug, done_count, total)
                    _consume_result(i, result, display)
                    if _digest_pause_requested(slug):
                        paused = True
                        break
            finally:
                if paused:
                    pool.shutdown(wait=False, cancel_futures=True)
                else:
                    pool.shutdown(wait=True)
            if paused:
                return _finish_paused_digest()

        if not paused:
            with ThreadPoolExecutor(max_workers=1) as sequential_pool:
                for row in sequential:
                    if _digest_pause_requested(slug):
                        return _finish_paused_digest()
                    i = int(row["i"])
                    filename = str(row["filename"])
                    done_count += 1
                    section_of = str(row.get("_section_of") or "").strip()
                    display = f"{section_of} -- {filename}" if section_of else filename
                    if emit_progress:
                        emit_live_view_preview(
                            (
                                f"Digesting section {done_count}/{total} of {section_of}..."
                                if section_of
                                else f"Digesting source {done_count}/{total}..."
                            ),
                            tool_name="digest_corpus",
                            progress_pct=int(100 * done_count / max(1, total)),
                        )
                        _emit_input_status(f"Studying: {display}")
                    _set_digest_job(slug, running=True, progress=f"{done_count}/{total}", current_file=display, total=total)
                    try:
                        digest_fut = sequential_pool.submit(
                            _digest_one_source,
                            slug=slug,
                            src=str(row["src"]),
                            filename=filename,
                            source_slug=str(row["source_slug"]),
                            old=row.get("old") if isinstance(row.get("old"), dict) else None,
                            update=bool(update),
                            is_url_input=bool(is_url_input),
                            pre_extracted_text=(
                                str(row.get("_pre_extracted_text"))
                                if row.get("_pre_extracted_text") is not None
                                else None
                            ),
                            parent_file_hash=(
                                str(row.get("_parent_file_hash")) if row.get("_parent_file_hash") else None
                            ),
                        )
                        result = digest_fut.result(timeout=per_source_timeout_s)
                    except Exception as exc:
                        result = {
                            "kind": "failed",
                            "entry": {
                                "filename": filename,
                                "path": str(row.get("src") or ""),
                                "slug": str(row.get("source_slug") or ""),
                                "status": "failed",
                                "error": f"digest timed out: {exc}"[:300],
                                "topics": [],
                                "chars_extracted": 0,
                                "chunks": 0,
                            },
                            "error": str(exc),
                        }
                    _mark_digest_wrap_up_if_done(slug, done_count, total)
                    _consume_result(i, result, display)
                    if _digest_pause_requested(slug):
                        return _finish_paused_digest()
    except _DigestPaused:
        return _complete_paused_digest(slug, label, planned, results_by_index)

    if _digest_pause_requested(slug):
        return _complete_paused_digest(slug, label, planned, results_by_index)

    _set_digest_phase(slug, "synthesizing", detail="Gathering source summaries…")

    new_sources: list[dict[str, Any]] = []
    summaries_for_synthesis: list[str] = []
    refreshed_paths: set[str] = set()
    if path_filter is not None:
        refreshed_paths = {os.path.abspath(c) for c in candidates}
    for row in planned:
        i = int(row["i"])
        result = results_by_index.get(i) or {}
        entry = result.get("entry")
        if isinstance(entry, dict):
            new_sources.append(entry)
        source_md = str(result.get("source_md") or "")
        if source_md:
            summary_body = _extract_summary_body(source_md)
            if summary_body:
                summaries_for_synthesis.append(f"### {row.get('filename')}\n{summary_body}")
    if path_filter is not None:
        refreshed_slugs = {str(s.get("slug") or "") for s in new_sources}
        gathered_extra = 0
        for old in old_sources:
            if not isinstance(old, dict):
                continue
            op = _manifest_source_fs_path(str(old.get("path") or ""))
            if op and op in refreshed_paths:
                continue
            new_sources.append(old)
            old_slug = str(old.get("slug") or "")
            if old_slug in refreshed_slugs:
                continue
            page = _source_page_path(slug, old_slug)
            if os.path.isfile(page):
                body = _extract_summary_body(_read_text_file(page))
                if body:
                    summaries_for_synthesis.append(f"### {old.get('filename')}\n{body}")
                    gathered_extra += 1
                    if gathered_extra % 20 == 0:
                        _set_digest_phase(
                            slug,
                            "synthesizing",
                            detail=f"Gathering source summaries ({gathered_extra})…",
                        )

    if success_count == 0 and fail_count == 0 and skipped_count > 0:
        needs_index_finalization = (
            force_finalize
            or prior_digest_status in ("partial", "failed")
            or not os.path.isfile(_synthesis_path(slug))
        )
        if not needs_index_finalization:
            manifest["updated_at"] = _iso_now()
            manifest["started_at"] = None
            manifest["sources"] = new_sources
            if new_sources:
                manifest["digest_status"] = "complete"
            _save_manifest(slug, manifest)
            _set_digest_job(slug, running=False, done=True, error="", summary="Corpus up to date; no source changes detected")
            try:
                from utils.embedding_chunking import corpus_semantic_index_state
                from utils.embedding_model import maybe_schedule_embedding_reindex

                tid = f"knowledge:{slug}"
                state = corpus_semantic_index_state(tid)
                if state in ("vectors_missing", "stale_chunker", "stale_identity", "reindexing"):
                    maybe_schedule_embedding_reindex(tid)
                    manifest["search_index_pending"] = True
                    _save_manifest(slug, manifest)
            except Exception:
                pass
            return {
                "ok": True,
                "slug": slug,
                "label": label,
                "message": f"No source changes detected. Corpus '{label}' is up to date.",
            }

    if success_count == 0:
        if fail_count > 0 or skipped_count == 0:
            manifest["sources"] = new_sources
            manifest["digest_status"] = "failed"
            manifest["started_at"] = None
            manifest["updated_at"] = _iso_now()
            _save_manifest(slug, manifest)
            _set_digest_job(slug, running=False, error="All files failed extraction or summarization.", done=True)
            return {"ok": False, "slug": slug, "error": "All files failed extraction or summarization."}

    old_source_slugs = {str(s.get("slug") or "") for s in old_sources}
    new_source_slugs = {str(s.get("slug") or "") for s in new_sources}
    for obsolete in sorted(old_source_slugs - new_source_slugs):
        if not obsolete:
            continue
        obsolete_path = _source_page_path(slug, obsolete)
        if os.path.isfile(obsolete_path):
            try:
                os.remove(obsolete_path)
            except OSError:
                pass

    try:
        llm_ctx = int(get_llm_context_length() or 4096)
    except Exception:
        llm_ctx = 4096
    context_chars = max(2000, min(llm_ctx * 3, 30000))
    synth_prompt = [
        {
            "role": "system",
            "content": (
                "Create synthesis markdown from source summaries.\n"
                "Include: Key Themes, Connections, Important Facts, and Notable Caveats."
            ),
        },
        {"role": "user", "content": ("\n\n".join(summaries_for_synthesis))[:context_chars]},
    ]
    if _digest_pause_requested(slug):
        return _complete_paused_digest(slug, label, planned, results_by_index)
    _set_digest_phase(slug, "synthesizing", detail="Writing corpus synthesis…")
    try:
        synthesis = _complete_with_fallback_timed(synth_prompt, max_tokens=900, temperature=0.2)
        synthesis_text = "# Synthesis\n\n" + synthesis.strip() + "\n"
        _write_text(_synthesis_path(slug), synthesis_text)

        topic_counts: dict[str, int] = defaultdict(int)
        for src in new_sources:
            if not isinstance(src, dict):
                continue
            for t in src.get("topics", []) or []:
                topic_counts[str(t)] += 1
        corpus_topics = [t for t, _ in sorted(topic_counts.items(), key=lambda kv: (-kv[1], kv[0].lower()))[:40]]
        broad_topics = _extract_broad_topics_from_synthesis(synthesis_text, limit=5)

        lines = [f"# {label}", "", "## Sources", ""]
        for src in new_sources:
            filename = str(src.get("filename") or "unknown")
            source_slug = str(src.get("slug") or "")
            status = str(src.get("status") or "")
            lines.append(f"- [{filename}](sources/{source_slug}.md) - {status}")
        lines += ["", "## Topics", ""]
        lines += [f"- {t}" for t in corpus_topics] or ["- (none)"]
        lines += ["", "## Broad Topics", ""]
        lines += [f"- {t}" for t in broad_topics] or ["- (none)"]
        lines.append("")
        _write_text(_index_path(slug), "\n".join(lines))
        raw_texts_for_index = _hydrate_raw_texts_for_indexing(slug, new_sources, raw_texts_by_slug)
        _set_digest_phase(slug, "indexing", detail="Search embeddings…")
        chunk_meta_by_slug = _index_knowledge_in_search_db(
            slug, label, synthesis_text, new_sources, raw_texts_for_index, ingest_profile=profile
        )
        semantic_index_ok = True
        try:
            from utils.embedding_model import embedding_enabled

            if embedding_enabled():
                import session_search_db as ss

                semantic_index_ok = ss.knowledge_semantic_index_complete_for_thread(
                    f"knowledge:{slug}"
                )
        except Exception:
            semantic_index_ok = False
        for src in new_sources:
            if not isinstance(src, dict):
                continue
            sid = str(src.get("slug") or "")
            if sid and sid in chunk_meta_by_slug:
                src["chunks"] = chunk_meta_by_slug[sid]
        graph_rebuild_ok = True
        try:
            from utils.knowledge_graph import rebuild_knowledge_graph_for_corpus

            _set_digest_phase(slug, "finalizing", detail="Knowledge graph…")
            graph_result = rebuild_knowledge_graph_for_corpus(
                slug, label, synthesis_text, new_sources, raw_texts_for_index
            )
            if graph_result.get("ok") is False:
                graph_rebuild_ok = False
                err = str(graph_result.get("error") or "graph_rebuild_failed")[:200]
                _debug_log(f"[knowledge] graph rebuild failed for {slug}: {err}", "WARN")
        except Exception as graph_exc:
            graph_rebuild_ok = False
            _debug_log(f"[knowledge] graph rebuild failed for {slug}: {graph_exc}", "WARN")

        connections_index_ok = True
        concept_graph_ok = True
        semantic_themes = _resolve_semantic_themes_for_finalize(slug)
        entity_bundle = _resolve_entities_for_finalize(slug, persist_db=False)
        entities = list(entity_bundle.get("entities") or [])
        claims = list(entity_bundle.get("claims") or [])
        contradictions = list(entity_bundle.get("contradictions") or [])
        if _broad_topics_need_semantic_backfill(broad_topics) and semantic_themes:
            broad_topics = [
                str(t.get("label") or "").strip()
                for t in semantic_themes[:5]
                if str(t.get("label") or "").strip()
            ]
            if broad_topics:
                _rewrite_index_broad_topics(slug, broad_topics)
        try:
            from utils.knowledge_connections_store import (
                materialize_connections_for_corpus,
                save_connections_index,
            )

            conn_data = materialize_connections_for_corpus(
                slug,
                label,
                synthesis_text,
                new_sources,
                broad_topics=broad_topics,
                corpus_topics=corpus_topics,
                semantic_themes=semantic_themes,
                entities=entities,
                claims=claims,
                contradictions=contradictions,
                ingest_profile=profile,
            )
            save_connections_index(slug, conn_data)
            try:
                from chat_storage import replace_corpus_entities_and_claims

                replace_corpus_entities_and_claims(slug, entities, claims)
            except Exception as ent_db_exc:
                _debug_log(
                    f"[knowledge] entity/claim DB sync failed after connections save for {slug}: {ent_db_exc}",
                    "WARN",
                )
            try:
                from utils.knowledge_graph import materialize_concept_graph_edges

                materialize_concept_graph_edges(slug)
            except Exception as graph_exc:
                concept_graph_ok = False
                _debug_log(f"[knowledge] concept graph failed for {slug}: {graph_exc}", "WARN")
        except Exception as conn_exc:
            connections_index_ok = False
            concept_graph_ok = False
            _debug_log(f"[knowledge] connections index failed for {slug}: {conn_exc}", "WARN")

        manifest["updated_at"] = _iso_now()
        manifest["started_at"] = None
        manifest["ingest_profile"] = profile
        manifest["topics"] = corpus_topics
        manifest["broad_topics"] = broad_topics
        manifest["sources"] = new_sources
        manifest["connections_index_ok"] = connections_index_ok
        manifest["concept_graph_ok"] = concept_graph_ok
        manifest["graph_rebuild_ok"] = graph_rebuild_ok
        digest_complete = fail_count == 0 and connections_index_ok
        manifest["digest_status"] = "partial" if (fail_count > 0 or not digest_complete) else "complete"
        manifest["search_index_pending"] = bool(not semantic_index_ok)
        _save_manifest(slug, manifest)
        invalidate_topic_index_cache()

        _set_digest_job(slug, running=False, done=True, error="", summary=f"{success_count}/{total} files digested")
        msg = f"Digested {success_count}/{total} files ({fail_count} failed, {skipped_count} skipped). Corpus '{label}' ready (slug: {slug})."
        if capped:
            msg += f" File scan hit cap of {_MAX_DIGEST_FILES}; some files were skipped."
        if failed_names:
            msg += " Failed: " + ", ".join(failed_names[:10])
        if not semantic_index_ok:
            try:
                from utils.embedding_model import maybe_schedule_embedding_reindex

                maybe_schedule_embedding_reindex(f"knowledge:{slug}")
            except Exception:
                pass
        digest_warnings: list[str] = []
        if not semantic_index_ok:
            digest_warnings.append(
                "semantic search index incomplete — background upgrade scheduled"
            )
        if not graph_rebuild_ok:
            digest_warnings.append("knowledge graph rebuild failed")
        if not connections_index_ok:
            digest_warnings.append("connections index failed — use Rebuild connections")
        elif not concept_graph_ok:
            digest_warnings.append("concept graph sync failed — use Rebuild connections")
        if digest_warnings:
            msg += " Warnings: " + "; ".join(digest_warnings) + "."
        return {"ok": True, "slug": slug, "label": label, "message": msg, "warnings": digest_warnings}
    except Exception as exc:
        manifest["sources"] = new_sources
        manifest["digest_status"] = "failed"
        manifest["started_at"] = None
        manifest["updated_at"] = _iso_now()
        _save_manifest(slug, manifest)
        invalidate_topic_index_cache()
        err = str(exc)
        _set_digest_job(slug, running=False, done=True, error=err, summary="")
        return {"ok": False, "slug": slug, "error": err}


def list_knowledge_corpora(*, repair: bool = True) -> list[dict[str, Any]]:
    """List corpora. ``repair`` gates manifest write-back (search_index_pending sync).

    Defaults ``True`` for background/maintenance callers; the HTTP ``GET /knowledge/list``
    route passes ``repair=False`` so a sidebar refresh computes display state without
    rewriting manifests on a read path.
    """
    _ensure_startup_manifest_repair()
    root = _knowledge_root()
    rows: list[dict[str, Any]] = []
    for entry in sorted(os.listdir(root)):
        corpus_dir = os.path.join(root, entry)
        # Ignore filesystem artifacts (e.g. .DS_Store) and non-directory entries.
        if entry.startswith(".") or not os.path.isdir(corpus_dir):
            continue
        slug = entry
        manifest = _load_manifest(slug)
        if manifest:
            manifest = _maybe_repair_corpus_manifest(slug, manifest)
        if not manifest:
            rows.append(
                {
                    "slug": slug,
                    "label": slug,
                    "source_path": "",
                    "updated_at": "",
                    "created_at": "",
                    "digest_status": "failed",
                    "source_count": 0,
                    "topics": [],
                    "last_error": "Corpus metadata is missing or unreadable. Re-digest or remove this corpus.",
                    "resumable": False,
                }
            )
            continue
        digest_status = str(manifest.get("digest_status") or "pending")
        with _digest_job_lock:
            if bool((_digest_jobs.get(slug) or {}).get("running")):
                digest_status = "running"
        semantic_index_state, manifest_dirty = _semantic_index_state_for_list(
            slug,
            manifest,
            digest_status=digest_status,
        )
        if _sync_search_index_pending_flag(manifest, semantic_index_state):
            manifest_dirty = True
        if manifest_dirty and repair:
            _save_manifest(slug, manifest)
        rows.append(
            {
                "slug": slug,
                "label": manifest.get("label") or slug,
                "source_path": manifest.get("source_path") or "",
                "updated_at": manifest.get("updated_at") or "",
                "created_at": manifest.get("created_at") or "",
                "digest_status": digest_status,
                "semantic_index_state": semantic_index_state,
                "search_index_pending": bool(manifest.get("search_index_pending")),
                "search_ready": str(semantic_index_state or "").strip().lower() == "ready",
                "source_count": len(manifest.get("sources") or []),
                "topics": manifest.get("topics") or [],
                "broad_topics": manifest.get("broad_topics") or [],
                "ingest_profile": manifest.get("ingest_profile") or "docs",
                "last_error": "",
                "resumable": _corpus_resumable(manifest, slug),
            }
        )
    return rows


def get_knowledge_status(slug: str, *, reconcile_jobs: bool = True) -> dict[str, Any]:
    """Knowledge corpus status. ``reconcile_jobs`` gates the zombie-job cleanup.

    Defaults to ``True`` so background maintenance (the heartbeat staleness sweep) keeps
    healing zombie/stale digest jobs. The HTTP ``GET /knowledge/{slug}/status`` route passes
    ``reconcile_jobs=False`` so a UI poll never stops an ingest or rewrites a manifest —
    stale jobs are still reported as not-running via the read-only ``_peek`` path.
    """
    _ensure_startup_manifest_repair()
    manifest = _load_manifest(slug) or {}
    manifest = _maybe_repair_corpus_manifest(slug, manifest)
    with _digest_job_lock:
        job = dict(_digest_jobs.get(slug, {}))
    if reconcile_jobs:
        job = _reconcile_digest_job_with_manifest(slug, manifest, job)
        manifest = _load_manifest(slug) or manifest
        job_running = bool(job.get("running", False))
    else:
        job_running = _peek_digest_job_running(slug, manifest, job)
    status = str(manifest.get("digest_status") or "unknown")
    if job_running:
        status = "running"
    progress = str(job.get("progress") or "")
    current_file = str(job.get("current_file") or "")
    # Stale in-memory job progress survives after manifest reaches complete/partial;
    # Swift treats any non-empty progress as "still running" (hours on 299/299 indexing).
    if not job_running:
        progress = ""
        current_file = ""
    err = str(job.get("error") or "")
    resumable = _corpus_resumable(manifest, slug) and not job_running
    if not err and resumable:
        err = "Digest incomplete — Continue ingest to finish (completed sections are skipped)."
    embedding_reindex: dict[str, Any] = {"state": "idle", "corpora_pending": 0, "current_corpus": ""}
    try:
        from utils.embedding_model import get_embedding_reindex_status

        embedding_reindex = get_embedding_reindex_status()
    except Exception:
        pass
    thread_id = f"knowledge:{slug}"
    chunker_version = ""
    embed_identity = ""
    semantic_index_state = "fts_only"
    search_index_encryption = "unknown"
    vec_backend = "bruteforce"
    try:
        import session_search_db as ss
        from utils.embedding_chunking import corpus_semantic_index_state

        conn = ss._safe_get_connection()
        if conn is not None:
            try:
                from utils.embedding_chunking import get_global_chunker_version

                chunker_version = get_global_chunker_version(conn)
            finally:
                conn.close()
        embed_identity = ss.get_thread_embed_identity(thread_id)
        per_thread_chunker = ss.get_thread_chunker_version(thread_id)
        if per_thread_chunker:
            chunker_version = per_thread_chunker
        semantic_index_state = corpus_semantic_index_state(
            thread_id,
            embedding_reindex_state=str(embedding_reindex.get("state") or "idle"),
        )
        search_index_encryption = (
            "plain" if ss.search_index_uses_plain_sqlite() else "sqlcipher"
        )
        vec_backend = ss.get_vec_backend_status()
    except Exception:
        vec_backend = "bruteforce"
    semantic_themes_count = 0
    entities_count = 0
    claims_count = 0
    contradictions_count = 0
    entities_state = "disabled"
    entities_algo_version = ""
    contradictions_state = "disabled"
    contradictions_algo_version = ""
    try:
        from utils.knowledge_connections_store import _validate_slug, load_connections_index
        from utils.knowledge_entities import ENTITIES_ALGO_VERSION
        from utils.knowledge_contradictions import (
            CONTRADICTIONS_ALGO_VERSION,
            visible_contradiction_rows,
        )

        norm = _validate_slug(slug)
        conn_index = load_connections_index(norm)
        if isinstance(conn_index, dict):
            semantic_rows = conn_index.get("semantic_themes")
            entity_rows = conn_index.get("entities")
            claim_rows = conn_index.get("claims")
            contradiction_rows = conn_index.get("contradictions")
            semantic_themes_count = len(semantic_rows) if isinstance(semantic_rows, list) else 0
            entities_count = len(entity_rows) if isinstance(entity_rows, list) else 0
            claims_count = len(claim_rows) if isinstance(claim_rows, list) else 0
            contradictions_count = len(
                visible_contradiction_rows(
                    contradiction_rows if isinstance(contradiction_rows, list) else None
                )
            )
        entities_state, entities_algo_version = _entities_state_for_status(
            norm,
            entities_count=entities_count,
            claims_count=claims_count,
        )
        contradictions_state, contradictions_algo_version = _contradictions_state_for_status(
            norm,
            contradictions_count=contradictions_count,
        )
        if not entities_algo_version:
            entities_algo_version = ENTITIES_ALGO_VERSION
        if not contradictions_algo_version:
            contradictions_algo_version = CONTRADICTIONS_ALGO_VERSION
    except (ValueError, Exception):
        semantic_themes_count = 0
        entities_count = 0
        claims_count = 0
        contradictions_count = 0
        entities_state = "disabled"
        contradictions_state = "disabled"
    if _sync_search_index_pending_flag(manifest, semantic_index_state):
        _save_manifest(slug, manifest)
    return {
        "slug": slug,
        "status": status,
        "running": job_running,
        "job_kind": str(job.get("job_kind") or JOB_KIND_DIGEST),
        "progress": progress,
        "current_file": current_file,
        "job_upgraded": int(job.get("upgraded") or 0),
        "job_skipped": int(job.get("skipped") or 0),
        "job_failed": int(job.get("failed") or 0),
        "summary_upgrade_code_rev": SUMMARY_UPGRADE_CODE_REV,
        "error": err,
        "summary": job.get("summary") or "",
        "resumable": resumable,
        "source_count": len(manifest.get("sources") or []),
        "embedding_reindex": embedding_reindex,
        "chunker_version": chunker_version,
        "embed_identity": embed_identity,
        "semantic_index_state": semantic_index_state,
        "search_index_pending": bool(manifest.get("search_index_pending")),
        "search_ready": str(semantic_index_state or "").strip().lower() == "ready",
        "search_index_encryption": search_index_encryption,
        "vec_backend": vec_backend,
        "semantic_themes_count": semantic_themes_count,
        "entities_count": entities_count,
        "claims_count": claims_count,
        "entities_state": entities_state,
        "entities_algo_version": entities_algo_version,
        "contradictions_count": contradictions_count,
        "contradictions_state": contradictions_state,
        "contradictions_algo_version": contradictions_algo_version,
    }


def start_digest_job(
    path: str,
    label: str,
    *,
    update: bool = False,
    ingest_profile: str | None = None,
    path_filter: set[str] | None = None,
    force_finalize: bool = False,
) -> dict[str, Any]:
    _ensure_startup_manifest_repair()
    slug, slug_conflict = _choose_corpus_slug(label, update)
    if slug_conflict and not update:
        return {"ok": False, "slug": slug, "error": f"A corpus with label '{label}' already exists. Use refresh/update."}
    with _digest_job_lock:
        running_jobs = sum(1 for j in _digest_jobs.values() if j.get("running"))
        if running_jobs >= _MAX_CONCURRENT_DIGESTS:
            return {"ok": False, "slug": slug, "error": "Too many concurrent digests running. Try again shortly."}
        existing = _digest_jobs.get(slug, {})
        if existing.get("running"):
            started_at = float(existing.get("started_at") or 0.0)
            if started_at and (time.time() - started_at) < _DIGEST_STALE_TTL:
                return {"ok": False, "slug": slug, "error": "digest already running"}
            _stop_digest_job(slug, error="Superseded by new digest request.")
    # Cross-process single-owner per corpus: ``_digest_jobs`` and the manifest
    # ``digest_status`` only dedupe within one process (and the manifest check is TOCTOU).
    # Two processes (backend + runner knowledge-watch, or backend + MCP/CLI digest) can
    # otherwise run concurrent digests racing the same manifest.json and source ingestion.
    from utils.process_lock import CrossProcessLock

    _xproc_lock = CrossProcessLock(f"digest_{slug}")
    if not _xproc_lock.try_acquire():
        return {"ok": False, "slug": slug, "error": "digest already running"}
    _clear_digest_pause_requested(slug)
    _set_digest_job(
        slug,
        running=True,
        progress="0/0",
        current_file="",
        error="",
        summary="",
        started_at=time.time(),
        pause_requested=False,
        aborted=False,
        job_kind=JOB_KIND_DIGEST,
    )

    def _runner() -> None:
        try:
            result = _digest_sync(
                path,
                label,
                update=update,
                emit_progress=False,
                ingest_profile=ingest_profile,
                path_filter=path_filter,
                force_finalize=force_finalize,
            )
            if result.get("paused"):
                _set_digest_job(
                    slug,
                    running=False,
                    done=True,
                    error="",
                    summary=str(result.get("message") or _DIGEST_PAUSE_MSG),
                )
                return
            _set_digest_job(
                slug,
                running=False,
                done=True,
                error="" if result.get("ok") else result.get("error", "digest failed"),
                summary=result.get("message", ""),
            )
            if result.get("ok"):
                try:
                    from utils.proactive_queue import append_proactive_queue

                    manifest = _load_manifest(slug) or {}
                    source_count = len(manifest.get("sources") or [])
                    corpus_label = str(manifest.get("label") or label or slug)
                    append_proactive_queue(
                        PROACTIVE_QUEUE_PATH,
                        (
                            f"I've finished studying {corpus_label} "
                            f"({source_count} sources). Ask me anything about it."
                        ),
                        source="knowledge_digest",
                        kind="notification",
                        dedupe_key=f"knowledge_digest:{slug}",
                    )
                except Exception:
                    pass
        except Exception as exc:
            _fail_running_digest_manifest(slug)
            _set_digest_job(slug, running=False, done=True, error=str(exc), summary="")
        finally:
            _xproc_lock.release()

    try:
        threading.Thread(target=_runner, daemon=True).start()
    except Exception as exc:
        # Thread creation failed: _runner (which releases the lock + marks the job done)
        # will never run. Release the cross-process lock and mark the job failed so the
        # corpus is not wedged in a phantom "running" state with a leaked lock.
        _log.warning("digest thread failed to start for %s", slug, exc_info=True)
        _xproc_lock.release()
        _fail_running_digest_manifest(slug)
        _set_digest_job(slug, running=False, done=True, error=str(exc), summary="")
        return {"ok": False, "slug": slug, "error": f"could not start digest: {exc}"}
    return {"ok": True, "slug": slug, "status": "running"}


def pause_digest_job(slug: str) -> dict[str, Any]:
    """Cooperatively pause a running ingest; checkpoints completed sources as ``partial``."""
    _ensure_startup_manifest_repair()
    manifest = _load_manifest(slug)
    if not manifest:
        return {"ok": False, "slug": slug, "error": "Corpus not found"}
    with _digest_job_lock:
        job = dict(_digest_jobs.get(slug, {}))
    manifest_running = str(manifest.get("digest_status") or "") == "running"
    job_running = bool(job.get("running"))
    if not manifest_running and not job_running:
        return {"ok": False, "slug": slug, "error": "No ingest running for this corpus."}
    if manifest_running and not job_running:
        _interrupt_running_digest_manifest(slug)
        manifest = _load_manifest(slug) or manifest
        if str(manifest.get("digest_status") or "") != "partial":
            manifest["digest_status"] = "partial" if manifest.get("sources") else "failed"
            manifest["started_at"] = None
            manifest["updated_at"] = _iso_now()
            _save_manifest(slug, manifest)
            invalidate_topic_index_cache()
        _clear_digest_pause_requested(slug)
        _set_digest_job(
            slug,
            running=False,
            done=True,
            error="",
            summary=_DIGEST_PAUSE_MSG,
            progress="",
            current_file="",
        )
        return {
            "ok": True,
            "slug": slug,
            "status": "paused",
            "message": _DIGEST_PAUSE_MSG,
        }
    _set_digest_pause_requested(slug)
    job_kind = str(job.get("job_kind") or JOB_KIND_DIGEST)
    if job_kind == JOB_KIND_SUMMARY_UPGRADE:
        pause_msg = "Stopping after the current source finishes…"
    else:
        pause_msg = "Pausing after the current section finishes…"
    return {
        "ok": True,
        "slug": slug,
        "status": "pausing",
        "message": pause_msg,
    }


def _canonical_fs_path(path: str) -> str:
    """Absolute, expanded, real path for manifest sources (sections → parent file)."""
    fs = _manifest_source_fs_path(path)
    if not fs:
        return ""
    try:
        return os.path.realpath(fs)
    except OSError:
        return fs


def _manifest_parent_key(path: str) -> str:
    """Stable parent key for grouping section rows (canonical when possible)."""
    raw = str(path or "")
    if "::section::" in raw:
        raw = raw.split("::section::", 1)[0]
    canon = _canonical_fs_path(raw)
    return canon or _manifest_source_fs_path(raw) or raw


def _entry_source_file_hash(entry: dict[str, Any]) -> str:
    for key in ("_source_file_hash", "_parent_content_hash"):
        value = str(entry.get(key) or "").strip()
        if value:
            return value
    return ""


def _direct_manifest_entry_is_stale(slug: str, entry: dict[str, Any]) -> tuple[bool, str]:
    """Return (is_stale, label).

    Uses on-disk file hash when available. When the file bytes are unchanged but
    ``content_hash_sha256`` (text digest) no longer matches extracted content, the
    entry is still stale — otherwise backfilled ``_source_file_hash`` would hide
    digest drift.
    """
    path = str(entry.get("path") or "")
    label = str(entry.get("filename") or os.path.basename(path) or path or "unknown")
    real = _canonical_fs_path(path)
    if not real or not os.path.isfile(real):
        return True, label
    stored_file_hash = _entry_source_file_hash(entry)
    try:
        live_file_hash = _sha256_file(real)
    except OSError:
        return True, label

    def _digest_matches_current_content(stored_digest: str) -> bool:
        if not stored_digest:
            return True
        if stored_digest == live_file_hash:
            return True
        cached = _load_raw_text_cache(slug, real)
        if cached is not None:
            text_hash = hashlib.sha256(cached.encode("utf-8", errors="ignore")).hexdigest()
            if stored_digest == text_hash:
                return True
        try:
            text = read_text_file_with_encoding_fallback(real)
            text_hash = hashlib.sha256(text.encode("utf-8", errors="ignore")).hexdigest()
            if stored_digest == text_hash:
                return True
        except Exception:
            pass
        return False

    if stored_file_hash:
        if live_file_hash != stored_file_hash:
            return True, label
        stored_digest = str(entry.get("content_hash_sha256") or "").strip()
        if stored_digest and not _digest_matches_current_content(stored_digest):
            return True, label
        return False, label

    stored = str(entry.get("content_hash_sha256") or "")
    if not stored:
        return False, label
    if _digest_matches_current_content(stored):
        return False, label
    return True, label


def _manifest_source_fs_path(path: str) -> str:
    """Normalize manifest source path to a real filesystem path (strip ``::section::`` suffix)."""
    p = str(path or "").strip()
    if not p:
        return ""
    if "::section::" in p:
        p = p.split("::section::", 1)[0]
    return os.path.abspath(os.path.expanduser(p))


def _validate_manifest_source_fs_path(path: str) -> tuple[str, str]:
    """Validate manifest replay path (DEF-089): dangerous/outside-workspace blocked on reindex."""
    fs = _manifest_source_fs_path(path)
    if not fs:
        return "", "empty manifest source path"
    try:
        real = os.path.realpath(fs)
    except OSError as exc:
        return "", f"manifest path resolution failed: {exc}"
    if is_dangerous_path(real):
        return "", f"blocked dangerous manifest path: {real}"
    if not _outside_workspace_digest_allowed():
        ws = _workspace_path()
        ws_real = os.path.realpath(os.path.expanduser(ws))
        if not _is_path_under_root(real, ws_real):
            return "", (
                f"manifest path outside workspace: {real}. "
                "Set app preference 'allow_knowledge_digest_outside_workspace' to allow this."
            )
    return real, ""


def _is_remote_source_path(path: str) -> bool:
    return bool(re.match(r"^https?://", str(path or "").strip(), flags=re.IGNORECASE))


def _manifest_entry_matches_stale_name(src: dict[str, Any], names: set[str]) -> bool:
    if not names or not isinstance(src, dict):
        return False
    lookup = {str(n) for n in names if str(n).strip()}
    lookup_lower = {n.lower() for n in lookup}
    filename = str(src.get("filename") or "")
    path = str(src.get("path") or "")
    fs = _manifest_source_fs_path(path)
    parent_base = os.path.basename(fs) if fs else ""
    for candidate in (filename, parent_base, os.path.basename(path)):
        if not candidate:
            continue
        if candidate in lookup or candidate.lower() in lookup_lower:
            return True
    return False


def _stale_refresh_path_filter(manifest: dict[str, Any], stale_names: set[str]) -> set[str]:
    """Map stale manifest sources to real filesystem paths (section virtual paths → parent file)."""
    out: set[str] = set()
    for s in manifest.get("sources") or []:
        if not isinstance(s, dict):
            continue
        if not _manifest_entry_matches_stale_name(s, stale_names):
            continue
        fs = _manifest_source_fs_path(str(s.get("path") or ""))
        if fs:
            out.add(fs)
    return out


def _prune_missing_manifest_sources(slug: str, missing_names: set[str]) -> int:
    """Drop manifest rows whose source files are missing on disk."""
    if not missing_names:
        return 0
    manifest = _load_manifest(slug)
    if not manifest:
        return 0
    kept: list[dict[str, Any]] = []
    removed_slugs: list[str] = []
    for src in manifest.get("sources") or []:
        if not isinstance(src, dict):
            continue
        if _is_remote_source_path(str(src.get("path") or "")):
            kept.append(src)
            continue
        if _manifest_entry_matches_stale_name(src, missing_names):
            sid = str(src.get("slug") or "")
            if sid:
                removed_slugs.append(sid)
            continue
        kept.append(src)
    if not removed_slugs:
        return 0
    manifest["sources"] = kept
    manifest["updated_at"] = _iso_now()
    _save_manifest(slug, manifest)
    for sid in removed_slugs:
        page = _source_page_path(slug, sid)
        if os.path.isfile(page):
            try:
                os.remove(page)
            except OSError:
                pass
    invalidate_topic_index_cache()
    return len(removed_slugs)


def _remaining_manifest_source_paths(manifest: dict[str, Any]) -> set[str]:
    out: set[str] = set()
    for src in manifest.get("sources") or []:
        if not isinstance(src, dict):
            continue
        fs = _manifest_source_fs_path(str(src.get("path") or ""))
        if fs and os.path.isfile(fs):
            out.add(fs)
    return out


def _sync_search_index_pending_flag(manifest: dict[str, Any], semantic_index_state: str) -> bool:
    """Clear search_index_pending once semantic index is ready."""
    if not bool(manifest.get("search_index_pending")):
        return False
    if str(semantic_index_state or "").strip().lower() != "ready":
        return False
    manifest["search_index_pending"] = False
    return True


def _semantic_index_state_for_list(
    slug: str,
    manifest: dict[str, Any],
    *,
    digest_status: str,
) -> tuple[str, bool]:
    """Resolve semantic index state for corpus list; cache on manifest when probed."""
    if str(digest_status or "") == "running":
        return "unknown", False
    try:
        from utils.embedding_model import get_embedding_reindex_status

        reindex_state = str((get_embedding_reindex_status() or {}).get("state") or "idle")
    except Exception:
        reindex_state = "idle"
    if reindex_state in ("reindexing", "preparing_model"):
        return "reindexing", False
    cached = str(manifest.get("semantic_index_state") or "").strip().lower()
    if cached and cached not in ("unknown", "") and not bool(manifest.get("search_index_pending")):
        return cached, False
    try:
        from utils.embedding_chunking import corpus_semantic_index_state

        state = str(
            corpus_semantic_index_state(
                f"knowledge:{slug}",
                embedding_reindex_state=reindex_state,
            )
            or "unknown"
        ).strip().lower() or "unknown"
    except Exception:
        state = cached or "unknown"
    dirty = False
    if state and state != "unknown" and manifest.get("semantic_index_state") != state:
        manifest["semantic_index_state"] = state
        dirty = True
    return state, dirty


def _corpus_slug_for_storage(slug: str) -> str:
    raw = str(slug or "").strip()
    if not raw:
        return ""
    try:
        from utils.knowledge_connections_store import _validate_slug

        return _validate_slug(raw)
    except ValueError:
        return _slugify(raw) or raw


def _purge_corpus_storage_indexes(slug: str) -> None:
    """Clear search index + graph rows for a corpus slug (manifest may still exist)."""
    norm = _corpus_slug_for_storage(slug)
    if not norm:
        return
    try:
        from utils.knowledge_graph import purge_knowledge_graph_for_corpus

        purge_knowledge_graph_for_corpus(norm)
    except Exception as exc:
        _log.warning("[knowledge] graph purge failed for %s: %s", norm, exc)
    try:
        from chat_storage import purge_corpus_entities_and_claims

        purge_corpus_entities_and_claims(norm)
    except Exception as exc:
        _log.warning("[knowledge] entity/claim purge failed for %s: %s", norm, exc)
    try:
        import session_search_db

        session_search_db.clear_thread_index(f"knowledge:{norm}", clear_fts=True, clear_vectors=True)
    except Exception as exc:
        _log.warning("[knowledge] search index clear failed for %s: %s", norm, exc)


def _apply_staleness_plan(slug: str, manifest: dict[str, Any]) -> dict[str, Any]:
    """Prune missing manifest rows and compute refresh paths."""
    stale = check_corpus_staleness(slug)
    stale_names = {str(n) for n in (stale.get("stale_files") or [])}
    missing_names = {str(n) for n in (stale.get("missing_files") or [])}
    pruned_count = 0
    if missing_names:
        pruned_count = _prune_missing_manifest_sources(slug, missing_names)
        if pruned_count:
            manifest = _load_manifest(slug) or manifest
    path_filter = _stale_refresh_path_filter(manifest, stale_names) if stale_names else None
    remaining = _remaining_manifest_source_paths(manifest)
    return {
        "manifest": manifest,
        "pruned_count": pruned_count,
        "path_filter": path_filter,
        "remaining": remaining,
        "has_stale": bool(stale_names),
    }


def _start_stale_refresh_from_plan(slug: str, plan: dict[str, Any]) -> dict[str, Any]:
    manifest = plan["manifest"]
    pruned_count = int(plan.get("pruned_count") or 0)
    path_filter = plan.get("path_filter")
    remaining = plan.get("remaining") or set()
    if not path_filter and pruned_count <= 0:
        return {"ok": True, "slug": slug, "status": "idle", "message": "No stale sources to refresh."}
    if path_filter:
        return start_digest_job(
            str(manifest.get("source_path") or ""),
            str(manifest.get("label") or slug),
            update=True,
            ingest_profile=str(manifest.get("ingest_profile") or "docs"),
            path_filter=path_filter,
            force_finalize=pruned_count > 0,
        )
    if not remaining:
        if manifest.get("sources"):
            return start_digest_job(
                str(manifest.get("source_path") or ""),
                str(manifest.get("label") or slug),
                update=True,
                ingest_profile=str(manifest.get("ingest_profile") or "docs"),
                force_finalize=True,
            )
        _purge_corpus_storage_indexes(slug)
        manifest["digest_status"] = "failed"
        manifest["search_index_pending"] = False
        manifest["updated_at"] = _iso_now()
        manifest["started_at"] = None
        _save_manifest(slug, manifest)
        invalidate_topic_index_cache()
        return {
            "ok": True,
            "slug": slug,
            "status": "idle",
            "message": f"Removed {pruned_count} missing source(s) from the corpus.",
            "pruned_missing": pruned_count,
        }
    return start_digest_job(
        str(manifest.get("source_path") or ""),
        str(manifest.get("label") or slug),
        update=True,
        ingest_profile=str(manifest.get("ingest_profile") or "docs"),
        path_filter=remaining,
        force_finalize=True,
    )


def _text_for_manifest_source(slug: str, entry: dict[str, Any], *, manifest: dict[str, Any] | None = None) -> str:
    """Re-extract source text for an existing manifest row (sections supported)."""
    src_path = str(entry.get("path") or "")
    is_url = bool(
        re.match(r"^https?://", str((manifest or {}).get("source_path") or ""), re.IGNORECASE)
    )
    real_path = src_path
    section_idx: int | None = None
    if "::section::" in src_path:
        parts = src_path.rsplit("::section::", 1)
        real_path = parts[0]
        try:
            section_idx = int(parts[1])
        except (ValueError, IndexError):
            section_idx = None
    if is_url:
        text = _fetch_url_text(real_path)
    else:
        if not os.path.isfile(real_path):
            return ""
        cached = _load_raw_text_cache(slug, real_path) if slug else None
        if cached:
            text = cached
        else:
            text = _extract_plaintext(real_path)
            if slug and text and text.strip():
                _save_raw_text_cache(slug, real_path, text)
    if not (text or "").strip():
        return ""
    if section_idx is not None:
        filename = str(entry.get("filename") or os.path.basename(real_path))
        sections = _detect_sections(text, filename)
        if sections and section_idx < len(sections):
            _, start, end = sections[section_idx]
            return text[start:end]
    return text


def _heal_manifest_for_summary_upgrade(slug: str, manifest: dict[str, Any]) -> dict[str, Any]:
    """Clear zombie digest_status=running when no digest worker owns the corpus."""
    manifest_status = str(manifest.get("digest_status") or "")
    if manifest_status != "running":
        return manifest
    with _digest_job_lock:
        job = dict(_digest_jobs.get(slug, {}))
    if job.get("running") and str(job.get("job_kind") or JOB_KIND_DIGEST) == JOB_KIND_DIGEST:
        return manifest
    healed = dict(manifest)
    healed["digest_status"] = "complete" if healed.get("sources") else "partial"
    healed["started_at"] = None
    healed["updated_at"] = _iso_now()
    _save_manifest(slug, healed)
    return healed


def _checkpoint_summary_upgrade_manifest(slug: str, manifest: dict[str, Any]) -> None:
    manifest["updated_at"] = _iso_now()
    _save_manifest(slug, manifest)
    invalidate_topic_index_cache()


def _finalize_summary_upgrade_side_effects(
    slug: str,
    manifest: dict[str, Any],
    *,
    synthesis_updated: bool,
) -> bool:
    """Reindex search (and connections when synthesis changed). Returns semantic_index_ok."""
    label = str(manifest.get("label") or slug)
    sources = [s for s in (manifest.get("sources") or []) if isinstance(s, dict)]
    synthesis_text = read_synthesis_text(slug) if os.path.isfile(_synthesis_path(slug)) else ""
    profile = str(manifest.get("ingest_profile") or "docs")
    _set_digest_phase(slug, "indexing", detail="Updating search index…")
    raw_texts = _hydrate_raw_texts_for_indexing(slug, sources, {})
    chunk_meta = _index_knowledge_in_search_db(
        slug, label, synthesis_text, sources, raw_texts, ingest_profile=profile
    )
    if chunk_meta:
        _apply_chunk_meta_to_manifest_sources(slug, sources, chunk_meta)
        manifest = _load_manifest(slug) or manifest
        sources = [s for s in (manifest.get("sources") or []) if isinstance(s, dict)]
    semantic_index_ok = True
    try:
        from utils.embedding_model import embedding_enabled

        if embedding_enabled():
            import session_search_db as ss

            semantic_index_ok = ss.knowledge_semantic_index_complete_for_thread(f"knowledge:{slug}")
    except Exception:
        semantic_index_ok = False
    if synthesis_updated and synthesis_text.strip():
        try:
            from utils.knowledge_connections_store import (
                materialize_connections_for_corpus,
                save_connections_index,
            )

            _set_digest_phase(slug, "finalizing", detail="Refreshing connections…")
            corpus_topics = list(manifest.get("topics") or [])
            broad_topics = list(manifest.get("broad_topics") or [])
            conn_data = materialize_connections_for_corpus(
                slug,
                label,
                synthesis_text,
                sources,
                broad_topics=broad_topics,
                corpus_topics=corpus_topics,
                ingest_profile=profile,
            )
            save_connections_index(slug, conn_data)
        except Exception as conn_exc:
            _debug_log(f"[knowledge] connections refresh after summary upgrade failed for {slug}: {conn_exc}", "WARN")
    manifest = _load_manifest(slug) or manifest
    manifest["search_index_pending"] = bool(not semantic_index_ok)
    manifest["updated_at"] = _iso_now()
    _save_manifest(slug, manifest)
    invalidate_topic_index_cache()
    if not semantic_index_ok:
        try:
            from utils.embedding_model import maybe_schedule_embedding_reindex

            maybe_schedule_embedding_reindex(f"knowledge:{slug}")
        except Exception:
            pass
    return semantic_index_ok


def upgrade_corpus_summaries_job(slug: str) -> dict[str, Any]:
    """Re-summarize sources with skipped/low-quality summaries; rebuild corpus synthesis."""
    _ensure_startup_manifest_repair()
    _prefer_utility_only_summaries()
    upgrade_token = _summary_upgrade_active.set(True)
    manifest = _load_manifest(slug)
    if not manifest:
        _summary_upgrade_active.reset(upgrade_token)
        return {"ok": False, "error": "Corpus not found"}
    manifest = _maybe_repair_corpus_manifest(slug, manifest)
    manifest = _heal_manifest_for_summary_upgrade(slug, manifest)
    sources = [s for s in (manifest.get("sources") or []) if isinstance(s, dict)]
    pending = []
    for entry in sources:
        source_slug = str(entry.get("slug") or "").strip()
        if not source_slug:
            continue
        page = _source_page_path(slug, source_slug)
        old_md = _read_text_file(page) if os.path.isfile(page) else ""
        if not (
            _summary_was_skipped(old_md)
            or _summary_contains_reasoning_trace(old_md)
            or _summary_needs_upgrade(old_md)
        ):
            continue
        pending.append((entry, old_md))
    if not pending:
        _summary_upgrade_active.reset(upgrade_token)
        return {
            "ok": True,
            "slug": slug,
            "status": "idle",
            "message": "All source summaries are already up to date.",
            "upgraded": 0,
            "failed": 0,
            "skipped": 0,
        }
    upgraded = 0
    failed = 0
    skipped = 0
    total = len(pending)
    _set_digest_job(
        slug,
        running=True,
        progress=f"0/{total}",
        current_file="Starting summary regeneration…",
        job_kind=JOB_KIND_SUMMARY_UPGRADE,
    )
    synthesis_updated = False
    try:
        for idx, (entry, _old_md) in enumerate(pending, 1):
            if _digest_pause_requested(slug):
                if upgraded > 0:
                    _checkpoint_summary_upgrade_manifest(slug, manifest)
                    _finalize_summary_upgrade_side_effects(slug, manifest, synthesis_updated=synthesis_updated)
                msg = (
                    f"Stopped after regenerating {upgraded} source summar"
                    f"{'y' if upgraded == 1 else 'ies'}."
                )
                _clear_digest_pause_requested(slug)
                _set_digest_job(slug, running=False, done=True, error="", summary=msg)
                return {
                    "ok": True,
                    "slug": slug,
                    "status": "idle",
                    "message": msg,
                    "upgraded": upgraded,
                    "failed": failed,
                    "skipped": skipped,
                    "stopped": True,
                }
            source_slug = str(entry.get("slug") or "")
            filename = str(entry.get("filename") or source_slug)
            src_path = str(entry.get("path") or "")
            _set_digest_job(
                slug,
                running=True,
                progress=f"{idx}/{total}",
                current_file=filename,
                job_kind=JOB_KIND_SUMMARY_UPGRADE,
                upgraded=upgraded,
                skipped=skipped,
                failed=failed,
            )
            text = _text_for_manifest_source(slug, entry, manifest=manifest)
            if not text.strip():
                failed += 1
                _emit_knowledge_digest_source_error(filename, "no extractable text")
                _set_digest_job(
                    slug,
                    running=True,
                    progress=f"{idx}/{total}",
                    current_file=f"No extractable text — {filename}",
                    job_kind=JOB_KIND_SUMMARY_UPGRADE,
                    upgraded=upgraded,
                    skipped=skipped,
                    failed=failed,
                )
                continue
            try:
                summary, chunks, summary_method = _summarize_source_for_upgrade(
                    filename,
                    text,
                    source_path=src_path,
                    slug=slug,
                    progress_index=idx,
                    progress_total=total,
                    upgraded=upgraded,
                    skipped=skipped,
                    failed=failed,
                )
            except Exception as exc:
                _debug_log(f"[knowledge] summary upgrade failed for {filename}: {exc}", "WARN")
                failed += 1
                _emit_knowledge_digest_source_error(filename, str(exc))
                _set_digest_job(
                    slug,
                    running=True,
                    progress=f"{idx}/{total}",
                    current_file=f"Failed — {filename}",
                    job_kind=JOB_KIND_SUMMARY_UPGRADE,
                    upgraded=upgraded,
                    skipped=skipped,
                    failed=failed,
                )
                continue
            persist_ok = _summary_acceptable_for_persist(
                summary_method, summary, for_upgrade=True
            )
            if not persist_ok:
                skipped += 1
                _debug_log(
                    f"[knowledge] Summary upgrade kept prior note for {filename} "
                    f"(method={summary_method}, chars={len((summary or '').strip())})",
                    "INFO",
                )
                _set_digest_job(
                    slug,
                    running=True,
                    progress=f"{idx}/{total}",
                    current_file=f"Kept prior note ({summary_method}) — {filename}",
                    job_kind=JOB_KIND_SUMMARY_UPGRADE,
                    upgraded=upgraded,
                    skipped=skipped,
                    failed=failed,
                )
                continue
            topics = _extract_topics(summary, limit=8)
            source_md = _render_source_markdown(
                filename, src_path, summary, topics, summary_method=summary_method
            )
            out_path = _source_page_path(slug, source_slug)
            _write_text(out_path, source_md)
            entry = dict(entry)
            entry["topics"] = topics
            entry["chunks"] = len(chunks)
            entry["chars_extracted"] = len(text)
            entry["status"] = "digested"
            upgraded += 1
            _set_digest_job(
                slug,
                running=True,
                progress=f"{idx}/{total}",
                current_file=f"Saved ({summary_method}) — {filename}",
                job_kind=JOB_KIND_SUMMARY_UPGRADE,
                upgraded=upgraded,
                skipped=skipped,
                failed=failed,
            )
            for i, src in enumerate(manifest.get("sources") or []):
                if isinstance(src, dict) and str(src.get("slug") or "") == source_slug:
                    manifest["sources"][i] = entry
                    break
            _checkpoint_summary_upgrade_manifest(slug, manifest)
        if upgraded > 0:
            if _digest_pause_requested(slug):
                _finalize_summary_upgrade_side_effects(slug, manifest, synthesis_updated=synthesis_updated)
                msg = (
                    f"Stopped after regenerating {upgraded} source summar"
                    f"{'y' if upgraded == 1 else 'ies'} (before corpus synthesis)."
                )
                _clear_digest_pause_requested(slug)
                _set_digest_job(slug, running=False, done=True, error="", summary=msg)
                return {
                    "ok": True,
                    "slug": slug,
                    "status": "idle",
                    "message": msg,
                    "upgraded": upgraded,
                    "failed": failed,
                    "skipped": skipped,
                    "stopped": True,
                }
            _set_digest_phase(slug, "synthesizing", detail="Rebuilding corpus synthesis…")
            summaries_for_synthesis: list[str] = []
            for src in manifest.get("sources") or []:
                if not isinstance(src, dict):
                    continue
                sid = str(src.get("slug") or "")
                if not sid:
                    continue
                page = _source_page_path(slug, sid)
                if not os.path.isfile(page):
                    continue
                body = _extract_summary_body(_read_text_file(page))
                if body and not _summary_was_skipped(body):
                    summaries_for_synthesis.append(f"### {src.get('filename')}\n{body}")
            if summaries_for_synthesis:
                try:
                    llm_ctx = int(get_llm_context_length() or 4096)
                except Exception:
                    llm_ctx = 4096
                context_chars = max(2000, min(llm_ctx * 3, 30000))
                synth_prompt = [
                    {
                        "role": "system",
                        "content": (
                            "Create synthesis markdown from source summaries.\n"
                            "Include: Key Themes, Connections, Important Facts, and Notable Caveats."
                        ),
                    },
                    {
                        "role": "user",
                        "content": ("\n\n".join(summaries_for_synthesis))[:context_chars],
                    },
                ]
                synthesis = _complete_with_fallback_timed(synth_prompt, max_tokens=900, temperature=0.2)
                _write_text(_synthesis_path(slug), "# Synthesis\n\n" + synthesis.strip() + "\n")
                synthesis_updated = True
            _checkpoint_summary_upgrade_manifest(slug, manifest)
            search_ok = _finalize_summary_upgrade_side_effects(
                slug, manifest, synthesis_updated=synthesis_updated
            )
        else:
            search_ok = True
        msg = f"Regenerated {upgraded} source summar{'y' if upgraded == 1 else 'ies'}."
        if skipped:
            msg += f" {skipped} kept prior notes (LLM unavailable)."
        if failed:
            msg += f" {failed} could not be summarized."
        if upgraded > 0 and not search_ok:
            msg += " Search index update is finishing in the background."
        _clear_digest_pause_requested(slug)
        _set_digest_job(slug, running=False, done=True, error="", summary=msg)
        return {
            "ok": True,
            "slug": slug,
            "status": "idle",
            "message": msg,
            "upgraded": upgraded,
            "failed": failed,
            "skipped": skipped,
        }
    except Exception as exc:
        if upgraded > 0:
            try:
                _checkpoint_summary_upgrade_manifest(slug, manifest)
                _finalize_summary_upgrade_side_effects(slug, manifest, synthesis_updated=synthesis_updated)
            except Exception as finalize_exc:
                _debug_log(
                    f"[knowledge] summary upgrade finalize after error for {slug}: {finalize_exc}",
                    "WARN",
                )
        _clear_digest_pause_requested(slug)
        _set_digest_job(slug, running=False, done=True, error=str(exc), summary="")
        return {"ok": False, "slug": slug, "error": str(exc)}
    finally:
        _summary_upgrade_active.reset(upgrade_token)


def start_upgrade_summaries_job(slug: str) -> dict[str, Any]:
    """Start background summary regeneration (non-blocking HTTP)."""
    _ensure_startup_manifest_repair()
    manifest = _load_manifest(slug)
    if not manifest:
        return {"ok": False, "slug": slug, "error": "Corpus not found"}
    manifest = _heal_manifest_for_summary_upgrade(slug, manifest)
    with _digest_job_lock:
        running_jobs = sum(1 for j in _digest_jobs.values() if j.get("running"))
        if running_jobs >= _MAX_CONCURRENT_DIGESTS:
            return {"ok": False, "slug": slug, "error": "Too many concurrent knowledge jobs running. Try again shortly."}
        existing = _digest_jobs.get(slug, {})
        if existing.get("running"):
            started_at = float(existing.get("started_at") or 0.0)
            if started_at and (time.time() - started_at) < _DIGEST_STALE_TTL:
                return {"ok": False, "slug": slug, "error": "summary regeneration already running"}
            _stop_digest_job(slug, error="Superseded by new summary regeneration request.")
    _clear_digest_pause_requested(slug)
    _set_digest_job(
        slug,
        running=True,
        progress="0/0",
        current_file="Regenerating summaries…",
        error="",
        summary="",
        started_at=time.time(),
        pause_requested=False,
        aborted=False,
        job_kind=JOB_KIND_SUMMARY_UPGRADE,
    )

    def _runner() -> None:
        try:
            result = upgrade_corpus_summaries_job(slug)
            if not result.get("ok"):
                _set_digest_job(
                    slug,
                    running=False,
                    done=True,
                    error=str(result.get("error") or "summary regeneration failed"),
                    summary="",
                )
                return
            if str(result.get("status") or "") == "idle":
                _set_digest_job(
                    slug,
                    running=False,
                    done=True,
                    error="",
                    summary=str(result.get("message") or "All source summaries are already up to date."),
                )
        except Exception as exc:
            _set_digest_job(slug, running=False, done=True, error=str(exc), summary="")

    threading.Thread(target=_runner, daemon=True).start()
    return {"ok": True, "slug": slug, "status": "running", "message": "Regenerating summaries…"}


def refresh_digest_job(slug: str, *, stale_only: bool = False) -> dict[str, Any]:
    _ensure_startup_manifest_repair()
    manifest = _load_manifest(slug)
    if not manifest:
        return {"ok": False, "error": "Corpus not found"}
    manifest = _maybe_repair_corpus_manifest(slug, manifest)
    if stale_only:
        plan = _apply_staleness_plan(slug, manifest)
        return _start_stale_refresh_from_plan(slug, plan)
    digest_status = str(manifest.get("digest_status") or "").lower()
    if digest_status == "complete":
        plan = _apply_staleness_plan(slug, manifest)
        if plan["has_stale"] or plan["pruned_count"] > 0:
            return _start_stale_refresh_from_plan(slug, plan)
        try:
            from utils.embedding_chunking import corpus_semantic_index_state
            from utils.embedding_model import (
                get_embedding_reindex_status,
                maybe_schedule_embedding_reindex,
            )

            reindex_state = str(
                (get_embedding_reindex_status() or {}).get("state") or "idle"
            )
            sem_state = corpus_semantic_index_state(
                f"knowledge:{slug}",
                embedding_reindex_state=reindex_state,
            )
        except Exception:
            sem_state = "unknown"
        if sem_state in (
            "vectors_missing",
            "stale_chunker",
            "stale_identity",
            "reindexing",
        ):
            try:
                maybe_schedule_embedding_reindex(f"knowledge:{slug}")
            except Exception:
                pass
            manifest = _load_manifest(slug) or manifest
            manifest["search_index_pending"] = True
            manifest["updated_at"] = _iso_now()
            _save_manifest(slug, manifest)
            return {
                "ok": True,
                "slug": slug,
                "status": "idle",
                "message": (
                    "Your files are up to date. Mugi is finishing search setup in the background."
                ),
                "semantic_index_state": sem_state,
            }
    return start_digest_job(
        str(manifest.get("source_path") or ""),
        str(manifest.get("label") or slug),
        update=True,
        ingest_profile=str(manifest.get("ingest_profile") or "docs"),
    )


def _clear_url_cache_for_manifest(manifest: dict[str, Any] | None) -> None:
    if not isinstance(manifest, dict):
        return
    urls: set[str] = set()
    source_path = str(manifest.get("source_path") or "").strip()
    if re.match(r"^https?://", source_path, flags=re.IGNORECASE):
        urls.add(source_path)
    for src in manifest.get("sources") or []:
        if not isinstance(src, dict):
            continue
        p = str(src.get("path") or "").strip()
        if re.match(r"^https?://", p, flags=re.IGNORECASE):
            urls.add(p)
    if not urls:
        return
    with _url_cache_lock:
        for u in urls:
            _url_cache.pop(u, None)


def delete_corpus(slug: str) -> bool:
    slug_raw = str(slug or "").strip()
    if not slug_raw:
        return False
    # Only allow plain names; never accept path separators or traversal.
    if os.path.basename(slug_raw) != slug_raw or (os.sep in slug_raw):
        return False
    root = _knowledge_root()
    path_raw = os.path.join(root, slug_raw)
    # Cleanup path for accidental artifact entries (e.g. .DS_Store) that are not corpora.
    if os.path.isfile(path_raw):
        try:
            os.remove(path_raw)
            invalidate_topic_index_cache()
            return True
        except Exception:
            return False
    slug_clean = _slugify(slug_raw)
    if not slug_clean:
        return False
    path_clean = os.path.join(root, slug_clean)
    target = path_raw if os.path.isdir(path_raw) else path_clean
    if not os.path.isdir(target):
        return False
    manifest = _load_manifest(slug_raw) or _load_manifest(slug_clean)
    import shutil

    shutil.rmtree(target)
    purge_slugs = {
        s
        for s in (
            _corpus_slug_for_storage(slug_clean),
            _corpus_slug_for_storage(slug_raw),
        )
        if s
    }
    for purge_slug in purge_slugs:
        _purge_corpus_storage_indexes(purge_slug)
    # Hard reset semantics: deleted corpus must not retain stale digest/job/cache state.
    with _digest_job_lock:
        _digest_jobs.pop(slug_raw, None)
        _digest_jobs.pop(slug_clean, None)
    _clear_url_cache_for_manifest(manifest)
    invalidate_topic_index_cache()
    return True


def _backfill_parent_content_hashes_in_manifest(manifest: dict[str, Any]) -> bool:
    """Set ``_parent_content_hash`` on section rows recovered without it (repair / legacy)."""
    if not isinstance(manifest, dict):
        return False
    by_parent: dict[str, list[dict[str, Any]]] = defaultdict(list)
    for src in manifest.get("sources") or []:
        if not isinstance(src, dict):
            continue
        path = str(src.get("path") or "")
        if "::section::" in path:
            by_parent[_manifest_parent_key(path)].append(src)
    changed = False
    for parent_key, entries in by_parent.items():
        if not parent_key or not os.path.isfile(parent_key):
            continue
        if all(str(e.get("_parent_content_hash") or "") for e in entries):
            continue
        try:
            parent_hash = _sha256_file(parent_key)
        except OSError:
            continue
        for entry in entries:
            if not str(entry.get("_parent_content_hash") or ""):
                entry["_parent_content_hash"] = parent_hash
                changed = True
    return changed


def _backfill_source_file_hashes_in_manifest(manifest: dict[str, Any]) -> bool:
    """Set ``_source_file_hash`` on whole-file manifest rows (legacy text-only digests)."""
    if not isinstance(manifest, dict):
        return False
    changed = False
    for src in manifest.get("sources") or []:
        if not isinstance(src, dict):
            continue
        path = str(src.get("path") or "")
        if "::section::" in path:
            continue
        if _entry_source_file_hash(src):
            continue
        real = _canonical_fs_path(path)
        if not real or not os.path.isfile(real):
            continue
        try:
            src["_source_file_hash"] = _sha256_file(real)
            changed = True
        except OSError:
            continue
    return changed


def check_corpus_staleness(slug: str) -> dict[str, Any]:
    manifest = _load_manifest(slug)
    if not manifest:
        return {"ok": False, "error": "Corpus not found", "stale_count": 0, "stale_files": [], "missing_files": []}
    changed = _backfill_parent_content_hashes_in_manifest(manifest)
    changed = _backfill_source_file_hashes_in_manifest(manifest) or changed
    if changed:
        _save_manifest(slug, manifest)
    stale_files: list[str] = []
    missing_files: list[str] = []
    by_parent: dict[str, list[dict[str, Any]]] = defaultdict(list)
    direct_entries: list[dict[str, Any]] = []
    for src in manifest.get("sources") or []:
        if not isinstance(src, dict):
            continue
        path = str(src.get("path") or "")
        if "::section::" in path:
            by_parent[_manifest_parent_key(path)].append(src)
        else:
            direct_entries.append(src)
    for src in direct_entries:
        path = str(src.get("path") or "")
        if _is_remote_source_path(path):
            continue
        real = _canonical_fs_path(path)
        if not real or not os.path.isfile(real):
            missing_files.append(str(src.get("filename") or path or "unknown"))
            continue
        is_stale, label = _direct_manifest_entry_is_stale(slug, src)
        if is_stale:
            stale_files.append(label)
    for parent_key, entries in by_parent.items():
        parent_label = os.path.basename(parent_key) or parent_key
        if not parent_key or not os.path.isfile(parent_key):
            missing_files.append(parent_label)
            continue
        try:
            parent_hash = _sha256_file(parent_key)
        except Exception:
            stale_files.append(parent_label)
            continue
        expected = ""
        for entry in entries:
            candidate = str(entry.get("_parent_content_hash") or "").strip()
            if candidate:
                expected = candidate
                break
        if not expected:
            continue
        if parent_hash != expected:
            stale_files.append(parent_label)
    stale_unique = sorted({str(x) for x in stale_files if str(x).strip()})
    missing_unique = sorted({str(x) for x in missing_files if str(x).strip()})
    return {
        "ok": True,
        "stale_count": len(stale_unique) + len(missing_unique),
        "stale_files": stale_unique,
        "missing_files": missing_unique,
    }


def invalidate_topic_index_cache() -> None:
    global _topic_index_cache, _topic_index_last_manifest_mtime, _topic_index_last_scan
    global _knowledge_catalog_cache, _knowledge_catalog_last_scan
    with _topic_index_lock:
        _topic_index_cache = {}
        _topic_index_last_manifest_mtime = 0.0
        _topic_index_last_scan = 0.0
        _knowledge_catalog_cache = None
        _knowledge_catalog_last_scan = 0.0


def _rebuild_topic_index_if_needed(force: bool = False) -> None:
    global _topic_index_cache, _topic_index_last_manifest_mtime, _topic_index_last_scan
    global _broad_topic_backfill_last_scan
    now = time.time()
    with _topic_index_lock:
        if not force and (now - _topic_index_last_scan) < _TOPIC_INDEX_RESCAN_SECONDS:
            return
        _topic_index_last_scan = now
        manifests: list[tuple[str, dict[str, Any], float]] = []
        max_mtime = 0.0
        root = _knowledge_root()
        for entry in os.listdir(root):
            path = _manifest_path(entry)
            if not os.path.isfile(path):
                continue
            mtime = os.path.getmtime(path)
            max_mtime = max(max_mtime, mtime)
            manifest = _load_manifest(entry)
            if manifest:
                manifests.append((entry, manifest, mtime))
        if (force or (now - _broad_topic_backfill_last_scan) >= _TOPIC_INDEX_RESCAN_SECONDS) and manifests:
            for i, (slug, manifest, _) in enumerate(manifests):
                if (manifest.get("broad_topics") or []):
                    continue
                broad_topics = _derive_broad_topics_from_manifest(manifest, limit=5)
                if not broad_topics:
                    continue
                manifest["broad_topics"] = broad_topics
                manifest["updated_at"] = _iso_now()
                _save_manifest(slug, manifest)
                updated_mtime = os.path.getmtime(_manifest_path(slug))
                manifests[i] = (slug, manifest, updated_mtime)
                max_mtime = max(max_mtime, updated_mtime)
            _broad_topic_backfill_last_scan = now
        if not force and max_mtime == _topic_index_last_manifest_mtime and _topic_index_cache:
            return
        index: dict[str, list[dict[str, str]]] = defaultdict(list)
        token_re, stopwords = _tokenizer_refs()
        stop = set(stopwords)

        def _append_term(term: str, slug: str, label: str) -> None:
            t = str(term).strip().lower()
            if not t:
                return
            existing = index.get(t) or []
            if any(str(e.get("slug") or "") == slug for e in existing):
                return
            index[t].append({"slug": slug, "label": label})

        for slug, manifest, _ in manifests:
            label = str(manifest.get("label") or slug)
            for topic in (manifest.get("topics") or []) + (manifest.get("broad_topics") or []):
                raw_topic = str(topic).strip()
                if not raw_topic:
                    continue
                _append_term(raw_topic, slug, label)
            for token in token_re.findall(label.lower()):
                if token in stop:
                    continue
                _append_term(token, slug, label)
        _topic_index_cache = dict(index)
        _topic_index_last_manifest_mtime = max_mtime


def get_knowledge_catalog_block(max_corpora: int = 5, max_topics: int = 5, max_chars: int = 400) -> str | None:
    global _knowledge_catalog_cache, _knowledge_catalog_last_scan
    now = time.time()
    with _topic_index_lock:
        if _knowledge_catalog_cache is not None and (now - _knowledge_catalog_last_scan) < _TOPIC_INDEX_RESCAN_SECONDS:
            return _knowledge_catalog_cache or None
    rows = list_knowledge_corpora()
    if not rows:
        with _topic_index_lock:
            _knowledge_catalog_cache = ""
            _knowledge_catalog_last_scan = now
        return None
    lines: list[str] = []
    used = 0
    # Reserve footer so total output stays within max_chars (agent_loop budget).
    footer_reserve = len(_KNOWLEDGE_CONTEXT_LANES_FOOTER) + 2
    corpus_cap = max(80, int(max_chars) - footer_reserve)
    for row in rows[:max_corpora]:
        label = str(row.get("label") or row.get("slug") or "").strip() or "Unknown corpus"
        source_count = int(row.get("source_count") or 0)
        topics = [str(t).strip() for t in ((row.get("broad_topics") or []) + (row.get("topics") or [])) if str(t).strip()]
        topic_preview = ", ".join(topics[:max_topics]) if topics else "(topics not yet extracted)"
        line = f"- {label} ({source_count} sources; topics: {topic_preview})"
        sep = 1 if lines else 0
        if used + sep + len(line) > corpus_cap:
            if lines:
                break
            room = max(0, corpus_cap - sep)
            if room <= 0:
                break
            line = line[:room]
        lines.append(line)
        used += sep + len(line)
    lines.append("")
    lines.append(_KNOWLEDGE_CONTEXT_LANES_FOOTER)
    out = "\n".join(lines).strip()
    with _topic_index_lock:
        _knowledge_catalog_cache = out
        _knowledge_catalog_last_scan = now
    return out or None


def find_relevant_knowledge(user_text: str, max_chars: int = 1200) -> str | None:
    text, _rows = find_relevant_knowledge_with_rows(user_text, max_chars=max_chars)
    return text


def find_relevant_knowledge_with_rows(
    user_text: str, max_chars: int = 1200
) -> tuple[str | None, list[dict[str, Any]]]:
    """Topic/synthesis fallback text plus structured trace rows for provenance."""
    if not (user_text or "").strip():
        return None, []
    _rebuild_topic_index_if_needed()
    with _topic_index_lock:
        if not _topic_index_cache:
            return None, []
        topic_index = dict(_topic_index_cache)
    token_re, stopwords = _tokenizer_refs()
    tokens = [t for t in token_re.findall(user_text.lower()) if t not in stopwords]
    if not tokens:
        return None, []
    token_set = set(tokens)
    matches: dict[str, set[str]] = defaultdict(set)
    for topic, corpora in topic_index.items():
        parts = [p for p in token_re.findall(topic.lower()) if p and p not in stopwords]
        if not parts:
            continue
        overlap = [p for p in parts if p in token_set]
        min_match = 1 if len(parts) <= 2 else 2
        if len(overlap) < min_match:
            continue
        for corpus in corpora:
            matches[corpus["slug"]].add(topic)
    if not matches:
        return None, []

    snippets: list[str] = []
    trace_rows: list[dict[str, Any]] = []
    used = 0
    for slug in list(matches.keys())[:_RELEVANT_SNIPPET_MAX_CORPORA]:
        manifest = _load_manifest(slug)
        if not manifest:
            continue
        label = str(manifest.get("label") or slug)
        synthesis = _read_text_file(_synthesis_path(slug)) if os.path.isfile(_synthesis_path(slug)) else ""
        synthesis = re.sub(r"(?is)^(?:#\s*Synthesis(?:\s+Markdown)?\s*\n+)+", "", synthesis).strip()
        if not synthesis:
            continue
        excerpt = synthesis[: max(80, min(450, max_chars - used))]
        line = f"[{label}] {excerpt.strip()}"
        if used + len(line) > max_chars:
            break
        snippets.append(line)
        trace_rows.append(
            {
                "corpus_slug": slug,
                "corpus_label": label,
                "source_name": "synthesis",
                "snippet": excerpt.strip(),
            }
        )
        used += len(line) + 1
        if used >= max_chars:
            break
    if not snippets:
        return None, []
    return "\n".join(snippets), trace_rows


_KNOWLEDGE_CONTEXT_LANES_FOOTER = (
    "Context lanes: studied corpora → ask_studied_corpus (cited answers) / query_knowledge (passage search) / "
    "search_memory_and_knowledge_corpora; live files → read_file / search_filesystem (Allowed Directories); "
    "code structure (callers, flows, impact) → GitNexus MCP — not the Knowledge Base digest."
)


def get_knowledge_context_lanes_footer() -> str:
    """Short agent-facing reminder of KB vs file tools vs GitNexus."""
    return _KNOWLEDGE_CONTEXT_LANES_FOOTER


def build_worker_knowledge_block(query_text: str, *, max_chars: int = 1000) -> str | None:
    """Relevance-gated studied-corpus context for kanban workers / council seats.

    Foreground chat already injects relevant-corpus context via
    ``find_relevant_knowledge_for_injection`` (gated to non-subagent turns). Subagents —
    notably council seats — historically got the ``query_knowledge`` / ``ask_studied_corpus``
    tools (when in their profile) but no signal about *whether* any corpus was relevant, so
    they either never queried or wandered into unrelated corpora.

    This returns a prompt block **only** when an ingested corpus is semantically relevant to
    *query_text* (the worker's task title + body). When nothing matches it returns ``None`` so
    the worker is never told corpora exist for an off-topic task — that is the anti-spinning
    guarantee: no relevant corpus → no nudge → the worker uses web search instead.
    """
    text = (query_text or "").strip()
    if not text:
        return None
    try:
        snippet, path, _rows = find_relevant_knowledge_for_injection(text, max_chars=max_chars)
    except Exception:
        return None
    if not snippet or path in ("absent", "empty"):
        return None
    lines = [
        "─── Ingested Knowledge (studied corpus relevant to this task) ───",
        "A studied corpus contains material relevant to this task. If your tools include "
        "query_knowledge or ask_studied_corpus, prefer them to retrieve grounded passages "
        "BEFORE search_web. If the corpus turns out not to cover the specifics you need, "
        "fall back to web search — do not loop knowledge queries.",
        "",
        "Relevant excerpts:",
        snippet.strip(),
    ]
    try:
        catalog = get_knowledge_catalog_block()
    except Exception:
        catalog = None
    if catalog:
        lines.append("")
        lines.append("Available corpora (pass the label as the query_knowledge corpus):")
        lines.append(catalog)
    return "\n".join(lines)


def _injection_hits_are_relevant(hits: list[dict[str, Any]], user_text: str) -> bool:
    """Relevance gate for ambient injection (mirrors the Library Ask gate).

    Ambient injection fires on every foreground turn without the user asking, so
    we only inject search hits that clear the inject fused-score floor *and* share
    at least one salient term with the user's recent text. Without this, the
    permissive global floor (``min_fused_score`` ~0.05) would dump up to 8 weakly
    related passages into the prompt on completely off-topic conversations.
    """
    if not hits:
        return False
    try:
        from utils.knowledge_retrieval import get_retrieval_config, inject_fused_score_floor

        floor = inject_fused_score_floor(get_retrieval_config())
    except Exception:
        floor = 0.10
    if not _hits_pass_score_floor(hits, floor):
        return False
    return _hits_contain_query_terms(hits, _ask_salient_terms(user_text))


def find_relevant_knowledge_for_injection(
    user_text: str, max_chars: int = 1200
) -> tuple[str | None, str, list[dict[str, Any]]]:
    """Ambient injection: hybrid search hits first, topic/synthesis fallback second.

    Search hits must pass :func:`_injection_hits_are_relevant` (score floor + query-term
    containment); otherwise we fall through to the topic/synthesis path rather than
    injecting weakly-related passages.

    Returns (snippet_text, path, trace_rows) where path is ``search``, ``topic_fallback``,
    ``absent``, or ``empty``.
    """
    text = (user_text or "").strip()
    if not text:
        return None, "empty", []
    try:
        from utils.embedding_model import startup_embedding_identity_sweep

        startup_embedding_identity_sweep()
    except Exception:
        pass
    cap = max(200, int(max_chars))
    hits = search_knowledge_hits(text, corpus=None, max_results=8, semantic=True, retrieval_mode="inject")
    if hits and _injection_hits_are_relevant(hits, text):
        body = format_knowledge_hits_as_text(hits, include_tip=False)
        if body.strip():
            if len(body) > cap:
                body = body[: cap - 1] + "…"
            return body.strip(), "search", list(hits)
    fallback, trace_rows = find_relevant_knowledge_with_rows(text, max_chars=cap)
    if fallback:
        return fallback, "topic_fallback", trace_rows
    return None, "absent", []


def get_knowledge_catalog_trace_metadata(max_labels: int = 16) -> dict[str, Any]:
    """Catalog metadata for provenance SSE (no catalog body text)."""
    rows = list_knowledge_corpora()
    labels = [
        str(r.get("label") or r.get("slug") or "").strip()
        for r in rows[:max_labels]
        if str(r.get("label") or r.get("slug") or "").strip()
    ]
    block = get_knowledge_catalog_block()
    return {
        "present": bool(block and block.strip()),
        "corpus_count": len(rows),
        "chars": len(block or ""),
        "corpus_labels": labels,
    }


def _knowledge_inspector_maybe_record_injection(
    *, path: str, trace_rows: list[dict[str, Any]], chars_injected: int
) -> None:
    try:
        from utils.knowledge_turn_trace import tls_get_trace

        tr = tls_get_trace()
        if tr:
            tr.record_injection(path=path, trace_rows=trace_rows, chars_injected=chars_injected)
    except Exception:
        pass


def _knowledge_inspector_maybe_record_catalog(meta: dict[str, Any]) -> None:
    try:
        from utils.knowledge_turn_trace import tls_get_trace

        tr = tls_get_trace()
        if tr:
            tr.record_catalog(
                present=bool(meta.get("present")),
                corpus_count=int(meta.get("corpus_count") or 0),
                chars=int(meta.get("chars") or 0),
                corpus_labels=list(meta.get("corpus_labels") or []),
            )
    except Exception:
        pass


def _knowledge_inspector_maybe_record_tool_query(
    *,
    tool: str,
    query: str,
    corpus_filter: str | None,
    hits: list[dict[str, Any]],
    outcome: str,
) -> None:
    try:
        from utils.knowledge_turn_trace import tls_get_trace

        tr = tls_get_trace()
        if tr:
            tr.record_tool_query(
                tool=tool,
                query=query,
                corpus_filter=corpus_filter,
                hits=hits,
                outcome=outcome,
            )
    except Exception:
        pass


def _knowledge_inspector_maybe_record_tool_read(
    *,
    corpus_slug: str,
    source_name: str,
    bytes_read: int,
    truncated: bool,
    offset: int = 0,
) -> None:
    try:
        from utils.knowledge_turn_trace import tls_get_trace

        tr = tls_get_trace()
        if tr:
            tr.record_tool_read(
                corpus_slug=corpus_slug,
                source_name=source_name,
                bytes_read=bytes_read,
                truncated=truncated,
                offset=offset,
            )
    except Exception:
        pass


def _knowledge_inspector_maybe_record_tool_list_sources(
    *, corpus_slug: str, corpus_label: str, source_count: int
) -> None:
    try:
        from utils.knowledge_turn_trace import tls_get_trace

        tr = tls_get_trace()
        if tr:
            tr.record_tool_list_sources(
                corpus_slug=corpus_slug,
                corpus_label=corpus_label,
                source_count=source_count,
            )
    except Exception:
        pass


def _knowledge_inspector_maybe_record_tool_combined(
    *, query: str, hit_count: int, outcome: str
) -> None:
    try:
        from utils.knowledge_turn_trace import tls_get_trace

        tr = tls_get_trace()
        if tr:
            tr.record_tool_combined(query=query, hit_count=hit_count, outcome=outcome)
    except Exception:
        pass


def _build_knowledge_corpus_index(
    slug: str,
    label: str,
    synthesis_text: str,
    sources: list[dict[str, Any]],
    raw_texts: dict[str, str] | None = None,
    *,
    progress_slug: str | None = None,
) -> tuple[list[dict[str, Any]], dict[str, list[dict[str, Any]]], list[dict[str, Any]]]:
    """Build FTS rows, manifest chunk anchors, and graph nodes with aligned embed chunks."""
    from utils.embedding_chunking import embedding_chunk_params
    from utils.knowledge_graph import _chunk_node_id

    chunk_size, overlap = embedding_chunk_params()
    rows: list[dict[str, Any]] = []
    graph_nodes: list[dict[str, Any]] = []
    chunk_meta_by_slug: dict[str, list[dict[str, Any]]] = {}
    row_id = 1
    prog = progress_slug or slug

    def _append_chunk_row(
        *,
        header: str,
        chunk_text: str,
        filename: str,
        source_slug: str,
        rel_path: str = "",
        repo_root: str = "",
        chunk_label: str,
        char_start: int | None = None,
        char_end: int | None = None,
        line_start: int | None = None,
        line_end: int | None = None,
        meta_sink: list[dict[str, Any]] | None = None,
    ) -> None:
        nonlocal row_id
        chunk_stripped = (chunk_text or "").strip()
        if len(chunk_stripped) < _INDEX_MIN_CHUNK_CHARS:
            return
        indexed_body = f"{header}\n{chunk_stripped}"
        try:
            from chat_storage import _looks_secret

            if _looks_secret(indexed_body):
                return
        except Exception:
            pass
        created = _kb_source_created_at(
            filename,
            char_start=char_start,
            char_end=char_end,
            line_start=line_start,
            line_end=line_end,
        )
        rows.append(
            {
                "role": "system",
                "content": indexed_body,
                "created_at": created if filename != "synthesis.md" else _iso_now(),
                "_row_id": row_id,
            }
        )
        graph_nodes.append(
            {
                "node_id": _chunk_node_id(slug, row_id),
                "corpus_slug": slug,
                "row_id": row_id,
                "source_slug": source_slug,
                "filename": filename,
                "label": chunk_label,
                "text": indexed_body[:12000],
                # Full body for graph embed (must match content_hash / search vector_chunks).
                "embed_text": indexed_body,
                # Same digest as session_search_db._upsert_vector_chunks (full body, errors=ignore).
                "content_hash": hashlib.sha256(
                    indexed_body.encode("utf-8", errors="ignore")
                ).hexdigest(),
                "rel_path": rel_path,
                "repo_root": repo_root,
            }
        )
        if meta_sink is not None:
            meta_sink.append(
                {
                    "row_id": row_id,
                    "char_start": char_start,
                    "char_end": char_end,
                    "line_start": line_start,
                    "line_end": line_end,
                }
            )
        row_id += 1

    syn = (synthesis_text or "").strip()
    if syn:
        for idx, chunk in enumerate(
            _chunk_text_anchored(syn, chunk_size=chunk_size, overlap=overlap), 1
        ):
            _append_chunk_row(
                header=f"[{label} > synthesis]",
                chunk_text=str(chunk.get("text") or ""),
                filename="synthesis.md",
                source_slug="synthesis",
                chunk_label=f"{label} > synthesis §chunk {idx}",
                char_start=chunk.get("char_start"),
                char_end=chunk.get("char_end"),
                line_start=chunk.get("line_start"),
                line_end=chunk.get("line_end"),
            )

    valid_sources = [s for s in sources if isinstance(s, dict) and str(s.get("slug") or "")]
    src_total = len(valid_sources)
    src_done = 0
    for src in sources:
        if not isinstance(src, dict):
            continue
        source_slug = str(src.get("slug") or "")
        filename = str(src.get("filename") or source_slug)
        if not source_slug:
            continue
        src_done += 1
        if prog and src_total and (src_done % 15 == 0 or src_done == src_total):
            _set_digest_phase(prog, "indexing", detail=f"Preparing search index ({src_done}/{src_total})…")
        rel_path = str(src.get("rel_path") or "")
        repo_root = str(src.get("repo_root") or "")
        raw = (raw_texts or {}).get(source_slug, "")
        if raw and len(raw) > 500:
            source_chunks: list[dict[str, Any]] = []
            for idx, chunk in enumerate(
                _chunk_text_anchored(raw, chunk_size=chunk_size, overlap=overlap), 1
            ):
                _append_chunk_row(
                    header=f"[{label} > {filename}]",
                    chunk_text=str(chunk.get("text") or ""),
                    filename=filename,
                    source_slug=source_slug,
                    rel_path=rel_path,
                    repo_root=repo_root,
                    chunk_label=f"{filename} §chunk {idx}",
                    char_start=chunk.get("char_start"),
                    char_end=chunk.get("char_end"),
                    line_start=chunk.get("line_start"),
                    line_end=chunk.get("line_end"),
                    meta_sink=source_chunks,
                )
            if source_chunks:
                chunk_meta_by_slug[source_slug] = source_chunks[:80]
        src_path = _source_page_path(slug, source_slug)
        if not os.path.isfile(src_path):
            continue
        body = _read_text_file(src_path)
        if not body.strip():
            continue
        page_chunks = _chunk_text_anchored(body, chunk_size=chunk_size, overlap=overlap)
        if not page_chunks:
            continue
        source_chunks = []
        for idx, chunk in enumerate(page_chunks, 1):
            _append_chunk_row(
                header=f"[{label} > {filename}]",
                chunk_text=str(chunk.get("text") or ""),
                filename=filename,
                source_slug=source_slug,
                rel_path=rel_path,
                repo_root=repo_root,
                chunk_label=f"{filename} §chunk {idx}",
                char_start=chunk.get("char_start"),
                char_end=chunk.get("char_end"),
                line_start=chunk.get("line_start"),
                line_end=chunk.get("line_end"),
                meta_sink=source_chunks,
            )
        if source_chunks:
            prior = chunk_meta_by_slug.get(source_slug) or []
            chunk_meta_by_slug[source_slug] = (prior + source_chunks)[:80]

    return rows, chunk_meta_by_slug, graph_nodes


def _index_knowledge_in_search_db(
    slug: str,
    label: str,
    synthesis_text: str,
    sources: list[dict[str, Any]],
    raw_texts: dict[str, str] | None = None,
    *,
    ingest_profile: str = "docs",
) -> dict[str, list[dict[str, Any]]]:
    del ingest_profile  # vector paths use embedding_chunk_params; digest keeps ingest_chunk_params
    chunk_meta_by_slug: dict[str, list[dict[str, Any]]] = {}
    thread_id = f"knowledge:{slug}"
    staging_thread_id = None
    try:
        import session_search_db as ss
        from utils.embedding_model import (
            begin_embedding_reindex_hold,
            embedding_enabled,
            embedding_work_should_defer,
            end_embedding_reindex_hold,
            load,
            maybe_schedule_embedding_reindex,
        )

        if embedding_work_should_defer():
            maybe_schedule_embedding_reindex(thread_id)
            return chunk_meta_by_slug

        rows, chunk_meta_by_slug, _nodes = _build_knowledge_corpus_index(
            slug, label, synthesis_text, sources, raw_texts, progress_slug=slug
        )
        if not rows:
            had_material = bool((synthesis_text or "").strip() or sources)
            existing_fts = ss.count_thread_fts_messages(thread_id)
            if had_material or existing_fts > 0:
                _debug_log(
                    f"[knowledge] search index skipped for {slug}: no indexable chunks "
                    f"(material={had_material}, existing_fts={existing_fts})",
                    "WARN",
                )
                if (
                    existing_fts > 0
                    and not embedding_work_should_defer()
                    and ss.reindex_vectors_from_existing_fts(thread_id)
                ):
                    _debug_log(
                        f"[knowledge] search index for {slug}: recovered vectors from existing FTS",
                        "INFO",
                    )
                elif not ss.knowledge_semantic_index_complete_for_thread(thread_id):
                    maybe_schedule_embedding_reindex(thread_id)
                return chunk_meta_by_slug
            _debug_log(f"[knowledge] search index skipped for {slug}: no indexable chunks", "WARN")
            ss.clear_thread_index(thread_id, clear_fts=True, clear_vectors=True)
            return chunk_meta_by_slug
        if embedding_enabled() and not load():
            _debug_log(
                f"[knowledge] search index deferred for {slug}: MLX embedder unavailable",
                "WARN",
            )
            maybe_schedule_embedding_reindex(thread_id)
            return chunk_meta_by_slug
        if embedding_work_should_defer():
            maybe_schedule_embedding_reindex(thread_id)
            return chunk_meta_by_slug
        staging_thread_id = ss.reindex_staging_thread_id(thread_id)
        ss.clear_thread_index(staging_thread_id, clear_fts=True, clear_vectors=True)
        _set_digest_phase(slug, "indexing", detail="Embedding search index…")
        begin_embedding_reindex_hold()
        try:
            _INDEX_BATCH = 64
            for start in range(0, len(rows), _INDEX_BATCH):
                if embedding_work_should_defer():
                    ss.clear_thread_index(
                        staging_thread_id, clear_fts=True, clear_vectors=True
                    )
                    maybe_schedule_embedding_reindex(thread_id)
                    return chunk_meta_by_slug
                ss.index_messages(
                    staging_thread_id,
                    rows[start : start + _INDEX_BATCH],
                    force_semantic=True,
                    pre_chunked=True,
                )
            if not ss.knowledge_semantic_index_complete_for_thread(staging_thread_id):
                _debug_log(
                    f"[knowledge] search index for {slug}: semantic index incomplete (staging)",
                    "WARN",
                )
                ss.clear_thread_index(staging_thread_id, clear_fts=True, clear_vectors=True)
                maybe_schedule_embedding_reindex(thread_id)
                return chunk_meta_by_slug
            if not ss.promote_staged_thread_index(staging_thread_id, thread_id):
                _debug_log(
                    f"[knowledge] search index for {slug}: staging promote failed",
                    "WARN",
                )
                ss.clear_thread_index(staging_thread_id, clear_fts=True, clear_vectors=True)
                maybe_schedule_embedding_reindex(thread_id)
                return chunk_meta_by_slug
            staging_thread_id = None
        finally:
            end_embedding_reindex_hold()
    except Exception as exc:
        _debug_log(f"[knowledge] search index failed for {slug}: {exc}", "WARN")
        try:
            if staging_thread_id:
                import session_search_db as ss

                ss.clear_thread_index(staging_thread_id, clear_fts=True, clear_vectors=True)
            from utils.embedding_model import maybe_schedule_embedding_reindex

            maybe_schedule_embedding_reindex(thread_id)
        except Exception as cleanup_exc:
            _debug_log(
                f"[knowledge] search index cleanup failed for {slug}: {cleanup_exc}",
                "WARN",
            )
    return chunk_meta_by_slug


def _apply_chunk_meta_to_manifest_sources(
    slug: str, sources: list[dict[str, Any]], chunk_meta_by_slug: dict[str, list[dict[str, Any]]]
) -> None:
    """Persist v2 row anchors onto manifest sources after index/reindex."""
    if not chunk_meta_by_slug:
        return
    manifest = _load_manifest(slug) or {}
    manifest_sources = manifest.get("sources")
    if not isinstance(manifest_sources, list):
        return
    updated = False
    for src in manifest_sources:
        if not isinstance(src, dict):
            continue
        sid = str(src.get("slug") or "")
        if sid and sid in chunk_meta_by_slug:
            src["chunks"] = chunk_meta_by_slug[sid]
            updated = True
    if updated:
        _save_manifest(slug, manifest)


@tool
def digest_corpus(path: str, label: str, update: bool = False, ingest_profile: str = "docs") -> str:
    """Study a file/folder and compile durable knowledge notes.

    ingest_profile: ``docs`` (default) or ``repo`` (skips code files under git roots).
    """
    _ensure_startup_manifest_repair()
    raw_path = (path or "").strip()
    is_url_input = bool(re.match(r"^https?://", raw_path, flags=re.IGNORECASE))
    if not is_url_input:
        resolved, resolve_err = safe_resolve_path(
            _workspace_path(),
            raw_path,
            allow_write=False,
            create_parents=False,
        )
        if resolve_err or not resolved:
            return f"Digest failed: {resolve_err or 'Invalid path'}"
        workspace = os.path.abspath(os.path.expanduser(_workspace_path())).rstrip(os.sep)
        target = os.path.abspath(os.path.expanduser(resolved))
        if not target.startswith(workspace + os.sep) and target != workspace and not _outside_workspace_digest_allowed():
            return (
                "Digest failed: path is outside workspace. "
                "Set app preference 'allow_knowledge_digest_outside_workspace' to true to allow this."
            )
    slug, _ = _choose_corpus_slug(label, bool(update))
    profile = normalize_ingest_profile(ingest_profile, source_path=raw_path)
    try:
        result = _digest_sync(
            path, label, update=bool(update), emit_progress=True, ingest_profile=profile
        )
    except BaseException:
        _fail_running_digest_manifest(slug)
        raise
    finally:
        _emit_input_status("")
    if result.get("ok"):
        return str(result.get("message") or "Digest complete.")
    return f"Digest failed: {result.get('error', 'unknown error')}"


def list_knowledge_text() -> str:
    corpora = list_knowledge_corpora()
    if not corpora:
        return "No knowledge corpora available."
    lines: list[str] = []
    for row in corpora:
        lines.append(
            f"- {row.get('label') or row.get('slug')} (slug: {row.get('slug')}, status: {row.get('digest_status')}, sources: {row.get('source_count')})"
        )
    return "\n".join(lines)


@tool
def list_knowledge() -> str:
    """List available compiled knowledge corpora."""
    return list_knowledge_text()


def _filter_corpora_by_name(corpora: list[dict[str, Any]], corpus: str | None) -> list[dict[str, Any]]:
    if not corpus:
        return corpora
    wanted = corpus.strip().lower()
    wanted_norm = re.sub(r"[\s_-]+", " ", wanted).strip()
    wanted_slug = re.sub(r"[\s_-]+", "_", wanted).strip("_")

    def _corpus_matches(c: dict[str, Any]) -> bool:
        slug = c["slug"].lower()
        label = str(c["label"]).lower()
        label_norm = re.sub(r"[\s_-]+", " ", label).strip()
        if slug == wanted or slug == wanted_slug or label == wanted or label_norm == wanted_norm:
            return True
        if wanted_norm in label_norm or wanted_norm in re.sub(r"_", " ", slug):
            return True
        return False

    return [c for c in corpora if _corpus_matches(c)]


def _hit_from_search_row(row: dict[str, Any], slug: str, label: str) -> dict[str, Any] | None:
    thread = f"knowledge:{slug}"
    if str(row.get("thread_id") or "") != thread:
        return None
    snippet = str(row.get("snippet") or "").replace(">>>", "").replace("<<<", "")
    source_name = ""
    source_slug = ""
    ca = str(row.get("created_at") or "")
    anchor = _parse_kb_created_at(ca)
    source_name = str(anchor.get("source_name") or "")
    if not source_name:
        m = re.search(r"\[([^\]]+)\s*>\s*([^\]]+)\]", snippet)
        if m:
            source_name = str(m.group(2)).strip()
    snippet = re.sub(r"^\s*\[[^\]]+\]\s*", "", snippet).strip()
    manifest = _load_manifest(slug) or {}
    for src in manifest.get("sources") or []:
        if not isinstance(src, dict):
            continue
        fn = str(src.get("filename") or "")
        if fn and (fn == source_name or str(src.get("slug") or "") == source_name):
            source_slug = str(src.get("slug") or "")
            source_name = fn or source_name
            break
    from utils.knowledge_retrieval import _search_row_base_score

    base = _search_row_base_score(row)
    score = float(base) if base is not None and base > 0 else None
    hit_id = f"{slug}#{source_slug or 'synthesis'}#{hash(snippet) & 0xFFFFFFFF}"
    out: dict[str, Any] = {
        "id": hit_id,
        "corpus_slug": slug,
        "corpus_label": label,
        "source_slug": source_slug,
        "source_name": source_name,
        "snippet": snippet,
        "thread_id": thread,
        "score": score,
        "base_score": base if base is not None else 0.0,
    }
    for internal_key in ("semantic_score", "rrf_score"):
        if row.get(internal_key) is not None:
            try:
                out[internal_key] = float(row[internal_key])
            except (TypeError, ValueError):
                pass
    if row.get("_source") is not None:
        out["search_source"] = str(row.get("_source"))
    for key in ("char_start", "char_end", "line_start", "line_end"):
        val = anchor.get(key)
        if val is not None:
            out[key] = val
    return out


def search_knowledge_hits(
    query: str,
    corpus: str | None = None,
    max_results: int = 8,
    *,
    semantic: bool = True,
    retrieval_mode: str = "tool",
) -> list[dict[str, Any]]:
    """Structured hybrid search over compiled knowledge corpora (multi-lane retrieval)."""
    from utils.knowledge_retrieval import retrieve_knowledge_hits

    mode = retrieval_mode if retrieval_mode in (
        "ask",
        "tool",
        "inject",
        "connections",
        "theme_detail",
        "unified",
    ) else "tool"
    return retrieve_knowledge_hits(
        query,
        corpus=corpus,
        max_results=max_results,
        mode=mode,  # type: ignore[arg-type]
        semantic=semantic,
    )


_UNIFIED_SEARCH_SCOPES = frozenset({"knowledge", "memory", "vault"})


def _parse_memory_search_text_to_hits(text: str, max_hits: int) -> list[dict[str, Any]]:
    """Turn ``search_memory`` fused blocks into structured unified-search hits."""
    import hashlib

    raw = (text or "").strip()
    if not raw or raw.startswith("No query") or raw.startswith("No matches"):
        return []
    if "Memory search refused" in raw:
        return []
    body = raw
    header_end = re.match(r"^Search results for .+?\):\n", body, re.DOTALL)
    if header_end:
        body = body[header_end.end() :]
    parts = re.split(r"\n\n---\n", body)
    hits: list[dict[str, Any]] = []
    for i, part in enumerate(parts):
        chunk = part.strip()
        if not chunk or chunk.startswith("Partial scan:"):
            continue
        lines = chunk.split("\n", 1)
        rel = lines[0].strip()
        if rel.startswith("[") and rel.endswith("]"):
            rel = rel[1:-1]
        snippet = lines[1].strip() if len(lines) > 1 else rel
        if not rel:
            continue
        digest = hashlib.sha256(f"{rel}:{snippet[:120]}".encode()).hexdigest()[:12]
        hits.append(
            {
                "id": f"memory:{digest}:{i}",
                "source_type": "memory",
                "title": rel,
                "subtitle": "",
                "snippet": snippet[:800],
                "memory_path": rel,
            }
        )
        if len(hits) >= max_hits:
            break
    return hits


def _vault_search_to_hits(query: str, *, max_notes: int = 10) -> list[dict[str, Any]]:
    from chat_storage import search_vault_captures

    out = search_vault_captures(query, limit_notes=max_notes, hits_per_note=1)
    hits: list[dict[str, Any]] = []
    for row in out.get("results") or []:
        if not isinstance(row, dict):
            continue
        note_id = str(row.get("note_id") or "").strip()
        if not note_id:
            continue
        title = str(row.get("title") or "").strip() or "Untitled note"
        hit_list = row.get("hits") if isinstance(row.get("hits"), list) else []
        snippet = ""
        if hit_list and isinstance(hit_list[0], dict):
            snippet = str(hit_list[0].get("snippet") or "").strip()
        project_id = str(row.get("project_id") or "").strip()
        hits.append(
            {
                "id": f"vault:{note_id}",
                "source_type": "vault",
                "title": title,
                "subtitle": "",
                "snippet": snippet,
                "vault_note_id": note_id,
                "vault_project_id": project_id,
            }
        )
    return hits


def unified_library_search(
    query: str,
    *,
    scopes: set[str] | None = None,
    max_knowledge: int = 12,
    max_memory: int = 8,
    max_vault: int = 10,
    semantic: bool = True,
) -> dict[str, Any]:
    """Search studied corpora, durable memory files, and Vault notes for Library UI."""
    q = (query or "").strip()
    if not q:
        return {
            "schema": "mugi.knowledge.unified_search.v1",
            "query": "",
            "scopes": [],
            "semantic": semantic,
            "hits": [],
        }
    active = {s.strip().lower() for s in (scopes or _UNIFIED_SEARCH_SCOPES) if s}
    if not active or "all" in active:
        active = set(_UNIFIED_SEARCH_SCOPES)
    else:
        active &= _UNIFIED_SEARCH_SCOPES
    if not active:
        active = set(_UNIFIED_SEARCH_SCOPES)

    hits: list[dict[str, Any]] = []
    if "knowledge" in active and int(max_knowledge) > 0:
        cap = max(1, min(int(max_knowledge), 20))
        for row in search_knowledge_hits(
            q, corpus=None, max_results=cap, semantic=semantic, retrieval_mode="unified"
        ):
            if not isinstance(row, dict):
                continue
            hit = dict(row)
            hit["source_type"] = "knowledge"
            hits.append(hit)
    if "memory" in active and int(max_memory) > 0:
        import tools.memory_tools as _mt

        cap = max(1, min(int(max_memory), 20))
        mem_text = _mt.search_memory.invoke(
            {
                "query": q,
                "max_results": cap,
                "include_archives": False,
                "semantic": semantic,
            },
        )
        hits.extend(_parse_memory_search_text_to_hits(str(mem_text or ""), cap))
    if "vault" in active and int(max_vault) > 0:
        cap = max(1, min(int(max_vault), 40))
        hits.extend(_vault_search_to_hits(q, max_notes=cap))

    return {
        "schema": "mugi.knowledge.unified_search.v1",
        "query": q,
        "scopes": sorted(active),
        "semantic": semantic,
        "hits": hits,
    }


# Last ">" before "]" splits corpus label (may contain ">") from source token.
_ASK_CITATION_RE = re.compile(r"\[([^\]]+)\s*>\s*([^\]]+)\]")
_ASK_NO_EVIDENCE_SENTINEL = "MUGI_KNOWLEDGE_INSUFFICIENT_EVIDENCE"


def _ask_salient_terms(question: str) -> set[str]:
    token_re, stopwords = _tokenizer_refs()
    return {
        t.lower()
        for t in token_re.findall((question or "").lower())
        if len(t) >= 3 and t.lower() not in stopwords
    }


def _hits_contain_query_terms(hits: list[dict[str, Any]], terms: set[str]) -> bool:
    if not terms:
        return True
    for h in hits:
        snip = str(h.get("snippet") or "").lower()
        if any(term in snip for term in terms):
            return True
    return False


def _hits_pass_score_floor(hits: list[dict[str, Any]], floor: float) -> bool:
    if not hits:
        return False
    return any(float(h.get("fused_score") or 0) >= floor for h in hits)


def _public_source_row(hit: dict[str, Any]) -> dict[str, Any]:
    return {
        "corpus_slug": hit.get("corpus_slug"),
        "corpus_label": hit.get("corpus_label"),
        "source_slug": hit.get("source_slug"),
        "source_name": hit.get("source_name"),
        "snippet": hit.get("snippet"),
        "fused_score": hit.get("fused_score"),
    }


def _finalize_ask_payload(
    base: dict[str, Any],
    *,
    status: str,
    answer: str = "",
    hits: list[dict[str, Any]] | None = None,
    cite_index: list[dict[str, Any]] | None = None,
    summarizer_method: str = "",
) -> dict[str, Any]:
    base["answer_status"] = status
    base["summarizer_method"] = summarizer_method or ""
    if hits is not None:
        base["sources"] = [_public_source_row(h) for h in hits]
    if status in ("no_hits", "no_evidence"):
        base["insufficient_evidence"] = True
        base["uncited_answer"] = False
        base["answer"] = answer
        base["citations"] = []
        return base
    if status == "generation_failed":
        base["insufficient_evidence"] = False
        base["uncited_answer"] = False
        base["answer"] = answer
        base["citations"] = []
        return base
    base["insufficient_evidence"] = False
    base["answer"] = answer
    base["uncited_answer"] = _ASK_CITATION_RE.search(answer) is None
    base["citations"] = _citations_from_answer(answer, cite_index or [])
    return base


def _build_ask_context_blocks(hits: list[dict[str, Any]]) -> tuple[str, list[dict[str, Any]]]:
    """Format search hits as LLM context; return (context_text, citation_index rows)."""
    blocks: list[str] = []
    index: list[dict[str, Any]] = []
    for i, h in enumerate(hits, 1):
        label = str(h.get("corpus_label") or h.get("corpus_slug") or "").strip()
        source_name = str(h.get("source_name") or "").strip()
        snippet = str(h.get("snippet") or "").strip()[:1200]
        corpus_slug = str(h.get("corpus_slug") or "").strip()
        source_slug = str(h.get("source_slug") or "").strip()
        cite_source = source_name or source_slug or "synthesis"
        cite_key = f"[{label} > {cite_source}]"
        blocks.append(f"{cite_key}\n{snippet}")
        index.append(
            {
                "ref_id": f"REF-{i}",
                "cite_key": cite_key,
                "corpus_slug": corpus_slug,
                "corpus_label": label,
                "source_slug": source_slug,
                "source_name": source_name,
                "snippet": snippet,
            }
        )
    return "\n\n---\n\n".join(blocks), index


def _ask_citation_source_matches(row: dict[str, Any], source: str) -> bool:
    """Match model citation source token to index row (filename, slug, or stem)."""
    want = (source or "").strip().lower()
    if not want:
        return False
    name = str(row.get("source_name") or "").strip().lower()
    slug = str(row.get("source_slug") or "").strip().lower()
    if name and name == want:
        return True
    if slug and slug == want:
        return True
    if name:
        base = name.rsplit("/", 1)[-1]
        if base == want or base.rsplit(".", 1)[0] == want:
            return True
    if slug and slug.rsplit(".", 1)[0] == want:
        return True
    return False


def _citations_from_answer(answer: str, index: list[dict[str, Any]]) -> list[dict[str, Any]]:
    """Structured citations referenced inline in the model answer."""
    found: list[dict[str, Any]] = []
    seen: set[str] = set()
    for m in _ASK_CITATION_RE.finditer(answer or ""):
        label = m.group(1).strip()
        source = m.group(2).strip()
        key = f"{label.lower()}|{source.lower()}"
        if key in seen:
            continue
        seen.add(key)
        matched: dict[str, Any] | None = None
        for row in index:
            if str(row.get("corpus_label") or "").lower() == label.lower() and _ask_citation_source_matches(
                row, source
            ):
                matched = row
                break
        if matched:
            found.append(
                {
                    "ref_id": matched.get("ref_id"),
                    "corpus_slug": matched.get("corpus_slug"),
                    "corpus_label": matched.get("corpus_label"),
                    "source_slug": matched.get("source_slug"),
                    "source_name": matched.get("source_name"),
                    "snippet": matched.get("snippet"),
                }
            )
        else:
            found.append(
                {
                    "ref_id": "",
                    "corpus_slug": "",
                    "corpus_label": label,
                    "source_slug": "",
                    "source_name": source,
                    "snippet": "",
                }
            )
    return found


def ask_knowledge_corpus(
    question: str,
    *,
    corpus: str | None = None,
    max_sources: int = 10,
    include_retrieval_meta: bool = False,
) -> dict[str, Any]:
    """Citation-required Q&A over studied corpora (Library Ask mode / HTTP)."""
    from utils.knowledge_retrieval import ask_fused_score_floor, get_retrieval_config, retrieval_lanes_used

    q = (question or "").strip()
    base: dict[str, Any] = {
        "schema": "mugi.knowledge.ask.v1",
        "question": q,
        "corpus": corpus,
        "answer": "",
        "answer_status": "no_hits",
        "citations": [],
        "sources": [],
        "insufficient_evidence": True,
        "uncited_answer": False,
        "summarizer_method": "",
    }
    if not q:
        base["error"] = "question required"
        return base

    cap = max(1, min(int(max_sources), 12))
    corpora = _filter_corpora_by_name(list_knowledge_corpora(), corpus)
    if not corpora:
        _record_knowledge_gap(q, corpus=corpus)
        return _finalize_ask_payload(
            base,
            status="no_hits",
            answer="No matching knowledge corpus found.",
        )

    hits = search_knowledge_hits(q, corpus=corpus, max_results=cap, semantic=True, retrieval_mode="ask")
    if include_retrieval_meta:
        base["retrieval_meta"] = {
            "hit_count": len(hits),
            "lanes_used": retrieval_lanes_used(hits),
        }

    terms = _ask_salient_terms(q)
    score_floor = ask_fused_score_floor(get_retrieval_config())

    if not hits or not _hits_pass_score_floor(hits, score_floor) or not _hits_contain_query_terms(hits, terms):
        _record_knowledge_gap(q, corpus=corpus)
        hint = ", ".join(sorted(terms)[:4]) if terms else q[:80]
        return _finalize_ask_payload(
            base,
            status="no_hits",
            answer=(
                f'No supporting excerpts matched "{q}". '
                f"Try Search for: {hint}. Add documents or rephrase."
            ),
            hits=hits or [],
        )

    context, cite_index = _build_ask_context_blocks(hits)
    system = (
        "You answer questions using ONLY the excerpts below from the operator's private Knowledge Library.\n"
        "Rules:\n"
        "- Every factual claim must include an inline citation exactly like [Corpus Label > source name] "
        "matching the excerpt headers.\n"
        "- Do not invent facts, sources, or citations not present in the excerpts.\n"
        f"- If the excerpts do not support an answer, respond with exactly: {_ASK_NO_EVIDENCE_SENTINEL}\n"
        "- Keep answers concise (under 400 words)."
    )
    user = f"Excerpts:\n\n{context}\n\n---\n\nQuestion: {q}\n\nAnswer with inline [Corpus > source] citations:"
    answer_raw, summarizer_method = _complete_with_method(
        [{"role": "system", "content": system}, {"role": "user", "content": user}],
        max_tokens=900,
        temperature=0.15,
    )
    answer = (answer_raw or "").strip()
    if not answer:
        return _finalize_ask_payload(
            base,
            status="generation_failed",
            answer="I could not generate an answer from the available excerpts.",
            hits=hits,
            summarizer_method=summarizer_method,
        )
    if _ASK_NO_EVIDENCE_SENTINEL in answer:
        clean = answer.replace(_ASK_NO_EVIDENCE_SENTINEL, "").strip()
        _record_knowledge_gap(q, corpus=corpus)
        msg = clean or (
            "The retrieved excerpts do not support a factual answer. "
            "Use Search or open sources in the library to verify."
        )
        return _finalize_ask_payload(
            base,
            status="no_evidence",
            answer=msg,
            hits=hits,
            summarizer_method=summarizer_method,
        )

    return _finalize_ask_payload(
        base,
        status="answered",
        answer=answer,
        hits=hits,
        cite_index=cite_index,
        summarizer_method=summarizer_method,
    )


def format_knowledge_hits_as_text(hits: list[dict[str, Any]], *, include_tip: bool = True) -> str:
    lines: list[str] = []
    for h in hits:
        label = str(h.get("corpus_label") or h.get("corpus_slug") or "")
        source_name = str(h.get("source_name") or "").strip()
        snippet = str(h.get("snippet") or "").strip()
        prefix = f"[{label} > {source_name}]" if source_name and source_name != "synthesis" else f"[{label}]"
        ls, le = h.get("line_start"), h.get("line_end")
        if ls is not None and le is not None:
            prefix += f" (L{ls}-{le})"
        lines.append(f"{prefix} {snippet}".strip())
    result = "\n\n".join(lines)
    if include_tip and hits:
        result += "\n\n(Tip: use read_knowledge_source with source_id matching a section name above for full text.)"
    return result


def parse_synthesis_connections_section(synthesis_text: str) -> list[str]:
    """Extract bullet lines from the Connections section of synthesis markdown."""
    text = (synthesis_text or "").strip()
    if not text:
        return []
    m = re.search(
        r"(?is)^##\s*Connections[^\n]*\n(.*?)(?=^##\s|\Z)",
        text,
        re.MULTILINE,
    )
    if not m:
        return []
    body = m.group(1)
    bullets: list[str] = []
    for line in body.splitlines():
        stripped = line.strip()
        if stripped.startswith(("-", "*", "•")):
            bullets.append(re.sub(r"^[-*•]\s*", "", stripped).strip())
        elif stripped and bullets:
            bullets[-1] = f"{bullets[-1]} {stripped}"
    return [b for b in bullets if b]


def get_knowledge_topic_index() -> dict[str, list[dict[str, str]]]:
    """Cross-corpus topic → [{slug, label}] map for Library connections UI."""
    _rebuild_topic_index_if_needed()
    with _topic_index_lock:
        return {k: list(v) for k, v in _topic_index_cache.items()}


def _topic_labels_for_source(
    manifest_sources: list[dict[str, Any]],
    source_slug: str,
    canonical_key: str,
    *,
    ingest_profile: str = "docs",
) -> list[str]:
    from utils.knowledge_topic_labels import classify_topic_label, normalize_theme_label

    labels: list[str] = []
    seen: set[str] = set()
    for src in manifest_sources:
        if str(src.get("slug") or "") != source_slug:
            continue
        filename = str(src.get("filename") or "")
        for raw in src.get("topics") or []:
            display, canon = normalize_theme_label(str(raw))
            if canon != canonical_key:
                continue
            if (
                classify_topic_label(
                    str(raw),
                    ingest_profile=ingest_profile,
                    source_filename=filename,
                )
                == "noise"
            ):
                continue
            key = display.lower()
            if key in seen:
                continue
            seen.add(key)
            labels.append(display)
        break
    return labels


def _thematic_search_query(theme_label: str) -> str:
    """Semantic query for theme membership — not a literal keyword search."""
    label = (theme_label or "").strip()
    if not label:
        return ""
    return (
        f"What does this material teach about {label}? "
        f"Passages, concepts, and themes related to {label}."
    )


def _connection_rationale(theme_label: str, snippet: str, *, membership: str) -> str:
    from utils.knowledge_topic_labels import extract_passage_locator

    theme = (theme_label or "").strip() or "this theme"
    if membership == "tagged":
        locator = extract_passage_locator(snippet)
        if locator:
            return f"Indexed under “{theme}” during digest ({locator})."
        return f"Indexed under “{theme}” during corpus digest."
    locator = extract_passage_locator(snippet)
    if locator:
        return f"Thematic match at {locator} — discusses “{theme}” in context."
    if snippet.strip():
        return f"Thematically related to “{theme}” (semantic retrieval, not keyword match)."
    return f"Related to “{theme}” via semantic search."


def _char_offset_for_snippet(corpus_slug: str, source_slug: str, snippet: str) -> int | None:
    needle = (snippet or "").strip()
    if len(needle) < 24:
        return None
    probe = needle[: min(120, len(needle))]
    try:
        manifest, entry, err = _resolve_corpus_and_source(corpus_slug, source_slug)
        if err or not entry:
            return None
        path = _source_page_path(corpus_slug, str(entry.get("slug") or source_slug))
        if not os.path.isfile(path):
            return None
        text = _read_text_file(path)
        idx = text.find(probe)
        if idx >= 0:
            return idx
        for line in needle.splitlines():
            line = line.strip()
            if len(line) < 20:
                continue
            idx = text.find(line[:80])
            if idx >= 0:
                return idx
    except Exception as exc:
        from tools._logging import _debug_log

        _debug_log(
            f"[knowledge] char_offset lookup failed for {corpus_slug}/{source_slug}: {exc}",
            "WARN",
        )
    return None


def _fetch_thematic_hits(display_label: str, corpus_filter: str | None) -> list[dict[str, Any]]:
    thematic_q = _thematic_search_query(display_label)
    if not thematic_q:
        return []
    hits = search_knowledge_hits(
        thematic_q,
        corpus=corpus_filter,
        max_results=48,
        semantic=True,
        retrieval_mode="theme_detail",
    )
    if not hits:
        from tools._logging import _debug_log

        _debug_log(
            f"[knowledge] thematic search returned no hits for theme {display_label!r}",
            "INFO",
        )
    return hits


def _index_thematic_hits_by_source(
    hits: list[dict[str, Any]],
) -> dict[tuple[str, str], dict[str, Any]]:
    out: dict[tuple[str, str], dict[str, Any]] = {}
    for hit in hits:
        corpus_slug = str(hit.get("corpus_slug") or "").strip()
        sid = str(hit.get("source_slug") or "").strip()
        if not corpus_slug or not sid:
            continue
        key = (corpus_slug, sid)
        prev = out.get(key)
        if prev is None or float(hit.get("fused_score") or 0) > float(prev.get("fused_score") or 0):
            out[key] = hit
    return out


def _snippet_for_source_from_hits(
    hits: list[dict[str, Any]],
    corpus_slug: str,
    source_slug: str,
) -> str:
    from utils.knowledge_topic_labels import sanitize_knowledge_snippet_for_display

    src_low = source_slug.lower()
    manifest = _load_manifest(corpus_slug) or {}
    manifest_sources = manifest.get("sources") if isinstance(manifest.get("sources"), list) else []
    filename_low = ""
    for src in manifest_sources:
        if str(src.get("slug") or "").lower() == src_low:
            filename_low = str(src.get("filename") or "").lower()
            break
    for hit in hits:
        hit_slug = str(hit.get("source_slug") or "").lower()
        hit_name = str(hit.get("source_name") or "").lower()
        if hit_slug == src_low or (filename_low and filename_low in hit_name):
            snip = sanitize_knowledge_snippet_for_display(str(hit.get("snippet") or ""))
            if snip:
                return snip[:600]
    return ""


def _summary_excerpt_fallback(
    corpus_slug: str,
    source_slug: str,
    query: str,
    topic_labels: list[str],
) -> str:
    try:
        body = read_knowledge_source_notes_body(
            corpus_slug,
            source_slug,
            summary_only=True,
            limit=8000,
        )
        text = str(body.get("text") or "")
        text = re.sub(r"^\[.+?\]\s*studied notes[^\n]*\n+", "", text, flags=re.MULTILINE)
        text = extract_summary_section_from_source_notes(text) or text
        needles = topic_labels or [query]
        for para in re.split(r"\n\s*\n", text):
            chunk = para.strip()
            if not chunk:
                continue
            for needle in needles:
                if needle.lower() in chunk.lower():
                    return chunk[:600]
    except Exception as exc:
        from tools._logging import _debug_log

        _debug_log(
            f"[knowledge] connection excerpt fallback failed for {corpus_slug}/{source_slug}: {exc}",
            "WARN",
        )
    return ""


def _connection_detail_for_source(
    corpus_slug: str,
    source_slug: str,
    query: str,
    topic_labels: list[str],
    *,
    membership: str = "tagged",
    theme_label: str = "",
    thematic_hits: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
    from utils.knowledge_topic_labels import extract_passage_locator

    excerpt = ""
    if thematic_hits is not None:
        excerpt = _snippet_for_source_from_hits(thematic_hits, corpus_slug, source_slug)
    if not excerpt:
        excerpt = _summary_excerpt_fallback(corpus_slug, source_slug, query, topic_labels)
    manifest = _load_manifest(corpus_slug) or {}
    manifest_sources = manifest.get("sources") if isinstance(manifest.get("sources"), list) else []
    src_low = source_slug.lower()
    locator = extract_passage_locator(excerpt)
    if not locator:
        for src in manifest_sources:
            if str(src.get("slug") or "").lower() == src_low:
                locator = extract_passage_locator(str(src.get("filename") or ""))
                break
    offset = _char_offset_for_snippet(corpus_slug, source_slug, excerpt) if excerpt else None
    label = theme_label or query
    return {
        "connection_excerpt": excerpt,
        "passage_locator": locator,
        "char_offset": offset,
        "connection_rationale": _connection_rationale(label, excerpt, membership=membership),
        "membership": membership,
    }


def _connection_excerpt_for_source(
    corpus_slug: str,
    source_slug: str,
    query: str,
    topic_labels: list[str],
) -> str:
    detail = _connection_detail_for_source(
        corpus_slug,
        source_slug,
        query,
        topic_labels,
        membership="tagged",
        theme_label=query,
        thematic_hits=_fetch_thematic_hits(query, corpus_slug),
    )
    return str(detail.get("connection_excerpt") or "")


def _append_semantic_theme_sources(
    merged: list[dict[str, Any]],
    seen: set[tuple[str, str]],
    *,
    display_label: str,
    hits_by_source: dict[tuple[str, str], dict[str, Any]],
) -> int:
    """Add sources from prefetched thematic hits that were not tagged at digest time."""
    from utils.knowledge_topic_labels import (
        build_source_entry,
        extract_passage_locator,
        sanitize_knowledge_snippet_for_display,
    )

    added = 0
    for (corpus_slug, sid), hit in sorted(
        hits_by_source.items(),
        key=lambda item: float(item[1].get("fused_score") or 0),
        reverse=True,
    ):
        key = (corpus_slug, sid)
        if key in seen:
            continue
        manifest = _load_manifest(corpus_slug) or {}
        manifest_sources = manifest.get("sources") if isinstance(manifest.get("sources"), list) else []
        src_row = next(
            (s for s in manifest_sources if isinstance(s, dict) and str(s.get("slug") or "") == sid),
            None,
        )
        if not src_row:
            from tools._logging import _debug_log

            _debug_log(
                f"[knowledge] thematic hit for missing manifest source {corpus_slug}/{sid}",
                "WARN",
            )
            continue
        row = dict(build_source_entry(src_row, corpus_slug))
        snip = sanitize_knowledge_snippet_for_display(str(hit.get("snippet") or ""))
        row["membership"] = "semantic"
        row["connection_excerpt"] = snip[:600]
        row["passage_locator"] = extract_passage_locator(snip) or extract_passage_locator(
            str(src_row.get("filename") or "")
        )
        row["char_offset"] = _char_offset_for_snippet(corpus_slug, sid, snip)
        row["connection_rationale"] = _connection_rationale(display_label, snip, membership="semantic")
        row["topic_labels"] = []
        seen.add(key)
        merged.append(row)
        added += 1
    return added


def _build_theme_detail_sources(
    intra_themes: list[dict[str, Any]],
    *,
    topic_canonical: str,
    topic_term: str,
    display_label: str,
    corpus_filter: str | None = None,
) -> tuple[list[dict[str, Any]], int, int]:
    from utils.knowledge_connections_store import source_rows_for_theme
    from utils.knowledge_topic_labels import extract_passage_locator

    merged: list[dict[str, Any]] = []
    seen: set[tuple[str, str]] = set()
    term_low = topic_term.lower()
    for theme_row in intra_themes:
        row_canon = str(theme_row.get("canonical_key") or "").lower()
        row_label = str(theme_row.get("label") or "").lower()
        if topic_canonical and row_canon != topic_canonical and row_label != term_low:
            continue
        corpus_slug = str(theme_row.get("corpus_slug") or "")
        manifest = _load_manifest(corpus_slug) or {}
        manifest_sources = manifest.get("sources") if isinstance(manifest.get("sources"), list) else []
        for src in source_rows_for_theme(theme_row, corpus_slug, manifest_sources):
            sid = str(src.get("slug") or "")
            if not sid:
                continue
            key = (corpus_slug, sid)
            if key in seen:
                continue
            seen.add(key)
            merged.append(dict(src))
    query = display_label or topic_term or topic_canonical
    tagged_count = len(merged)
    thematic_hits = _fetch_thematic_hits(display_label, corpus_filter)
    hits_by_source = _index_thematic_hits_by_source(thematic_hits)
    for row in merged:
        corpus_slug = str(row.get("corpus_slug") or "")
        sid = str(row.get("slug") or "")
        manifest = _load_manifest(corpus_slug) or {}
        profile = str(manifest.get("ingest_profile") or "docs")
        manifest_sources = manifest.get("sources") if isinstance(manifest.get("sources"), list) else []
        labels = _topic_labels_for_source(
            manifest_sources,
            sid,
            topic_canonical,
            ingest_profile=profile,
        )
        if not labels and display_label:
            labels = [display_label]
        detail = _connection_detail_for_source(
            corpus_slug,
            sid,
            query,
            labels,
            membership="tagged",
            theme_label=display_label,
            thematic_hits=thematic_hits,
        )
        row["topic_labels"] = labels
        row["membership"] = "tagged"
        row["connection_excerpt"] = detail.get("connection_excerpt") or ""
        row["passage_locator"] = detail.get("passage_locator") or extract_passage_locator(
            str(row.get("filename") or "")
        )
        row["char_offset"] = detail.get("char_offset")
        row["connection_rationale"] = detail.get("connection_rationale") or ""
    semantic_added = _append_semantic_theme_sources(
        merged,
        seen,
        display_label=display_label,
        hits_by_source=hits_by_source,
    )
    merged.sort(
        key=lambda r: (
            0 if str(r.get("membership") or "") == "tagged" else 1,
            str(r.get("filename") or r.get("slug") or "").lower(),
        )
    )
    return merged, tagged_count, semantic_added


def build_knowledge_connections_payload(
    *,
    topic: str | None = None,
    query: str | None = None,
    corpus: str | None = None,
    max_corpora: int = 10,
    related: bool = False,
    theme_detail: bool = False,
) -> dict[str, Any]:
    """Shared topics, structured edges, and optional semantic related hits."""
    from utils.knowledge_connections_store import (
        _SLUG_RE,
        _sidebar_row_matches_active_topic,
        _topic_source_slug_keys,
        build_shared_topics_from_connections,
        corpora_missing_connections_index,
        list_all_connection_edges,
    )
    from utils.knowledge_topic_labels import normalize_theme_label, sanitize_knowledge_snippet_for_display

    filter_term = (query or "").strip().lower()
    topic_term = (topic or "").strip()
    topic_canonical = ""
    if topic_term:
        _, topic_canonical = normalize_theme_label(topic_term)
        if not topic_canonical:
            topic_canonical = topic_term.lower()

    corpus_filter = (corpus or "").strip() or None
    shared_filter = topic_canonical or filter_term
    shared_topics = build_shared_topics_from_connections(
        corpus_filter=corpus_filter,
        shared_filter=shared_filter,
        max_corpora=max_corpora,
    )

    intra_themes, structured_edges = list_all_connection_edges(
        corpus_filter=corpus_filter,
        topic_filter=topic,
        query_filter=query if not topic else None,
    )

    topic_source_keys: set[tuple[str, str]] = set()
    if topic_term:
        topic_source_keys = _topic_source_slug_keys(
            intra_themes,
            topic_canonical=topic_canonical,
            topic_term=topic_term,
        )

    theme_detail_obj: dict[str, Any] | None = None
    if topic_term and theme_detail:
        matching = [t for t in intra_themes if str(t.get("canonical_key") or "").lower() == topic_canonical]
        if not matching:
            matching = [
                t
                for t in intra_themes
                if str(t.get("label") or "").lower() == topic_term.lower()
                or str(t.get("canonical_key") or "").lower() == topic_canonical
            ]
        primary = matching[0] if matching else None
        display_label = str((primary or {}).get("label") or topic_term)
        detail_sources, tagged_count, semantic_count = _build_theme_detail_sources(
            intra_themes,
            topic_canonical=topic_canonical,
            topic_term=topic_term,
            display_label=display_label,
            corpus_filter=corpus_filter,
        )
        theme_edges = [
            e
            for e in structured_edges
            if str(e.get("kind") or "") == "topic_cooccurrence"
            and str(e.get("theme_key") or e.get("src") or "").lower() == topic_canonical
        ]
        theme_detail_obj = {
            "label": display_label,
            "canonical_key": topic_canonical or topic_term.lower(),
            "sources": detail_sources,
            "source_count": len(detail_sources)
            or (primary or {}).get("source_count")
            or len((primary or {}).get("source_slugs") or []),
            "tagged_source_count": tagged_count,
            "semantic_source_count": semantic_count,
            "edges": theme_edges,
        }

    connection_bullets: list[dict[str, Any]] = []
    for edge in structured_edges:
        if str(edge.get("kind") or "") != "synthesis_link":
            continue
        connection_bullets.append(
            {
                "corpus_slug": edge.get("corpus_slug"),
                "corpus_label": edge.get("corpus_label"),
                "text": edge.get("rationale") or "",
            }
        )
    if not connection_bullets and not topic_term:
        corpora = list_knowledge_corpora()
        slug_to_label = {str(c["slug"]): str(c.get("label") or c["slug"]) for c in corpora}
        for c in corpora[:max_corpora]:
            slug = str(c["slug"])
            if corpus_filter:
                cf = corpus_filter.strip().lower()
                if _SLUG_RE.match(cf):
                    if slug.lower() != cf:
                        continue
                elif cf not in str(c.get("label") or "").lower():
                    continue
            if filter_term and not topic_term:
                label_low = str(c.get("label") or "").lower()
                if filter_term not in slug.lower() and filter_term not in label_low:
                    topics = [str(t).lower() for t in (c.get("topics") or []) + (c.get("broad_topics") or [])]
                    if not any(filter_term in t for t in topics):
                        continue
            synth_path = _synthesis_path(slug)
            if not os.path.isfile(synth_path):
                continue
            synthesis = _read_text_file(synth_path)
            for bullet in parse_synthesis_connections_section(synthesis):
                connection_bullets.append(
                    {
                        "corpus_slug": slug,
                        "corpus_label": slug_to_label.get(slug, slug),
                        "text": bullet,
                    }
                )

    related_hits: list[dict[str, Any]] = []
    graph_neighbors: list[dict[str, Any]] = []
    related_searched = False
    search_q = (query or topic or "").strip()
    if related and search_q:
        related_searched = True
        related_hits = search_knowledge_hits(
            search_q,
            corpus=corpus_filter,
            max_results=6,
            semantic=True,
            retrieval_mode="connections",
        )
        cleaned_related: list[dict[str, Any]] = []
        for hit in related_hits:
            row = dict(hit)
            row["snippet"] = sanitize_knowledge_snippet_for_display(str(row.get("snippet") or ""))
            cleaned_related.append(row)
        related_hits = cleaned_related
        if related_hits:
            try:
                from utils.knowledge_graph import focus_node_for_source, list_knowledge_graph_neighbors

                top = related_hits[0]
                slug = str(top.get("corpus_slug") or "")
                src = str(top.get("source_slug") or top.get("source_name") or "")
                focus = focus_node_for_source(slug, src) if slug and src else None
                if not focus and slug and not src:
                    focus = focus_node_for_source(slug, "synthesis") or f"knowledge:{slug}:chunk:1"
                if focus:
                    label = slug_to_label.get(slug, slug)
                    graph_neighbors = []
                    for nb in list_knowledge_graph_neighbors(
                        focus, corpus_slug=slug or None, limit=8, include_suggested=False
                    ):
                        row = dict(nb)
                        row["corpus_label"] = label
                        graph_neighbors.append(row)
                    if len(graph_neighbors) < 8:
                        seen_nb = {str(r.get("neighbor_id") or "") for r in graph_neighbors}
                        for nb in list_knowledge_graph_neighbors(
                            focus, corpus_slug=slug or None, limit=8, include_suggested=True
                        ):
                            if str(nb.get("status") or "") != "suggested":
                                continue
                            oid = str(nb.get("neighbor_id") or "")
                            if not oid or oid in seen_nb:
                                continue
                            seen_nb.add(oid)
                            row = dict(nb)
                            row["corpus_label"] = label
                            graph_neighbors.append(row)
                            if len(graph_neighbors) >= 8:
                                break
            except Exception:
                graph_neighbors = []

    missing_index = corpora_missing_connections_index(corpus_filter=corpus_filter)
    semantic_themes: list[dict[str, Any]] = []
    entities_payload: list[dict[str, Any]] = []
    claims_payload: list[dict[str, Any]] = []
    contradictions_payload: list[dict[str, Any]] = []
    entities_state = "disabled"
    contradictions_state = "disabled"
    try:
        from utils.knowledge_connections_store import (
            _ACCEPTED_SCHEMAS,
            _claim_matches_filter,
            _claims_row_cap,
            _contradiction_matches_filter,
            _contradictions_row_cap,
            _entities_row_cap,
            _filter_corpora_rows,
            _topic_matches_filter,
            load_connections_index,
        )
        from utils.knowledge_entities import entities_enabled, get_entities_config
        from utils.knowledge_contradictions import contradictions_enabled, visible_contradiction_rows

        entities_enabled_flag = entities_enabled(get_entities_config())
        contradictions_enabled_flag = contradictions_enabled()
        entities_state = "disabled" if not entities_enabled_flag else "empty"
        contradictions_state = (
            "disabled" if not contradictions_enabled_flag else "empty"
        )
        entity_cap = _entities_row_cap()
        claim_cap = _claims_row_cap()
        contradiction_cap = _contradictions_row_cap()
        index_has_entity_rows = False
        index_has_contradiction_rows = False
        entities_cap_reached = False
        claims_cap_reached = False
        contradictions_cap_reached = False
        row_filter_kwargs = (
            {"topic_low": topic_canonical, "query_low": ""}
            if topic_term
            else {"topic_low": "", "query_low": filter_term}
        )

        if corpus_filter:
            corpora_scope = _filter_corpora_rows(list_knowledge_corpora(), corpus_filter)
        else:
            corpora_scope = list_knowledge_corpora()
        for c in corpora_scope[:max_corpora]:
            slug_row = str(c.get("slug") or "")
            if not slug_row:
                continue
            conn_data = load_connections_index(slug_row)
            if not isinstance(conn_data, dict):
                continue
            if str(conn_data.get("schema") or "") not in _ACCEPTED_SCHEMAS:
                continue
            raw_entities = conn_data.get("entities")
            raw_claims = conn_data.get("claims")
            raw_contradictions = conn_data.get("contradictions")
            if (isinstance(raw_entities, list) and raw_entities) or (
                isinstance(raw_claims, list) and raw_claims
            ):
                index_has_entity_rows = True
            if visible_contradiction_rows(
                raw_contradictions if isinstance(raw_contradictions, list) else None
            ):
                index_has_contradiction_rows = True
            for row in conn_data.get("semantic_themes") or []:
                if not isinstance(row, dict):
                    continue
                if topic_term or filter_term:
                    if not _sidebar_row_matches_active_topic(
                        row,
                        corpus_slug=slug_row,
                        topic_source_keys=topic_source_keys,
                        topic_low=row_filter_kwargs.get("topic_low", ""),
                        query_low=row_filter_kwargs.get("query_low", ""),
                    ):
                        continue
                enriched = dict(row)
                enriched.setdefault("corpus_slug", str(conn_data.get("corpus_slug") or slug_row))
                semantic_themes.append(enriched)
            for row in conn_data.get("entities") or []:
                if not isinstance(row, dict):
                    continue
                if entities_cap_reached or len(entities_payload) >= entity_cap:
                    entities_cap_reached = True
                    break
                if topic_term or filter_term:
                    if not _sidebar_row_matches_active_topic(
                        row,
                        corpus_slug=slug_row,
                        topic_source_keys=topic_source_keys,
                        topic_low=row_filter_kwargs.get("topic_low", ""),
                        query_low=row_filter_kwargs.get("query_low", ""),
                    ):
                        continue
                enriched = dict(row)
                enriched.setdefault("corpus_slug", str(conn_data.get("corpus_slug") or slug_row))
                entities_payload.append(enriched)
            for row in conn_data.get("claims") or []:
                if not isinstance(row, dict):
                    continue
                if claims_cap_reached or len(claims_payload) >= claim_cap:
                    claims_cap_reached = True
                    break
                if topic_term or filter_term:
                    if not _sidebar_row_matches_active_topic(
                        row,
                        corpus_slug=slug_row,
                        topic_source_keys=topic_source_keys,
                        topic_low=row_filter_kwargs.get("topic_low", ""),
                        query_low=row_filter_kwargs.get("query_low", ""),
                        claim_row=True,
                    ):
                        continue
                enriched = dict(row)
                enriched.setdefault("corpus_slug", str(conn_data.get("corpus_slug") or slug_row))
                claims_payload.append(enriched)
            if contradictions_enabled_flag:
                for row in visible_contradiction_rows(conn_data.get("contradictions")):
                    if not isinstance(row, dict):
                        continue
                    if contradictions_cap_reached or len(contradictions_payload) >= contradiction_cap:
                        contradictions_cap_reached = True
                        break
                    if topic_term or filter_term:
                        if not _contradiction_matches_filter(row, **row_filter_kwargs):
                            continue
                    enriched = dict(row)
                    enriched.setdefault("corpus_slug", str(conn_data.get("corpus_slug") or slug_row))
                    slug_map = _corpus_source_slug_map(slug_row)
                    enriched_rows = _enrich_contradiction_source_names(
                        [enriched],
                        slug_map=slug_map,
                        corpus_slug=slug_row,
                    )
                    if enriched_rows:
                        contradictions_payload.append(enriched_rows[0])
        if entities_enabled_flag and (entities_payload or claims_payload):
            entities_state = "ready"
        elif (
            entities_enabled_flag
            and entities_state == "empty"
            and index_has_entity_rows
            and (topic_term or filter_term)
        ):
            entities_state = "ready"
        if contradictions_enabled_flag and contradictions_payload:
            contradictions_state = "ready"
        elif (
            contradictions_enabled_flag
            and contradictions_state == "empty"
            and index_has_contradiction_rows
            and (topic_term or filter_term)
        ):
            contradictions_state = "ready"
    except Exception as load_exc:
        _debug_log(
            f"[knowledge] connections payload load failed: {load_exc}",
            "WARN",
        )
        semantic_themes = []
        entities_payload = []
        claims_payload = []
        contradictions_payload = []
        entities_state = "disabled"
        contradictions_state = "disabled"

    out: dict[str, Any] = {
        "shared_topics": shared_topics,
        "intra_corpus_themes": intra_themes,
        "semantic_themes": semantic_themes,
        "entities": entities_payload,
        "claims": claims_payload,
        "entities_state": entities_state,
        "contradictions": contradictions_payload,
        "contradictions_state": contradictions_state,
        "edges": structured_edges,
        "connection_bullets": connection_bullets[:40],
        "related_hits": related_hits,
        "related_searched": related_searched,
        "graph_neighbors": graph_neighbors,
        "missing_connections_index": missing_index,
    }
    if theme_detail_obj is not None:
        out["theme_detail"] = theme_detail_obj
    return out


def rebuild_connections_index(slug: str) -> dict[str, Any]:
    """Rebuild connections.json for one corpus without re-ingesting sources."""
    from utils.knowledge_connections_store import (
        _validate_slug,
        materialize_connections_for_corpus,
        save_connections_index,
    )

    norm = _validate_slug(slug)
    manifest = _load_manifest(norm)
    if not manifest:
        raise ValueError("manifest missing")
    if str(manifest.get("digest_status") or "").lower() == "running":
        raise RuntimeError("digest running")
    synth_path = _synthesis_path(norm)
    synthesis_text = _read_text_file(synth_path) if os.path.isfile(synth_path) else ""
    sources = manifest.get("sources") or []
    if not isinstance(sources, list):
        sources = []

    label = str(manifest.get("label") or norm)
    profile = str(manifest.get("ingest_profile") or "docs")
    broad_topics = (
        manifest.get("broad_topics") if isinstance(manifest.get("broad_topics"), list) else []
    )
    semantic_themes = _resolve_semantic_themes_for_finalize(norm)
    entity_bundle = _resolve_entities_for_finalize(norm)
    entities = list(entity_bundle.get("entities") or [])
    claims = list(entity_bundle.get("claims") or [])
    contradictions = list(entity_bundle.get("contradictions") or [])
    if _broad_topics_need_semantic_backfill(broad_topics) and semantic_themes:
        broad_topics = [
            str(t.get("label") or "").strip()
            for t in semantic_themes[:5]
            if str(t.get("label") or "").strip()
        ]
        if broad_topics:
            manifest["broad_topics"] = broad_topics
            _rewrite_index_broad_topics(norm, broad_topics)
    conn_data = materialize_connections_for_corpus(
        norm,
        label,
        synthesis_text,
        sources,
        broad_topics=broad_topics or None,
        corpus_topics=manifest.get("topics") if isinstance(manifest.get("topics"), list) else None,
        semantic_themes=semantic_themes,
        entities=entities,
        claims=claims,
        contradictions=contradictions,
        ingest_profile=profile,
    )
    save_connections_index(norm, conn_data)
    graph_ok = True
    try:
        from utils.knowledge_graph import materialize_concept_graph_edges

        materialize_concept_graph_edges(norm)
    except Exception as exc:
        graph_ok = False
        _debug_log(f"[knowledge] concept graph failed after connections rebuild for {norm}: {exc}", "WARN")
    manifest["connections_index_ok"] = True
    manifest["concept_graph_ok"] = graph_ok
    manifest["updated_at"] = _iso_now()
    _save_manifest(norm, manifest)
    invalidate_topic_index_cache()
    return {
        "ok": True,
        "schema": conn_data.get("schema"),
        "theme_count": len(conn_data.get("topics") or []),
        "semantic_theme_count": len(conn_data.get("semantic_themes") or []),
        "entity_count": len(conn_data.get("entities") or []),
        "claim_count": len(conn_data.get("claims") or []),
        "contradiction_count": len(conn_data.get("contradictions") or []),
        "edge_count": len(conn_data.get("edges") or []),
        "graph_ok": graph_ok,
    }


def search_knowledge_related_in_corpus(
    slug: str,
    *,
    query: str | None = None,
    source_id: str | None = None,
    max_results: int = 6,
) -> list[dict[str, Any]]:
    """Semantic/FTS neighbors within one corpus (powers Related inspector)."""
    manifest = _load_manifest(slug)
    if not manifest:
        return []
    q = (query or "").strip()
    if not q and source_id:
        q = str(source_id).strip()
    if not q:
        return []
    hits = search_knowledge_hits(
        q, corpus=slug, max_results=max_results, semantic=True, retrieval_mode="tool"
    )
    if source_id:
        sid = source_id.strip().lower()
        hits = [
            h
            for h in hits
            if sid not in (str(h.get("source_slug") or "").lower(), str(h.get("source_name") or "").lower())
        ]
    return hits


def _graph_neighbor_to_hit(nb: dict[str, Any], *, slug: str, label: str) -> dict[str, Any]:
    kind = str(nb.get("link_kind") or "link")
    title = str(nb.get("title") or nb.get("filename") or nb.get("neighbor_id") or "")
    conf = nb.get("confidence")
    snippet = f"[graph {kind}] {title}"
    if conf is not None:
        try:
            snippet += f" · {float(conf):.2f}"
        except (TypeError, ValueError):
            pass
    return {
        "id": str(nb.get("neighbor_id") or ""),
        "corpus_slug": slug,
        "corpus_label": label,
        "source_slug": str(nb.get("source_slug") or ""),
        "source_name": str(nb.get("filename") or ""),
        "snippet": snippet,
        "via": "graph",
    }


def search_knowledge_related_with_graph(
    slug: str,
    *,
    query: str | None = None,
    source_id: str | None = None,
    max_results: int = 6,
) -> list[dict[str, Any]]:
    """Related inspector: graph neighbors first, semantic search fallback."""
    manifest = _load_manifest(slug)
    if not manifest:
        return []
    from utils.knowledge_graph import focus_node_for_source, list_knowledge_graph_neighbors

    label = str(manifest.get("label") or slug)
    cap = max(1, min(int(max_results), 20))
    sid = str(source_id or "").strip()
    focus = focus_node_for_source(slug, sid) if sid else None
    if not focus and not sid:
        focus = focus_node_for_source(slug, "synthesis") or f"knowledge:{slug}:chunk:1"

    merged: list[dict[str, Any]] = []
    seen: set[str] = set()

    for nb in list_knowledge_graph_neighbors(focus, corpus_slug=slug, limit=cap, include_suggested=False):
        oid = str(nb.get("neighbor_id") or "")
        if not oid or oid == focus or oid in seen:
            continue
        seen.add(oid)
        merged.append(_graph_neighbor_to_hit(nb, slug=slug, label=label))
        if len(merged) >= cap:
            return merged

    if len(merged) < cap:
        for nb in list_knowledge_graph_neighbors(focus, corpus_slug=slug, limit=cap, include_suggested=True):
            if str(nb.get("status") or "") != "suggested":
                continue
            oid = str(nb.get("neighbor_id") or "")
            if not oid or oid == focus or oid in seen:
                continue
            seen.add(oid)
            merged.append(_graph_neighbor_to_hit(nb, slug=slug, label=label))
            if len(merged) >= cap:
                return merged

    for hit in search_knowledge_related_in_corpus(
        slug, query=query, source_id=source_id, max_results=cap
    ):
        hid = str(hit.get("id") or "")
        if not hid or hid in seen:
            continue
        seen.add(hid)
        row = dict(hit)
        row["via"] = "search"
        merged.append(row)
        if len(merged) >= cap:
            break
    return merged


def read_knowledge_gaps(
    limit: int = 30,
    *,
    include_resolved: bool = False,
    workspace: str | None = None,
) -> list[dict[str, Any]]:
    """Recent unanswered knowledge queries from ``knowledge_gaps.jsonl``."""
    cap = max(1, min(int(limit), 100))
    gaps_path = _knowledge_gaps_path(workspace)
    rows = _parse_knowledge_gap_file(gaps_path)
    if not include_resolved:
        rows = [r for r in rows if not _gap_row_is_resolved(r)]
    return rows[-cap:]


def extract_summary_section_from_source_notes(notes_md: str) -> str:
    """Return the ## Summary body from a studied source notes page."""
    text = (notes_md or "").strip()
    if not text:
        return ""
    m = re.search(
        r"(?is)^##\s*Summary\s*\n(.*?)(?=^##\s|\Z)",
        text,
        re.MULTILINE,
    )
    body = m.group(1).strip() if m else text
    return polish_knowledge_summary_body(body)


def read_knowledge_source_notes_body(
    corpus: str,
    source_id: str,
    offset: int = 0,
    limit: int = 5000,
    *,
    summary_only: bool = False,
) -> dict[str, Any]:
    """HTTP-friendly reader for studied notes under sources/{slug}.md."""
    manifest, entry, err = _resolve_corpus_and_source(corpus, source_id)
    if err:
        return {
            "text": err,
            "offset": offset,
            "limit": limit,
            "total_chars": None,
            "next_offset": None,
        }
    assert manifest is not None and entry is not None
    corpus_slug = str(manifest.get("slug") or "")
    source_slug = str(entry.get("slug") or "")
    path = _source_page_path(corpus_slug, source_slug)
    filename = str(entry.get("filename") or source_id)
    if not os.path.isfile(path):
        return {
            "text": f"No studied notes for source '{filename}'.",
            "offset": offset,
            "limit": limit,
            "total_chars": None,
            "next_offset": None,
        }
    raw = _read_text_file(path)
    text = extract_summary_section_from_source_notes(raw) if summary_only else raw
    limit = max(500, min(int(limit), 20_000))
    offset = max(0, int(offset))
    total = len(text)
    window = text[offset : offset + limit]
    next_offset = None
    if offset + len(window) < total:
        next_offset = offset + len(window)
    header = f"[{filename}] studied notes chars {offset}–{offset + len(window)} of {total}"
    if next_offset is not None:
        header += f" (next: offset={next_offset})"
    body = f"{header}\n\n{window}" if window else f"{header}\n\n"
    return {
        "text": body,
        "offset": offset,
        "limit": limit,
        "total_chars": total,
        "next_offset": next_offset,
    }


def read_knowledge_source_body(
    corpus: str,
    source_id: str,
    offset: int = 0,
    limit: int = 5000,
) -> dict[str, Any]:
    """HTTP-friendly wrapper around read_knowledge_source pagination."""
    text = read_knowledge_source.invoke(
        {"corpus": corpus, "source_id": source_id, "offset": offset, "limit": limit}
    )
    total = None
    next_offset = None
    m = re.search(r"chars (\d+)–(\d+) of (\d+)", text)
    if m:
        total = int(m.group(3))
        end = int(m.group(2))
        if end < total:
            next_offset = end
    return {
        "text": text,
        "offset": offset,
        "limit": limit,
        "total_chars": total,
        "next_offset": next_offset,
    }


def _query_knowledge_text(
    query: str,
    corpus: str | None = None,
    max_results: int = 8,
    *,
    hits: list[dict[str, Any]] | None = None,
    record_trace: bool = True,
) -> str:
    """Format ``query_knowledge`` output; optional precomputed ``hits`` skip a second search."""
    q = (query or "").strip()
    if not q:
        return "No query provided."
    cap = max(1, min(int(max_results), 20))
    corpora = _filter_corpora_by_name(list_knowledge_corpora(), corpus)
    if not corpora:
        _record_knowledge_gap(q, corpus=corpus)
        return "No matching knowledge corpus found."

    resolved_hits = hits if hits is not None else search_knowledge_hits(
        q, corpus=corpus, max_results=cap, semantic=True, retrieval_mode="tool",
    )
    if resolved_hits:
        text = format_knowledge_hits_as_text(resolved_hits, include_tip=True)
        if len(text) > 3000:
            text = text[:3000].rstrip() + "\n...[truncated]"
        if record_trace:
            _knowledge_inspector_maybe_record_tool_query(
                tool="query_knowledge",
                query=q,
                corpus_filter=corpus,
                hits=resolved_hits,
                outcome="hits",
            )
        return text

    _record_knowledge_gap(q, corpus=corpus)
    if record_trace:
        _knowledge_inspector_maybe_record_tool_query(
            tool="query_knowledge",
            query=q,
            corpus_filter=corpus,
            hits=[],
            outcome="miss",
        )
    return f"No compiled knowledge matched '{q}'."


def _format_ask_studied_corpus_text(payload: dict[str, Any]) -> str:
    status = str(payload.get("answer_status") or "")
    answer = str(payload.get("answer") or "").strip()
    lines: list[str] = []
    if status in ("no_hits", "no_evidence"):
        lines.append(f"Knowledge Q&A ({status}): {answer}")
        sources = payload.get("sources") or []
        if sources:
            lines.append("")
            lines.append("Retrieved sources (for manual review):")
            lines.extend(format_knowledge_hits_as_text(sources, include_tip=False).splitlines()[:12])
        lines.append("")
        lines.append("Try query_knowledge for raw search or read_knowledge_source for full text.")
        return "\n".join(lines).strip()
    if status == "generation_failed":
        return f"Knowledge Q&A failed to generate an answer: {answer}"
    lines.append(answer)
    cites = payload.get("citations") or []
    if cites:
        lines.append("")
        lines.append("Citations:")
        for i, c in enumerate(cites, 1):
            label = c.get("corpus_label") or c.get("corpus_slug") or "Corpus"
            src = c.get("source_name") or c.get("source_slug") or "source"
            lines.append(f"{i}. [{label} > {src}]")
    elif payload.get("uncited_answer"):
        lines.append("")
        lines.append("(Answer may lack inline citations — verify against sources.)")
    lines.append("")
    lines.append("Use read_knowledge_source with corpus and source_id for full passages.")
    return "\n".join(lines).strip()


@tool
def ask_studied_corpus(
    question: str,
    corpus: str | None = None,
    max_sources: int = 10,
) -> str:
    """Answer a question using only studied knowledge corpora with inline source citations.

    Prefer this over query_knowledge when the user wants a synthesized answer, not raw search snippets.
    Use query_knowledge to locate passages; read_knowledge_source for full document text.
    """
    q = (question or "").strip()
    if not q:
        return "No question provided."
    payload = ask_knowledge_corpus(q, corpus=corpus, max_sources=max_sources)
    status = str(payload.get("answer_status") or "")
    hits = payload.get("sources") or []
    if status in ("no_hits", "no_evidence"):
        _record_knowledge_gap(q, corpus=corpus)
    _knowledge_inspector_maybe_record_tool_query(
        tool="ask_studied_corpus",
        query=q,
        corpus_filter=corpus,
        hits=hits if isinstance(hits, list) else [],
        outcome=status,
    )
    return _format_ask_studied_corpus_text(payload)


@tool
def query_knowledge(query: str, corpus: str | None = None, max_results: int = 8) -> str:
    """Search compiled knowledge notes by keyword with source attribution."""
    return _query_knowledge_text(query, corpus, max_results, record_trace=True)


@tool
def explore_knowledge_connections(
    topic_or_query: str,
    max_corpora: int = 5,
) -> str:
    """Explore cross-document and cross-corpus links in studied knowledge notes.

    Surfaces shared topics, synthesis connection bullets, and search hits related to the query.
    Use when the user wants relationships between documents they might not have noticed.
    """
    q = (topic_or_query or "").strip()
    if not q:
        return "Provide a topic or search phrase to explore connections."
    payload = build_knowledge_connections_payload(topic=q, query=q, max_corpora=max(1, min(int(max_corpora), 10)), related=True)
    lines: list[str] = [f"Knowledge connections for: {q}", ""]
    themes = payload.get("intra_corpus_themes") or []
    if themes:
        lines.append("## Themes in corpora")
        for row in themes[:20]:
            label = row.get("label") or "?"
            corp = row.get("corpus_label") or row.get("corpus_slug") or ""
            slugs = row.get("source_slugs") or []
            names = ", ".join(
                str(s.get("filename") or s.get("slug") or "")
                for s in (row.get("sources") or [])[:5]
            )
            if not names and slugs:
                names = ", ".join(str(s) for s in slugs[:5])
            lines.append(f"- [{corp}] {label} ({len(slugs)} sources: {names})")
        lines.append("")
    edges = payload.get("edges") or []
    if edges:
        lines.append("## Structured connections")
        for edge in edges[:20]:
            kind = edge.get("kind") or "link"
            corp = edge.get("corpus_label") or edge.get("corpus_slug") or ""
            src_names = ", ".join(
                str(s.get("filename") or s.get("slug") or "")
                for s in (edge.get("sources") or [])[:4]
            )
            suffix = f" [{src_names}]" if src_names else ""
            lines.append(f"- [{corp}] ({kind}) {edge.get('rationale') or edge.get('src')}{suffix}")
        lines.append("")
    shared = payload.get("shared_topics") or []
    if shared:
        lines.append("## Shared topics across corpora")
        for row in shared:
            topic = row.get("topic") or "?"
            names = ", ".join(
                f"{e.get('label') or e.get('slug')}" for e in (row.get("corpora") or [])[:5]
            )
            lines.append(f"- {topic}: {names}")
        lines.append("")
    bullets = payload.get("connection_bullets") or []
    if bullets:
        lines.append("## Synthesis connections")
        for b in bullets[:15]:
            lines.append(f"- [{b.get('corpus_label')}] {b.get('text')}")
        lines.append("")
    contradictions = payload.get("contradictions") or []
    if contradictions and payload.get("contradictions_state") == "ready":
        lines.append("## Heuristic contradictions (cross-source, not verified)")
        for row in contradictions[:10]:
            label = row.get("entity_label") or row.get("entity_key") or "?"
            kind = row.get("kind") or "?"
            ca = row.get("claim_a") if isinstance(row.get("claim_a"), dict) else {}
            cb = row.get("claim_b") if isinstance(row.get("claim_b"), dict) else {}
            lines.append(
                f"- {label} ({kind}): "
                f"[{ca.get('source_slug')}] {ca.get('text') or ''} vs "
                f"[{cb.get('source_slug')}] {cb.get('text') or ''}"
            )
        lines.append("")
    related = payload.get("related_hits") or []
    if related:
        lines.append("## Related content")
        lines.append(format_knowledge_hits_as_text(related, include_tip=False))
    graph_chunks = payload.get("graph_neighbors") or []
    if graph_chunks:
        lines.append("")
        lines.append("## Chunk graph neighbors")
        for row in graph_chunks[:10]:
            lines.append(
                f"- [{row.get('corpus_label')}] {row.get('title') or row.get('neighbor_id')} "
                f"({row.get('link_kind')}, {row.get('direction')})"
            )
    if len(lines) <= 2:
        return f"No cross-corpus connections found for '{q}'. Try query_knowledge for literal search."
    return "\n".join(lines).strip()


_READ_SOURCE_CHUNK = 5000


_SOURCE_MATCH_STOPWORDS = frozenset(
    "a an the in on at to for of is it and or by with from this that"
    .split()
)


def _source_match_score(query_words: set[str], s: dict[str, Any]) -> float:
    """Score a source entry against query words.  Higher = better match.

    Returns 0.0 for no overlap.
    """
    s_name = str(s.get("filename") or "").lower()
    s_slug = str(s.get("slug") or "").lower()
    s_name_norm = re.sub(r"[\s_/-]+", " ", s_name).strip()
    target_words = set(s_name_norm.split()) | set(s_slug.replace("_", " ").split())
    target_words -= _SOURCE_MATCH_STOPWORDS
    if not query_words or not target_words:
        return 0.0
    overlap = query_words & target_words
    if not overlap:
        return 0.0
    return len(overlap) / len(query_words)


def _resolve_corpus_and_source(
    corpus: str, source: str
) -> tuple[dict[str, Any] | None, dict[str, Any] | None, str]:
    """Find a corpus manifest and a source entry within it.

    Returns (manifest, source_entry, error_message).
    """
    corpora = list_knowledge_corpora()
    wanted = corpus.strip().lower()
    wanted_norm = re.sub(r"[\s_-]+", " ", wanted).strip()
    wanted_slug = re.sub(r"[\s_-]+", "_", wanted).strip("_")

    def _cm(c: dict[str, Any]) -> bool:
        slug = c["slug"].lower()
        label = str(c["label"]).lower()
        label_norm = re.sub(r"[\s_-]+", " ", label).strip()
        if slug == wanted or slug == wanted_slug or label == wanted or label_norm == wanted_norm:
            return True
        if wanted_norm in label_norm or wanted_norm in re.sub(r"_", " ", slug):
            return True
        return False

    matches = [c for c in corpora if _cm(c)]
    if not matches:
        return None, None, f"No corpus matching '{corpus}'."
    slug = str(matches[0]["slug"])
    manifest = _load_manifest(slug)
    if not manifest:
        return None, None, f"Corpus '{corpus}' has no readable manifest."
    sources = manifest.get("sources") or []
    if not sources:
        return manifest, None, f"Corpus '{corpus}' has no digested sources."

    src_q = (source or "").strip().lower()
    src_q_norm = re.sub(r"[\s_-]+", " ", src_q).strip()

    def _sm(s: dict[str, Any]) -> bool:
        s_slug = str(s.get("slug") or "").lower()
        s_name = str(s.get("filename") or "").lower()
        s_name_norm = re.sub(r"[\s_-]+", " ", s_name).strip()
        if s_slug == src_q or s_name == src_q or s_name_norm == src_q_norm:
            return True
        if src_q_norm in s_name_norm or src_q_norm in s_slug:
            return True
        return False

    matched = [s for s in sources if _sm(s)]
    if matched:
        return manifest, matched[0], ""

    # Fallback: word-level scoring — pick the source with the highest keyword overlap
    query_words = set(src_q_norm.split()) - _SOURCE_MATCH_STOPWORDS
    if query_words:
        scored = [(s, _source_match_score(query_words, s)) for s in sources]
        scored.sort(key=lambda x: x[1], reverse=True)
        if scored[0][1] >= 0.5:
            return manifest, scored[0][0], ""
        # Show the top suggestions if no strong match
        suggestions = [
            str(s.get("filename") or s.get("slug"))
            for s, sc in scored[:5] if sc > 0.0
        ]
        if suggestions:
            suggestion_text = "Closest matches: " + " | ".join(suggestions) + "\n\n"
        else:
            suggestion_text = ""
    else:
        suggestion_text = ""

    available = " | ".join(str(s.get("filename") or s.get("slug")) for s in sources)
    return manifest, None, (
        f"No source matching '{source}' in corpus '{corpus}'. "
        f"{suggestion_text}"
        f"Available sources: {available}"
    )


@tool
def read_knowledge_source(
    corpus: str,
    source_id: str = "",
    offset: int = 0,
    limit: int = _READ_SOURCE_CHUNK,
) -> str:
    """Read raw original text from a knowledge corpus source (chapter, file, section).

    Use this when the summary from query_knowledge is not detailed enough and you
    need the actual source material. Returns a paginated window of the extracted text.

    When source_id is omitted or empty, lists all available sources in the corpus
    so you can pick the right one.

    Args:
        corpus: Corpus name or slug (e.g. "SAS Survival Handbook").
        source_id: Source/section name or keyword within the corpus (e.g. "Filtering", "Finding Water"). Omit to list available sources.
        offset: Character offset to start reading from (default 0).
        limit: Max characters to return (default 5000, max 20000).
    """
    if not (corpus or "").strip():
        return "corpus is required."
    source = (source_id or "").strip()
    if not source:
        corpora = list_knowledge_corpora()
        wanted = corpus.strip().lower()
        wanted_norm = re.sub(r"[\s_-]+", " ", wanted).strip()
        wanted_slug = re.sub(r"[\s_-]+", "_", wanted).strip("_")
        matched_c = None
        for c in corpora:
            slug = c["slug"].lower()
            label = str(c["label"]).lower()
            label_norm = re.sub(r"[\s_-]+", " ", label).strip()
            if slug == wanted or slug == wanted_slug or label == wanted or label_norm == wanted_norm:
                matched_c = c
                break
            if wanted_norm in label_norm or wanted_norm in re.sub(r"_", " ", slug):
                matched_c = c
                break
        if not matched_c:
            return f"No corpus matching '{corpus}'."
        manifest = _load_manifest(str(matched_c["slug"]))
        if not manifest:
            return f"Corpus '{corpus}' has no readable manifest."
        sources = manifest.get("sources") or []
        if not sources:
            return f"Corpus '{corpus}' has no digested sources."
        _knowledge_inspector_maybe_record_tool_list_sources(
            corpus_slug=str(matched_c["slug"]),
            corpus_label=str(matched_c.get("label") or matched_c["slug"]),
            source_count=len(sources),
        )
        lines = [f"Sources in '{matched_c['label']}' ({len(sources)} total):"]
        for i, s in enumerate(sources):
            fn = s.get("filename") or s.get("slug") or f"source_{i}"
            lines.append(f"  [{i}] {fn}")
        return "\n".join(lines)
    limit = max(500, min(int(limit), 20_000))
    offset = max(0, int(offset))

    manifest, entry, err = _resolve_corpus_and_source(corpus, source)
    if err:
        return err
    assert manifest is not None and entry is not None

    src_path = str(entry.get("path") or "")
    is_url = bool(re.match(r"^https?://", str(manifest.get("source_path") or ""), re.IGNORECASE))

    real_path = src_path
    section_idx: int | None = None
    if "::section::" in src_path:
        parts = src_path.rsplit("::section::", 1)
        real_path = parts[0]
        try:
            section_idx = int(parts[1])
        except (ValueError, IndexError):
            section_idx = None

    try:
        if is_url:
            text = _fetch_url_text(real_path)
        else:
            if not os.path.isfile(real_path):
                return f"Original file no longer exists: {real_path}"
            corpus_slug = str(manifest.get("slug") or "")
            cached = _load_raw_text_cache(corpus_slug, real_path) if corpus_slug else None
            if cached:
                text = cached
            else:
                text = _extract_plaintext(real_path)
                if corpus_slug and text and text.strip():
                    _save_raw_text_cache(corpus_slug, real_path, text)
    except Exception as exc:
        return f"Failed to extract text from source: {exc}"

    if not (text or "").strip():
        return "No text could be extracted from the original source."

    if section_idx is not None:
        filename = str(entry.get("filename") or os.path.basename(real_path))
        sections = _detect_sections(text, filename)
        if sections and section_idx < len(sections):
            _title, start, end = sections[section_idx]
            text = text[start:end]
        else:
            _debug_log(
                f"[knowledge] read_knowledge_source: section {section_idx} not found "
                f"(detected {len(sections)} sections); returning full text",
                "WARN",
            )

    total = len(text)
    window = text[offset : offset + limit]
    if not window.strip():
        return f"Offset {offset} is beyond the end of the source ({total} chars total)."

    header = f"[{entry.get('filename') or entry.get('slug')}] chars {offset}–{offset + len(window)} of {total}"
    if offset + limit < total:
        header += f" (next: offset={offset + len(window)})"
    corpus_slug = str(manifest.get("slug") or "")
    source_name = str(entry.get("filename") or entry.get("slug") or "")
    _knowledge_inspector_maybe_record_tool_read(
        corpus_slug=corpus_slug,
        source_name=source_name,
        bytes_read=len(window),
        truncated=offset + limit < total,
        offset=offset,
    )
    return f"{header}\n\n{window}"


_COMBINED_MEMORY_KNOWLEDGE_CHAR_CAP = 100_000


@tool
def search_memory_and_knowledge_corpora(
    query: str,
    max_memory_results: int = 8,
    max_corpus_hits: int = 8,
    corpus: str | None = None,
) -> str:
    """Search long-term file memory and studied knowledge corpora in one combined step.

    Runs the same paths as ``search_memory`` (hub, shards, summaries) plus ``query_knowledge``
    over compiled corpora. Output is capped (~100k characters); the tail may be truncated with a notice.
    Treat memory hits as potentially sensitive; do not paste private memory into shared channels.
    """
    import tools.memory_tools as _mt

    q = (query or "").strip()
    if not q:
        return "No query provided. Please provide a search term."
    cap_k = max(1, min(int(max_corpus_hits), 20))
    kb_hits = search_knowledge_hits(
        q, corpus=corpus, max_results=cap_k, semantic=True, retrieval_mode="tool",
    )
    _knowledge_inspector_maybe_record_tool_combined(
        query=q,
        hit_count=len(kb_hits),
        outcome="hits" if kb_hits else "miss",
    )
    mem = _mt.search_memory.invoke(
        {"query": q, "max_results": max_memory_results, "include_archives": False},
    )
    corp = _query_knowledge_text(
        q, corpus, max_corpus_hits, hits=kb_hits, record_trace=False,
    )
    low = (corp or "").lower()
    if (
        not corp.strip()
        or "no compiled knowledge" in low
        or "no matching knowledge corpus" in low
    ):
        combined = f"{mem}\n\n(Studied corpora: no additional hits for this query.)"
    else:
        combined = f"{mem}\n\n─── Studied corpora ───\n{corp}"
    cap = _COMBINED_MEMORY_KNOWLEDGE_CHAR_CAP
    if len(combined) > cap:
        marker = "\n\n[truncated: combined memory+knowledge output exceeded character cap; narrow the query.]\n"
        keep = max(0, cap - len(marker))
        combined = combined[:keep] + marker
    return combined


__all__ = [
    "KNOWLEDGE_SCHEMA_VERSION",
    "digest_corpus",
    "list_knowledge",
    "query_knowledge",
    "ask_studied_corpus",
    "read_knowledge_source",
    "explore_knowledge_connections",
    "search_knowledge_hits",
    "unified_library_search",
    "ask_knowledge_corpus",
    "format_knowledge_hits_as_text",
    "parse_synthesis_connections_section",
    "get_knowledge_topic_index",
    "build_knowledge_connections_payload",
    "rebuild_connections_index",
    "search_knowledge_related_in_corpus",
    "read_knowledge_gaps",
    "resolve_knowledge_gap",
    "read_knowledge_source_body",
    "read_knowledge_source_notes_body",
    "extract_summary_section_from_source_notes",
    "search_memory_and_knowledge_corpora",
    "find_relevant_knowledge",
    "find_relevant_knowledge_with_rows",
    "find_relevant_knowledge_for_injection",
    "get_knowledge_catalog_trace_metadata",
    "get_knowledge_context_lanes_footer",
    "list_knowledge_corpora",
    "get_knowledge_status",
    "start_digest_job",
    "refresh_digest_job",
    "pause_digest_job",
    "delete_corpus",
    "check_corpus_staleness",
    "invalidate_topic_index_cache",
    "get_knowledge_catalog_block",
]

