# curiosity_engine.py — Background curiosity loop and research prefetch.
#
# Curiosity: reads memory/insights.md + recent summaries, uses an LLM (subagent)
# to generate 1-2 specific follow-up questions about unresolved threads, then
# writes them to the proactive queue for the user's next session.
#
# Research prefetch: reads insights for recurring patterns (3+ mentions), spawns
# a research subagent to pre-fetch context, saves results to memory/prefetched/.
#
# Feed-topic matching: given new feed items and the user's interests from insights,
# uses an LLM to identify relevant items worth surfacing.
from __future__ import annotations

import json
import os
import re
import threading
import unicodedata
from datetime import datetime, timedelta
from urllib.parse import urlparse

from utils.llm_output_sanitize import (
    extract_structured_tail,
    strip_markdown_fences,
    strip_think_tags,
    strip_untagged_cot,
)
from utils.path_safety import safe_resolve_path

_MAX_INPUT_CHARS = 20_000
_MAX_QUESTION_CHARS = 1000
_MAX_PREFETCH_CHARS = 8000
_MAX_FEED_MATCH_CHARS = 2000
_REASONING_PREFIX_DENYLIST = (
    "okay",
    "let me",
    "first,",
    "the user",
    "i need to",
)
_QUESTION_START_HINTS = ("what", "why", "how", "should", "could", "would", "is", "are", "can", "do", "does")
_PREFETCH_WRITE_LOCK = threading.Lock()

# ---------------------------------------------------------------------------
# Shared helpers
# ---------------------------------------------------------------------------

def _extract_questions_from_output(result: str) -> list[str]:
    normalized = unicodedata.normalize("NFC", (result or ""))
    questions: list[str] = []
    fallback_questions: list[str] = []
    for line in normalized.splitlines():
        line = line.strip()
        if not line:
            continue
        if line.startswith("Q:"):
            q = line[2:].strip()
            if q and len(q) < _MAX_QUESTION_CHARS:
                questions.append(q)
            continue

        lower_line = line.lower()
        if (
            line.endswith("?")
            and len(line) < _MAX_QUESTION_CHARS
            and not line.startswith("#")
            and len(line) >= 12
            and not lower_line.startswith(_REASONING_PREFIX_DENYLIST)
        ):
            fallback_questions.append(line)

    if questions:
        return questions[:5]
    if fallback_questions:
        return fallback_questions[:5]

    # Second-pass salvage for prose-heavy responses with no explicit Q: lines.
    tail = extract_structured_tail(normalized)
    chunks = re.split(r"(?<=[?])\s+", tail[-3000:])
    recovered: list[str] = []
    for chunk in chunks:
        c = chunk.strip()
        if not c.endswith("?"):
            continue
        if c.startswith("#") or len(c) < 12 or len(c) >= _MAX_QUESTION_CHARS:
            continue
        low = c.lower()
        if low.startswith(_REASONING_PREFIX_DENYLIST):
            continue
        if not any(low.startswith(h + " ") for h in _QUESTION_START_HINTS):
            continue
        recovered.append(c)
        if len(recovered) >= 5:
            break
    return recovered


def _read_file_safe(path: str, max_chars: int = _MAX_INPUT_CHARS) -> str:
    if not path:
        return ""
    workspace = _get_workspace()
    if workspace:
        resolved, err = safe_resolve_path(workspace, path, allow_write=False, create_parents=False)
        if err or not resolved:
            return ""
        path = resolved
    if not os.path.isfile(path):
        return ""
    try:
        with open(path, "r", encoding="utf-8") as f:
            return f.read(max_chars)
    except Exception:
        return ""


def _get_workspace() -> str | None:
    try:
        from core import get_workspace_path
        return get_workspace_path()
    except Exception:
        return None


def _get_curiosity_config() -> dict:
    try:
        from core import get_config
        return get_config().get("curiosity") or {}
    except Exception:
        return {}


def _get_curiosity_scoring_config() -> dict:
    try:
        from core import get_config

        return get_config().get("curiosity_scoring") or {}
    except Exception:
        return {}


def _knowledge_context_hint(max_items: int = 6) -> str:
    try:
        from tools.knowledge_tools import list_knowledge_corpora
    except Exception:
        return ""
    corpora = list_knowledge_corpora()
    if not corpora:
        return ""
    lines = ["### Knowledge corpora (for gap awareness)"]
    for row in corpora[:max_items]:
        label = str(row.get("label") or row.get("slug") or "").strip()
        topics = [str(t).strip() for t in (row.get("topics") or []) if str(t).strip()]
        if topics:
            lines.append(f"- {label}: {', '.join(topics[:5])}")
        else:
            lines.append(f"- {label}: (topics not yet extracted)")
    return "\n".join(lines)


def _knowledge_gap_hint(workspace: str, max_items: int = 8) -> str:
    from tools.knowledge_tools import read_knowledge_gaps

    cap = max(1, min(int(max_items), 100))
    parsed: list[dict] = []
    for row in read_knowledge_gaps(limit=cap, workspace=workspace):
        if not isinstance(row, dict):
            continue
        q = str(row.get("query") or "").strip()
        if not q:
            continue
        parsed.append(
            {
                "query": q[:300],
                "corpus": str(row.get("corpus") or "").strip()[:120],
                "ts": str(row.get("ts") or "").strip()[:40],
            }
        )
    if not parsed:
        return ""
    out = ["### Knowledge gaps from prior conversations"]
    for row in parsed:
        corpus = row["corpus"]
        suffix = f" (corpus: {corpus})" if corpus else ""
        out.append(f"- {row['query']}{suffix}")
    return "\n".join(out)


def _llm_call(system_prompt: str, user_prompt: str, thread_prefix: str, on_log=None) -> str | None:
    """Run a tool-free subagent LLM call. Returns response text or None."""
    cfg = _get_curiosity_config()
    model = (cfg.get("model") or "").strip() or None
    base_url = (cfg.get("base_url") or "").strip() or None
    api_key = (cfg.get("api_key") or "").strip() or None
    temp = float(cfg.get("temperature", 0.2))
    max_tokens = int(cfg.get("max_tokens", 512))

    messages = [
        {"role": "user", "content": f"{system_prompt}\n\n---\n\n{user_prompt}"},
    ]

    try:
        from agent_loop import run_raw_agent

        collector: list[str] = []
        error_ref: list[str] = []

        def _on_chunk(text):
            if text:
                collector.append(text)

        def _on_error(err):
            error_ref.append(str(err))

        from tool_policy import context_from_prefs_and_flags
        _bg_policy_ctx = context_from_prefs_and_flags(background_automation=True, subagent_mode=True)

        returned = run_raw_agent(
            messages=messages,
            thread_id=f"{thread_prefix}_{datetime.now().strftime('%Y%m%d_%H%M')}",
            config=None,
            on_chunk=_on_chunk,
            on_error=_on_error,
            on_log=on_log,
            research_mode=False,
            subagent_mode=True,
            model_override=model,
            base_url_override=base_url,
            api_key_override=api_key,
            tools_override=[],
            temperature=temp,
            max_tokens=max_tokens,
            stop=cfg.get("stop"),
            response_format=cfg.get("response_format") if isinstance(cfg.get("response_format"), dict) else None,
            policy_context=_bg_policy_ctx,
        )

        text = ((returned or "").strip() or "".join(collector).strip())
        text = strip_untagged_cot(strip_markdown_fences(strip_think_tags(text)))
        if error_ref and not text:
            if on_log:
                on_log("ERROR", f"[{thread_prefix}] LLM error: {error_ref[0]}")
            return None

        return text if text else None

    except Exception as e:
        if on_log:
            on_log("ERROR", f"[{thread_prefix}] LLM call failed: {e}")
        return None


def _normalize_curiosity_text(text: str) -> str:
    return re.sub(r"\s+", " ", str(text or "").strip().lower())


def _user_model_context_for_curiosity(workspace: str, *, max_chars: int = 1800) -> str:
    """Compact User Model snapshot when agent_plane snippet is unavailable."""
    try:
        from drive_engine import _user_model_discovery_context

        return _user_model_discovery_context(workspace, max_chars=max_chars)
    except Exception:
        return ""


def filter_open_loops_for_curiosity(
    questions: list[str],
    *,
    observation_anchor: str,
    observation_emitted: bool,
    um_phrases: list[str] | None = None,
) -> list[str]:
    """Drop open-loop rows that duplicate an emitted observation or User Model phrases."""
    from drive_engine import _jaccard_similarity

    anchor = str(observation_anchor or "").strip()
    anchor_norm = _normalize_curiosity_text(anchor)
    phrases = [str(p or "").strip() for p in (um_phrases or []) if str(p or "").strip()]
    kept: list[str] = []
    for raw in questions or []:
        q = str(raw or "").strip()
        if not q:
            continue
        if phrases and any(
            len(phrase) >= 12 and _jaccard_similarity(q, phrase) >= 0.55 for phrase in phrases
        ):
            continue
        if observation_emitted and anchor:
            q_norm = _normalize_curiosity_text(q)
            if q_norm and anchor_norm and q_norm == anchor_norm:
                continue
            if len(anchor) >= 12 and _jaccard_similarity(q, anchor) >= 0.55:
                continue
        kept.append(q)
    return kept


def _gather_context(workspace: str, include_insights: bool = True) -> str:
    """Gather context via agent_plane."""
    plane_enabled = True
    try:
        from utils.agent_plane import get_agent_plane_config

        plane_enabled = bool(get_agent_plane_config().get("enabled", True))
    except Exception:
        plane_enabled = True

    snippet = ""
    try:
        from utils.agent_plane import build_wake_context_snippet

        snippet = build_wake_context_snippet(
            workspace=workspace,
            wake_kind="curiosity",
            task_name="growth_check",
        )
        if snippet and snippet.strip():
            if not include_insights:
                snippet = re.sub(
                    r"### Consolidated Insights.*?(?=\n### |\Z)",
                    "",
                    snippet,
                    flags=re.DOTALL,
                ).strip()
    except Exception:
        snippet = ""

    if not plane_enabled or not (snippet or "").strip():
        um_hint = _user_model_context_for_curiosity(workspace)
        if um_hint:
            if (snippet or "").strip():
                snippet = f"{um_hint}\n\n{snippet}".strip()
            else:
                snippet = um_hint

    extras: list[str] = []
    gap_hint = _knowledge_gap_hint(workspace)
    if gap_hint:
        extras.append(gap_hint)
    kb_hint = _knowledge_context_hint()
    if kb_hint:
        extras.append(kb_hint)
    if extras:
        suffix = "\n\n".join(extras)
        snippet = f"{snippet}\n\n{suffix}".strip() if snippet and snippet.strip() else suffix

    return snippet[:_MAX_INPUT_CHARS] if snippet else ""


# ---------------------------------------------------------------------------
# 1. Curiosity question generation
# ---------------------------------------------------------------------------

_CURIOSITY_SYSTEM = (
    "You are a curiosity engine for a personal AI assistant. Your job is to review "
    "the assistant's memory, insights, and recent activity, then generate 1-2 short, "
    "specific follow-up questions or nudges about things that seem unresolved, "
    "half-finished, or worth revisiting.\n\n"
    "Rules:\n"
    "- Each question must reference a SPECIFIC topic, decision, or task from the provided context.\n"
    "- Questions should be things the USER would find helpful or thought-provoking.\n"
    "- Do NOT generate generic questions like 'How are you?' or 'Anything new?'\n"
    "- Do NOT generate questions about topics the user clearly finished or resolved.\n"
    "- Keep each question to 1-2 sentences.\n"
    "- Output ONLY the questions, one per line, prefixed with 'Q: '.\n"
    "- If nothing warrants a question, output exactly: NOTHING_TO_ASK"
)

_CURIOSITY_USER_TEMPLATE = (
    "Review the following memory and context. Generate 1-2 specific follow-up "
    "questions about unresolved threads, half-finished tasks, or topics worth revisiting.\n\n"
    "---\n\n{context}\n\n---"
)


def generate_curiosity_questions(
    workspace: str | None = None,
    on_log=None,
) -> list[str]:
    """Generate curiosity questions from memory/insights. Returns list of question strings."""
    if workspace is None:
        workspace = _get_workspace()
    if not workspace:
        return []
    try:
        from activity_bus import emit_activity_event
        emit_activity_event(
            {
                "source": "curiosity",
                "state": "started",
                "summary": "Curiosity question generation started",
                "job_id": "curiosity_questions",
            }
        )
    except Exception:
        pass

    cfg = _get_curiosity_config()
    if not cfg.get("enabled", True):
        if on_log:
            on_log("INFO", "[curiosity] disabled in config")
        return []

    context = _gather_context(workspace)
    if not context.strip():
        if on_log:
            on_log("INFO", "[curiosity] no context to generate questions from")
        return []

    if on_log:
        on_log("INFO", f"[curiosity] gathered {len(context)} chars of context")

    user_prompt = _CURIOSITY_USER_TEMPLATE.format(context=context)
    result = _llm_call(_CURIOSITY_SYSTEM, user_prompt, "curiosity", on_log)

    if not result or "NOTHING_TO_ASK" in result:
        if on_log:
            on_log("INFO", "[curiosity] no questions generated (nothing to ask)")
        try:
            from activity_bus import emit_activity_event
            emit_activity_event(
                {
                    "source": "curiosity",
                    "state": "completed",
                    "summary": "Curiosity loop completed (no questions)",
                    "job_id": "curiosity_questions",
                }
            )
        except Exception:
            pass
        return []

    questions = _extract_questions_from_output(result)
    if result.strip() and not questions and on_log:
        preview = result[:200].replace("\n", "\\n")
        on_log("INFO", f"[curiosity] non-empty model output yielded no questions: {preview}")

    if on_log:
        on_log("INFO", f"[curiosity] generated {len(questions)} question(s)")
    try:
        from activity_bus import emit_activity_event
        emit_activity_event(
            {
                "source": "curiosity",
                "state": "completed",
                "summary": f"Curiosity generated {len(questions)} question(s)",
                "job_id": "curiosity_questions",
            }
        )
    except Exception:
        pass

    return questions[:3]


# ---------------------------------------------------------------------------
# 2. Research prefetch
# ---------------------------------------------------------------------------

_PREFETCH_EXTRACT_SYSTEM = (
    "You are a topic extraction engine. Given consolidated memory insights, "
    "identify topics that appear 3 or more times or that the user has shown "
    "sustained interest in. For each, output a single concise search query "
    "that would find the latest, most useful information.\n\n"
    "Rules:\n"
    "- Output one line per topic, formatted as: TOPIC: <short label> | QUERY: <search query>\n"
    "- Maximum 3 topics.\n"
    "- Only include topics with clear recurring interest (not one-off mentions).\n"
    "- If no topics qualify, output exactly: NO_RECURRING_TOPICS"
)

_PREFETCH_EXTRACT_USER = (
    "Extract recurring topics from these insights and memory:\n\n"
    "---\n\n{context}\n\n---"
)

_PREFETCH_TOPIC_RE = re.compile(r"TOPIC:\s*(.+?)\s*\|\s*QUERY:\s*(.+)", re.IGNORECASE)
_PREFETCH_RELAXED_RE = re.compile(r"TOPIC\s*[:\-]?\s*(.+?)\s*(?:\||\s+)QUERY\s*[:\-]?\s*(.+)", re.IGNORECASE)


def _query_looks_unsafe(raw_query: str) -> bool:
    q = (raw_query or "").strip()
    if not q:
        return True
    if "file://" in q.lower():
        return True
    m = re.search(r"https?://[^\s]+", q)
    if not m:
        return False
    try:
        host = (urlparse(m.group(0)).hostname or "").lower()
    except Exception:
        return True
    return host in {"localhost", "127.0.0.1", "0.0.0.0"}


def _get_prefetched_dir(workspace: str) -> str:
    d = os.path.join(workspace, "memory", "prefetched")
    os.makedirs(d, exist_ok=True)
    return d


def run_research_prefetch(
    workspace: str | None = None,
    on_log=None,
) -> list[str]:
    """Identify recurring topics from insights and prefetch research for each.

    Returns list of file paths written to memory/prefetched/.
    """
    if workspace is None:
        workspace = _get_workspace()
    if not workspace:
        return []
    try:
        from activity_bus import emit_activity_event
        emit_activity_event(
            {
                "source": "curiosity",
                "state": "started",
                "summary": "Research prefetch started",
                "job_id": "research_prefetch",
            }
        )
    except Exception:
        pass

    cfg = _get_curiosity_config()
    if not cfg.get("prefetch_enabled", True):
        if on_log:
            on_log("INFO", "[prefetch] disabled in config")
        return []

    context = _gather_context(workspace)
    if not context.strip():
        if on_log:
            on_log("INFO", "[prefetch] no context for topic extraction")
        return []

    user_prompt = _PREFETCH_EXTRACT_USER.format(context=context)
    extract_result = _llm_call(_PREFETCH_EXTRACT_SYSTEM, user_prompt, "prefetch_extract", on_log)

    if not extract_result or "NO_RECURRING_TOPICS" in extract_result:
        if on_log:
            on_log("INFO", "[prefetch] no recurring topics found")
        try:
            from activity_bus import emit_activity_event
            emit_activity_event(
                {
                    "source": "curiosity",
                    "state": "completed",
                    "summary": "Research prefetch completed (no recurring topics)",
                    "job_id": "research_prefetch",
                }
            )
        except Exception:
            pass
        return []

    topics: list[tuple[str, str]] = []
    cleaned_extract = strip_untagged_cot(strip_markdown_fences(strip_think_tags(extract_result)))
    scanned_lines = 0
    for line in cleaned_extract.splitlines():
        stripped = line.strip()
        if not stripped:
            continue
        scanned_lines += 1
        m = _PREFETCH_TOPIC_RE.match(stripped)
        if not m:
            m = _PREFETCH_RELAXED_RE.match(stripped)
        if m:
            label = m.group(1).strip()
            query = m.group(2).strip()
            if label and query and not _query_looks_unsafe(query):
                topics.append((label, query))

    if not topics:
        if on_log:
            preview = cleaned_extract[:300].replace("\n", "\\n")
            on_log("INFO", f"[prefetch] no parseable topics from extraction (lines={scanned_lines}, preview={preview})")
        return []

    prefetch_dir = _get_prefetched_dir(workspace)
    written_files: list[str] = []
    scoring_cfg = _get_curiosity_scoring_config()
    scoring_enabled = bool(scoring_cfg.get("enabled", True))
    score_threshold = float(scoring_cfg.get("score_threshold", 0.15))
    half_life_hours = float(scoring_cfg.get("half_life_hours", 48.0))
    interest_decay = float(scoring_cfg.get("interest_decay", 0.9))
    interest_model = None
    gap_tracker = None
    if scoring_enabled:
        try:
            from curiosity_scoring import InterestModel, KnowledgeGapTracker

            interest_model = InterestModel(workspace=workspace, decay=interest_decay)
            gap_tracker = KnowledgeGapTracker(workspace=workspace)
        except Exception as scoring_err:
            scoring_enabled = False
            if on_log:
                on_log("WARN", f"[prefetch] curiosity scoring init failed: {scoring_err}")

    for label, query in topics[:3]:
        if scoring_enabled and interest_model is not None and gap_tracker is not None:
            try:
                from curiosity_scoring import calculate_curiosity_score

                score = calculate_curiosity_score(
                    topic=label,
                    interest_model=interest_model,
                    gap_tracker=gap_tracker,
                    prefetch_dir=prefetch_dir,
                    half_life_hours=half_life_hours,
                )
                if score < score_threshold:
                    if on_log:
                        on_log(
                            "INFO",
                            f"[prefetch] skipping '{label}' — curiosity score {score:.3f} below threshold {score_threshold:.3f}",
                        )
                    continue
            except Exception as scoring_err:
                if on_log:
                    on_log("WARN", f"[prefetch] curiosity scoring failed for '{label}': {scoring_err}")

        if on_log:
            on_log("INFO", f"[prefetch] researching: {label} — {query}")

        try:
            from subagent_runner import run_subagent
            from tool_policy import context_from_prefs_and_flags
            _prefetch_policy_ctx = context_from_prefs_and_flags(unattended_external=True, background_automation=True, subagent_mode=True)
            result = run_subagent(
                task_description=(
                    f"Research the following topic and provide a concise, factual summary "
                    f"with the latest information, best practices, and key resources.\n\n"
                    f"Topic: {label}\nSearch query: {query}"
                ),
                agent_type="research",
                timeout_seconds=int(cfg.get("prefetch_timeout_seconds", 180)),
                policy_context=_prefetch_policy_ctx,
            )
        except Exception as e:
            if on_log:
                on_log("ERROR", f"[prefetch] research subagent failed for '{label}': {e}")
            continue

        if not result or "timed out" in result.lower() or "error" in result.lower()[:50]:
            if on_log:
                on_log("WARN", f"[prefetch] no usable result for '{label}'")
            continue

        if len(result) > _MAX_PREFETCH_CHARS:
            result = result[:_MAX_PREFETCH_CHARS] + "\n\n[... truncated]"

        safe_label = re.sub(r"[^a-zA-Z0-9_-]", "_", label.lower())[:40]
        datestamp = datetime.now().strftime("%Y%m%d")
        fname = f"{datestamp}_{safe_label}.md"
        fpath = os.path.join(prefetch_dir, fname)

        header = (
            f"# Prefetched Research: {label}\n"
            f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
            f"**Query:** {query}\n\n---\n\n"
        )

        try:
            from utils.path_safety import secure_write_file
            with _PREFETCH_WRITE_LOCK:
                secure_write_file(fpath, header + result, mode=0o600)
        except Exception:
            try:
                with _PREFETCH_WRITE_LOCK:
                    with open(fpath, "w", encoding="utf-8") as f:
                        f.write(header + result)
                    os.chmod(fpath, 0o600)
            except Exception as write_err:
                if on_log:
                    on_log("ERROR", f"[prefetch] write failed for '{label}': {write_err}")
                continue

        written_files.append(fpath)
        if on_log:
            on_log("INFO", f"[prefetch] saved {len(result)} chars to {fname}")

        if interest_model is not None:
            try:
                interest_model.update(label, 0.7, source="agent")
            except Exception:
                pass

    try:
        from activity_bus import emit_activity_event
        emit_activity_event(
            {
                "source": "curiosity",
                "state": "completed",
                "summary": f"Research prefetch completed ({len(written_files)} file(s))",
                "job_id": "research_prefetch",
            }
        )
    except Exception:
        pass

    return written_files


# ---------------------------------------------------------------------------
# 3. Feed-to-curiosity topic matching
# ---------------------------------------------------------------------------

_FEED_MATCH_SYSTEM = (
    "You are a relevance matcher. Given the user's interests (from consolidated "
    "memory insights) and a list of new feed items, identify which items are "
    "relevant to the user's current projects, interests, or recurring topics.\n\n"
    "Also note items that connect to topics the assistant has been researching "
    "or curious about. For these, add the tag SHARED: to indicate mutual interest.\n\n"
    "Rules:\n"
    "- Only flag items that have a CLEAR connection to the user's interests.\n"
    "- For each relevant item, output one line: RELEVANT: <feed item title> — <why it matters to the user>\n"
    "- For shared-interest items: SHARED: <feed item title> — <why it connects to mutual interests>\n"
    "- If no items are relevant, output exactly: NO_RELEVANT_ITEMS\n"
    "- Be selective. Quality over quantity."
)

_FEED_MATCH_USER = (
    "## User's interests and context\n\n{interests}\n\n"
    "## New feed items\n\n{feed_items}\n\n"
    "Which feed items are relevant to this user?"
)

_FEED_RELEVANT_RE = re.compile(r"RELEVANT:\s*(.+?)(?:\s*—\s*|\s*-\s*)(.+)", re.IGNORECASE)
_FEED_SHARED_RE = re.compile(r"SHARED:\s*(.+?)(?:\s*—\s*|\s*-\s*)(.+)", re.IGNORECASE)


def match_feeds_to_interests(
    new_feed_items: list[dict],
    workspace: str | None = None,
    on_log=None,
) -> list[dict]:
    """Match new feed items against user interests from insights.

    Args:
        new_feed_items: list of dicts with at least 'title' and optionally 'link', 'summary'.

    Returns list of dicts: [{"title": ..., "reason": ..., "link": ...}, ...]
    """
    if not new_feed_items:
        return []

    if workspace is None:
        workspace = _get_workspace()
    if not workspace:
        return []

    cfg = _get_curiosity_config()
    if not cfg.get("feed_matching_enabled", True):
        if on_log:
            on_log("INFO", "[feed_match] disabled in config")
        return []

    insights_path = os.path.join(workspace, "memory", "insights.md")
    insights = _read_file_safe(insights_path, 4000)
    if not insights.strip():
        if on_log:
            on_log("INFO", "[feed_match] no insights to match against")
        return []

    feed_lines = []
    for item in new_feed_items[:20]:
        title = (item.get("title") or "").strip()
        summary = (item.get("summary") or "")[:200].strip()
        link = (item.get("link") or "").strip()
        if title:
            line = f"- {title}"
            if summary:
                line += f" — {summary}"
            if link:
                line += f" ({link})"
            feed_lines.append(line)

    if not feed_lines:
        return []

    feed_text = "\n".join(feed_lines)
    user_prompt = _FEED_MATCH_USER.format(interests=insights.strip(), feed_items=feed_text)
    result = _llm_call(_FEED_MATCH_SYSTEM, user_prompt, "feed_match", on_log)

    if not result or "NO_RELEVANT_ITEMS" in result:
        if on_log:
            on_log("INFO", "[feed_match] no relevant items found")
        return []

    matches = []
    for line in result.splitlines():
        stripped = line.strip()
        shared = False
        m = _FEED_SHARED_RE.match(stripped)
        if m:
            shared = True
        else:
            m = _FEED_RELEVANT_RE.match(stripped)
        if m:
            title_match = m.group(1).strip()
            reason = m.group(2).strip()
            link = ""
            for item in new_feed_items:
                if (item.get("title") or "").strip().lower() in title_match.lower():
                    link = item.get("link", "")
                    break
            entry = {
                "title": title_match[:200],
                "reason": reason[:300],
                "link": link,
            }
            if shared:
                entry["shared"] = True
                entry["reason"] = f"I came across this — it connects to something we have both been interested in: {reason[:250]}"
            matches.append(entry)

    if on_log:
        on_log("INFO", f"[feed_match] found {len(matches)} relevant item(s)")

    return matches[:5]


# ---------------------------------------------------------------------------
# 4. Insight injection: read latest insight block for agent_loop
# ---------------------------------------------------------------------------

_INSIGHT_MAX_AGE_HOURS = 48
_INSIGHT_MAX_CHARS = 800


def get_fresh_insight_for_injection(workspace: str | None = None) -> str | None:
    """Return the latest consolidation insight if it's fresh enough for injection.

    Returns a short string suitable for appending to the system prompt, or None.
    """
    if workspace is None:
        workspace = _get_workspace()
    if not workspace:
        return None

    insights_path = os.path.join(workspace, "memory", "insights.md")
    content = _read_file_safe(insights_path, 10000)
    if not content.strip():
        return None

    blocks = re.split(r"(?=## Consolidation — \d{4}-\d{2}-\d{2} \d{2}:\d{2})", content)
    if not blocks:
        return None

    latest = blocks[-1].strip()
    if not latest:
        return None

    ts_match = re.match(r"## Consolidation — (\d{4}-\d{2}-\d{2} \d{2}:\d{2})", latest)
    if ts_match:
        try:
            ts = datetime.strptime(ts_match.group(1), "%Y-%m-%d %H:%M")
            age_hours = (datetime.now() - ts).total_seconds() / 3600
            if age_hours > _INSIGHT_MAX_AGE_HOURS:
                return None
        except ValueError:
            pass

    takeaways = ""
    for section_name in ("Actionable Takeaways", "Recurring Patterns"):
        pattern = rf"## {section_name}\s*\n((?:- .+\n?)+)"
        m = re.search(pattern, latest)
        if m:
            takeaways += m.group(0).strip() + "\n\n"

    if not takeaways.strip():
        return None

    try:
        from utils.memory_promotion import classify_memory_entry, PromotionClass
        filtered_lines = []
        for ln in takeaways.splitlines():
            if ln.strip().startswith("- "):
                decision = classify_memory_entry(ln)
                if decision.classification == PromotionClass.BLOCKED:
                    continue
            filtered_lines.append(ln)
        takeaways = "\n".join(filtered_lines).strip()
    except Exception:
        pass

    if not takeaways.strip():
        return None

    if len(takeaways) > _INSIGHT_MAX_CHARS:
        takeaways = takeaways[:_INSIGHT_MAX_CHARS].rstrip() + "…"

    return takeaways.strip()


def get_prefetched_context(workspace: str | None = None, max_total_chars: int = 2000) -> str | None:
    """Return a summary of available prefetched research files for system prompt injection."""
    if workspace is None:
        workspace = _get_workspace()
    if not workspace:
        return None

    prefetch_dir = os.path.join(workspace, "memory", "prefetched")
    if not os.path.isdir(prefetch_dir):
        return None

    cutoff = datetime.now() - timedelta(hours=_INSIGHT_MAX_AGE_HOURS)
    files = []
    try:
        for fname in os.listdir(prefetch_dir):
            if not fname.endswith(".md"):
                continue
            fpath = os.path.join(prefetch_dir, fname)
            try:
                mtime = os.path.getmtime(fpath)
                if datetime.fromtimestamp(mtime) > cutoff:
                    files.append((fname, fpath))
            except OSError:
                continue
    except OSError:
        return None

    if not files:
        return None

    parts = []
    total = 0
    for fname, fpath in sorted(files, reverse=True)[:3]:
        content = _read_file_safe(fpath, 800)
        if content.strip():
            parts.append(f"[{fname}] {content.strip()[:600]}")
            total += len(parts[-1])
            if total > max_total_chars:
                break

    if not parts:
        return None

    return "Pre-researched context available in memory/prefetched/:\n" + "\n".join(parts)
