# tools/research.py — Deep research: search, read/analyze, synthesize, save, follow-ups, final report.
# All path handling via utils.path_safety. Uses tools.web_tools (search_web, search_web_public, read_webpage).
from __future__ import annotations

import json
import os
import re
import time
import ipaddress
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.parse import urlparse

from langchain_core.tools import tool
from pydantic import BaseModel, Field, field_validator

from agent_loop_config import _read_pref, get_backend_base_url, get_llmster_model, get_research_limits
from research_run_state import (
    bind_state_to_scratch,
    load_state_for_scratch,
    merge_url,
    normalize_source_url,
    save_state,
)
from tools._logging import _debug_log, _emit_input_status
from tools.job_stream_schema import (
    open_job_stream,
    update_job_stream,
    close_job_stream,
    error_job_stream,
    make_capabilities,
    JobCapabilities,
)
from tools.live_view_visuals import emit_live_view_preview
from tools.plugin_stream_controls import (
    set_control_value as _set_stream_control_value,
    get_control_value as _get_stream_control_value,
    get_action_count as _get_stream_action_count,
    clear_stream_state as _clear_stream_control_state,
)
from tools.web_tools import SearchError, SearchSuccess
from utils.path_safety import get_user_output_dir, safe_resolve_path, secure_write_file


# ─── CRAAP source credibility scoring ────────────────────────────────────────
# Deterministic, no-LLM scoring based on Currency, Authority, and Purpose signals.
# Inspired by the CRAAP framework used in OpenFang's Researcher Hand.

_AUTHORITY_TIER1 = frozenset({
    # Government / intergovernmental
    ".gov", ".gov.uk", ".gc.ca", ".europa.eu",
    # Academic / research
    ".edu", ".ac.uk",
    # Major reference / encyclopaedia
    "wikipedia.org", "britannica.com",
    # Scientific publishers
    "nature.com", "science.org", "cell.com", "nejm.org", "thelancet.com",
    "pubmed.ncbi.nlm.nih.gov", "scholar.google.com", "arxiv.org",
    "doi.org", "nih.gov", "who.int", "cdc.gov",
    # Quality journalism
    "reuters.com", "apnews.com", "bbc.com", "bbc.co.uk",
    "theguardian.com", "nytimes.com", "ft.com", "washingtonpost.com",
    "economist.com", "bloomberg.com",
})

_COMMERCIAL_SIGNALS = re.compile(
    r"(?:sponsored|advertisement|buy now|add to cart|subscribe to (?:get|access)|sign up for (?:free|premium))",
    re.IGNORECASE,
)

_RECENCY_PATTERN = re.compile(r"\b(202[3-9]|2030)\b")


def _score_source_credibility(url: str, content: str) -> dict:
    """Score a source using three CRAAP dimensions (deterministic, no LLM).

    Returns a dict with 'authority', 'currency', 'purpose', 'total' (0–9) and 'label'.
    """
    authority = 1  # default: unknown domain
    try:
        parsed = urlparse(url)
        host = (parsed.netloc or "").lower().lstrip("www.")
        # Tier-1 check: exact match or suffix match (e.g. example.gov)
        if host in _AUTHORITY_TIER1 or any(host.endswith(t) for t in _AUTHORITY_TIER1):
            authority = 3
        elif host.endswith(".org"):
            authority = 2  # .org is higher than unknown commercial
    except Exception:
        pass

    # Currency (0–3): recent year found in content → higher score
    currency = 1  # default
    if content:
        scan = content[:5000]
        matches = _RECENCY_PATTERN.findall(scan)
        if len(matches) >= 3:
            currency = 3
        elif len(matches) >= 1:
            currency = 2

    # Purpose (0–3): heavy commercial signals → lower score
    purpose = 3
    if content and _COMMERCIAL_SIGNALS.search(content[:3000]):
        purpose = 1

    total = authority + currency + purpose  # 3–9
    if total >= 7:
        label = "High"
    elif total >= 5:
        label = "Medium"
    else:
        label = "Low"

    return {"authority": authority, "currency": currency, "purpose": purpose,
            "total": total, "label": label}


def _extract_source_url_from_analysis(analysis: str) -> str:
    """Pull the URL out of a read_and_analyze_page output (last '## Source' section)."""
    if not analysis:
        return ""
    # The tool appends "\n\n## Source\n\nURL" at the end
    m = re.search(r"##\s+Source\s*\n+\s*(https?://\S+)", analysis, re.IGNORECASE)
    if m:
        return m.group(1).strip()
    return ""


def _source_link_label(url: str, index: int) -> str:
    """Return a short label for a source URL (e.g. domain) for use in [label](url)."""
    if not url or not isinstance(url, str):
        return f"Source {index + 1}"
    url = url.strip()
    try:
        parsed = urlparse(url)
        netloc = (parsed.netloc or "").strip()
        if netloc:
            return netloc.lstrip("www.")
    except Exception:
        pass
    return f"Source {index + 1}"


def _clean_source_url(url: str) -> str:
    """Normalize/sanitize source URLs before report output."""
    if not isinstance(url, str):
        return ""
    u = str(url).strip().strip("<>").strip("\"'`")
    if not u:
        return ""
    # Repair common LLM formatting artifacts like "pmc. ncbi. nlm. nih. gov".
    u = re.sub(r"\s+", "", u)
    if not u.startswith(("http://", "https://")):
        return ""
    normalized = normalize_source_url(u)
    try:
        p = urlparse(normalized)
    except Exception:
        return ""
    if p.scheme not in ("http", "https") or not p.netloc:
        return ""
    if " " in p.netloc:
        return ""
    return normalized


def _is_public_http_url(url: str) -> bool:
    try:
        p = urlparse((url or "").strip())
    except Exception:
        return False
    if p.scheme not in ("http", "https"):
        return False
    host = (p.hostname or "").strip().lower()
    if not host:
        return False
    if host in {"localhost", "127.0.0.1", "0.0.0.0"}:
        return False
    try:
        ip = ipaddress.ip_address(host)
        if ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_multicast:
            return False
    except ValueError:
        pass
    return True


def _sanitize_research_findings(raw: str) -> str:
    """Remove common junk from concatenated research tool output so the report reads cleanly.
    - Strip ```json ... ``` blocks (raw search result arrays).
    - Remove ## Key facts sections that are only bullet lists of numbers/short tokens."""
    if not raw or not isinstance(raw, str):
        return raw
    text = raw.strip()
    # Remove markdown JSON code blocks (often from deep_web_search output).
    text = re.sub(r"```\s*json\s*\n[\s\S]*?```", "", text, flags=re.IGNORECASE)
    # Remove any placeholder paragraphs that slipped in (trim messages).
    text = re.sub(
        r"\n*\[Research result from an earlier round omitted[\s\S]*?\]\s*\n*",
        "\n",
        text,
    )
    text = re.sub(
        r"\n*\[Large tool result omitted[\s\S]*?\]\s*\n*",
        "\n",
        text,
    )
    # Remove scratch/truncation placeholders that must never appear in final reports.
    # These can slip in when the model echoes history truncation (e.g. "[Content written to scratch file...]"
    # or "[Detailed synthesis of ... written to scratch file on disk; truncated from history to save context.]").
    text = re.sub(
        r"\n*\[[\s\S]*?written to scratch file[\s\S]*?(?:truncated from history|save context)[\s\S]*?\]\s*\n*",
        "\n",
        text,
        flags=re.IGNORECASE,
    )
    # Remove ## Key facts sections where content is mostly short bullets (numbers, single words).
    pattern = re.compile(r"\n##\s+Key facts\s*\n([\s\S]*?)(?=\n##\s|\n---|\Z)", re.IGNORECASE)
    def replace_key_facts(match):
        section_body = match.group(1)
        lines = [ln.strip() for ln in section_body.splitlines() if ln.strip()]
        short = sum(1 for ln in lines if re.match(r"^[-*]\s*.{0,15}$", ln) or len(ln) <= 20)
        if lines and short >= 0.9 * len(lines):
            return "\n"
        return match.group(0)
    text = pattern.sub(replace_key_facts, text)
    # Collapse multiple blank lines.
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip()


def _extract_scratch_path(text: str) -> str | None:
    if not isinstance(text, str):
        return None
    m = re.search(r"Scratch file:\s*([^\s(]+)", text, re.IGNORECASE)
    if not m:
        return None
    out = (m.group(1) or "").strip().rstrip(")")
    return os.path.abspath(os.path.expanduser(out)) if out else None


def _extract_open_questions_from_synthesis(text: str) -> list[str]:
    if not isinstance(text, str) or not text.strip():
        return []
    m = re.search(r"###\s+Open questions\s*\n([\s\S]*?)(?:\n###\s|\Z)", text, re.IGNORECASE)
    if not m:
        return []
    lines = []
    for ln in m.group(1).splitlines():
        ln = ln.strip()
        if ln.startswith("- "):
            q = ln[2:].strip()
            if q and q.lower() != "none noted.":
                lines.append(q)
    return lines[:6]


def _extract_contradictions_from_synthesis(text: str) -> list[str]:
    if not isinstance(text, str) or not text.strip():
        return []
    m = re.search(r"###\s+Contradictions[\s\S]*?\n([\s\S]*?)(?:\n###\s|\Z)", text, re.IGNORECASE)
    if not m:
        return []
    lines = []
    for ln in m.group(1).splitlines():
        ln = ln.strip()
        if ln.startswith("- "):
            val = ln[2:].strip()
            if val and "no clear contradictions" not in val.lower():
                lines.append(val)
    return lines[:6]


def _extract_recommendation_from_synthesis(text: str) -> str:
    if not isinstance(text, str) or not text.strip():
        return ""
    m = re.search(r"###\s+Recommendation\s*\n([\s\S]*?)(?:\n###\s|\Z)", text, re.IGNORECASE)
    if not m:
        return ""
    rec = " ".join(x.strip() for x in m.group(1).splitlines() if x.strip())
    return rec[:400]


def _llm_followup_queries(current_findings: str, original_query: str, synthesis_text: str = "") -> list[str]:
    """Optional pref-gated LLM follow-up generation (fallback to heuristics on failure)."""
    if not bool(_read_pref("research_llm_followups", False)):
        return []
    if not (current_findings or synthesis_text or original_query):
        return []
    try:
        from openai import OpenAI
    except Exception:
        return []
    try:
        base_url = get_backend_base_url()
        model = get_llmster_model() or "local-model"
        client = OpenAI(base_url=base_url, api_key="lm-studio")
        prompt = (
            "Return exactly 5 short web-search queries, one per line, no numbering.\n\n"
            f"Original research question:\n{(original_query or '').strip()[:500]}\n\n"
            f"Current findings:\n{(current_findings or '').strip()[:2500]}\n\n"
            f"Synthesis:\n{(synthesis_text or '').strip()[:2500]}\n"
        )
        resp = client.chat.completions.create(
            model=model,
            temperature=0.2,
            max_tokens=220,
            messages=[
                {"role": "system", "content": "You generate focused search-engine queries for unresolved research gaps."},
                {"role": "user", "content": prompt},
            ],
            timeout=20,
        )
        text = (((resp.choices or [None])[0].message.content) or "").strip()
        queries = []
        for ln in text.splitlines():
            ln = re.sub(r"^\s*[\-\d\.\)]*\s*", "", ln).strip().strip("`")
            if len(ln) >= 4:
                queries.append(ln)
        out: list[str] = []
        seen: set[str] = set()
        for q in queries:
            k = q.lower()
            if k in seen:
                continue
            seen.add(k)
            out.append(q)
            if len(out) >= 6:
                break
        return out
    except Exception:
        return []



# Optional tab_id for scratchpad (app can set via set_research_tab_id). If None, we use "default".
_research_tab_id: str | None = None


def set_research_tab_id(tab_id: str | None) -> None:
    """Set the tab id used when writing research to a scratchpad (e.g. current tab when agent runs)."""
    global _research_tab_id
    _research_tab_id = tab_id


# ─── Helpers ─────────────────────────────────────────────────────────────────

_RJ_PLUGIN_ID = "research_job"
_RJ_PLUGIN_NAME = "Research Job"


class ResearchJobStream:
    """Lightweight wrapper around job_stream_schema for research tools.

    Eliminates repeated plugin_id / plugin_name / stream_id boilerplate.
    Use as a context manager for auto-cleanup of control state.
    """

    def __init__(
        self,
        *,
        tool_name: str,
        title: str,
        controls: list[dict] | None = None,
        metadata: dict | None = None,
        capabilities: JobCapabilities | None = None,
    ):
        self.stream_id = open_job_stream(
            plugin_id=_RJ_PLUGIN_ID,
            plugin_name=_RJ_PLUGIN_NAME,
            title=title,
            job_type=tool_name,
            controls=controls or [],
            metadata=metadata or {},
            capabilities=capabilities,
        )
        self._cancel_count = 0

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        _clear_stream_control_state(self.stream_id)
        return False

    def update(self, message: str, *, phase: str | None = None,
               progress_pct: float | None = None,
               metrics: dict[str, float] | None = None,
               extra: dict | None = None,
               status: str = "running") -> None:
        update_job_stream(
            plugin_id=_RJ_PLUGIN_ID,
            plugin_name=_RJ_PLUGIN_NAME,
            stream_id=self.stream_id,
            message=message,
            phase=phase,
            progress_pct=progress_pct,
            metrics=metrics,
            extra=extra,
            status=status,
        )

    def close(self, status: str, message: str, *,
              metrics: dict[str, float] | None = None,
              extra: dict | None = None) -> None:
        close_job_stream(
            plugin_id=_RJ_PLUGIN_ID,
            plugin_name=_RJ_PLUGIN_NAME,
            stream_id=self.stream_id,
            status=status,
            message=message,
            metrics=metrics,
            extra=extra,
        )

    def error(self, message: str, *, phase: str | None = None,
              extra: dict | None = None) -> None:
        error_job_stream(
            plugin_id=_RJ_PLUGIN_ID,
            plugin_name=_RJ_PLUGIN_NAME,
            stream_id=self.stream_id,
            message=message,
            phase=phase,
            extra=extra,
        )

    def cancel_requested(self) -> bool:
        """Check if the user pressed cancel since the last check."""
        n = _get_stream_action_count(self.stream_id, "cancel_job")
        cancelled = n > self._cancel_count
        self._cancel_count = n
        return cancelled

    def get_control(self, control_id: str, default=None):
        return _get_stream_control_value(self.stream_id, control_id, default)

    def set_control(self, control_id: str, value) -> None:
        _set_stream_control_value(self.stream_id, control_id, value)

_SEARCH_RATE_LIMIT_BACKOFF_CAP = 30  # seconds


def _invoke_search(query: str, time_filter: str | None) -> SearchSuccess | SearchError:
    """Call search_web first; on retryable failure try search_web_public. Returns SearchSuccess or SearchError."""
    from tools.web_tools import search_web, search_web_public
    kwargs = {"query": query.strip(), "time_range": time_filter}
    out = search_web.invoke(kwargs)
    if isinstance(out, SearchError) and out.retryable:
        out = search_web_public.invoke(kwargs)
        # A 429 on the public fallback is usually transient; one bounded
        # backoff retry keeps a rate-limit blip from zeroing out the only
        # gather path (council researcher batches died on exactly this).
        if (
            isinstance(out, SearchError)
            and out.retryable
            and out.status_code == 429
        ):
            wait = min(int(out.retry_after_seconds or 12), _SEARCH_RATE_LIMIT_BACKOFF_CAP)
            time.sleep(max(wait, 1))
            out = search_web_public.invoke(kwargs)
    return out


def _open_research_job_stream(*, tool_name: str, title: str, controls: list[dict] | None = None, metadata: dict | None = None) -> str:
    """Backwards-compat shim — prefer ResearchJobStream context manager."""
    return open_job_stream(
        plugin_id=_RJ_PLUGIN_ID, plugin_name=_RJ_PLUGIN_NAME,
        title=title, job_type=tool_name, controls=controls or [], metadata=metadata or {},
    )


def _job_cancel_requested(stream_id: str, last_action_count: int) -> tuple[bool, int]:
    """Backwards-compat shim — prefer ResearchJobStream.cancel_requested()."""
    n = _get_stream_action_count(stream_id, "cancel_job")
    return (n > last_action_count), n


def _parse_search_text(text: str, max_results: int) -> list[dict]:
    """Parse SearXNG-style 'Title: ... URL: ... Snippet: ...' blocks into list of dicts."""
    results = []
    # Blocks separated by double newline; each block has Title/URL/Snippet lines
    blocks = re.split(r"\n\n+", text)
    for block in blocks:
        if len(results) >= max_results:
            break
        title = url = snippet = None
        for line in block.splitlines():
            if line.startswith("Title:"):
                title = line[6:].strip()
            elif line.startswith("URL:"):
                url = line[4:].strip()
            elif line.startswith("Snippet:"):
                snippet = line[8:].strip()
        if url or title:
            results.append({
                "title": title or "No title",
                "url": url or "No URL",
                "snippet": snippet or "No snippet",
                "date": "N/A",
            })
    return results


def _sanitize_topic(topic: str) -> str:
    """Safe folder segment: alphanumeric, underscores, single dashes; max 80 chars."""
    s = re.sub(r"[^\w\s-]", "", (topic or "").strip())
    s = re.sub(r"\s+", "_", s).strip("_") or "research"
    return s[:80]


def _write_research_folder(topic: str, report_content: str, sources: list[str]) -> tuple[str | None, str]:
    """
    Create folder ~/Documents/Mugi/research/<safe_topic>_<date>, write report.md.
    report_content is the full markdown (findings + ## Sources + list). Returns (absolute_folder_path, error_message).
    """
    base = get_user_output_dir()
    safe_topic = _sanitize_topic(topic)
    date = datetime.now().strftime("%Y-%m-%d")
    relative = f"research/{safe_topic}_{date}"
    abs_path, err = safe_resolve_path(base, relative, allow_write=True, create_parents=True)
    if err or not abs_path:
        return None, err or "Failed to resolve path"
    try:
        os.makedirs(abs_path, exist_ok=True)
    except OSError as e:
        return None, str(e)
    report_path = os.path.join(abs_path, "report.md")
    try:
        secure_write_file(report_path, (report_content or "").strip(), mode=0o644)
    except Exception as e:
        return None, f"Failed to write report.md: {e}"
    return abs_path, ""


def _write_research_to_scratchpad(
    folder_path: str,
    topic: str,
    report_content: str,
    tab_id: str | None = None,
) -> str:
    """
    Create a new scratchpad with the research report and a clickable link to the folder.
    Uses tab_id or _research_tab_id or "default". Returns empty string on success, error message on failure.
    """
    tid = tab_id or _research_tab_id or "default"
    try:
        from chat_storage import ensure_encryption_ready, create_scratchpad
        ensure_encryption_ready()
        # Clickable filesystem link at top (file:// URL opens in Finder on macOS)
        try:
            file_url = Path(os.path.abspath(folder_path)).resolve().as_uri()
        except Exception:
            file_url = "file://" + os.path.abspath(folder_path).replace("\\", "/")
        link_line = f"[📁 Open research folder]({file_url})"
        body = link_line + "\n\n" + (report_content or "").strip()
        date = datetime.now().strftime("%Y-%m-%d")
        title = f"Research: {topic or 'research'} ({date})"
        create_scratchpad(tid, title=title, body=body)
        return ""
    except Exception as e:
        return str(e)


def _find_latest_scratch_for_topic(topic: str) -> str:
    """Best-effort lookup for an existing scratch file for this topic."""
    base = get_user_output_dir()
    scratch_dir, err = safe_resolve_path(base, "research/_scratch", allow_write=False, create_parents=False)
    if err or not scratch_dir or not os.path.isdir(scratch_dir):
        return ""
    safe_topic = _sanitize_topic(topic or "research")
    prefix = f"{safe_topic}_"
    try:
        candidates = [
            os.path.join(scratch_dir, name)
            for name in os.listdir(scratch_dir)
            if name.startswith(prefix) and name.endswith(".md")
        ]
    except Exception:
        return ""
    if not candidates:
        return ""
    try:
        candidates.sort(key=lambda p: os.path.getmtime(p), reverse=True)
    except Exception:
        pass
    return candidates[0]


# ─── Tool 1: deep_web_search ──────────────────────────────────────────────────

class DeepWebSearchInput(BaseModel):
    """Input for deep_web_search."""

    query: str = Field(..., description="Search query for deep research (e.g. 'AI agent safety 2026').")
    num_results: int = Field(10, description="Maximum number of results to return (default 10).")
    time_filter: str | None = Field(None, description="Optional recency: 'day', 'week', 'month', or 'year'.")
    iteration_hint: int | None = Field(None, description="Optional iteration counter from caller for Live View display.")

    @field_validator("num_results", mode="before")
    @classmethod
    def _coerce_num_results(cls, v):
        if isinstance(v, str):
            s = v.strip()
            if re.fullmatch(r"-?\d+", s):
                return int(s)
        return v


@tool(args_schema=DeepWebSearchInput)
def deep_web_search(query: str, num_results: int = 10, time_filter: str | None = None, iteration_hint: int | None = None) -> str:
    """Run a deep web search for research — use this instead of search_web for any research or investigation task. Returns structured markdown plus a JSON block (title, snippet, url, date) for each result. Pass time_filter ('day'/'week'/'month'/'year') for recent content. Falls back to search_web_public if local engine is down. Example: deep_web_search(query='LLM agent safety guidelines 2025', num_results=8, time_filter='year')."""
    _debug_log(f"[DEBUG deep_web_search] query={query!r}, num_results={num_results}")
    _emit_input_status("Searching…")
    q = (query or "").strip()
    if not q:
        return "Search requires a non-empty query."
    emit_live_view_preview(
        f"Searching: {q[:50]}{'…' if len(q) > 50 else ''}",
        tool_name="deep_web_search",
    )
    if num_results < 1:
        num_results = 10
    if num_results > 20:
        num_results = 20
    tr = None
    if time_filter and isinstance(time_filter, str):
        tr = time_filter.strip().lower()
        if tr not in ("day", "week", "month", "year"):
            tr = None

    with ResearchJobStream(
        tool_name="deep_web_search",
        title="Research: Deep Web Search",
        controls=[
            {"id": "num_results", "type": "slider", "label": "Max Results", "min": 1, "max": 20, "step": 1, "default": int(num_results)},
            {"id": "time_filter", "type": "select", "label": "Time Filter", "options": ["none", "day", "week", "month", "year"], "default": tr or "none"},
            {"id": "cancel", "type": "button", "label": "Cancel Search", "action_id": "cancel_job", "style": "destructive"},
        ],
        capabilities=make_capabilities(cancel=True, progress=True, metrics=True),
        metadata={
            "query": q,
            "search_limit": int(get_research_limits().get("max_search_rounds", 8)),
            "iteration": int(iteration_hint or 0),
        },
    ) as js:
        js.set_control("num_results", int(num_results))
        js.set_control("time_filter", tr or "none")
        started_at = time.time()
        try:
            js.update(f"Searching for '{q[:80]}'", phase="search_start", progress_pct=5, metrics={"num_results": float(num_results)})
            if js.cancel_requested():
                js.close("stopped", "Search cancelled by user before execution.")
                _emit_input_status("Done")
                return "Search cancelled by user."

            live_num = js.get_control("num_results", int(num_results))
            try:
                num_results = max(1, min(20, int(float(live_num))))
            except Exception:
                pass
            live_tf = str(js.get_control("time_filter", tr or "none") or "none").strip().lower()
            tr = None if live_tf in {"", "none"} else (live_tf if live_tf in {"day", "week", "month", "year"} else tr)

            raw = _invoke_search(q, tr)
            if isinstance(raw, SearchError):
                msg = raw.message or "No search results returned."
                js.close("error", msg)
                _emit_input_status("Done")
                return msg
            if not raw.text and not (raw.results and len(raw.results)):
                js.close("completed", "No search results returned.")
                _emit_input_status("Done")
                return "No search results returned."
            if raw.results and len(raw.results) > 0:
                results = [{"title": r["title"], "url": r["url"], "snippet": r["snippet"], "date": "N/A"} for r in raw.results[:num_results]]
            else:
                results = _parse_search_text(raw.text, num_results)
            if not results:
                js.close("completed", "No parsed search results available.")
                _emit_input_status("Done")
                return raw.text or "No search results returned."
            lines = ["## Search results\n"]
            for i, r in enumerate(results, 1):
                lines.append(f"{i}. **{r['title']}**")
                lines.append(f"   - URL: {r['url']}")
                lines.append(f"   - Snippet: {r['snippet']}")
                lines.append("")
            lines.append("\n```json\n")
            lines.append(json.dumps(results, indent=2))
            lines.append("\n```")
            emit_live_view_preview(f"Found {len(results)} result(s)", tool_name="deep_web_search")
            js.close("completed", f"Search complete: {len(results)} results.", metrics={
                "results_count": float(len(results)),
                "elapsed_seconds": float(round(time.time() - started_at, 2)),
            })
            _emit_input_status("Done")
            return "\n".join(lines)
        except Exception as e:
            js.error(str(e), phase="search_error")
            _emit_input_status("Done")
            return f"Search failed: {e}"


# ─── Helper: table extraction (used by read_and_analyze_page) ─────────────

def _extract_tables(text: str, max_tables: int = 3, max_chars_per_table: int = 500) -> str:
    """Extract markdown tables from *text*. Returns up to *max_tables* tables under a heading."""
    tables: list[str] = []
    current_rows: list[str] = []
    in_table = False
    has_separator = False
    for line in text.splitlines():
        stripped = line.strip()
        if stripped.startswith("|") and stripped.endswith("|") and stripped.count("|") >= 3:
            current_rows.append(stripped)
            if re.match(r"^\|[\s\-:|]+\|$", stripped):
                has_separator = True
            in_table = True
        else:
            if in_table and has_separator and len(current_rows) >= 3:
                table_str = "\n".join(current_rows)[:max_chars_per_table]
                tables.append(table_str)
                if len(tables) >= max_tables:
                    break
            current_rows = []
            in_table = False
            has_separator = False
    if in_table and has_separator and len(current_rows) >= 3 and len(tables) < max_tables:
        tables.append("\n".join(current_rows)[:max_chars_per_table])
    if not tables:
        return ""
    parts = ["## Tables\n"]
    for t in tables:
        parts.append(t)
        parts.append("")
    return "\n".join(parts)


# ─── Tool 2: read_and_analyze_page ──────────────────────────────────────────

class ReadAndAnalyzePageInput(BaseModel):
    """Input for read_and_analyze_page."""

    url: str = Field(..., description="URL of the page to read (should come from prior search results).")
    focus_areas: list[str] | None = Field(None, description="Optional list of focus areas, e.g. ['key statistics', 'criticisms', 'future implications'].")

    @field_validator("focus_areas", mode="before")
    @classmethod
    def _coerce_focus_areas(cls, v):
        if isinstance(v, str):
            s = v.strip()
            if not s:
                return None
            try:
                loaded = json.loads(s)
            except Exception:
                loaded = None
            if isinstance(loaded, list):
                return [str(x).strip() for x in loaded if str(x).strip()]
            return [s]
        return v


@tool(args_schema=ReadAndAnalyzePageInput)
def read_and_analyze_page(url: str, focus_areas: list[str] | None = None) -> str:
    """Read a single page and produce a structured analysis: summary, key facts, and source citation. Use URLs from deep_web_search results. Optionally pass focus_areas to structure the summary (e.g. key statistics, criticisms, future implications). Returns markdown with facts, quotes, and data highlights; no LLM is called inside the tool. Example: read_and_analyze_page(url='https://example.com/article', focus_areas=['key statistics', 'criticisms'])."""
    _debug_log(f"[DEBUG read_and_analyze_page] url={url!r}")
    _emit_input_status("Reading webpage…")
    with ResearchJobStream(
        tool_name="read_and_analyze_page",
        title="Research: Read and Analyze Page",
        controls=[
            {"id": "include_tables", "type": "toggle", "label": "Include Tables", "default": True},
            {"id": "cancel", "type": "button", "label": "Cancel Analysis", "action_id": "cancel_job", "style": "destructive"},
        ],
        capabilities=make_capabilities(cancel=True, progress=True, phases=True),
        metadata={"url": (url or "").strip()},
    ) as js:
        js.set_control("include_tables", True)
        started_at = time.time()
        try:
            if not _is_public_http_url(url):
                js.close("error", "URL blocked by safety policy.")
                _emit_input_status("Done")
                return "Blocked URL. Only public http(s) URLs are allowed."
            js.update("Reading webpage content…", phase="read_start", progress_pct=5)
            from tools import web_tools
            content = web_tools.read_webpage.invoke({"url": re.sub(r"\s+", "", url or "")})
            if js.cancel_requested():
                js.close("stopped", "Page analysis cancelled by user.")
                _emit_input_status("Done")
                return "Page analysis cancelled by user."
            _emit_input_status("Analyzing…")
            if (
                not content
                or "Could not read" in content
                or "451" in content
                or "Failed to retrieve" in content
                or (isinstance(content, str) and len(content) < 100)
            ):
                js.close("error", "Failed to read page content.")
                _emit_input_status("Done")
                return content or "Failed to read page."

            text = content[:50000]
            js.update("Extracting section-aware summary and key facts…", phase="extract_sections", progress_pct=45, metrics={"content_chars": float(len(text))})

            _HEADING_RE = re.compile(
                r"^(?:#{1,4}\s+.+|[A-Z][A-Z \-:]{8,}|.{5,80}:\s*)$",
                re.MULTILINE,
            )
            heading_positions = [(m.start(), m.group()) for m in _HEADING_RE.finditer(text)]
            summary_parts: list[str] = []
            _SUMMARY_BUDGET = 5000
            _PER_SECTION_CAP = 600

            if heading_positions:
                for idx, (pos, heading) in enumerate(heading_positions):
                    if len("\n\n".join(summary_parts)) >= _SUMMARY_BUDGET:
                        break
                    end_pos = heading_positions[idx + 1][0] if idx + 1 < len(heading_positions) else len(text)
                    section_body = text[pos + len(heading):end_pos].strip()
                    first_para = ""
                    for para in re.split(r"\n{2,}|\n(?=[A-Z])", section_body):
                        para = para.strip()
                        if len(para) > 30:
                            first_para = para[:_PER_SECTION_CAP]
                            break
                    if first_para:
                        summary_parts.append(f"**{heading.strip().lstrip('#').strip()}**: {first_para}")

            if len("\n\n".join(summary_parts)) < _SUMMARY_BUDGET // 2 and len(text) > _PER_SECTION_CAP:
                num_samples = max(1, (_SUMMARY_BUDGET - len("\n\n".join(summary_parts))) // _PER_SECTION_CAP)
                step = max(1, len(text) // (num_samples + 1))
                for i in range(num_samples):
                    if len("\n\n".join(summary_parts)) >= _SUMMARY_BUDGET:
                        break
                    start = i * step
                    boundary = text.find(". ", start)
                    if boundary != -1 and boundary - start < 300:
                        start = boundary + 2
                    chunk = text[start:start + _PER_SECTION_CAP].strip()
                    if len(chunk) > 30 and chunk not in summary_parts:
                        summary_parts.append(chunk)

            summary = "\n\n".join(summary_parts)[:_SUMMARY_BUDGET] if summary_parts else text[:3000]
            numbers = re.findall(r"\b\d{1,3}(?:,\d{3})*(?:\.\d+)?%?\b|\b\d+\.\d+\b", text)
            quotes = re.findall(r'"([^"]{20,200})"', text)
            current_heading = ""
            bullets: list[str] = []
            for line in text.splitlines():
                stripped = line.strip()
                if _HEADING_RE.match(stripped):
                    current_heading = stripped.lstrip("#").strip()
                elif re.match(r"^[\-\*•]\s+", stripped) or re.match(r"^\d+[\.\)]\s+", stripped):
                    if 10 < len(stripped) < 400:
                        prefix = f"{current_heading}: " if current_heading else ""
                        bullets.append(f"{prefix}{stripped}")

            key_facts: list[str] = []
            for n in numbers[:15]:
                if n not in key_facts:
                    key_facts.append(n)
            for q in quotes[:5]:
                key_facts.append(f'Quote: "{q}"')
            for b in bullets[:15]:
                key_facts.append(b)

            include_tables = bool(js.get_control("include_tables", True))
            tables_section = _extract_tables(text) if include_tables else ""
            _RAW_EXCERPT_START = min(2000, len(text) // 3)
            _RAW_EXCERPT_CAP = 2000
            raw_excerpt = text[_RAW_EXCERPT_START:_RAW_EXCERPT_START + _RAW_EXCERPT_CAP].strip()

            out = ["## Summary\n", summary, "\n\n## Key facts\n"]
            if focus_areas:
                out.append("\nFocus areas: " + ", ".join(focus_areas) + "\n")
            for f in key_facts[:25]:
                out.append(f"- {f}")
            if tables_section:
                out.append(f"\n\n{tables_section}")
            if raw_excerpt:
                out.append("\n\n## Raw content (excerpt)\n")
                out.append(raw_excerpt)
            out.append(f"\n\n## Source\n\n{url.strip()}")
            final_output = "\n".join(out)
            _emit_input_status("Done")
            try:
                from urllib.parse import urlparse as _urlparse
                _host = _urlparse(url).netloc or url[:60]
                emit_live_view_preview(
                    f"Analyzed: {_host}",
                    tool_name="read_and_analyze_page",
                    preview_text=summary[:300] if summary else None,
                    url=url,
                )
            except Exception:
                pass
            js.close("completed", "Page analysis complete.", metrics={
                "elapsed_seconds": float(round(time.time() - started_at, 2)),
                "key_facts_count": float(min(len(key_facts), 25)),
                "tables_included": 1.0 if bool(tables_section) else 0.0,
            })
            return final_output
        except Exception as e:
            js.error(str(e), phase="analysis_error")
            _emit_input_status("Done")
            return f"Failed to analyze page: {e}"


# ─── Tool 3: analyze_multiple_sources ─────────────────────────────────────────

class AnalyzeMultipleSourcesInput(BaseModel):
    """Input for analyze_multiple_sources."""

    analysis_outputs: list[str] = Field(..., description="List of analysis strings from read_and_analyze_page.")
    original_query: str = Field(..., description="The original research question.")


@tool(args_schema=AnalyzeMultipleSourcesInput)
def analyze_multiple_sources(analysis_outputs: list[str], original_query: str) -> str:
    """Compare multiple source analyses to identify consensus, contradictions, open questions, and confidence. Use after reading 4–8 sources with read_and_analyze_page. Returns a synthesis paragraph, bullet points, and a recommendation (more research needed vs sufficient). Purely deterministic comparison; no LLM inside the tool. Example JSON: {\"analysis_outputs\": [\"<read_and_analyze_page output>\", \"...\"], \"original_query\": \"your question\"}. If the list is under the wrong key, **sources** is accepted as an alias for **analysis_outputs**; **query** / **topic** can stand in for **original_query**."""
    _debug_log(f"[DEBUG analyze_multiple_sources] {len(analysis_outputs or [])} outputs")
    _emit_input_status("Synthesizing…")
    outputs = [s for s in (analysis_outputs or []) if isinstance(s, str) and s.strip()]
    with ResearchJobStream(
        tool_name="analyze_multiple_sources",
        title="Research: Synthesize Sources",
        controls=[
            {"id": "max_findings", "type": "slider", "label": "Max Findings Per Section", "min": 3, "max": 10, "step": 1, "default": 5},
            {"id": "cancel", "type": "button", "label": "Cancel Synthesis", "action_id": "cancel_job", "style": "destructive"},
        ],
        capabilities=make_capabilities(cancel=True, progress=True, metrics=True),
        metadata={"input_sources": len(outputs)},
    ) as js:
        js.set_control("max_findings", 5)
        started_at = time.time()
        try:
            if not outputs:
                js.close("error", "No analysis outputs provided.")
                _emit_input_status("Done")
                return "No analysis outputs provided. Run read_and_analyze_page on several URLs first."

            if js.cancel_requested():
                js.close("stopped", "Synthesis cancelled by user.")
                _emit_input_status("Done")
                return "Synthesis cancelled by user."

            max_items_raw = js.get_control("max_findings", 5)
            try:
                max_items = max(3, min(10, int(float(max_items_raw))))
            except Exception:
                max_items = 5

            js.update(f"Synthesizing {len(outputs)} source analyses…", phase="synthesize", progress_pct=45, metrics={"sources_count": float(len(outputs))})

            emit_live_view_preview(f"Synthesizing {len(outputs)} source(s)…", tool_name="analyze_multiple_sources")
            query = (original_query or "").strip() or "Research question"
            all_numbers: list[tuple[int, str]] = []
            for i, text in enumerate(outputs):
                for m in re.findall(r"\b\d{1,3}(?:,\d{3})*(?:\.\d+)?%?\b", text):
                    all_numbers.append((i, m))
            number_counts: dict[str, list[int]] = {}
            for i, n in all_numbers:
                number_counts.setdefault(n, []).append(i)
            contradictions = [n for n, idx in number_counts.items() if len(set(idx)) > 1 and len(idx) >= 2]
            consensus_candidates = [n for n, idx in number_counts.items() if len(idx) >= len(outputs) // 2]
            open_questions = []
            if len(outputs) < 4:
                open_questions.append("Few sources; consider adding more for higher confidence.")
            if not consensus_candidates and not contradictions:
                open_questions.append("No strong consensus or clear contradictions; topic may need more targeted searches.")
            confidence = "low" if (len(outputs) < 4 or open_questions) else "medium" if contradictions else "high"
            recommendation = (
                "More research recommended (gaps or contradictions)."
                if confidence == "low" or contradictions
                else "Sufficient for a summary answer."
            )
            lines = [
                "## Synthesis",
                "",
                f"**Original question:** {query}",
                "",
                f"Compared {len(outputs)} source(s).",
                "",
                "### Consensus",
                "",
            ]
            if consensus_candidates:
                for c in consensus_candidates[:max_items]:
                    lines.append(f"- {c} (repeated across sources)")
            else:
                lines.append("- No strong consensus identified.")
            lines.extend(["", "### Contradictions / differing data", ""])
            if contradictions:
                for d in contradictions[:max_items]:
                    lines.append(f"- Differing values: {d}")
            else:
                lines.append("- No clear contradictions.")
            lines.extend(["", "### Open questions", ""])
            for q in open_questions[:max_items]:
                lines.append(f"- {q}")
            if not open_questions:
                lines.append("- None noted.")
            lines.extend(["", "### Confidence", "", f"- **{confidence}**", "", "### Recommendation", "", recommendation])

            credibility_lines = ["", "### Source credibility (Authority / Currency / Purpose — each 1–3)", ""]
            any_scored = False
            for i, text in enumerate(outputs):
                src_url = _extract_source_url_from_analysis(text)
                if src_url:
                    score = _score_source_credibility(src_url, text)
                    label = _source_link_label(src_url, i)
                    credibility_lines.append(
                        f"- **{label}** ({src_url[:80]}): "
                        f"{score['label']} ({score['total']}/9) — "
                        f"Authority {score['authority']}, Currency {score['currency']}, Purpose {score['purpose']}"
                    )
                    any_scored = True
            if any_scored:
                lines.extend(credibility_lines)

            emit_live_view_preview("Synthesis complete", tool_name="analyze_multiple_sources")
            _emit_input_status("Done")
            result = "\n".join(lines)
            js.close("completed", "Source synthesis complete.", metrics={
                "consensus_items": float(min(len(consensus_candidates), max_items)),
                "contradictions": float(min(len(contradictions), max_items)),
                "elapsed_seconds": float(round(time.time() - started_at, 2)),
            })
            return result
        except Exception as e:
            js.error(str(e), phase="synthesis_error")
            _emit_input_status("Done")
            return f"Synthesis failed: {e}"


# ─── Tool: append_research_scratch ─────────────────────────────────────────────

class AppendResearchScratchInput(BaseModel):
    """Input for append_research_scratch."""

    topic: str = Field(..., description="Short topic label (same as you will use for the final report).")
    content: str = Field(..., description="Findings or notes from this batch to append.")


@tool(args_schema=AppendResearchScratchInput)
def append_research_scratch(topic: str, content: str) -> str:
    """Append research findings to a temporary scratch file for this topic. Write DETAILED paragraphs with concrete facts, names, numbers, data points, and direct quotes from sources. Do NOT write vague summaries — preserve the specific details, comparisons, and recommendations you found. Use after each batch of search/read: append → more search/read → append again. When research is **complete**, call **generate_final_research_report(topic, sources=..., scratch_path=<this path>)** once—pass this path as scratch_path so the report uses the file contents. Returns the scratch file path."""
    _emit_input_status("Appending to research scratch…")
    emit_live_view_preview("Appending to research scratch…", tool_name="append_research_scratch")
    base = get_user_output_dir()
    safe_topic = _sanitize_topic(topic)
    date = datetime.now().strftime("%Y-%m-%d")
    relative = f"research/_scratch/{safe_topic}_{date}.md"
    abs_path, err = safe_resolve_path(base, relative, allow_write=True, create_parents=True)
    if err or not abs_path:
        _emit_input_status("Done")
        return f"Failed to resolve scratch path: {err or 'unknown'}"
    # Strip truncation placeholders so they never get into the scratch file (and thus never into the final report).
    content_to_append = _sanitize_research_findings(content or "").strip()
    if not content_to_append:
        _emit_input_status("Done")
        return f"No content to append (placeholder or empty). Scratch file unchanged: {abs_path}"
    try:
        sep = "\n\n---\n\n"
        had_content = os.path.isfile(abs_path) and os.path.getsize(abs_path) > 0
        with open(abs_path, "a", encoding="utf-8") as f:
            if had_content:
                f.write(sep)
            f.write(content_to_append)
    except OSError as e:
        _emit_input_status("Done")
        return f"Failed to append: {e}"
    # Keep per-run state JSON beside scratch markdown when available.
    try:
        st = load_state_for_scratch(abs_path)
        bind_state_to_scratch(st, abs_path)
        st.append_count = int(st.append_count or 0) + 1
        st.last_phase = "deepening"
        save_state(st)
    except Exception:
        pass
    emit_live_view_preview("Appended to scratch", tool_name="append_research_scratch")
    _emit_input_status("Done")
    return f"Appended. Scratch file: {abs_path} (when research is complete, call generate_final_research_report with scratch_path={abs_path!r})."


# ─── Tool 4: save_research_findings ───────────────────────────────────────────

class SaveResearchFindingsInput(BaseModel):
    """Input for save_research_findings."""

    topic: str = Field(..., description="Short topic label, e.g. 'AI agent safety 2026'.")
    findings: str = Field(..., description="Findings to save. For quick or intermediate save only; for final report use generate_final_research_report when research is complete.")
    sources: list[str] = Field(..., description="List of source URLs.")


@tool(args_schema=SaveResearchFindingsInput)
def save_research_findings(topic: str, findings: str, sources: list[str]) -> str:
    """Save research findings to disk and to a new scratchpad (quick or intermediate save). For the **final** deliverable use **generate_final_research_report** once when research is complete. Creates ~/Documents/Mugi/research/<safe_topic>_<YYYY-MM-DD>/ with report.md and a scratchpad link."""
    _debug_log(f"[DEBUG save_research_findings] topic={topic!r}")
    _emit_input_status("Saving research…")
    emit_live_view_preview("Saving research findings…", tool_name="save_research_findings")
    sources_list = sources or []
    source_lines = [
        f"- [{_source_link_label(u, i)}]({u})" for i, u in enumerate(sources_list)
    ]
    findings_clean = _sanitize_research_findings(findings or "").strip()
    report_content = (findings_clean or "Research findings saved.") + "\n\n## Sources\n\n" + "\n".join(source_lines)
    abs_path, err = _write_research_folder(topic or "research", report_content, sources or [])
    if err:
        _emit_input_status("Done")
        return f"Failed to save research: {err}"
    scratch_err = _write_research_to_scratchpad(abs_path, topic or "research", report_content)
    emit_live_view_preview(f"Research saved: {os.path.basename(abs_path)}", tool_name="save_research_findings")
    _emit_input_status("Done")
    out = f"Research saved to {abs_path}"
    if scratch_err:
        out += f"; scratchpad failed: {scratch_err}"
    else:
        out += " (folder and scratchpad created)."
    return out


# ─── Tool 5: suggest_follow_up_queries ───────────────────────────────────────

class SuggestFollowUpQueriesInput(BaseModel):
    """Input for suggest_follow_up_queries."""

    current_findings: str = Field(..., description="Current findings or synthesis text.")
    original_query: str = Field(..., description="Original research question.")
    synthesis_text: str | None = Field(
        None,
        description="Optional output from analyze_multiple_sources. When provided, open questions/contradictions are used to generate gap-targeted follow-up queries.",
    )


@tool(args_schema=SuggestFollowUpQueriesInput)
def suggest_follow_up_queries(current_findings: str, original_query: str, synthesis_text: str | None = None) -> str:
    """Suggest 3–6 precise follow-up search queries to close gaps or deepen research. Use when analyze_multiple_sources recommends more research or when key aspects (e.g. recent data, criticisms) are missing. Returns a markdown list of queries to pass to deep_web_search. Example: suggest_follow_up_queries(current_findings='...', original_query='Risks of AI agents')."""
    _debug_log("[DEBUG suggest_follow_up_queries]")
    emit_live_view_preview("Suggesting follow-up queries…", tool_name="suggest_follow_up_queries")
    query = (original_query or "").strip()
    findings = (current_findings or "").strip()
    synthesis = (synthesis_text or "").strip()
    suggestions: list[str] = []
    # Gap-aware first so they survive the per-run cap when synthesis is provided.
    for qn in _extract_open_questions_from_synthesis(synthesis):
        qn_terms = " ".join(re.findall(r"\w{3,}", qn)[:8])
        if qn_terms:
            suggestions.append(f"{query} {qn_terms}")
    for c in _extract_contradictions_from_synthesis(synthesis):
        c_terms = " ".join(re.findall(r"\w{2,}", c)[:8])
        if c_terms:
            suggestions.append(f"{query} reconcile {c_terms}")
    rec = _extract_recommendation_from_synthesis(synthesis)
    if "more research recommended" in rec.lower():
        suggestions.append(f"{query} latest data 2024 2025")
        suggestions.append(f"{query} counterarguments evidence")
    # Heuristic breadth: recency, statistics, criticism, study, comparison
    base_terms = re.findall(r"\w{3,}", query)[:5]
    if base_terms:
        base = " ".join(base_terms)
        suggestions.append(f"{base} recent 2024 2025")
        suggestions.append(f"{base} statistics data")
        suggestions.append(f"{base} criticism risks")
        suggestions.append(f"{base} study research")
        suggestions.append(f"{base} comparison benchmark")
        suggestions.append(f"{base} guidelines best practices")
    else:
        suggestions.extend(
            [
                f"{query} recent",
                f"{query} statistics",
                f"{query} criticism",
                f"{query} study",
            ]
        )
    # Optional LLM-generated candidates when user enabled this behavior.
    suggestions.extend(_llm_followup_queries(findings, query, synthesis))
    # Deduplicate and cap at 6
    seen = set()
    unique = []
    for s in suggestions:
        n = s.lower().strip()
        if n not in seen and len(unique) < 6:
            seen.add(n)
            unique.append(s)
    lines = ["## Suggested follow-up queries\n", "Use these with deep_web_search to close gaps or deepen:\n"]
    for i, q in enumerate(unique, 1):
        lines.append(f"{i}. `{q}`")
    result = "\n".join(lines)
    emit_live_view_preview("Follow-up queries ready", tool_name="suggest_follow_up_queries", preview_text=result[:400] if result else None)
    return result


# ─── Tool 6: reflect_research_state ───────────────────────────────────────────

class ReflectResearchStateInput(BaseModel):
    """Input for reflect_research_state."""

    original_query: str = Field(..., description="Original user research question.")
    current_findings: str = Field(
        default="",
        description="Current findings or synthesis markdown.",
    )
    scratch_path: str | None = Field(
        None,
        description="Optional scratch file path from append_research_scratch. If provided and readable, its text is used as reflection context.",
    )


@tool(args_schema=ReflectResearchStateInput)
def reflect_research_state(original_query: str, current_findings: str = "", scratch_path: str | None = None) -> str:
    """Create a concise research reflection: supported claims, open gaps, and next queries."""
    text = (current_findings or "").strip()
    if (scratch_path or "").strip():
        base = get_user_output_dir()
        resolved, _err = safe_resolve_path(base, (scratch_path or "").strip(), allow_write=False, create_parents=False)
        if resolved and os.path.isfile(resolved):
            try:
                with open(resolved, "r", encoding="utf-8", errors="replace") as f:
                    text = f.read()
            except Exception:
                pass
    synthesis = text[:20000]
    gaps = _extract_open_questions_from_synthesis(synthesis)
    contradictions = _extract_contradictions_from_synthesis(synthesis)
    followups = suggest_follow_up_queries.invoke(
        {
            "current_findings": (current_findings or synthesis[:8000]),
            "original_query": original_query,
            "synthesis_text": synthesis,
        }
    )
    lines = [
        "## Reflection",
        "",
        f"**Original question:** {original_query.strip() or 'Research question'}",
        "",
        "### Claims supported",
    ]
    claims = []
    for sentence in re.split(r"(?<=\.)\s+", _sanitize_research_findings(synthesis))[:40]:
        sentence = sentence.strip()
        if len(sentence) >= 70:
            claims.append(sentence[:260] + ("…" if len(sentence) > 260 else ""))
        if len(claims) >= 4:
            break
    if claims:
        for c in claims:
            lines.append(f"- {c}")
    else:
        lines.append("- Not enough detailed synthesis yet.")
    lines.extend(["", "### Gaps", ""])
    if gaps:
        for g in gaps[:6]:
            lines.append(f"- {g}")
    else:
        lines.append("- No explicit gaps detected; verify recency and source diversity.")
    if contradictions:
        lines.extend(["", "### Contradictions to resolve", ""])
        for c in contradictions[:6]:
            lines.append(f"- {c}")
    lines.extend(["", "### Proposed next queries", "", followups])
    out = "\n".join(lines)
    emit_live_view_preview("Research reflection ready", tool_name="reflect_research_state", preview_text=out[:400])
    return out


# ─── Tool 7: plan_research_subquestions ──────────────────────────────────────

class PlanResearchSubquestionsInput(BaseModel):
    """Input for plan_research_subquestions."""

    topic: str = Field(..., description="Main topic/question to break into sub-questions.")
    max_questions: int = Field(5, description="How many sub-questions to return (3-8).")


_MAX_PLAN_TOPIC_SNIPPET_CHARS = 280


def _topic_snippet_for_planning(topic: str) -> str:
    """Normalize topic text for templated sub-questions (full phrase, capped)."""
    t = (topic or "").strip()
    if not t:
        return ""
    t = re.sub(r"\s+", " ", t)
    if len(t) > _MAX_PLAN_TOPIC_SNIPPET_CHARS:
        t = t[: _MAX_PLAN_TOPIC_SNIPPET_CHARS - 1].rstrip() + "…"
    return t


@tool(args_schema=PlanResearchSubquestionsInput)
def plan_research_subquestions(topic: str, max_questions: int = 5) -> str:
    """Generate focused sub-questions to guide a multi-step research run."""
    t = (topic or "").strip()
    if not t:
        return "Please provide a non-empty research topic."
    count = max(3, min(int(max_questions or 5), 8))
    base = _topic_snippet_for_planning(t)
    if not base:
        return "Please provide a non-empty research topic."
    candidates = [
        f"What is the current state of {base} in 2024-2026?",
        f"What are the strongest supporting arguments for {base}?",
        f"What are the strongest criticisms or risks of {base}?",
        f"What quantitative benchmarks or metrics are available for {base}?",
        f"How do top alternatives compare with {base}?",
        f"What implementation constraints or prerequisites exist for {base}?",
        f"What does recent expert or academic guidance say about {base}?",
        f"What open questions remain unresolved about {base}?",
    ]
    lines = ["## Research sub-questions", ""]
    for i, q in enumerate(candidates[:count], 1):
        lines.append(f"{i}. {q}")
    result = "\n".join(lines)
    emit_live_view_preview("Sub-questions planned", tool_name="plan_research_subquestions", preview_text=result[:350])
    return result


# ─── Tool 8: generate_final_research_report ──────────────────────────────────

class GenerateFinalResearchReportInput(BaseModel):
    """Input for generate_final_research_report."""

    topic: str = Field(..., description="Short topic label for the report and folder name.")
    findings: str = Field(default="", description="Findings text if not using scratch_path. Ignored when scratch_path is provided and readable.")
    sources: list[str] | None = Field(None, description="Optional list of source URLs; if omitted, sources section will be empty.")
    scratch_path: str | None = Field(None, description="Path to the scratch file from append_research_scratch. When provided, the tool reads this file and uses its contents as the report—pass this so the report has the full accumulated content, not a summary.")


@tool(args_schema=GenerateFinalResearchReportInput)
def generate_final_research_report(topic: str, findings: str = "", sources: list[str] | None = None, scratch_path: str | None = None) -> str:
    """Call **once** when research is complete—this is the **final** step. Prefer passing **scratch_path** (the path returned by append_research_scratch): the tool will read that file and use it as the full report content. If you pass scratch_path, you do not need to pass findings. Produces report.md + scratchpad at ~/Documents/Mugi/research/<topic>_<date>/ with Summary then Full report."""
    _debug_log(f"[DEBUG generate_final_research_report] topic={topic!r}")
    _emit_input_status("Formatting report…")
    emit_live_view_preview(
        f"Writing report: {topic or 'research'}…",
        tool_name="generate_final_research_report",
    )
    with ResearchJobStream(
        tool_name="generate_final_research_report",
        title="Research: Final Report",
        controls=[
            {"id": "include_summary", "type": "toggle", "label": "Include Summary Section", "default": True},
            {"id": "cancel", "type": "button", "label": "Cancel Report", "action_id": "cancel_job", "style": "destructive"},
        ],
        capabilities=make_capabilities(cancel=True, progress=True),
        metadata={"topic": topic or "research"},
    ) as js:
        js.set_control("include_summary", True)
        started_at = time.time()
        try:
            js.update(f"Preparing final report for '{topic or 'research'}'…", phase="prepare_report", progress_pct=20)
            if js.cancel_requested():
                js.close("stopped", "Report generation cancelled by user.")
                _emit_input_status("Done")
                return "Report generation cancelled by user."

            findings_text = (findings or "").strip()
            _state_sources: list[str] = []
            scratch_input = (scratch_path or "").strip().strip("\"'")
            if not scratch_input and not findings_text:
                scratch_input = _find_latest_scratch_for_topic(topic or "research")
            if scratch_input:
                base = get_user_output_dir()
                resolved, _err = safe_resolve_path(base, scratch_input, allow_write=False, create_parents=False)
                if resolved and os.path.isfile(resolved):
                    try:
                        with open(resolved, "r", encoding="utf-8", errors="replace") as f:
                            findings_text = f.read()
                        try:
                            st = load_state_for_scratch(resolved)
                            for row in st.urls_registered:
                                u = str((row or {}).get("url") or "").strip()
                                if u and u.startswith("http"):
                                    _state_sources.append(u)
                        except Exception:
                            pass
                    except Exception:
                        findings_text = (findings or "").strip() or "Scratch read error – partial findings only."
                elif not findings_text:
                    findings_text = "No scratch content found—using partial findings."

            raw = _sanitize_research_findings(findings_text)
            if not raw.strip():
                js.close("error", "No report content available.")
                _emit_input_status("Done")
                return (
                    "Cannot generate an empty report. Provide findings text or pass a valid scratch_path "
                    "from append_research_scratch."
                )
            include_summary = bool(js.get_control("include_summary", True))
            if not raw.startswith("#") and include_summary:
                sections = [s.strip() for s in re.split(r"\n---+\n", raw) if s.strip()]
                key_findings: list[str] = []
                for sec in sections:
                    candidates = re.split(r"(?<=\.)\s+", sec)
                    for cand in candidates:
                        cand = cand.strip()
                        if len(cand) < 50:
                            continue
                        cand_words = set(cand.lower().split())
                        is_dup = False
                        for existing in key_findings:
                            existing_words = set(existing.lower().split())
                            overlap = len(cand_words & existing_words) / max(len(cand_words | existing_words), 1)
                            if overlap > 0.6:
                                is_dup = True
                                break
                        if not is_dup:
                            if len(cand) > 250:
                                cand = cand[:247] + "…"
                            key_findings.append(cand)
                        if len(key_findings) >= 5:
                            break
                    if len(key_findings) >= 5:
                        break
                key_findings = key_findings[:5]
                summary_parts = []
                if key_findings:
                    summary_parts.append("### Key findings\n")
                    for kf in key_findings:
                        summary_parts.append(f"- {kf}")
                else:
                    summary_parts.append(raw[:200].strip() + ("…" if len(raw) > 200 else ""))
                formatted = "\n".join(
                    [
                        "# Research report",
                        "",
                        "## Summary",
                        "",
                        "\n".join(summary_parts),
                        "",
                        "## Full report",
                        "",
                        raw,
                    ]
                )
            else:
                formatted = raw

            sources_list = list(sources or [])
            if not sources_list and _state_sources:
                sources_list = _state_sources
            # Stable dedupe while preserving order.
            deduped_sources: list[str] = []
            seen_sources: set[str] = set()
            for u in sources_list:
                su = _clean_source_url(str(u or ""))
                if not su or su in seen_sources:
                    continue
                seen_sources.add(su)
                deduped_sources.append(su)
            sources_list = deduped_sources
            source_lines = [f"- [{_source_link_label(u, i)}]({u})" for i, u in enumerate(sources_list)]
            report_content = formatted + "\n\n## Sources\n\n" + "\n".join(source_lines)
            abs_path, err = _write_research_folder(topic or "research", report_content, sources_list)
            if err:
                js.close("error", f"Failed to save report: {err}")
                _emit_input_status("Done")
                return f"Failed to save report: {err}"

            scratch_err = _write_research_to_scratchpad(abs_path, topic or "research", report_content)
            preview = (report_content[:500] + "…") if len(report_content) > 500 else report_content
            emit_live_view_preview(
                f"Report saved: {os.path.basename(abs_path)}",
                tool_name="generate_final_research_report",
                preview_text=preview.strip() or None,
            )
            _emit_input_status("Done")
            out = f"Report saved to {abs_path}"
            if scratch_err:
                out += f"; scratchpad failed: {scratch_err}"
            else:
                out += " (folder and scratchpad created)."
            js.close("completed", "Final report generated.", metrics={"elapsed_seconds": float(round(time.time() - started_at, 2))})
            return out
        except Exception as e:
            js.error(str(e), phase="report_error")
            _emit_input_status("Done")
            return f"Failed to generate report: {e}"


__all__ = [
    "deep_web_search",
    "read_and_analyze_page",
    "analyze_multiple_sources",
    "append_research_scratch",
    "save_research_findings",
    "suggest_follow_up_queries",
    "reflect_research_state",
    "plan_research_subquestions",
    "generate_final_research_report",
    "set_research_tab_id",
]
