#!/usr/bin/env python3
"""Run the tool-chaining behavioral eval suite and write the CI artifact.

Two execution modes:

- ``--self-test`` — uses a deterministic fake executor (NO model). Validates the harness
  plumbing (scenario filter, grading, artifact write) offline; this is what unit CI exercises.
- default (live) — runs each scenario against the **live local model** via ``run_raw_agent``,
  capturing the tool chain from ``on_tool_event`` and a best-effort salvage signal from the
  loop's log stream. Requires a running model + MLX/LLM backend; run locally/manually.

The output artifact (JSON) is what the Phase 1 CI gate (``scripts/check_salvage_slo.py``,
to be added) reads to enforce the salvage SLO. This script itself always exits 0 unless it
fails to produce an artifact — gating is a separate concern by design (a measurement tool must
not double as a policy gate).

Usage:
    python scripts/run_tool_chaining_evals.py --self-test
    python scripts/run_tool_chaining_evals.py --offline --out evals/artifacts/tool_chaining_latest.json
    python scripts/run_tool_chaining_evals.py --model my-model
"""
from __future__ import annotations

import argparse
import os
import sys
import uuid

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from evals.scenarios import default_scenarios, filter_scenarios
from evals.tool_chaining import (
    ExecutorResult,
    ToolChainScenario,
    run_suite,
    write_artifact,
)

DEFAULT_ARTIFACT = os.path.join(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
    "evals",
    "artifacts",
    "tool_chaining_latest.json",
)

# Substrings in the agent loop's log/status stream that indicate the harness had to fall back
# to salvage/synthesis/guard-retry to finalize — a best-effort signal for the foreground live
# path (the authoritative, structured signal is the kanban run `handoff_source`, used elsewhere).
_SALVAGE_LOG_SIGNALS = (
    "synthesiz",       # synthesize_* contract/handoff
    "salvage",
    "honest stop",
    "guard retry",
    "close synthesis retry",
    "missing contract",
    "xml-only",
    "parser mismatch",
)


def _self_test_executor(scenario: ToolChainScenario) -> ExecutorResult:
    """Deterministic fake: pretends the agent ran the required chain and answered cleanly.

    Used to validate the harness end-to-end without a model. It does NOT prove model behavior —
    it proves the suite runner, grading, and artifact format work.
    """
    answer = " ".join(scenario.success_substrings) if scenario.success_substrings else "done"
    return ExecutorResult(
        tools_called=list(scenario.required_tools),
        answer=answer,
        completed=True,
        salvaged=False,
        model="self-test",
    )


def _make_live_executor(model: str):
    """Build a live executor closure. Imports the agent loop lazily so ``--self-test`` and
    ``--help`` work without the (heavy) model stack importable.
    """

    def _live_executor(scenario: ToolChainScenario) -> ExecutorResult:
        import shutil
        import tempfile
        from unittest.mock import patch

        import tools.file_ops as file_ops
        from agent_loop import run_raw_agent
        from core.config import get_config

        tools_called: list[str] = []
        chunks: list[str] = []
        final_holder: list[str] = []
        errored: list[str] = []
        salvage_hit: list[bool] = [False]

        def on_tool_event(ev: dict) -> None:
            if ev.get("phase") == "start":
                name = str(ev.get("tool_name") or "").strip()
                if name:
                    tools_called.append(name)

        def on_chunk(text: str) -> None:
            if text:
                chunks.append(text)

        def on_finished(final: str) -> None:
            final_holder.append(final or "")

        def on_error(msg: str) -> None:
            errored.append(str(msg or ""))

        def on_log(level: str, msg: str) -> None:
            low = (msg or "").lower()
            if any(sig in low for sig in _SALVAGE_LOG_SIGNALS):
                salvage_hit[0] = True

        # The file tools (tools.file_ops.read_file / list_directory / write_text_file)
        # resolve every path against the module-level ``_DEFAULT_WORKSPACE`` constant — NOT
        # ``core.config.get_workspace_path()`` — and utils.path_safety rejects locations
        # outside the real workspace (e.g. a bare /tmp dir is "protected/dangerous"). So a
        # plausible-looking ``patch(get_workspace_path -> /tmp)`` silently does nothing: the
        # setup files stay invisible and every scenario scores 0%. To actually isolate the run
        # we (1) create the scratch workspace UNDER the real workspace root so path-safety
        # accepts it, and (2) patch ``file_ops._DEFAULT_WORKSPACE`` (the value the tools use).
        # The get_workspace_path patches stay too, for any tool that goes through
        # ``_configured_workspace_dir()`` instead.
        real_ws_root = os.path.abspath(os.path.expanduser(file_ops._DEFAULT_WORKSPACE))
        os.makedirs(real_ws_root, exist_ok=True)
        workspace = tempfile.mkdtemp(prefix=f".eval_{scenario.id}_", dir=real_ws_root)
        try:
            for rel, content in (scenario.setup_files or {}).items():
                abs_path = os.path.join(workspace, rel)
                os.makedirs(os.path.dirname(abs_path) or workspace, exist_ok=True)
                with open(abs_path, "w", encoding="utf-8") as fh:
                    fh.write(content)

            messages = [{"role": "user", "content": scenario.task}]
            thread_id = f"eval_{scenario.id}_{uuid.uuid4().hex[:8]}"
            # Isolate the workspace so read/write tools resolve against the scenario's files
            # and never touch the operator's real workspace.
            with patch.object(file_ops, "_DEFAULT_WORKSPACE", workspace), \
                 patch("core.config.get_workspace_path", return_value=workspace), \
                 patch("core.get_workspace_path", return_value=workspace):
                try:
                    run_raw_agent(
                        messages,
                        thread_id,
                        get_config(),
                        on_chunk,
                        on_log=on_log,
                        on_error=on_error,
                        on_finished=on_finished,
                        on_tool_event=on_tool_event,
                        research_mode="search_read_synthesize" in scenario.tags,
                        model_override=model or None,
                    )
                except Exception as exc:  # a live crash is a non-completion, recorded loudly
                    return ExecutorResult(
                        tools_called=tools_called,
                        answer="".join(chunks),
                        completed=False,
                        salvaged=salvage_hit[0],
                        model=model,
                        error=f"{type(exc).__name__}: {exc}",
                    )
        finally:
            shutil.rmtree(workspace, ignore_errors=True)

        answer = (final_holder[0] if final_holder else "".join(chunks)).strip()
        completed = bool(answer) and not errored
        return ExecutorResult(
            tools_called=tools_called,
            answer=answer,
            completed=completed,
            salvaged=salvage_hit[0],
            model=model,
            error=errored[0] if errored else "",
        )

    return _live_executor


def build_arg_parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(description="Run the tool-chaining behavioral eval suite.")
    p.add_argument("--out", default=DEFAULT_ARTIFACT, help="artifact JSON output path")
    p.add_argument("--self-test", action="store_true",
                   help="use the deterministic fake executor (no model) to validate plumbing")
    p.add_argument("--offline", action="store_true",
                   help="exclude network scenarios (shortcut for --exclude-tag network)")
    p.add_argument("--include-tag", action="append", default=[],
                   help="only run scenarios with this tag (repeatable)")
    p.add_argument("--exclude-tag", action="append", default=[],
                   help="skip scenarios with this tag (repeatable)")
    p.add_argument("--model", default="", help="model override for the live executor")
    return p


def main(argv: list[str] | None = None) -> int:
    args = build_arg_parser().parse_args(argv)

    exclude = list(args.exclude_tag)
    if args.offline and "network" not in exclude:
        exclude.append("network")

    scenarios = filter_scenarios(
        default_scenarios(),
        include_tags=args.include_tag or None,
        exclude_tags=exclude or None,
    )
    if not scenarios:
        print("No scenarios matched the given tag filters.", file=sys.stderr)
        return 0

    if args.self_test:
        executor = _self_test_executor
        model = "self-test"
    else:
        executor = _make_live_executor(args.model)
        model = args.model or "(live)"

    report = run_suite(scenarios, executor, model=model)
    path = write_artifact(report, args.out)

    print(f"Tool-chaining eval ({'self-test' if args.self_test else 'live'}) — {report.total} scenarios")
    print(f"  chain_success:               {report.chain_success}/{report.total} "
          f"({report.chain_success_rate:.0%})")
    print(f"  completed_without_salvage:   {report.completed_without_salvage}/{report.total} "
          f"({report.completion_without_salvage_rate:.0%})")
    print(f"  salvage_rate (of completed): {report.salvage_rate:.0%}")
    print(f"  artifact: {path}")
    for o in report.outcomes:
        flags = []
        if not o.completed:
            flags.append("incomplete")
        if not o.chain_satisfied:
            flags.append("chain-miss")
        if not o.answer_ok:
            flags.append("answer-miss")
        if o.salvaged:
            flags.append("salvaged")
        status = "OK" if o.chain_success and o.completed_without_salvage else ",".join(flags) or "OK"
        print(f"    - {o.scenario_id}: {status}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
