"""
Eval harness for skill text evolution.

Runs candidate SKILL.md text against a set of eval tasks using
run_raw_agent in subagent_mode.  Returns a scalar aggregate score
(mean of per-task scores 0–1) suitable for DSPy/GEPA optimization.

Eval tasks live in JSONL: each line has id, user_message, rubric,
and optional expected_substrings / must_not_call_tool fields.
"""
from __future__ import annotations

import json
import os
import re
import uuid
from typing import Any, Callable

_EVAL_THREAD_PREFIX = "skill_eval_"


def _log(msg: str, level: str = "WARNING") -> None:
    try:
        from tools._logging import _debug_log
        _debug_log(f"[evolution/skill_eval_harness] {msg}", level)
    except Exception:
        pass


# ---------------------------------------------------------------------------
# Eval task loading
# ---------------------------------------------------------------------------

def load_eval_tasks(path: str) -> list[dict]:
    """Load eval tasks from a JSONL file. Skips malformed lines."""
    tasks: list[dict] = []
    if not path or not os.path.isfile(path):
        return tasks
    with open(path, "r", encoding="utf-8") as f:
        for lineno, line in enumerate(f, 1):
            line = line.strip()
            if not line:
                continue
            try:
                task = json.loads(line)
                if not isinstance(task, dict):
                    continue
                if "user_message" not in task:
                    _log(f"Eval task line {lineno} missing user_message, skipping")
                    continue
                task.setdefault("id", f"task_{lineno}")
                tasks.append(task)
            except json.JSONDecodeError as exc:
                _log(f"Eval task line {lineno} JSON error: {exc}")
    return tasks


def resolve_eval_tasks_path(config: dict | None = None) -> str:
    """Resolve the eval tasks JSONL path from config or defaults."""
    if config is None:
        from core.config import get_text_evolution_config
        config = get_text_evolution_config()
    custom = (config.get("eval_tasks_path") or "").strip()
    if custom:
        expanded = os.path.expanduser(custom)
        if os.path.isabs(expanded):
            return expanded
    # Default: workspace-relative
    try:
        from persistence import get_workspace_path
        workspace = get_workspace_path()
        if workspace:
            return os.path.join(workspace, "memory", "evolution-tasks.jsonl")
    except Exception:
        pass
    return os.path.join(os.path.expanduser("~"), ".mugi", "evolution", "eval_tasks.jsonl")


def generate_eval_tasks_for_skill(skill_name: str, skill_text: str) -> list[dict]:
    """Auto-generate basic eval tasks from a SKILL.md's content.

    Produces tasks that test whether an agent following the procedure
    addresses the core scenarios described in the skill. No LLM needed;
    this is a deterministic extraction from the markdown structure.
    """
    tasks: list[dict] = []
    lines = skill_text.strip().splitlines()

    title = ""
    steps: list[str] = []
    for line in lines:
        stripped = line.strip()
        if stripped.startswith("# ") and not title:
            title = stripped.lstrip("# ").strip()
        elif stripped and (stripped[0].isdigit() or stripped.startswith("- ")):
            step_text = stripped.lstrip("0123456789.-) ").strip()
            if len(step_text) > 10:
                steps.append(step_text)

    topic = title or skill_name.replace("-", " ")

    tasks.append({
        "id": f"auto_overview_{skill_name}",
        "user_message": f"I need help with: {topic}. Walk me through the full process step by step.",
        "rubric": f"Response should cover the key steps of the {topic} workflow in a logical order.",
    })

    tasks.append({
        "id": f"auto_brief_{skill_name}",
        "user_message": f"Give me a brief summary of how to {topic.lower()}.",
        "rubric": f"Response should accurately summarize the {topic} procedure without missing critical steps.",
    })

    if steps:
        first_step = steps[0]
        tasks.append({
            "id": f"auto_firststep_{skill_name}",
            "user_message": f"I'm about to start: {topic}. What should I do first?",
            "expected_substrings": [first_step[:40].lower().split()[0]] if first_step.split() else [],
            "rubric": f"Response should address the first step: {first_step[:100]}",
        })

    if len(steps) >= 3:
        mid = steps[len(steps) // 2]
        tasks.append({
            "id": f"auto_midstep_{skill_name}",
            "user_message": f"I'm partway through {topic.lower()}. I've done the initial steps. What comes next?",
            "rubric": f"Response should cover middle/later steps of the workflow, including: {mid[:100]}",
        })

    tasks.append({
        "id": f"auto_troubleshoot_{skill_name}",
        "user_message": f"Something went wrong while doing {topic.lower()}. How should I troubleshoot?",
        "rubric": f"Response should provide relevant troubleshooting guidance for the {topic} workflow.",
    })

    return tasks


def ensure_eval_tasks_for_skill(
    skill_name: str, skill_text: str, config: dict | None = None,
) -> list[dict]:
    """Load eval tasks, auto-generating them if the file is missing or empty.

    Writes generated tasks to the default eval tasks path so they persist
    and the user can edit/extend them.
    """
    tasks_path = resolve_eval_tasks_path(config)
    tasks = load_eval_tasks(tasks_path)

    if tasks:
        return tasks

    generated = generate_eval_tasks_for_skill(skill_name, skill_text)
    if not generated:
        return []

    # Write to disk so they persist and user can review/edit
    try:
        parent = os.path.dirname(tasks_path)
        if parent:
            os.makedirs(parent, exist_ok=True)
        with open(tasks_path, "w", encoding="utf-8") as f:
            for task in generated:
                f.write(json.dumps(task, ensure_ascii=False) + "\n")
        _log(f"Auto-generated {len(generated)} eval tasks at {tasks_path}", "INFO")
    except Exception as exc:
        _log(f"Failed to write auto-generated eval tasks: {exc}", "ERROR")

    return generated


# ---------------------------------------------------------------------------
# Single-task evaluation
# ---------------------------------------------------------------------------

def _build_eval_preamble(skill_markdown: str) -> str:
    """Build the system preamble that injects the candidate skill text."""
    return (
        "You are an AI assistant being evaluated on how well you follow a workflow procedure.\n\n"
        "─── Procedure to follow ───\n"
        f"{skill_markdown}\n"
        "─── End procedure ───\n\n"
        "Follow the procedure above to handle the user's request. "
        "Be thorough and precise."
    )


def _run_single_eval(
    skill_markdown: str,
    task: dict,
    config: dict,
    stop_check: Callable[[], bool] | None = None,
) -> dict:
    """Run one eval task with the candidate skill text.

    Returns {"score": float 0-1, "task_id": str, "reply": str, "error": str|None}.
    """
    thread_id = f"{_EVAL_THREAD_PREFIX}{uuid.uuid4().hex[:12]}"
    task_id = task.get("id", "unknown")

    messages = [
        {"role": "system", "content": _build_eval_preamble(skill_markdown)},
        {"role": "user", "content": task["user_message"]},
    ]

    reply_parts: list[str] = []

    def on_chunk(text: str) -> None:
        reply_parts.append(text)

    try:
        from agent_loop import run_raw_agent
        from tool_policy import context_from_prefs_and_flags
        from subagent_runner import IterationBudget

        budget_total = int(config.get("eval_iteration_budget_total", 15))
        iteration_budget = IterationBudget(total=budget_total)

        policy_context = context_from_prefs_and_flags(
            background_automation=True,
            subagent_mode=True,
            plan_mode=False,
        )

        eval_model = (config.get("eval_model") or "").strip() or None
        eval_base_url = (config.get("base_url") or "").strip() or None
        eval_api_key = (config.get("api_key") or "").strip() or None

        run_raw_agent(
            messages=messages,
            thread_id=thread_id,
            config=None,
            on_chunk=on_chunk,
            stop_check=stop_check,
            subagent_mode=True,
            tools_override=[],
            policy_context=policy_context,
            iteration_budget=iteration_budget,
            model_override=eval_model,
            base_url_override=eval_base_url,
            api_key_override=eval_api_key,
        )
    except Exception as exc:
        _log(f"Eval run failed for task {task_id}: {exc}", "ERROR")
        return {"score": 0.0, "task_id": task_id, "reply": "", "error": str(exc)}

    reply = "".join(reply_parts)
    score = _judge_reply(reply, task)

    return {"score": score, "task_id": task_id, "reply": reply[:500], "error": None}


# ---------------------------------------------------------------------------
# Scoring / judging
# ---------------------------------------------------------------------------

def _judge_reply(reply: str, task: dict) -> float:
    """Score a reply against task criteria. Returns 0.0–1.0."""
    scores: list[float] = []

    # Substring checks
    expected = task.get("expected_substrings", [])
    if expected:
        hits = sum(1 for s in expected if s.lower() in reply.lower())
        scores.append(hits / len(expected))

    # Must-not-call-tool (always passes for tool-free eval, but we check reply text)
    forbidden_tools = task.get("must_not_call_tool", [])
    if forbidden_tools:
        penalty = sum(1 for t in forbidden_tools if t in reply)
        scores.append(max(0.0, 1.0 - penalty / max(len(forbidden_tools), 1)))

    # Rubric-based: use LLM judge if rubric present and model available
    rubric = task.get("rubric", "")
    if rubric:
        rubric_score = _llm_judge(reply, rubric, task.get("user_message", ""))
        if rubric_score is not None:
            scores.append(rubric_score)

    if not scores:
        return 0.5 if reply.strip() else 0.0

    return sum(scores) / len(scores)


def _llm_judge(reply: str, rubric: str, user_message: str) -> float | None:
    """Use the eval model as a 0–1 judge. Returns None on failure."""
    try:
        from core.config import get_text_evolution_config
        te_cfg = get_text_evolution_config()
        model = (te_cfg.get("eval_model") or "").strip()
        base_url = (te_cfg.get("base_url") or "").strip()
        api_key = (te_cfg.get("api_key") or "").strip()

        from agent_loop import get_backend_base_url
        if not base_url:
            base_url = get_backend_base_url()
        if not api_key:
            api_key = "lm-studio"

        import openai
        client = openai.OpenAI(base_url=base_url, api_key=api_key)

        judge_prompt = (
            "You are evaluating an AI assistant's response.\n\n"
            f"User request: {user_message[:1000]}\n\n"
            f"Rubric: {rubric}\n\n"
            f"Assistant response:\n{reply[:2000]}\n\n"
            "Score the response from 0.0 to 1.0 based on the rubric. "
            "Reply with ONLY a JSON object: {\"score\": <float>}"
        )

        resp = client.chat.completions.create(
            model=model or "local-model",
            messages=[{"role": "user", "content": judge_prompt}],
            temperature=0.0,
            max_tokens=50,
        )
        text = (resp.choices[0].message.content or "").strip()
        match = re.search(r'"score"\s*:\s*([\d.]+)', text)
        if match:
            return max(0.0, min(1.0, float(match.group(1))))
    except Exception as exc:
        _log(f"LLM judge failed: {exc}", "ERROR")
    return None


# ---------------------------------------------------------------------------
# Full evaluation run
# ---------------------------------------------------------------------------

def evaluate_skill(
    skill_markdown: str,
    tasks: list[dict],
    config: dict | None = None,
    stop_check: Callable[[], bool] | None = None,
) -> dict:
    """Run all eval tasks and return aggregate results.

    Returns {"mean_score": float, "scores": list[dict], "n_tasks": int}.
    """
    if config is None:
        from core.config import get_text_evolution_config
        config = get_text_evolution_config()

    if not tasks:
        return {"mean_score": 0.0, "scores": [], "n_tasks": 0}

    results: list[dict] = []
    for task in tasks:
        if stop_check and stop_check():
            break
        result = _run_single_eval(skill_markdown, task, config, stop_check)
        results.append(result)

    valid_scores = [r["score"] for r in results if r["error"] is None]
    mean = sum(valid_scores) / len(valid_scores) if valid_scores else 0.0

    return {
        "mean_score": mean,
        "scores": results,
        "n_tasks": len(results),
    }
