#!/usr/bin/env python3
"""
Persistent workspace and agent config (~/.mugi/).

- Ensures ~/.mugi/ and ~/.mugi/workspace exist.
- Loads config from ~/.mugi/mugi_config.toml (tomllib); falls back to defaults if missing.
- Expands ~ in path values.
- HEARTBEAT.md template includes notification rules (urgent → macOS notification + queue; medium → queue only; low → log only).
"""
import os
import re
import copy
import json
import hashlib
import shutil
import logging
import secrets
import threading
import tempfile
import time
from contextlib import contextmanager
from typing import Any
from datetime import datetime

_log = logging.getLogger(__name__)

try:
    import tomllib
except ImportError:
    import tomli as tomllib  # type: ignore[no-redef]
try:
    import tomli_w
except ImportError:
    tomli_w = None

# --- Paths ---
MUGI_DIR = os.path.expanduser("~/.mugi")
MUGI_WORKSPACE = os.path.expanduser("~/.mugi/workspace")
MUGI_CONFIG_PATH = os.path.join(MUGI_DIR, "mugi_config.toml")

# Default Hugging Face repo for the embedded utility model (background summarization, etc.).
# Gemma 4 E2B QAT 8-bit installs via mlx-vlm (no Prism/Xcode build) and matches the
# recommended chat QAT line. Bonsai remains opt-in.
DEFAULT_UTILITY_MODEL_REPO = "mlx-community/gemma-4-E2B-it-qat-8bit"

# Prior defaults superseded by QAT weights; remapped in get_utility_config().
LEGACY_UTILITY_MODEL_REPOS = frozenset({
    "mlx-community/gemma-4-e2b-it-8bit",
})

# Backward compatibility: alias for code that still imports old names (deprecated).
MYAGENT_DIR = MUGI_DIR
MYAGENT_WORKSPACE = MUGI_WORKSPACE
MYAGENT_CONFIG_PATH = MUGI_CONFIG_PATH

# Background agent IPC: Unix domain socket (chmod 600). Only used when background runner is enabled.
SOCKET_PATH = os.path.join(MUGI_DIR, "agent.sock")
# Proactive notifications from background heartbeats; UI polls this (chmod 600 when created).
PROACTIVE_QUEUE_PATH = os.path.join(MUGI_DIR, "proactive_queue.jsonl")
# LaunchAgent stdout/stderr logs (created by launchd; runner ensures dir exists for optional file logging).
LAUNCHD_LOGS_DIR = os.path.join(MUGI_DIR, "launchd_logs")
# All application and tool logs (debug, Kraken, scratchpad, messaging, etc.). Use this so logs are in one place regardless of how the app is run.
MUGI_LOGS_DIR = os.path.join(MUGI_DIR, "logs")
# Kraken OCR: dedicated venv created and managed by the app (separate from agent venv).
KRAKEN_VENV_DIR = os.path.join(MUGI_DIR, "kraken-venv")
# Standalone Python 3.12 (python-build-standalone) for creating Kraken venv when system has no 3.12.
STANDALONE_PYTHON312_DIR = os.path.join(MUGI_DIR, "python-3.12")
# Standalone Python 3.11 for optional tool/venv installs (for example Chatterbox).
STANDALONE_PYTHON311_DIR = os.path.join(MUGI_DIR, "python-3.11")
# Isolated Homebrew for Mugi: installs here so agent can brew install without affecting system.
MUGI_HOMEBREW_PREFIX = os.path.join(MUGI_DIR, "homebrew")
# Optional Chatterbox TTS runtime (isolated from backend venv).
CHATTERBOX_VENV_DIR = os.path.join(MUGI_DIR, "chatterbox-venv")
CHATTERBOX_VOICES_DIR = os.path.join(MUGI_DIR, "chatterbox_voices")
CHATTERBOX_HF_HOME = os.path.join(MUGI_DIR, "huggingface", "chatterbox")
# Optional Hyperframes HTML-to-video runtime (Node.js + Chromium under Mugi's tree).
HYPERFRAMES_DIR = os.path.join(MUGI_DIR, "hyperframes")
HYPERFRAMES_PROJECTS_DIR = os.path.join(MUGI_DIR, "hyperframes", "projects")
HYPERFRAMES_RENDERS_DIR = os.path.join(MUGI_DIR, "hyperframes", "renders")
# User-facing plugins directory: plugins are Python packages with a plugin.toml manifest.
MUGI_PLUGINS_DIR = os.path.join(MUGI_DIR, "plugins")
# Dashboard-only plugins (kind=dashboard); folder names must be dash_* (see plugin_manager discovery).
MUGI_DASHBOARD_PLUGINS_DIR = os.path.join(MUGI_DIR, "dashboard_plugins")
MUGI_PLUGINS_GUIDE_PATH = os.path.join(MUGI_DIR, "PLUGINS.md")
MUGI_PLUGINS_GUIDE_BACKUP_PATH = os.path.join(MUGI_DIR, "PLUGINS.md.bak")
# Isolated venvs for pip/pipx-installed CLI tools: each program gets ~/.mugi/cli-venvs/<name>/.
CLI_VENVS_DIR = os.path.join(MUGI_DIR, "cli-venvs")
# Proactive nudge: turns-since-last-nudge per thread (persisted across run_raw_agent invocations).
PROACTIVE_NUDGE_STATE_PATH = os.path.join(MUGI_DIR, "proactive_nudge_state.json")
# Agent/network errors from main app (cleared on each app launch).
AGENT_NETWORK_LOG_PATH = os.path.join(MUGI_DIR, "logs", "agent_network.log")
# Main chat activity timestamp (float seconds since epoch). Written by backend.
MAIN_ACTIVITY_TS_PATH = os.path.join(MUGI_DIR, "main_activity_ts.txt")
# Main chat activity state JSON written atomically by backend:
# {"busy": bool, "ts": float}. mugi_runner reads this to gate background runs.
MAIN_ACTIVITY_STATE_PATH = os.path.join(MUGI_DIR, "main_activity_state.json")
# Last explicit user-input timestamp (float seconds since epoch). Updated on user send/regenerate/edit actions.
MAIN_USER_INPUT_TS_PATH = os.path.join(MUGI_DIR, "main_user_input_ts.txt")


def read_main_chat_busy(*, max_age_s: float = 7200.0) -> bool:
    """True when the backend marked a foreground main-chat agent run as busy."""
    try:
        if not os.path.isfile(MAIN_ACTIVITY_STATE_PATH):
            return False
        with open(MAIN_ACTIVITY_STATE_PATH, "r", encoding="utf-8") as f:
            obj = json.load(f)
        if not isinstance(obj, dict):
            return False
        ts = float(obj.get("ts", 0.0))
        if ts <= 0 or (time.time() - ts) > max_age_s:
            return False
        return bool(obj.get("busy", False))
    except Exception:
        return False
# macOS loopback bridge URL + auth for LaunchAgent `mugi_runner` (re-read each heartbeat).
# Written atomically by the Swift app when the embedded backend starts; removed on backend stop.
MACOS_BRIDGE_RUNNER_ENV_PATH = os.path.join(MUGI_DIR, ".macos_bridge_runner_env.json")

# Current agent thread_id for the active run_raw_agent (used by working buffer path
# so each conversation has its own buffer).  Thread-local so concurrent subagents
# on different OS threads don't clobber each other's IDs.
_thread_local = threading.local()

_PLUGINS_GUIDE_VERSION_RE = re.compile(r"PLUGINS_GUIDE_VERSION:\s*(\d+)")


def _extract_plugins_guide_version(text: str) -> int:
    if not isinstance(text, str):
        return 0
    m = _PLUGINS_GUIDE_VERSION_RE.search(text)
    if not m:
        return 0
    try:
        return int(m.group(1))
    except Exception:
        return 0


def _plugin_slug(name: str) -> str:
    slug = re.sub(r"[^a-z0-9_]", "_", (name or "").strip().lower())
    return slug.strip("_") or "unnamed"


def _parse_semver_tuple(version: str) -> tuple[int, int, int]:
    raw = str(version or "").strip().split("-", 1)[0]
    parts = [p for p in raw.split(".") if p != ""]
    ints: list[int] = []
    for part in parts[:3]:
        try:
            ints.append(int(part))
        except Exception:
            ints.append(0)
    while len(ints) < 3:
        ints.append(0)
    return tuple(ints[:3])


def ensure_bundled_plugins() -> list[str]:
    """Sync first-party bundled plugins into ~/.mugi/plugins and auto-approve."""
    src_root = os.path.join(os.path.dirname(__file__), "templates", "bundled_plugins")
    if not os.path.isdir(src_root):
        return []
    os.makedirs(MUGI_PLUGINS_DIR, exist_ok=True)
    os.makedirs(MUGI_DASHBOARD_PLUGINS_DIR, exist_ok=True)
    try:
        import shutil
        from plugin_security import (
            PluginManifest,
            get_plugin_approval_state,
            hash_plugin_directory,
            set_plugin_approval,
        )
    except Exception:
        return []

    installed: list[str] = []
    marker_name = ".mugi_bundled_plugin"
    for entry in sorted(os.listdir(src_root)):
        src_dir = os.path.join(src_root, entry)
        src_toml = os.path.join(src_dir, "plugin.toml")
        if not os.path.isdir(src_dir) or not os.path.isfile(src_toml):
            continue
        try:
            manifest = PluginManifest.from_toml(src_toml)
        except Exception:
            continue
        slug = _plugin_slug(manifest.name)
        dst_dir = os.path.join(MUGI_PLUGINS_DIR, slug)
        dst_toml = os.path.join(dst_dir, "plugin.toml")
        marker_path = os.path.join(dst_dir, marker_name)
        should_copy = not os.path.isdir(dst_dir)

        if not should_copy:
            # Do not overwrite user-created plugin directories at the same slug.
            if not os.path.isfile(marker_path):
                continue
            src_ver = _parse_semver_tuple(manifest.version)
            dst_ver = (0, 0, 0)
            try:
                dst_manifest = PluginManifest.from_toml(dst_toml)
                dst_ver = _parse_semver_tuple(dst_manifest.version)
            except Exception:
                dst_ver = (0, 0, 0)
            should_copy = src_ver > dst_ver

        if not should_copy:
            continue

        try:
            if os.path.isdir(dst_dir):
                shutil.rmtree(dst_dir)
            shutil.copytree(src_dir, dst_dir)
            secure_atomic_write(os.path.join(dst_dir, marker_name), "bundled=true\n")
            plugin_hash = hash_plugin_directory(dst_dir)
            state = get_plugin_approval_state(slug)
            enabled = bool(state.get("enabled", True)) if isinstance(state, dict) else True
            set_plugin_approval(slug, enabled=enabled, approved_hash=plugin_hash)
            installed.append(slug)
        except Exception:
            continue
    return installed


def ensure_bundled_dashboard_plugins() -> list[str]:
    """Sync first-party bundled dashboard plugins (kind=dashboard) into ~/.mugi/dashboard_plugins/."""
    src_root = os.path.join(os.path.dirname(__file__), "templates", "bundled_dashboard_plugins")
    if not os.path.isdir(src_root):
        return []
    os.makedirs(MUGI_DASHBOARD_PLUGINS_DIR, exist_ok=True)
    try:
        import shutil
        from plugin_security import (
            PluginManifest,
            get_plugin_approval_state,
            hash_plugin_directory,
            set_plugin_approval,
        )
    except Exception:
        return []

    installed: list[str] = []
    marker_name = ".mugi_bundled_plugin"
    for entry in sorted(os.listdir(src_root)):
        src_dir = os.path.join(src_root, entry)
        src_toml = os.path.join(src_dir, "plugin.toml")
        if not os.path.isdir(src_dir) or not os.path.isfile(src_toml):
            continue
        if not entry.startswith("dash_"):
            continue
        try:
            manifest = PluginManifest.from_toml(src_toml)
        except Exception:
            continue
        # Folder name under templates/bundled_dashboard_plugins must be the canonical dash_* slug.
        slug = entry if entry.startswith("dash_") else _plugin_slug(manifest.name)
        if not slug.startswith("dash_"):
            slug = "dash_" + slug.lstrip("_")
        dst_dir = os.path.join(MUGI_DASHBOARD_PLUGINS_DIR, slug)
        dst_toml = os.path.join(dst_dir, "plugin.toml")
        marker_path = os.path.join(dst_dir, marker_name)
        should_copy = not os.path.isdir(dst_dir)

        if not should_copy:
            if not os.path.isfile(marker_path):
                continue
            src_ver = _parse_semver_tuple(manifest.version)
            dst_ver = (0, 0, 0)
            try:
                dst_manifest = PluginManifest.from_toml(dst_toml)
                dst_ver = _parse_semver_tuple(dst_manifest.version)
            except Exception:
                dst_ver = (0, 0, 0)
            should_copy = src_ver > dst_ver

        if not should_copy:
            continue

        try:
            if os.path.isdir(dst_dir):
                shutil.rmtree(dst_dir)
            shutil.copytree(src_dir, dst_dir)
            secure_atomic_write(os.path.join(dst_dir, marker_name), "bundled=true\n")
            plugin_hash = hash_plugin_directory(dst_dir)
            state = get_plugin_approval_state(slug)
            enabled = bool(state.get("enabled", True)) if isinstance(state, dict) else True
            set_plugin_approval(slug, enabled=enabled, approved_hash=plugin_hash)
            installed.append(slug)
        except Exception:
            continue
    return installed


def set_current_agent_thread_id(thread_id: str | None) -> None:
    """Set the thread_id for the current agent run (used by working buffer path). Clear when run ends."""
    _thread_local.agent_thread_id = thread_id


def get_current_agent_thread_id() -> str | None:
    """Return the thread_id for the current agent run, or None."""
    return getattr(_thread_local, "agent_thread_id", None)


@contextmanager
def agent_thread_scope(thread_id: str | None):
    """Bind :func:`get_current_agent_thread_id` for the **current OS thread**.

    The agent sets the id on the main thread at the start of ``run_raw_agent``.
    Parallel tool batches use :class:`~concurrent.futures.ThreadPoolExecutor`;
    worker threads do not inherit ``threading.local`` from the coordinator, so
    tools that resolve the conversation via ``get_current_agent_thread_id`` (for
    example ``update_task_plan``) must run inside this scope on those workers.
    """
    prev = get_current_agent_thread_id()
    set_current_agent_thread_id(thread_id)
    try:
        yield
    finally:
        set_current_agent_thread_id(prev)


# --- Security utilities ---

def get_proactive_nudge_turns(thread_id: str) -> int:
    """Return the number of conversation turns since the last proactive nudge for this thread. 0 if never nudged or missing."""
    if not thread_id or not isinstance(thread_id, str):
        return 0
    path = PROACTIVE_NUDGE_STATE_PATH
    try:
        if not os.path.isfile(path):
            return 0
        with open(path, "r", encoding="utf-8") as f:
            data = json.load(f)
        return int(data.get(thread_id, 0))
    except Exception:
        return 0


def set_proactive_nudge_turns(thread_id: str, turns: int) -> None:
    """Persist turns-since-last-nudge for this thread so the next run_raw_agent can continue counting."""
    if not thread_id or not isinstance(thread_id, str):
        return
    path = PROACTIVE_NUDGE_STATE_PATH
    try:
        data = {}
        if os.path.isfile(path):
            with open(path, "r", encoding="utf-8") as f:
                data = json.load(f)
        data[thread_id] = max(0, int(turns))
        os.makedirs(os.path.dirname(path), exist_ok=True)
        secure_atomic_write(path, json.dumps(data, indent=0))
    except Exception:
        _log.debug("Failed to persist proactive nudge turns", exc_info=True)


def secure_atomic_write(path: str, data_str: str) -> None:
    """Write data_str to path atomically with 0o600 permissions.
    Uses tempfile + os.replace so a crash mid-write never corrupts the target."""
    path = os.path.abspath(os.path.expanduser(path))
    try:
        from utils.user_model import enforce_low_level_user_model_write_guard

        err = enforce_low_level_user_model_write_guard(path, mode="overwrite")
        if err:
            raise PermissionError(err)
    except PermissionError:
        raise
    d = os.path.dirname(path) or "."
    os.makedirs(d, exist_ok=True)
    fd, tmp = tempfile.mkstemp(dir=d, prefix=".sec_", suffix=".tmp")
    try:
        os.fchmod(fd, 0o600)
        with os.fdopen(fd, "w", encoding="utf-8") as f:
            f.write(data_str)
            f.flush()
            os.fsync(f.fileno())
        os.replace(tmp, path)
    except BaseException:
        try:
            os.unlink(tmp)
        except OSError:
            pass
        raise


def _valid_runner_bridge_url(raw: str) -> bool:
    """Same constraints as tools._macos_bridge: loopback http only, fixed bridge port."""
    from urllib.parse import urlparse

    u = (raw or "").strip()
    if not u:
        return False
    try:
        p = urlparse(u)
    except Exception:
        return False
    if p.scheme.lower() != "http":
        return False
    if (p.hostname or "").lower() not in ("127.0.0.1", "localhost"):
        return False
    if p.port not in (8324, 8325):
        return False
    if p.username or p.password:
        return False
    return True


def apply_macos_bridge_runner_env() -> bool:
    """
    Load ~/.mugi/.macos_bridge_runner_env.json into os.environ for the LaunchAgent runner.

    The Swift app writes this file when the embedded Python backend starts (same values as
    MUGI_CALENDAR_URL / MUGI_AUTH_TOKEN passed to mugi_backend) and deletes it when the
    backend stops. Call before each heartbeat task so credentials stay valid after backend
    restarts without restarting launchd.

    Returns True if both variables were set from a valid file. On missing/invalid file,
    clears MUGI_CALENDAR_URL and MUGI_AUTH_TOKEN so stale values are not reused.
    """
    path = MACOS_BRIDGE_RUNNER_ENV_PATH

    def _clear_bridge_env() -> None:
        os.environ.pop("MUGI_CALENDAR_URL", None)
        os.environ.pop("MUGI_AUTH_TOKEN", None)

    if not os.path.isfile(path):
        _clear_bridge_env()
        return False
    try:
        with open(path, "r", encoding="utf-8") as f:
            data = json.load(f)
    except Exception:
        _log.debug("macos bridge runner env: unreadable JSON", exc_info=True)
        _clear_bridge_env()
        return False
    if not isinstance(data, dict):
        _clear_bridge_env()
        return False
    url = str(data.get("MUGI_CALENDAR_URL") or "").strip()
    token = str(data.get("MUGI_AUTH_TOKEN") or "").strip()
    if not _valid_runner_bridge_url(url) or not token or len(token) > 512:
        _clear_bridge_env()
        return False
    os.environ["MUGI_CALENDAR_URL"] = url.rstrip("/")
    os.environ["MUGI_AUTH_TOKEN"] = token
    return True


def append_feedback_entry(workspace_path: str, thread_id: str, message_index: int, feedback: str, snippet: str) -> str:
    """Append one thumbs up/down entry to workspace/memory/feedback.md. Returns '' on success, error message otherwise."""
    if not workspace_path or not isinstance(workspace_path, str):
        return "Invalid workspace path"
    memory_dir = os.path.join(workspace_path, "memory")
    path = os.path.join(memory_dir, "feedback.md")
    try:
        os.makedirs(memory_dir, exist_ok=True)
    except OSError as e:
        return f"Could not create memory dir: {e}"
    snippet = (snippet or "").strip().replace("\n", " ").replace("\r", " ")[:180]
    snippet = snippet.replace('"', '\\"')
    emoji = "👍" if (feedback or "").strip().lower() == "up" else "👎"
    now = datetime.now().strftime("%Y-%m-%d %H:%M")
    line = f'- **{now}** {emoji} — "{snippet}" (thread: {thread_id}, msg #{message_index})\n'
    try:
        content = ""
        try:
            with open(path, "r", encoding="utf-8") as f:
                content = f.read()
        except FileNotFoundError:
            pass
        if not content.strip():
            content = "# Feedback (thumbs up / down on your replies)\n\n"
        content = content.rstrip() + "\n" + line
        secure_atomic_write(path, content)
        return ""
    except Exception as e:
        return str(e)


_KEYRING_DISCORD_SERVICE = "mugi-discord"
_KEYRING_DISCORD_ACCOUNT = "bot_token"

def keychain_get_discord_token():
    """Read Discord bot token from macOS Keychain. Returns empty string if not found."""
    try:
        import keyring
        val = keyring.get_password(_KEYRING_DISCORD_SERVICE, _KEYRING_DISCORD_ACCOUNT)
        return val or ""
    except Exception:
        return ""

def keychain_set_discord_token(token):
    """Store Discord bot token in macOS Keychain."""
    try:
        import keyring
        if token:
            keyring.set_password(_KEYRING_DISCORD_SERVICE, _KEYRING_DISCORD_ACCOUNT, token)
        else:
            try:
                keyring.delete_password(_KEYRING_DISCORD_SERVICE, _KEYRING_DISCORD_ACCOUNT)
            except Exception:
                _log.debug("Failed to delete Discord token from keychain", exc_info=True)
    except Exception:
        _log.debug("Failed to set Discord token in keychain", exc_info=True)


_KEYRING_IMAP_SERVICE = "mugi-imap"
_KEYRING_IMAP_ACCOUNT = "password"


def _imap_account_key(account_id: str) -> str:
    account = str(account_id or "").strip()
    return account if account else _KEYRING_IMAP_ACCOUNT


def keychain_get_imap_password():
    """Read IMAP password from macOS Keychain. Returns empty string if not found."""
    try:
        import keyring
        val = keyring.get_password(_KEYRING_IMAP_SERVICE, _KEYRING_IMAP_ACCOUNT)
        return val or ""
    except Exception:
        return ""


def keychain_set_imap_password(password):
    """Store IMAP password in macOS Keychain."""
    try:
        import keyring
        if password:
            keyring.set_password(_KEYRING_IMAP_SERVICE, _KEYRING_IMAP_ACCOUNT, password)
        else:
            try:
                keyring.delete_password(_KEYRING_IMAP_SERVICE, _KEYRING_IMAP_ACCOUNT)
            except Exception:
                _log.debug("Failed to delete IMAP password from keychain", exc_info=True)
    except Exception:
        _log.debug("Failed to set IMAP password in keychain", exc_info=True)


def keychain_get_imap_password_for_account(account_id: str) -> str:
    """Read IMAP password for one account id. Returns '' if missing."""
    try:
        import keyring
        val = keyring.get_password(_KEYRING_IMAP_SERVICE, _imap_account_key(account_id))
        return val or ""
    except Exception:
        return ""


def keychain_set_imap_password_for_account(account_id: str, password: str) -> None:
    """Store IMAP password for one account id. Empty password clears it."""
    account = _imap_account_key(account_id)
    try:
        import keyring
        if password:
            keyring.set_password(_KEYRING_IMAP_SERVICE, account, password)
        else:
            try:
                keyring.delete_password(_KEYRING_IMAP_SERVICE, account)
            except Exception:
                _log.debug("Failed to delete IMAP password from keychain", exc_info=True)
    except Exception:
        _log.debug("Failed to set IMAP password in keychain", exc_info=True)


def keychain_delete_imap_password_for_account(account_id: str) -> None:
    """Delete IMAP password for one account id."""
    keychain_set_imap_password_for_account(account_id, "")


# ── Webhook Keychain helpers (Phase 2a) ───────────────────────────────────────
_KEYRING_WEBHOOK_SERVICE = "mugi-webhook"
_KEYRING_WEBHOOK_PRIMARY_ACCOUNT = "primary_token"
_KEYRING_WEBHOOK_HMAC_PREFIX = "hmac."

# Outbound webhook HMAC secrets live in a separate Keychain service so we can
# enumerate and purge them independently from inbound webhook secrets.
_KEYRING_OUTBOUND_WEBHOOK_SERVICE = "mugi-outbound-webhook"
_KEYRING_OUTBOUND_WEBHOOK_ACCOUNT_PREFIX = "hmac."


def keychain_get_webhook_token() -> str:
    """Read the primary webhook token from macOS Keychain. '' if absent."""
    try:
        import keyring
        return keyring.get_password(_KEYRING_WEBHOOK_SERVICE, _KEYRING_WEBHOOK_PRIMARY_ACCOUNT) or ""
    except Exception:
        return ""


def keychain_set_webhook_token(token: str) -> None:
    """Store the primary webhook token in macOS Keychain. Empty clears it."""
    try:
        import keyring
        if token:
            keyring.set_password(_KEYRING_WEBHOOK_SERVICE, _KEYRING_WEBHOOK_PRIMARY_ACCOUNT, token)
        else:
            try:
                keyring.delete_password(_KEYRING_WEBHOOK_SERVICE, _KEYRING_WEBHOOK_PRIMARY_ACCOUNT)
            except Exception:
                _log.debug("Failed to delete webhook token", exc_info=True)
    except Exception:
        _log.debug("Failed to set webhook token", exc_info=True)


def keychain_get_webhook_hmac(source: str) -> str:
    """Read the per-source HMAC secret from macOS Keychain. '' if absent."""
    try:
        import keyring
        account = _KEYRING_WEBHOOK_HMAC_PREFIX + str(source or "")
        return keyring.get_password(_KEYRING_WEBHOOK_SERVICE, account) or ""
    except Exception:
        return ""


def keychain_set_webhook_hmac(source: str, secret: str) -> None:
    """Store a per-source HMAC secret in Keychain. Empty secret clears it."""
    try:
        import keyring
        account = _KEYRING_WEBHOOK_HMAC_PREFIX + str(source or "")
        if secret:
            keyring.set_password(_KEYRING_WEBHOOK_SERVICE, account, secret)
        else:
            try:
                keyring.delete_password(_KEYRING_WEBHOOK_SERVICE, account)
            except Exception:
                _log.debug("Failed to delete webhook HMAC", exc_info=True)
    except Exception:
        _log.debug("Failed to set webhook HMAC", exc_info=True)


def keychain_delete_webhook_hmac(source: str) -> None:
    """Delete per-source HMAC secret."""
    keychain_set_webhook_hmac(source, "")


def generate_secure_token(nbytes: int = 32) -> str:
    """Return a URL-safe random token suitable for the primary webhook token."""
    import secrets as _secrets
    return _secrets.token_urlsafe(max(16, int(nbytes)))


# ── Webhook source directory (name list only; secrets stay in Keychain) ──────
WEBHOOK_SOURCES_FILE = os.path.join(MUGI_DIR, "webhook_sources.json")


_WEBHOOK_CAPABILITIES = ("notify_only", "full_turn")


def _webhook_sources_read() -> list[dict]:
    """Return the ordered list of known webhook source names (no secrets).

    The ``capability`` field was added in Phase 2b; rows written before it
    existed are treated as ``full_turn`` to preserve backward compatibility
    with users who already configured inbound sources.
    """
    try:
        if not os.path.exists(WEBHOOK_SOURCES_FILE):
            return []
        with open(WEBHOOK_SOURCES_FILE, "r", encoding="utf-8") as f:
            data = json.load(f)
        out: list[dict] = []
        if isinstance(data, list):
            for row in data:
                if isinstance(row, dict) and isinstance(row.get("source"), str):
                    cap = str(row.get("capability") or "").strip().lower()
                    if cap not in _WEBHOOK_CAPABILITIES:
                        cap = "full_turn"
                    out.append({
                        "source": row["source"],
                        "created_at": str(row.get("created_at") or ""),
                        "capability": cap,
                    })
        return out
    except Exception:
        return []


def _webhook_sources_write(rows: list[dict]) -> None:
    try:
        os.makedirs(MUGI_DIR, exist_ok=True)
        secure_atomic_write(WEBHOOK_SOURCES_FILE, json.dumps(rows, indent=2))
    except Exception:
        _log.debug("Failed to write webhook source directory", exc_info=True)


def webhook_sources_list() -> list[dict]:
    """Public: read-only list of webhook source records (name + created_at)."""
    return _webhook_sources_read()


def webhook_sources_add(source: str, *, capability: str = "notify_only") -> None:
    """Record *source* in the local source directory (idempotent, no secrets).

    ``capability`` must be one of ``notify_only`` or ``full_turn``. New sources
    default to ``notify_only`` so freshly-configured webhooks cannot trigger a
    full agent turn without explicit user upgrade; rows missing the field are
    preserved as ``full_turn`` by the reader to avoid silently downgrading
    existing Phase 2a setups.
    """
    s = str(source or "").strip()
    if not s:
        return
    cap = str(capability or "").strip().lower()
    if cap not in _WEBHOOK_CAPABILITIES:
        cap = "notify_only"
    rows = _webhook_sources_read()
    if any(r.get("source") == s for r in rows):
        return
    rows.append({"source": s, "created_at": _now_iso_utc(), "capability": cap})
    _webhook_sources_write(rows)


def webhook_sources_set_capability(source: str, capability: str) -> bool:
    """Update capability for an existing source. Returns True on success."""
    s = str(source or "").strip()
    cap = str(capability or "").strip().lower()
    if not s or cap not in _WEBHOOK_CAPABILITIES:
        return False
    rows = _webhook_sources_read()
    updated = False
    for row in rows:
        if row.get("source") == s:
            row["capability"] = cap
            updated = True
            break
    if updated:
        _webhook_sources_write(rows)
    return updated


def webhook_source_capability(source: str) -> str:
    """Return the capability for *source*. Defaults to ``full_turn`` (legacy)."""
    s = str(source or "").strip()
    for row in _webhook_sources_read():
        if row.get("source") == s:
            cap = str(row.get("capability") or "").strip().lower()
            if cap in _WEBHOOK_CAPABILITIES:
                return cap
    return "full_turn"


def webhook_sources_remove(source: str) -> None:
    s = str(source or "").strip()
    if not s:
        return
    rows = [r for r in _webhook_sources_read() if r.get("source") != s]
    _webhook_sources_write(rows)


# ── Outbound webhook registry (Phase 2b §2) ───────────────────────────────────
OUTBOUND_WEBHOOKS_FILE = os.path.join(MUGI_DIR, "outbound_webhooks.json")

_SLUG_RE = re.compile(r"^[a-zA-Z0-9_-]{1,32}$")


def _outbound_webhooks_read() -> list[dict]:
    """Return the list of configured outbound webhooks. Secrets stay in Keychain."""
    try:
        if not os.path.exists(OUTBOUND_WEBHOOKS_FILE):
            return []
        with open(OUTBOUND_WEBHOOKS_FILE, "r", encoding="utf-8") as f:
            data = json.load(f)
    except Exception:
        return []
    out: list[dict] = []
    if not isinstance(data, list):
        return out
    for row in data:
        if not isinstance(row, dict):
            continue
        slug = str(row.get("slug") or "").strip()
        url = str(row.get("url") or "").strip()
        if not slug or not _SLUG_RE.match(slug) or not url:
            continue
        out.append({
            "slug": slug,
            "url": url,
            "label": str(row.get("label") or "")[:80],
            "created_at": str(row.get("created_at") or ""),
        })
    return out


def _outbound_webhooks_write(rows: list[dict]) -> None:
    try:
        os.makedirs(MUGI_DIR, exist_ok=True, mode=0o700)
        secure_atomic_write(OUTBOUND_WEBHOOKS_FILE, json.dumps(rows, indent=2))
    except Exception:
        _log.debug("Failed to write outbound webhook registry", exc_info=True)


def outbound_webhooks_list() -> list[dict]:
    """Public read-only listing of outbound webhook registrations."""
    return _outbound_webhooks_read()


def outbound_webhooks_get(slug: str) -> dict | None:
    s = str(slug or "").strip()
    for row in _outbound_webhooks_read():
        if row.get("slug") == s:
            return dict(row)
    return None


def outbound_webhooks_add(slug: str, url: str, *, label: str = "") -> dict:
    """Register an outbound webhook. Returns the stored row (minus secret)."""
    s = str(slug or "").strip()
    u = str(url or "").strip()
    if not s or not _SLUG_RE.match(s):
        raise ValueError("invalid slug; must match [a-zA-Z0-9_-]{1,32}")
    if not u:
        raise ValueError("url is required")
    from urllib.parse import urlparse
    parsed = urlparse(u)
    if parsed.scheme not in ("http", "https"):
        raise ValueError("url scheme must be http or https")
    if parsed.scheme == "http" and parsed.hostname not in (
        "127.0.0.1", "::1", "localhost"
    ):
        raise ValueError("http is only allowed for loopback; use https for remote hosts")
    rows = _outbound_webhooks_read()
    if any(r.get("slug") == s for r in rows):
        raise ValueError(f"outbound webhook slug '{s}' already registered")
    row = {
        "slug": s,
        "url": u,
        "label": str(label or "")[:80],
        "created_at": _now_iso_utc(),
    }
    rows.append(row)
    _outbound_webhooks_write(rows)
    return row


def outbound_webhooks_remove(slug: str) -> bool:
    s = str(slug or "").strip()
    if not s:
        return False
    rows = _outbound_webhooks_read()
    new_rows = [r for r in rows if r.get("slug") != s]
    if len(new_rows) == len(rows):
        return False
    _outbound_webhooks_write(new_rows)
    keychain_delete_outbound_webhook_hmac(s)
    return True


def keychain_get_outbound_webhook_hmac(slug: str) -> str:
    try:
        import keyring
        account = _KEYRING_OUTBOUND_WEBHOOK_ACCOUNT_PREFIX + str(slug or "")
        return keyring.get_password(_KEYRING_OUTBOUND_WEBHOOK_SERVICE, account) or ""
    except Exception:
        return ""


def keychain_set_outbound_webhook_hmac(slug: str, secret: str) -> None:
    try:
        import keyring
        account = _KEYRING_OUTBOUND_WEBHOOK_ACCOUNT_PREFIX + str(slug or "")
        if secret:
            keyring.set_password(_KEYRING_OUTBOUND_WEBHOOK_SERVICE, account, secret)
        else:
            try:
                keyring.delete_password(_KEYRING_OUTBOUND_WEBHOOK_SERVICE, account)
            except Exception:
                _log.debug("Failed to delete outbound webhook HMAC", exc_info=True)
    except Exception:
        _log.debug("Failed to set outbound webhook HMAC", exc_info=True)


def keychain_delete_outbound_webhook_hmac(slug: str) -> None:
    keychain_set_outbound_webhook_hmac(slug, "")


def _now_iso_utc() -> str:
    from datetime import datetime, timezone
    return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")


# ── Generic API-key Keychain helpers ──────────────────────────────────────────
_KEYRING_API_KEY_SERVICE_PREFIX = "mugi-apikey"
_API_KEY_SECTIONS = ("subagent", "summarization", "consolidation", "curiosity", "drives", "text_evolution")


def keychain_get_api_key(section: str) -> str:
    """Read an LLM API key from macOS Keychain for *section*. Returns '' if absent."""
    try:
        import keyring
        return keyring.get_password(f"{_KEYRING_API_KEY_SERVICE_PREFIX}-{section}", "api_key") or ""
    except Exception:
        return ""


def keychain_set_api_key(section: str, key: str) -> None:
    """Store an LLM API key in macOS Keychain for *section*. Empty key deletes the entry."""
    try:
        import keyring
        service = f"{_KEYRING_API_KEY_SERVICE_PREFIX}-{section}"
        if key:
            keyring.set_password(service, "api_key", key)
        else:
            try:
                keyring.delete_password(service, "api_key")
            except Exception:
                _log.debug("Failed to delete API key from keychain", exc_info=True)
    except Exception:
        _log.debug("Failed to set API key in keychain", exc_info=True)


def _migrate_api_keys_to_keychain(config: dict) -> bool:
    """One-time migration: move api_key values from TOML sections into Keychain.

    Returns True if any key was migrated (caller should re-save TOML without keys).
    """
    migrated = False
    for section in _API_KEY_SECTIONS:
        val = (config.get(section) or {}).get("api_key", "")
        if isinstance(val, str) and val.strip():
            keychain_set_api_key(section, val.strip())
            config[section]["api_key"] = ""
            migrated = True
    return migrated


def _inject_keychain_api_keys(config: dict) -> None:
    """Overlay Keychain API keys onto config sections (in-memory only)."""
    for section in _API_KEY_SECTIONS:
        kc_val = keychain_get_api_key(section)
        if kc_val:
            config.setdefault(section, {})["api_key"] = kc_val


DEFAULT_AGENT_CONFIG = {
    "general": {
        "name": "Assistant",
        "workspace_path": "~/.mugi/workspace",
        "evolution_enabled": False,
    },
    "personality_files": {
        "files": ["PERSONA.md", "AGENTS.md", "SESSION-STATE.md"],
        "inject_mode": "lazy",
    },
    "debug": {
        "enabled": False,
        "trace_conversations": False,
        "trace_dir": "",
    },
    "onboarding": {
        "enabled": True,
        "done": False,
        # Swift feature-tour sheet; separate from [onboarding] done (agent/bootstrap).
        "welcome_tour_completed": False,
    },
    "setup": {
        "wizard_done": False,
    },
    "summaries": {
        "enabled": True,
        "daily_interval_hours": 24,
        "weekly_day": "Sunday",
    },
    "memory": {
        "soft_cap_characters": 18000,
        "hard_cap_characters": 24000,
        "archive_when_exceeds": True,
    },
    "search": {
        "use_local_docker": False,
        "base_url": "http://192.168.1.111:8888",
        "port": 8888,
        "public_fallback_url": "https://searx.party",
    },
    "web": {
        # read_webpage: for these host rules, always attempt Playwright (if installed) after HTTP 200, same as challenge/sparse heuristics.
        "playwright_fallback_hosts": [
            "x.com",
            "twitter.com",
        ],
    },
    "heartbeat": {
        "standard_interval_minutes": 30,
        "memory_maintenance_interval_minutes": 360,
        "growth_check_interval_minutes": 1440,
        "consciousness_interval_minutes": 360,
        "consolidation_interval_minutes": 720,
        "curiosity_interval_minutes": 240,
        "research_prefetch_interval_minutes": 720,
        "drive_cycle_interval_minutes": 180,
        "notification_sound": True,
    },
    "paths": {
        "allowed_dirs": [],
    },
    "subagent": {
        "enabled": True,
        "timeout_seconds": 300,
        "max_concurrent": 3,
        "max_background": 5,
        "max_retries": 1,
        "model": "",
        "base_url": "",
        "api_key": "",
        "subagent_first_routing": False,
    },
    "summarization": {
        "enabled": True,
        "trigger_fraction": 0.6,
        "keep_messages": 20,
        "model": "",
        "base_url": "",
        "api_key": "",
    },
    "consolidation": {
        "enabled": True,
        "lookback_days": 7,
        "merge_to_memory": False,
        "model": "",
        "base_url": "",
        "api_key": "",
    },
    "curiosity": {
        "enabled": True,
        "prefetch_enabled": True,
        "feed_matching_enabled": True,
        "prefetch_timeout_seconds": 180,
        "prefetch_max_age_hours": 24,
        "model": "",
        "base_url": "",
        "api_key": "",
    },
    "curiosity_scoring": {
        "enabled": True,
        "interest_decay": 0.9,
        "score_threshold": 0.15,
        "half_life_hours": 48.0,
    },
    "drives": {
        "enabled": True,
        "discovery_model": "",
        "base_url": "",
        "api_key": "",
        "escalation_step": 0.1,
        "max_active_drives": 5,
        "visibility_threshold": 0.3,
        "urgency_threshold": 0.6,
        "critical_threshold": 0.9,
        "max_age_days": 14,
    },
    "utility": {
        "enabled": True,
        "model_repo": DEFAULT_UTILITY_MODEL_REPO,
        "completion_critic_enabled": False,
        "completion_critic_max_retries": 1,
    },
    "mesh": {
        "autonomous_social_enabled": True,
        "mesh_rapport_memory_enabled": True,
        "mesh_rapport_digest_in_main_chat": True,
        "mesh_rapport_context_max_chars": 1800,
        "mesh_rapport_digest_max_chars": 1200,
        "mesh_rapport_digest_style": "normal",
        "mesh_rapport_global_max_bytes": 40 * 1024 * 1024,
        "mesh_rapport_per_record_max_bytes": 1024 * 1024,
        "mesh_rapport_stale_days": 30,
        "mesh_rapport_export_max_records": 200,
        "mesh_rapport_export_max_bytes": 4 * 1024 * 1024,
        "mesh_rapport_export_per_minute": 30,
        "idle_conversation_interval_seconds": 900,
        "daily_autonomous_conversation_cap": 20,
        "autonomous_llm_calls_per_hour": 24,
        "max_followups_per_social_conversation": 2,
        "user_active_pause_debounce_ms": 3000,
        "long_idle_nudge_minutes": 120,
        "mesh_llm_max_seconds": 120,
        "pending_result_max_total": 20,
        "pending_result_max_per_peer": 5,
        "mesh_logical_message_dedupe_enabled": True,
        "mesh_logical_message_dedupe_ttl_seconds": 259200,
        "mesh_logical_message_dedupe_max_entries": 8192,
        "message_max_bytes": 131072,
        "pending_result_max_chars": 8000,
    },
    "hooks": {
        "enabled": True,
        "rate_limit_per_minute": 10,
        "allowed_webhook_urls": [],
    },
    "scheduling": {
        "allowed_models": [],
        "default_timezone": "",
    },
    "skills": {
        "bundled_path": "",
    },
    "agent_loop_tool_context": {
        "pressure_trim_enabled": True,
        "tier2_reduce_other_keep_last_n": False,
        "soft_stub_enabled": False,
        "tool_format_reminder_enabled": False,
        "auto_tool_reminder_pressure": 0.92,
        "ephemeral_parse_retry_enabled": False,
        "allow_quick_panel_dashboard_pressure": False,
        "ephemeral_tail_max_nudge_messages": 12,
        "ephemeral_tail_max_nudge_chars": 0,
        "log_pressure_trim_stats": True,
    },
    "agent_loop_tier0": {
        # Optional extra tool names merged into tier-0 (same pattern as QUICK_PANEL_TIER_MERGE_TOOL_NAMES).
        "merge_tool_names": [],
    },
    "agent_loop_router": {
        "enabled": False,
        "model": "",
        "base_url": "",
    },
    "text_evolution": {
        "enabled": False,
        "min_interval_hours": 24,
        "min_eval_tasks": 3,
        "max_metric_calls_per_day": 50,
        "max_pending_proposals": 10,
        "allow_on_battery": False,
        "eval_iteration_budget_total": 15,
        "eval_tasks_path": "",
        "eval_model": "",
        "optimizer_model": "",
        "base_url": "",
        "api_key": "",
    },
    "dashboard": {
        # Symbols for the bundled Markets tile (`dashboard_bundled.feeds.feed_stocks`), comma-separated.
        "stock_symbols": "AAPL,MSFT,GOOG",
        # Optional IANA timezone override for the clock tile; blank uses system local.
        "timezone": "",
    },
}

# Personality and memory templates — loaded from templates/*.md at runtime.
# The templates/ directory is the single source of truth for all template content.
_TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), "templates")

# Template filenames for workspace creation/reset only. Order controls which files get created.
# Prompt load order comes from config personality_files.files — see load_personality_prompt().
_TEMPLATE_FILENAMES = [
    "PERSONA.md",
    "USER.md",
    "BOOTSTRAP.md",
    "AGENTS.md",
    "SESSION-STATE.md",
    "HEARTBEAT.md",
    "MEMORY.md",
    "TOOLS.md",
]

_EVOLUTION_TEMPLATE_FILENAMES = [
    "CONSTITUTION.md",
    "JOURNAL.md",
]


def _read_template(filename: str) -> str:
    """Read a template file from the templates/ directory. Returns empty string on failure."""
    path = os.path.join(_TEMPLATES_DIR, filename)
    try:
        with open(path, "r", encoding="utf-8") as f:
            return f.read().strip()
    except Exception:
        return ""


class _TemplateLazyDict(dict):
    """Dict that loads template content from disk on first access, then caches."""

    def __init__(self, filenames: list[str]):
        super().__init__()
        self._filenames = filenames
        self._loaded = False

    def _ensure_loaded(self):
        if not self._loaded:
            for fn in self._filenames:
                content = _read_template(fn)
                if content:
                    super().__setitem__(fn, content)
            self._loaded = True

    def __getitem__(self, key):
        self._ensure_loaded()
        return super().__getitem__(key)

    def __contains__(self, key):
        self._ensure_loaded()
        return super().__contains__(key)

    def __iter__(self):
        self._ensure_loaded()
        return iter(self._filenames)

    def items(self):
        self._ensure_loaded()
        return [(fn, super(_TemplateLazyDict, self).__getitem__(fn)) for fn in self._filenames if fn in self]

    def keys(self):
        self._ensure_loaded()
        return [fn for fn in self._filenames if fn in self]

    def values(self):
        self._ensure_loaded()
        return [super(_TemplateLazyDict, self).__getitem__(fn) for fn in self._filenames if fn in self]

    def __len__(self):
        self._ensure_loaded()
        return super().__len__()

    def get(self, key, default=None):
        self._ensure_loaded()
        return super().get(key, default)


TEMPLATE_FILES = _TemplateLazyDict(_TEMPLATE_FILENAMES)
EVOLUTION_TEMPLATE_FILES = _TemplateLazyDict(_EVOLUTION_TEMPLATE_FILENAMES)

DEFAULT_CONFIG_TOML = '''# Mugi config - edit paths and personality files as needed.
[general]
name = "Assistant"
workspace_path = "~/.mugi/workspace"
evolution_enabled = false   # false = simplified assistant (no constitution/JOURNAL/consciousness heartbeat tooling)

[personality_files]
files = ["PERSONA.md", "AGENTS.md", "SESSION-STATE.md"]
inject_mode = "lazy"  # lazy = capped turn-1 inject; full = legacy whole-file inject; auto resolved per turn

[debug]
enabled = false  # master switch for verbose debug logging
trace_conversations = false  # only active when debug.enabled is also true
trace_dir = ""  # optional: override trace output directory

[onboarding]
enabled = true  # toggles first-time setup ritual
done = false    # set true after agent identity/bootstrap complete (see sync + /onboarding/complete)
welcome_tour_completed = false  # Swift welcome slides; reset separately from done when only personality resets

[setup]
wizard_done = false  # set true after first-run setup wizard completes

[summaries]
enabled = true
daily_interval_hours = 24
weekly_day = "Sunday"

[memory]
soft_cap_characters = 18000
hard_cap_characters = 24000
archive_when_exceeds = true
# memory_search_max_bytes_per_file = 200000
# memory_search_max_total_bytes = 1500000
# memory_search_max_shards_scan = 200
# memory_search_max_semantic_chunks = 600
# memory_search_fusion_weight_lexical = 1.0
# memory_search_fusion_weight_semantic = 0.85
# memory_search_fusion_weight_recency = 0.35
# memory_search_per_source_char_cap = 1200
# memory_search_semantic_min_score = 0.05

[search]
use_local_docker = false
base_url = "http://192.168.1.111:8888"
port = 8888
public_fallback_url = "https://searx.party"

# read_webpage: hostnames (registrable domain) whose pages usually need a real browser.
# A rule like "twitter.com" matches mobile.twitter.com, etc. Add domains here when HTTP-only fetch is insufficient.
[web]
playwright_fallback_hosts = [ "x.com", "twitter.com" ]

[heartbeat]
standard_interval_minutes = 30
memory_maintenance_interval_minutes = 360
growth_check_interval_minutes = 1440
consciousness_interval_minutes = 360
drive_cycle_interval_minutes = 180
notification_sound = true

[drives]
enabled = true
discovery_model = ""
base_url = ""
api_key = ""
escalation_step = 0.1
max_active_drives = 5
visibility_threshold = 0.3
urgency_threshold = 0.6
critical_threshold = 0.9
max_age_days = 14

[mesh]
autonomous_social_enabled = true
idle_conversation_interval_seconds = 900
daily_autonomous_conversation_cap = 20
autonomous_llm_calls_per_hour = 24
max_followups_per_social_conversation = 2
user_active_pause_debounce_ms = 3000
long_idle_nudge_minutes = 120
mesh_llm_max_seconds = 120
pending_result_max_total = 20
pending_result_max_per_peer = 5
message_max_bytes = 131072
pending_result_max_chars = 8000

[hooks]
enabled = true
rate_limit_per_minute = 10
allowed_webhook_urls = []

[scheduling]
allowed_models = []
default_timezone = ""

[skills]
bundled_path = ""
'''


_WORKSPACE_GITIGNORE = """# Generated by Mugi - do not commit transient/derived memory files
memory/summaries/
memory/archives/
memory/working-buffer.md
memory/feedback.md
memory/recurring-patterns.md
memory/proactive-ideas.md
memory/drives.json
memory/open_loops.jsonl
memory/growth-events.md
memory/tool_errors.json
DREAMS.md
knowledge/
skills/.usage.json
skills/.curator_state.json
skills/.lifecycle_state.json
skills/.archive/
"""


def ensure_workspace_git(workspace_path: str, config=None):
    """
    If evolution mode is enabled and workspace has no .git, run git init,
    write .gitignore, and set local user identity so commits work without global gitconfig.
    """
    if not config or not (config.get("general") or {}).get("evolution_enabled", False):
        return
    git_dir = os.path.join(workspace_path, ".git")
    if os.path.isdir(git_dir):
        return
    try:
        import subprocess
        subprocess.run(
            ["git", "init"],
            cwd=workspace_path,
            check=True,
            capture_output=True,
            timeout=10,
        )
        gitignore_path = os.path.join(workspace_path, ".gitignore")
        with open(gitignore_path, "w", encoding="utf-8") as f:
            f.write(_WORKSPACE_GITIGNORE)
        for key, val in [("user.name", "Mugi Workspace"), ("user.email", "mugi@local")]:
            subprocess.run(
                ["git", "config", key, val],
                cwd=workspace_path,
                capture_output=True,
                timeout=5,
            )
        _debug_print(config or {}, "[Mugi] Initialized git in workspace")
    except Exception as e:
        _debug_print(config or {}, f"[Mugi] workspace git init failed: {e}")


_IDENTITY_NAME_RE = re.compile(r"\*\*Name:\*\*[ \t]+\S", re.IGNORECASE)


def _identity_file_has_real_content(path: str) -> bool:
    """Return True if an identity/user file has been filled in (not blank template).

    Checks:
      1. File has a ``**Name:** <non-blank>`` field.
      2. File is not cross-contaminated (IDENTITY.md shouldn't start with ``# USER.md`` etc.).
    """
    try:
        with open(path, "r", encoding="utf-8") as f:
            text = f.read(2048)
    except Exception:
        return False
    if not text.strip():
        return False
    basename_upper = os.path.basename(path).upper()
    first_line = text.lstrip().split("\n", 1)[0].strip()
    if basename_upper == "IDENTITY.MD" and first_line.startswith("# USER.md"):
        return False
    if basename_upper == "USER.MD" and first_line.startswith("# IDENTITY.md"):
        return False
    if basename_upper == "PERSONA.MD" and first_line.startswith("# USER.md"):
        return False
    return bool(_IDENTITY_NAME_RE.search(text))


def _strip_md_title(text: str, titles: tuple[str, ...]) -> str:
    lines = (text or "").splitlines()
    if not lines:
        return ""
    first = lines[0].strip().lower()
    for t in titles:
        tl = t.lower()
        if first == tl or first.startswith(tl.split()[0]):
            rest = lines[1:]
            while rest and not rest[0].strip():
                rest = rest[1:]
            return "\n".join(rest).strip()
    return (text or "").strip()


def _migrate_legacy_persona_if_needed(workspace_path: str) -> None:
    """One-time synthesis of PERSONA.md from legacy IDENTITY.md + SOUL.md."""
    persona_path = os.path.join(workspace_path, "PERSONA.md")
    if os.path.isfile(persona_path):
        return
    id_path = os.path.join(workspace_path, "IDENTITY.md")
    soul_path = os.path.join(workspace_path, "SOUL.md")
    id_text = ""
    soul_text = ""
    if os.path.isfile(id_path):
        try:
            with open(id_path, "r", encoding="utf-8") as f:
                id_text = f.read().strip()
        except Exception:
            id_text = ""
    if os.path.isfile(soul_path):
        try:
            with open(soul_path, "r", encoding="utf-8") as f:
                soul_text = f.read().strip()
        except Exception:
            soul_text = ""
    if not id_text and not soul_text:
        return
    parts: list[str] = ["# PERSONA.md - Who you are", ""]
    if id_text:
        body = _strip_md_title(id_text, ("# IDENTITY.md", "# IDENTITY.md - Who Am I?", "# IDENTITY"))
        if body:
            if "## Identity" in body:
                parts.append(body)
            else:
                parts.extend(["## Identity", "", body])
    if soul_text:
        body = _strip_md_title(soul_text, ("# SOUL.md", "# SOUL.md - Who You Are"))
        if body:
            parts.extend(["", "## Soul", "", body])
    merged = "\n".join(parts).strip() + "\n"
    try:
        from persistence import secure_atomic_write

        secure_atomic_write(persona_path, merged)
        _log.debug("Migrated legacy IDENTITY/SOUL into PERSONA.md")
    except Exception:
        _log.debug("PERSONA migration failed", exc_info=True)


def sync_onboarding_done_if_bootstrap_removed(config: dict | None, workspace_path: str) -> bool:
    """Set ``[onboarding] done = true`` in config and persist when bootstrap is gone.

    When the agent (or user) removes BOOTSTRAP.md but the flag was still false, the next
    ``ensure_mugi_workspace`` would recreate BOOTSTRAP from the template. This sync prevents
    that. The agent must not write ``mugi_config.toml``; only this path (and ``delete_file``)
    updates the flag.

    Returns True if config was updated and written to disk.
    """
    if not isinstance(config, dict) or not workspace_path:
        return False
    ob = config.get("onboarding")
    if isinstance(ob, dict) and ob.get("done") is True:
        return False
    if any(
        os.path.isfile(os.path.join(workspace_path, n))
        for n in ("BOOTSTRAP.md", "BOOTSTRAP.MD")
    ):
        return False
    user_path = os.path.join(workspace_path, "USER.md")
    persona_path = os.path.join(workspace_path, "PERSONA.md")
    id_path = os.path.join(workspace_path, "IDENTITY.md")
    if not os.path.isfile(user_path):
        return False
    persona_ok = os.path.isfile(persona_path) and _identity_file_has_real_content(persona_path)
    legacy_ok = os.path.isfile(id_path) and _identity_file_has_real_content(id_path)
    if not (persona_ok or legacy_ok):
        _debug_print(
            config,
            "[Mugi] sync_onboarding: PERSONA.md (or legacy IDENTITY.md) still blank; not marking done",
        )
        return False
    if not _identity_file_has_real_content(user_path):
        _debug_print(
            config,
            "[Mugi] sync_onboarding: USER.md still has blank/template content; not marking done",
        )
        return False
    ob = config.setdefault("onboarding", {})
    ob["done"] = True
    ob["welcome_tour_completed"] = True
    _save_agent_config(config)
    return True


def ensure_mugi_workspace(config=None):
    """
    Create ~/.mugi/, workspace dir, workspace/memory/, and template files if missing.
    If config is provided, use config's workspace_path; otherwise MUGI_WORKSPACE.
    Loops TEMPLATE_FILES and creates each file in workspace if it doesn't exist.
    """
    os.makedirs(MUGI_DIR, exist_ok=True, mode=0o700)
    workspace_path = None
    if config and isinstance(config.get("general"), dict):
        workspace_path = (config["general"].get("workspace_path") or "").strip()
    if not workspace_path:
        workspace_path = MUGI_WORKSPACE
    else:
        workspace_path = os.path.expanduser(workspace_path)
    os.makedirs(workspace_path, exist_ok=True)
    memory_dir = os.path.join(workspace_path, "memory")
    os.makedirs(memory_dir, exist_ok=True)
    os.makedirs(os.path.join(memory_dir, "summaries"), exist_ok=True)
    os.makedirs(os.path.join(memory_dir, "archives"), exist_ok=True)
    # Create working-buffer.md and recurring-patterns.md stubs if missing
    working_buf = os.path.join(memory_dir, "working-buffer.md")
    if not os.path.exists(working_buf):
        try:
            with open(working_buf, "w", encoding="utf-8") as f:
                f.write("# Working Buffer (Danger Zone Log)\n**Status:** INACTIVE\n")
            _debug_print(config or {}, "[Mugi] Created working-buffer.md")
        except Exception as e:
            _log.warning("Failed to create working-buffer.md: %s", e)
    patterns_path = os.path.join(memory_dir, "recurring-patterns.md")
    if not os.path.exists(patterns_path):
        try:
            with open(patterns_path, "w", encoding="utf-8") as f:
                f.write("# Recurring Patterns\n\nTrack repeated user requests here. Propose automation at 3+ occurrences.\n")
            _debug_print(config or {}, "[Mugi] Created recurring-patterns.md")
        except Exception as e:
            _log.warning("Failed to create recurring-patterns.md: %s", e)
    drives_path = os.path.join(memory_dir, "drives.json")
    if not os.path.exists(drives_path):
        try:
            with open(drives_path, "w", encoding="utf-8") as f:
                f.write(
                    '{\n  "active_drives": [],\n  "resolved_drives": [],\n  "expired_drives": [],\n'
                    '  "stats": {\n    "total_discovered": 0,\n    "total_resolved": 0,\n'
                    '    "total_expired": 0,\n    "longest_active_days": 0\n  }\n}\n'
                )
            _debug_print(config or {}, "[Mugi] Created drives.json")
        except Exception as e:
            _log.warning("Failed to create drives.json: %s", e)
    growth_events_path = os.path.join(memory_dir, "growth-events.md")
    if not os.path.exists(growth_events_path):
        try:
            with open(growth_events_path, "w", encoding="utf-8") as f:
                f.write("# Growth Events\n")
            _debug_print(config or {}, "[Mugi] Created growth-events.md")
        except Exception as e:
            _log.warning("Failed to create growth-events.md: %s", e)
    tool_errors_path = os.path.join(memory_dir, "tool_errors.json")
    if not os.path.exists(tool_errors_path):
        try:
            with open(tool_errors_path, "w", encoding="utf-8") as f:
                f.write("[]\n")
            _debug_print(config or {}, "[Mugi] Created tool_errors.json")
        except Exception as e:
            _log.warning("Failed to create tool_errors.json: %s", e)
    os.makedirs(MUGI_PLUGINS_DIR, exist_ok=True)
    _plugins_guide_src = os.path.join(os.path.dirname(__file__), "templates", "PLUGINS.md")
    _plugins_guide_dst = MUGI_PLUGINS_GUIDE_PATH
    if os.path.isfile(_plugins_guide_src):
        try:
            import shutil

            src_text = ""
            src_ver = 0
            try:
                with open(_plugins_guide_src, "r", encoding="utf-8") as f:
                    src_text = f.read()
                src_ver = _extract_plugins_guide_version(src_text)
            except Exception:
                src_text = ""
                src_ver = 0

            if not os.path.exists(_plugins_guide_dst):
                shutil.copy2(_plugins_guide_src, _plugins_guide_dst)
                _debug_print(config or {}, "[Mugi] Created PLUGINS.md guide")
            else:
                dst_text = ""
                dst_ver = 0
                try:
                    with open(_plugins_guide_dst, "r", encoding="utf-8") as f:
                        dst_text = f.read()
                    dst_ver = _extract_plugins_guide_version(dst_text)
                except Exception:
                    dst_text = ""
                    dst_ver = 0
                # Only auto-refresh guides that are clearly versioned and stale.
                # If user removed marker or edited custom content, leave in place.
                should_refresh = bool(src_ver > 0 and dst_ver > 0 and src_ver > dst_ver)
                if should_refresh:
                    try:
                        shutil.copy2(_plugins_guide_dst, MUGI_PLUGINS_GUIDE_BACKUP_PATH)
                        _debug_print(config or {}, "[Mugi] Backed up PLUGINS.md guide to PLUGINS.md.bak")
                    except Exception:
                        _debug_print(config or {}, "[Mugi] Could not create PLUGINS.md backup before refresh")
                    shutil.copy2(_plugins_guide_src, _plugins_guide_dst)
                    _debug_print(config or {}, f"[Mugi] Refreshed PLUGINS.md guide (v{dst_ver} → v{src_ver})")
        except Exception as e:
            _log.warning("Failed to create or refresh PLUGINS.md: %s", e)
    if isinstance(config, dict):
        sync_onboarding_done_if_bootstrap_removed(config, workspace_path)
    _migrate_legacy_persona_if_needed(workspace_path)
    onboarding_done = config and isinstance(config.get("onboarding"), dict) and config["onboarding"].get("done", False)
    memory_just_created = False
    for filename, content in TEMPLATE_FILES.items():
        # Do not recreate BOOTSTRAP.md after onboarding is complete (agent deleted it on purpose)
        if filename == "BOOTSTRAP.md" and onboarding_done:
            continue
        path = os.path.join(workspace_path, filename)
        if not os.path.exists(path):
            try:
                with open(path, "w", encoding="utf-8") as f:
                    f.write(content.strip())
                _debug_print(config or {}, f"[Mugi] Created template: {filename}")
                if filename == "MEMORY.md":
                    memory_just_created = True
            except Exception as e:
                _log.warning("Failed to create template %s: %s", filename, e)
    if memory_just_created:
        try:
            from utils.memory_shard_layout import ensure_sharded_bootstrap_new_workspace

            ensure_sharded_bootstrap_new_workspace(workspace_path)
            _debug_print(config or {}, "[Mugi] Sharded memory bootstrap applied for new MEMORY.md")
        except Exception as e:
            _log.warning("Sharded memory bootstrap failed: %s", e)
    # Evolution mode: create CONSTITUTION.md and JOURNAL.md, ensure workspace has .git
    if (config or {}).get("general", {}).get("evolution_enabled", False):
        ensure_workspace_git(workspace_path, config)
        for filename, content in EVOLUTION_TEMPLATE_FILES.items():
            path = os.path.join(workspace_path, filename)
            if not os.path.exists(path):
                try:
                    with open(path, "w", encoding="utf-8") as f:
                        f.write(content.strip())
                    _debug_print(config or {}, f"[Mugi] Created evolution template: {filename}")
                except Exception as e:
                    _log.warning("Failed to create evolution template %s: %s", filename, e)
    _debug_print(config or {}, f"[Mugi] workspace ensured: {workspace_path}")


# Files preserved during identity-only reset (user data that shouldn't be wiped).
_IDENTITY_RESET_PRESERVE = frozenset({"MEMORY.md", "SESSION-STATE.md", "HEARTBEAT.md"})


def clear_phase2_transient_mugi_state(mugi_dir: str | None = None) -> None:
    """Remove Phase 2 / 2a control state under ~/.mugi.

    Cleared:
      - Phase 2 idle-tick state and proactive salience fingerprints.
      - Phase 2a unified task ledger (``task_ledger.jsonl`` + ``.meta.json``).
      - Phase 2a delivery queue (``delivery_queue.jsonl`` + ``.meta.json``).
      - Webhook Keychain entries: primary token and per-source HMACs.

    Does **not** delete ``proactive_queue.jsonl`` (may hold unread
    notifications). Called on full factory reset; safe to call if files are
    missing.
    """
    root = os.path.expanduser(mugi_dir or MUGI_DIR)
    for name in (
        "idle_tick_claim.json",
        "idle_tick_state.json",
        "proactive_salience_fingerprints.json",
        "task_ledger.jsonl",
        "task_ledger.meta.json",
        "delivery_queue.jsonl",
        "delivery_queue.meta.json",
        # Phase 2b sidecar state — safe to drop on full reset.
        "timers.jsonl",
        "outbound_webhooks.json",
        "file_watcher_state.json",
        "calendar_proximity_state.json",
        "ledger_reflection_state.json",
    ):
        path = os.path.join(root, name)
        if os.path.isfile(path):
            try:
                os.remove(path)
                _log.info("Removed Phase 2 transient file: %s", path)
            except OSError as e:
                _log.warning("Could not remove %s: %s", path, e)
    for lock_name in (
        "proactive_queue.jsonl.lock",
        "task_ledger.jsonl.lock",
        "delivery_queue.jsonl.lock",
        "timers.jsonl.lock",
    ):
        lock_path = os.path.join(root, lock_name)
        if os.path.isfile(lock_path):
            try:
                os.remove(lock_path)
            except OSError:
                pass

    # Webhook Keychain entries — primary token + all known per-source HMACs.
    try:
        keychain_set_webhook_token("")
    except Exception:
        _log.debug("Could not clear webhook primary token from Keychain", exc_info=True)
    _cleared: set[str] = set()
    try:
        for row in _webhook_sources_read():
            s = str(row.get("source") or "").strip()
            if not s or s in _cleared:
                continue
            try:
                keychain_delete_webhook_hmac(s)
                _cleared.add(s)
            except Exception:
                _log.debug("Could not clear HMAC for webhook source %r", s, exc_info=True)
    except Exception:
        _log.debug("Could not enumerate webhook source directory", exc_info=True)
    try:
        from core.config import get_webhooks_config as _get_webhooks_config
        cfg = _get_webhooks_config() or {}
        per_source = (cfg.get("per_source") or {})
        for source_name in list(per_source.keys()):
            s = str(source_name)
            if s in _cleared:
                continue
            try:
                keychain_delete_webhook_hmac(s)
                _cleared.add(s)
            except Exception:
                _log.debug(
                    "Could not clear HMAC for webhook source %r", source_name, exc_info=True
                )
    except Exception:
        _log.debug("Skipping per-source HMAC reset (config unavailable)", exc_info=True)
    try:
        if os.path.exists(WEBHOOK_SOURCES_FILE):
            os.remove(WEBHOOK_SOURCES_FILE)
    except Exception:
        _log.debug("Could not remove webhook source directory file", exc_info=True)

    # Outbound-webhook HMAC secrets (Phase 2b §2). Purge from the dedicated
    # Keychain service; the registry file was removed above.
    try:
        for row in _outbound_webhooks_read():
            slug = str(row.get("slug") or "").strip()
            if not slug:
                continue
            try:
                keychain_delete_outbound_webhook_hmac(slug)
            except Exception:
                _log.debug(
                    "Could not clear outbound webhook HMAC for slug %r", slug, exc_info=True
                )
    except Exception:
        _log.debug("Could not enumerate outbound webhook registry", exc_info=True)


def reset_personality_files(config, preserve_user_data=False):
    """
    Delete template files in the workspace and re-create them with original content.

    preserve_user_data: when True, performs an identity-only reset — MEMORY.md,
    SESSION-STATE.md, HEARTBEAT.md, and USER.md are backed up before reset and
    restored afterward so accumulated knowledge is not lost.  When False (default),
    performs a full factory reset.
    """
    if not config or not isinstance(config.get("general"), dict):
        return
    workspace = (config["general"].get("workspace_path") or "").strip()
    if not workspace:
        workspace = MUGI_WORKSPACE
    else:
        workspace = os.path.expanduser(workspace)

    backups: dict[str, str] = {}
    if preserve_user_data:
        for filename in list(_IDENTITY_RESET_PRESERVE) + ["USER.md", "PERSONA.md"]:
            path = os.path.join(workspace, filename)
            if os.path.isfile(path):
                try:
                    with open(path, "r", encoding="utf-8") as f:
                        backups[filename] = f.read()
                    _debug_print(config, f"[Mugi] Backed up before reset: {filename}")
                except Exception as e:
                    _debug_print(config, f"[Mugi] Could not back up {filename}: {e}")

    for filename in TEMPLATE_FILES:
        path = os.path.join(workspace, filename)
        if os.path.exists(path):
            try:
                os.remove(path)
                _debug_print(config, f"[Mugi] Removed for reset: {filename}")
            except OSError as e:
                _debug_print(config, f"[Mugi] Could not remove {filename}: {e}")
    for filename, content in TEMPLATE_FILES.items():
        path = os.path.join(workspace, filename)
        try:
            with open(path, "w", encoding="utf-8") as f:
                f.write(content.strip())
            _debug_print(config, f"[Mugi] Re-created template: {filename}")
        except Exception as e:
            _log.warning("Failed to write template %s: %s", filename, e)

    # Restore backed-up user data over the fresh templates.
    for filename, content in backups.items():
        path = os.path.join(workspace, filename)
        try:
            with open(path, "w", encoding="utf-8") as f:
                f.write(content)
            _debug_print(config, f"[Mugi] Restored from backup: {filename}")
        except Exception as e:
            _debug_print(config, f"[Mugi] Could not restore {filename}: {e}")

    if (config or {}).get("general", {}).get("evolution_enabled", False):
        for filename in EVOLUTION_TEMPLATE_FILES:
            path = os.path.join(workspace, filename)
            if os.path.exists(path):
                try:
                    os.remove(path)
                    _debug_print(config, f"[Mugi] Removed for reset: {filename}")
                except OSError as e:
                    _debug_print(config, f"[Mugi] Could not remove {filename}: {e}")
        for filename, content in EVOLUTION_TEMPLATE_FILES.items():
            path = os.path.join(workspace, filename)
            try:
                with open(path, "w", encoding="utf-8") as f:
                    f.write(content.strip())
                _debug_print(config, f"[Mugi] Re-created evolution template: {filename}")
            except Exception as e:
                _log.warning("Failed to write evolution template %s: %s", filename, e)

    if not preserve_user_data:
        knowledge_dir = os.path.join(workspace, "knowledge")
        if os.path.isdir(knowledge_dir):
            try:
                shutil.rmtree(knowledge_dir, ignore_errors=True)
                _debug_print(config, "[Mugi] Cleared workspace knowledge/ during full reset.")
            except Exception as e:
                _debug_print(config, f"[Mugi] Could not clear knowledge/: {e}")

        # Dreaming audit/by-product artifacts are derived — remove on full reset so
        # a fresh identity doesn't inherit stale scored-promotion history.
        dreams_path = os.path.join(workspace, "DREAMS.md")
        if os.path.isfile(dreams_path):
            try:
                os.remove(dreams_path)
                _debug_print(config, "[Mugi] Removed DREAMS.md during full reset.")
            except OSError as e:
                _debug_print(config, f"[Mugi] Could not remove DREAMS.md: {e}")
        dreams_dir = os.path.join(workspace, "memory", "dreams")
        if os.path.isdir(dreams_dir):
            try:
                shutil.rmtree(dreams_dir, ignore_errors=True)
                _debug_print(config, "[Mugi] Cleared workspace memory/dreams/ during full reset.")
            except Exception as e:
                _debug_print(config, f"[Mugi] Could not clear memory/dreams/: {e}")

        # Skills lifecycle/curator transient artifacts.
        skills_dir = os.path.join(workspace, "skills")
        for filename in (".usage.json", ".curator_state.json", ".lifecycle_state.json"):
            path = os.path.join(skills_dir, filename)
            if os.path.isfile(path):
                try:
                    os.remove(path)
                    _debug_print(config, f"[Mugi] Removed {filename} during full reset.")
                except OSError as e:
                    _debug_print(config, f"[Mugi] Could not remove {filename}: {e}")
        archive_dir = os.path.join(skills_dir, ".archive")
        if os.path.isdir(archive_dir):
            try:
                shutil.rmtree(archive_dir, ignore_errors=True)
                _debug_print(config, "[Mugi] Cleared skills/.archive/ during full reset.")
            except Exception as e:
                _debug_print(config, f"[Mugi] Could not clear skills/.archive/: {e}")

        clear_phase2_transient_mugi_state()


def write_default_config_if_missing():
    """Write mugi_config.toml from defaults if the file does not exist."""
    if os.path.isfile(MUGI_CONFIG_PATH):
        return
    os.makedirs(MUGI_DIR, exist_ok=True, mode=0o700)
    try:
        with open(MUGI_CONFIG_PATH, "w", encoding="utf-8") as f:
            f.write(DEFAULT_CONFIG_TOML)
        _log.info("Created default config: %s", MUGI_CONFIG_PATH)
    except Exception as e:
        _log.warning("Failed to write default config: %s", e)


def expand_config_paths(config):
    """Expand ~ and $HOME in path-like string values (in-place)."""
    if not isinstance(config, dict):
        return config
    for key, value in list(config.items()):
        if isinstance(value, str) and ("path" in key.lower() or key == "workspace_path"):
            config[key] = os.path.expanduser(value)
        elif isinstance(value, dict):
            expand_config_paths(value)
        elif isinstance(value, list):
            for i, item in enumerate(value):
                if isinstance(item, str) and item.startswith("~"):
                    value[i] = os.path.expanduser(item)
    return config


def _debug_print(config, msg):
    """Log msg at DEBUG level if config.debug.enabled is True."""
    if config.get("debug", {}).get("enabled", False):
        _log.debug(msg)


def _save_agent_config(config):
    """
    Write config dict to MUGI_CONFIG_PATH as TOML.
    Uses tomli_w.dump if available; otherwise writes sections manually.
    """
    if not config or not isinstance(config, dict):
        return
    os.makedirs(MUGI_DIR, exist_ok=True, mode=0o700)
    out = {}
    for section, values in config.items():
        if section in ("general", "personality_files", "debug", "onboarding", "setup",
                       "summaries", "messaging", "search", "web", "heartbeat", "paths", "email",
                       "drives", "subagent", "summarization", "consolidation", "curiosity",
                       "curiosity_scoring",
                       "mesh",
                       "agent_plane", "session_search",
                       "memory", "utility",
                       "hooks", "scheduling", "skills", "text_evolution",
                       "dashboard", "agent_loop_tool_context", "agent_loop_tier0",
                       "agent_loop_router",
                       "kanban", "council", "knowledge",
                       "timers", "file_watcher", "ledger_reflection",
                       "weather") and isinstance(values, dict):
            out[section] = dict(values)
    # Strip API keys from TOML output — they belong in Keychain.
    for section in _API_KEY_SECTIONS:
        if section in out and "api_key" in out[section]:
            kc_val = keychain_get_api_key(section)
            if kc_val:
                out[section]["api_key"] = ""
    if tomli_w is not None:
        try:
            import io
            buf = io.BytesIO()
            tomli_w.dump(out, buf)
            secure_atomic_write(MUGI_CONFIG_PATH, buf.getvalue().decode("utf-8"))
            try:
                from core.config import _sync_runtime_debug_state
                _sync_runtime_debug_state(config)
            except Exception:
                pass
            return
        except Exception as e:
            _log.warning("tomli_w save error: %s", e)
    # Fallback: manual TOML write (no tomli_w)
    lines = ["# Mugi config\n"]
    for section, values in sorted(out.items()):
        if not isinstance(values, dict):
            continue
        lines.append(f"[{section}]\n")
        for k, v in sorted(values.items()):
            if v is None:
                continue
            if isinstance(v, dict):
                # Nested table (e.g. messaging.discord): skip here; emitted below
                continue
            if isinstance(v, bool):
                lines.append(f"{k} = {str(v).lower()}\n")
            elif isinstance(v, list):
                items = ", ".join(json.dumps(str(x)) for x in v)
                lines.append(f"{k} = [{items}]\n")
            elif isinstance(v, str):
                lines.append(f"{k} = {json.dumps(v)}\n")
            else:
                lines.append(f"{k} = {v}\n")
        lines.append("\n")
        # Nested table: [section.sub]
        if section == "messaging":
            discord_vals = values.get("discord")
            if isinstance(discord_vals, dict):
                lines.append("[messaging.discord]\n")
                for k, v in sorted(discord_vals.items()):
                    if v is None:
                        continue
                    if isinstance(v, bool):
                        lines.append(f"{k} = {str(v).lower()}\n")
                    elif isinstance(v, list):
                        items = ", ".join(json.dumps(str(x)) for x in v)
                        lines.append(f"{k} = [{items}]\n")
                    elif isinstance(v, str):
                        lines.append(f"{k} = {json.dumps(v)}\n")
                    else:
                        lines.append(f"{k} = {v}\n")
                lines.append("\n")
    try:
        secure_atomic_write(MUGI_CONFIG_PATH, "".join(lines))
    except Exception as e:
        _log.warning("Config save error: %s", e)
        return

    try:
        from core.config import _sync_runtime_debug_state
        _sync_runtime_debug_state(config)
    except Exception:
        pass


def _coerce_toml_bool(value: Any, default: bool = False) -> bool:
    """Normalize values read from TOML to bool. tomllib only yields bool for ``true``/``false``;
    hand-edited files may use ``1``/``0`` (int). Without this, debug flags reset on every
    ``reload_config`` (including each agent turn)."""
    if isinstance(value, bool):
        return value
    if isinstance(value, (int, float)):
        return value != 0
    if isinstance(value, str):
        s = value.strip().lower()
        if s in ("1", "true", "yes", "on"):
            return True
        if s in ("0", "false", "no", "off", ""):
            return False
    return default


def load_agent_config() -> dict[str, Any]:
    """
    Load agent config from ~/.mugi/mugi_config.toml using tomllib.
    Expands paths (~), handles missing config with defaults. Debug prints when config.debug.enabled is true.
    Returns a dict with 'general', 'personality_files', 'debug', and any other sections.
    """
    write_default_config_if_missing()
    config = copy.deepcopy(DEFAULT_AGENT_CONFIG)
    if not os.path.isfile(MUGI_CONFIG_PATH):
        _debug_print(config, "[Mugi] config file missing, using defaults")
        config.setdefault("summaries", {}).update({
            "enabled": True,
            "daily_interval_hours": 24,
            "weekly_day": "Sunday",
        })
        expand_config_paths(config)
        ensure_mugi_workspace(config)
        return config
    try:
        with open(MUGI_CONFIG_PATH, "rb") as f:
            data = tomllib.load(f)
        if isinstance(data, dict):
            for section, values in data.items():
                if isinstance(values, dict):
                    config.setdefault(section, {}).update(values)
                else:
                    config[section] = values
        dbg_raw = data.get("debug") or {}
        debug_val = (dbg_raw.get("enabled", False) if isinstance(dbg_raw, dict) else False)
        config.setdefault("debug", {})["enabled"] = _coerce_toml_bool(debug_val, default=False)
        onb = data.get("onboarding", {})
        config.setdefault("onboarding", {})["enabled"] = bool(onb.get("enabled", True)) if isinstance(onb.get("enabled"), bool) else True
        done_val = bool(onb.get("done", False)) if isinstance(onb.get("done"), bool) else False
        config.setdefault("onboarding", {})["done"] = done_val
        wt = onb.get("welcome_tour_completed")
        if isinstance(wt, bool):
            config.setdefault("onboarding", {})["welcome_tour_completed"] = wt
        else:
            # Legacy configs: if agent onboarding was already complete, assume the Swift tour was dismissed.
            config.setdefault("onboarding", {})["welcome_tour_completed"] = done_val
        sum_cfg = data.get("summaries", {})
        config.setdefault("summaries", {}).update({
            "enabled": bool(sum_cfg.get("enabled", True)) if isinstance(sum_cfg.get("enabled"), bool) else True,
            "daily_interval_hours": int(sum_cfg.get("daily_interval_hours", 24)) if isinstance(sum_cfg.get("daily_interval_hours"), (int, float)) else 24,
            "weekly_day": str(sum_cfg.get("weekly_day", "Sunday")) if sum_cfg.get("weekly_day") else "Sunday",
        })
        expand_config_paths(config)
        _debug_print(config, f"[Mugi] config loaded: {MUGI_CONFIG_PATH}")
        _debug_print(config, f"[Mugi] general.name = {config.get('general', {}).get('name', '?')}")
        _debug_print(config, f"[Mugi] general.workspace_path = {config.get('general', {}).get('workspace_path', '?')}")
        _debug_print(config, f"[Mugi] personality_files.files = {config.get('personality_files', {}).get('files', [])}")
        ensure_mugi_workspace(config)
    except Exception as e:
        _log.warning("Config load error, using defaults: %s", e)
        expand_config_paths(config)
        ensure_mugi_workspace(config)
    # Migrate plaintext API keys from TOML to Keychain (one-time).
    if _migrate_api_keys_to_keychain(config):
        _save_agent_config(config)
        _debug_print(config, "[Mugi] Migrated API keys from TOML to Keychain")
    _inject_keychain_api_keys(config)
    return config


def set_setup_wizard_done(done: bool = True) -> None:
    """Set setup.wizard_done in mugi_config.toml. Called when setup wizard completes."""
    config = load_agent_config()
    config.setdefault("setup", {})["wizard_done"] = bool(done)
    from core.config import save_config
    save_config(config)


# Soft cap for single personality file (chars).
# SOUL/CONSTITUTION remain untruncated; large operational logs/instructions are capped.
# Caps are tight to keep total system-prompt overhead viable on ≤100K context windows.
PERSONALITY_FILE_SOFT_CAP = 2500
PERSONALITY_FILE_TRUNCATE_TO = 2300
_PERSONALITY_NO_TRUNCATE = frozenset({"SOUL.md", "CONSTITUTION.md"})
_PERSONALITY_CAP_OVERRIDES = {
    "AGENTS.md": 6500,
    "PERSONA.md": 4500,
    "JOURNAL.md": 1500,
    "MEMORY.md": 2000,
}

# In-memory cache for concatenated personality prompt (invalidated when any file mtime/size or list changes).
_personality_cache: str | None = None
_personality_cache_key: str | None = None


_LAZY_INJECT_VERSION = "1"
_LAZY_USER_NOTES_CAP = 500
_LAZY_PERSONA_SOUL_CAP = 1500
_LAZY_AGENTS_STANDING_CAP = 2000


def get_personality_inject_mode(config) -> str:
    """Return ``lazy`` or ``full`` from config (default lazy)."""
    if not config or not isinstance(config, dict):
        return "lazy"
    raw = (config.get("personality_files") or {}).get("inject_mode", "lazy")
    mode = str(raw or "lazy").strip().lower()
    return mode if mode in ("lazy", "full") else "lazy"


def should_use_lazy_personality(config, workspace_path: str) -> bool:
    """Disk probe: lazy inject when identity exists and bootstrap is absent."""
    if get_personality_inject_mode(config) == "full":
        return False
    ws = os.path.expanduser(workspace_path or "")
    if not ws:
        return False
    bootstrap_exists = any(
        os.path.isfile(os.path.join(ws, n)) for n in ("BOOTSTRAP.MD", "BOOTSTRAP.md")
    )
    if bootstrap_exists:
        return False
    persona_path = os.path.join(ws, "PERSONA.md")
    id_path = os.path.join(ws, "IDENTITY.md")
    persona_ok = os.path.isfile(persona_path) and _identity_file_has_real_content(persona_path)
    legacy_ok = os.path.isfile(id_path) and _identity_file_has_real_content(id_path)
    return persona_ok or legacy_ok


def _personality_file_block(workspace_path: str, filename: str, *, cap: int | None = None) -> str:
    """Load one workspace personality file as a labeled block (no cache)."""
    if filename.startswith("/") or filename.startswith("~"):
        filepath = os.path.expanduser(filename)
    else:
        filepath = os.path.join(workspace_path, os.path.basename(filename))
    display_name = os.path.basename(filepath)
    label = f"─── {display_name} ───"
    if not os.path.isfile(filepath):
        return f"{label}\n<file does not exist yet>\n"
    try:
        with open(filepath, "r", encoding="utf-8") as f:
            content = f.read()
    except OSError:
        return f"{label}\n<error reading file>\n"
    if not content.strip():
        return f"{label}\n<empty>\n"
    if cap is not None and len(content) > cap:
        if display_name in _PERSONALITY_CAP_OVERRIDES:
            hint = f"read_file('{filepath}')"
            content = content[:cap] + f"\n… [truncated — use {hint} for full instructions]"
        else:
            content = content[:cap] + "\n… [truncated — read full file if needed]"
    return f"{label}\n{content}\n"


def load_personality_prompt(config, file_list_override=None, mode: str = "auto"):
    """
    Read personality files from config; return concatenated string for system prompt.
    If file_list_override is provided, use it instead of config.personality_files.files.
    Otherwise files = config.personality_files.files (fall back to []). Does NOT read MEMORY/.
    For each file: path sanitized (absolute / or ~ use as-is; else join workspace_path);
    non-.md/.MD skipped with [WARN]. exists+content → "─── FILENAME ───\\n{content}\\n" (soft cap);
    exists+empty → "─── FILENAME ───\\n<empty>\\n"; missing → "─── FILENAME ───\\n<file does not exist yet>\\n".
    Uses in-memory cache keyed by workspace_path + sorted file list + (path, mtime, size) per file; rebuilds on change.
    """
    global _personality_cache, _personality_cache_key
    if not config or not isinstance(config, dict):
        return ""
    workspace_path = (config.get("general") or {}).get("workspace_path")
    if not workspace_path:
        return ""
    workspace_path = os.path.expanduser(workspace_path)
    if file_list_override is not None:
        files = list(file_list_override) if isinstance(file_list_override, (list, tuple)) else []
    else:
        files = config.get("personality_files", {}).get("files", [])
    if not isinstance(files, list):
        files = []

    inject_mode = str(mode or "auto").strip().lower()
    if inject_mode == "auto":
        inject_mode = "lazy" if should_use_lazy_personality(config, workspace_path) else "full"
    elif inject_mode not in ("lazy", "full"):
        inject_mode = "full"
    elif inject_mode == "lazy" and not should_use_lazy_personality(config, workspace_path):
        inject_mode = "full"

    # Build cache key: workspace + file list order (matters for prompt) + (resolved path, mtime, size) per file.
    # If stat() fails for any file, key stays None → we always rebuild.
    md_files = [f.strip() for f in files if f and isinstance(f, str) and (f.strip().lower().endswith(".md"))]
    cache_key = None
    key_parts = [workspace_path, "|".join(md_files), f"mode:{inject_mode}", _LAZY_INJECT_VERSION]
    for filename in md_files:
        if filename.startswith("/") or filename.startswith("~"):
            filepath = os.path.expanduser(filename)
        else:
            filepath = os.path.join(workspace_path, filename)
        if not os.path.exists(filepath) or not os.path.isfile(filepath):
            key_parts.append(f"{filepath}:missing")
            continue
        try:
            st = os.stat(filepath)
            key_parts.append(f"{filepath}:{st.st_mtime}:{st.st_size}")
        except OSError:
            cache_key = None
            break
    else:
        cache_key = hashlib.sha256("".join(key_parts).encode()).hexdigest()

    if cache_key is not None and cache_key == _personality_cache_key and _personality_cache is not None:
        _debug_print(config, "[Mugi] Personality prompt cache hit")
        return _personality_cache

    _debug_print(config, "[Mugi] Personality prompt cache miss → rebuilding")
    if inject_mode == "lazy" and file_list_override is None:
        from utils.profile_markdown import build_lazy_personality_block

        result = build_lazy_personality_block(
            workspace_path,
            config,
            files,
            sync_config=config,
            persona_soul_cap=_LAZY_PERSONA_SOUL_CAP,
            user_notes_cap=_LAZY_USER_NOTES_CAP,
            agents_standing_cap=_LAZY_AGENTS_STANDING_CAP,
        )
        _personality_cache = result
        _personality_cache_key = cache_key
        _debug_print(config, f"[Mugi] Lazy personality prompt length: {len(result)} chars")
        return result

    parts = []
    for filename in files:
        if not filename or not isinstance(filename, str):
            continue
        filename_stripped = filename.strip()
        if not filename_stripped.lower().endswith(".md"):
            if config.get("debug", {}).get("enabled", False):
                _log.warning("Skipping non-MD personality file: %s", filename_stripped)
            continue
        if filename_stripped.startswith("/") or filename_stripped.startswith("~"):
            filepath = os.path.expanduser(filename_stripped)
        else:
            filepath = os.path.join(workspace_path, filename_stripped)
        display_name = os.path.basename(filepath)
        label = f"─── {display_name} ───"
        if not os.path.exists(filepath):
            parts.append(f"{label}\n<file does not exist yet>\n")
            continue
        if not os.path.isfile(filepath):
            parts.append(f"{label}\n<not a file>\n")
            continue
        try:
            with open(filepath, "r", encoding="utf-8") as f:
                content = f.read()
            if not content.strip():
                parts.append(f"{label}\n<empty>\n")
            else:
                if display_name in ("IDENTITY.md", "PERSONA.md"):
                    from core.config import parse_identity_name_from_markdown

                    name_val = parse_identity_name_from_markdown(content)
                    if name_val:
                        config.setdefault("general", {})["name"] = name_val
                if display_name not in _PERSONALITY_NO_TRUNCATE:
                    cap = int(_PERSONALITY_CAP_OVERRIDES.get(display_name, PERSONALITY_FILE_SOFT_CAP))
                    if len(content) > cap:
                        if display_name in _PERSONALITY_CAP_OVERRIDES:
                            hint = f"read_file('{filepath}')"
                            content = content[:cap] + f"\n… [truncated — use {hint} for full instructions]"
                        else:
                            content = content[:PERSONALITY_FILE_TRUNCATE_TO] + "\n… [truncated — read full file if needed]"
                parts.append(f"{label}\n{content}\n")
        except Exception:
            parts.append(f"{label}\n<error reading file>\n")
    result = "".join(parts)
    _personality_cache = result
    _personality_cache_key = cache_key
    _debug_print(config, f"[MyAgent] Personality prompt length: {len(result)} chars")
    return result


def invalidate_personality_cache() -> None:
    """Clear in-memory personality prompt cache after profile file writes."""
    global _personality_cache, _personality_cache_key
    _personality_cache = None
    _personality_cache_key = None


def get_memory_file_path(config, period="daily"):
    """
    Return path to dated memory file in workspace/memory/summaries/.
    period="daily" -> memory/summaries/YYYY-MM-DD.md
    period="weekly" -> memory/summaries/YYYY-Www.md (ISO week).
    """
    if not config or not isinstance(config, dict):
        return None
    workspace_path = (config.get("general") or {}).get("workspace_path")
    if not workspace_path:
        return None
    workspace_path = os.path.expanduser(workspace_path)
    summaries_dir = os.path.join(workspace_path, "memory", "summaries")
    if period == "weekly":
        return os.path.join(summaries_dir, datetime.now().strftime("%Y-W%W") + ".md")
    return os.path.join(summaries_dir, datetime.now().strftime("%Y-%m-%d") + ".md")


def weekday_for(day_name):
    """Return Python weekday (0=Monday, 6=Sunday) for day name."""
    d = (day_name or "Sunday").strip().lower()
    return {"monday": 0, "tuesday": 1, "wednesday": 2, "thursday": 3, "friday": 4, "saturday": 5, "sunday": 6}.get(d, 6)
