#!/usr/bin/env bash
# Run release gate, cool down MLX, restart backend if needed, then stability soak.
#
# Usage:
#   ./scripts/release_gate_then_soak.sh
#   MLX_COOLDOWN_MINUTES=20 ./scripts/release_gate_then_soak.sh
#
# Logs to /tmp/mugi_gate_soak_<timestamp>.log
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$REPO_ROOT"

VENV_PYTHON="${HOME}/.mugi/venv/bin/python"
PORT=8321
COOLDOWN_MIN="${MLX_COOLDOWN_MINUTES:-15}"
SOAK_HOURS="${SOAK_HOURS:-2}"
TS="$(date +%Y%m%d_%H%M%S)"
LOG="/tmp/mugi_gate_soak_${TS}.log"
GATE_JSON="/tmp/release_gate_${TS}.json"
SOAK_JSON="/tmp/soak_${TS}.json"

exec > >(tee -a "$LOG") 2>&1

echo "mugi gate+soak run started at $(date)"
echo "log=$LOG"
echo "gate_json=$GATE_JSON"
echo "soak_json=$SOAK_JSON"
echo "mlx_cooldown_minutes=$COOLDOWN_MIN soak_hours=$SOAK_HOURS"

_health_ok() {
  curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1
}

_wait_backend() {
  local tries="${1:-90}"
  for ((i = 1; i <= tries; i++)); do
    if _health_ok; then
      echo "backend healthy on port ${PORT} (attempt ${i})"
      return 0
    fi
    sleep 5
  done
  echo "ERROR: backend not healthy on port ${PORT} after ${tries} attempts" >&2
  return 1
}

_start_backend() {
  if _health_ok; then
    echo "backend already running on port ${PORT}"
    return 0
  fi
  echo "starting backend on port ${PORT}..."
  local token
  token="$("$VENV_PYTHON" -c "import secrets; print(secrets.token_hex(32))")"
  MUGI_AUTH_TOKEN="$token" "$VENV_PYTHON" mugi_backend.py --port "$PORT" &
  local pid=$!
  echo "$pid" >/tmp/mugi_backend_gate_soak.pid
  echo "backend pid=$pid"
  _wait_backend 90
}

_stop_mlx() {
  echo "stopping MLX server (if running)..."
  "$VENV_PYTHON" - <<'PY' || true
import sys
sys.path.insert(0, ".")
try:
    from app_mlx import mlx_server_running, stop_mlx_server
    if mlx_server_running():
        stop_mlx_server()
        print("MLX server stopped")
    else:
        print("MLX server was not running")
except Exception as exc:
    print(f"MLX stop helper: {exc}")
PY
  # Best-effort: terminate orphan mlx_vlm listeners on the MLX port.
  if command -v lsof >/dev/null 2>&1; then
    for pid in $(lsof -ti :8000 2>/dev/null || true); do
      echo "sending SIGTERM to pid ${pid} on :8000"
      kill -TERM "$pid" 2>/dev/null || true
    done
  fi
}

echo ""
echo "========== PHASE 1: RELEASE GATE =========="
_start_backend
if ! "$VENV_PYTHON" scripts/release_gate.py --json "$GATE_JSON"; then
  echo "RELEASE GATE FAILED — skipping soak. See $LOG and $GATE_JSON"
  exit 1
fi

echo ""
echo "========== PHASE 2: MLX COOLDOWN (${COOLDOWN_MIN} min) =========="
_stop_mlx
for ((m = 1; m <= COOLDOWN_MIN; m++)); do
  echo "cooldown ${m}/${COOLDOWN_MIN} min ($(date +%H:%M:%S))"
  sleep 60
done

echo ""
echo "========== PHASE 3: RESTART BACKEND FOR SOAK =========="
# test-swift / live batteries may have killed the standalone backend.
if ! _health_ok; then
  _start_backend
else
  echo "backend still healthy after gate"
fi

echo ""
echo "========== PHASE 4: STABILITY SOAK (${SOAK_HOURS}h) =========="
if ! "$VENV_PYTHON" scripts/soak.py --hours "$SOAK_HOURS" --json "$SOAK_JSON"; then
  echo "SOAK FAILED. See $LOG and $SOAK_JSON"
  exit 1
fi

echo ""
echo "GATE+SOAK: PASS at $(date)"
echo "log=$LOG"
echo "gate_json=$GATE_JSON"
echo "soak_json=$SOAK_JSON"
