458 lines
18 KiB
Python
458 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DATA_ROOT = ROOT / ".data_uat_preflight"
|
|
DEFAULT_EVIDENCE_DIR = ROOT / "docs" / "uat" / "evidence"
|
|
|
|
SERVICE_SPECS = [
|
|
{"name": "auth", "module": "services.auth_service.app:app", "port": 48001},
|
|
{"name": "audit", "module": "services.audit_service.app:app", "port": 48002},
|
|
{"name": "customer", "module": "services.customer_service.app:app", "port": 48003},
|
|
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 48004},
|
|
{"name": "routing", "module": "services.routing_service.app:app", "port": 48005},
|
|
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 48006},
|
|
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 48007},
|
|
{"name": "kb", "module": "services.kb_service.app:app", "port": 48008},
|
|
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 48009},
|
|
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 48010},
|
|
{"name": "gateway", "module": "gateway.app:app", "port": 48080},
|
|
]
|
|
|
|
REQUIRED_SERVICES = [
|
|
"auth",
|
|
"audit",
|
|
"customer",
|
|
"interaction",
|
|
"routing",
|
|
"voice",
|
|
"telegram",
|
|
"kb",
|
|
"reporting",
|
|
"supervisor",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class CheckResult:
|
|
name: str
|
|
ok: bool
|
|
details: str
|
|
|
|
|
|
def utc_now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def now_compact() -> str:
|
|
return utc_now().strftime("%Y%m%d_%H%M%S")
|
|
|
|
|
|
def iso_now() -> str:
|
|
return utc_now().isoformat()
|
|
|
|
|
|
async def wait_for_health(base_url: str, retries: int = 80, delay: float = 0.2) -> None:
|
|
async with httpx.AsyncClient(timeout=2) as client:
|
|
for _ in range(retries):
|
|
try:
|
|
response = await client.get(f"{base_url}/health")
|
|
if response.status_code == 200:
|
|
return
|
|
except Exception:
|
|
pass
|
|
await asyncio.sleep(delay)
|
|
raise RuntimeError(f"Service not healthy: {base_url}/health")
|
|
|
|
|
|
def start_one_service(spec: dict[str, Any], data_dir: Path) -> subprocess.Popen:
|
|
env = os.environ.copy()
|
|
env["CC_DATA_DIR"] = str(data_dir)
|
|
|
|
if spec["name"] == "gateway":
|
|
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:48001"
|
|
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:48002"
|
|
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:48003"
|
|
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:48004"
|
|
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:48005"
|
|
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:48006"
|
|
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:48007"
|
|
env["KB_SERVICE_URL"] = "http://127.0.0.1:48008"
|
|
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:48009"
|
|
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:48010"
|
|
|
|
cmd = [
|
|
sys.executable,
|
|
"-m",
|
|
"uvicorn",
|
|
spec["module"],
|
|
"--host",
|
|
"127.0.0.1",
|
|
"--port",
|
|
str(spec["port"]),
|
|
]
|
|
return subprocess.Popen(
|
|
cmd,
|
|
cwd=str(ROOT),
|
|
env=env,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
|
|
|
|
async def start_services_sequential(data_dir: Path) -> list[subprocess.Popen]:
|
|
data_dir.mkdir(parents=True, exist_ok=True)
|
|
processes: list[subprocess.Popen] = []
|
|
for spec in SERVICE_SPECS:
|
|
proc = start_one_service(spec, data_dir)
|
|
processes.append(proc)
|
|
await wait_for_health(f"http://127.0.0.1:{spec['port']}")
|
|
return processes
|
|
|
|
|
|
def stop_services(processes: list[subprocess.Popen]) -> None:
|
|
for proc in processes:
|
|
if proc.poll() is not None:
|
|
continue
|
|
try:
|
|
proc.terminate()
|
|
except Exception:
|
|
pass
|
|
|
|
for proc in processes:
|
|
try:
|
|
proc.wait(timeout=2)
|
|
except Exception:
|
|
try:
|
|
proc.kill()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
async def run_checks(base_url: str) -> tuple[list[CheckResult], dict[str, Any]]:
|
|
checks: list[CheckResult] = []
|
|
artifacts: dict[str, Any] = {}
|
|
admin = {"X-User": "admin", "X-Role": "admin"}
|
|
supervisor = {"X-User": "supervisor", "X-Role": "supervisor"}
|
|
operator = {"X-User": "operator", "X-Role": "operator"}
|
|
analyst = {"X-User": "analyst", "X-Role": "analyst"}
|
|
|
|
async with httpx.AsyncClient(base_url=base_url, timeout=10) as client:
|
|
try:
|
|
health = await client.get("/health")
|
|
ok = health.status_code == 200
|
|
checks.append(CheckResult("Gateway health", ok, f"status={health.status_code}"))
|
|
if not ok:
|
|
return checks, artifacts
|
|
except Exception as exc:
|
|
checks.append(CheckResult("Gateway health", False, f"exception={exc}"))
|
|
return checks, artifacts
|
|
|
|
try:
|
|
registry = await client.get("/registry")
|
|
data = registry.json() if registry.status_code == 200 else {}
|
|
services = data.get("services", {})
|
|
missing = [name for name in REQUIRED_SERVICES if name not in services]
|
|
ok = registry.status_code == 200 and not missing
|
|
details = f"status={registry.status_code}"
|
|
if missing:
|
|
details += f", missing={missing}"
|
|
checks.append(CheckResult("Service registry", ok, details))
|
|
if not ok:
|
|
return checks, artifacts
|
|
except Exception as exc:
|
|
checks.append(CheckResult("Service registry", False, f"exception={exc}"))
|
|
return checks, artifacts
|
|
|
|
try:
|
|
login = await client.post("/proxy/auth/auth/login", json={"username": "admin", "password": "admin123"})
|
|
ok = login.status_code == 200 and "access_token" in login.json()
|
|
checks.append(CheckResult("Auth login", ok, f"status={login.status_code}"))
|
|
except Exception as exc:
|
|
checks.append(CheckResult("Auth login", False, f"exception={exc}"))
|
|
return checks, artifacts
|
|
|
|
try:
|
|
queue = await client.post(
|
|
"/proxy/routing/queues",
|
|
headers=admin,
|
|
json={
|
|
"name": f"UAT Queue {now_compact()}",
|
|
"description": "UAT preflight queue",
|
|
"rules": [
|
|
{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 30},
|
|
{"channel": "telegram", "priority": 3, "strategy": "round_robin", "sla_seconds": 45},
|
|
],
|
|
},
|
|
)
|
|
queue_ok = queue.status_code == 200
|
|
queue_id = queue.json().get("queue_id") if queue_ok else ""
|
|
artifacts["queue_id"] = queue_id
|
|
checks.append(CheckResult("Queue create", queue_ok, f"status={queue.status_code}, queue_id={queue_id}"))
|
|
except Exception as exc:
|
|
checks.append(CheckResult("Queue create", False, f"exception={exc}"))
|
|
return checks, artifacts
|
|
|
|
try:
|
|
customer = await client.post(
|
|
"/proxy/customer/customers",
|
|
json={
|
|
"display_name": "UAT Preflight Customer",
|
|
"phones": ["+77019990000"],
|
|
"preferred_phone": "+77019990000",
|
|
"tags": ["uat", "preflight"],
|
|
},
|
|
)
|
|
customer_ok = customer.status_code == 200
|
|
customer_id = customer.json().get("customer_id") if customer_ok else ""
|
|
artifacts["customer_id"] = customer_id
|
|
checks.append(
|
|
CheckResult("Customer create", customer_ok, f"status={customer.status_code}, customer_id={customer_id}")
|
|
)
|
|
except Exception as exc:
|
|
checks.append(CheckResult("Customer create", False, f"exception={exc}"))
|
|
return checks, artifacts
|
|
|
|
interaction_id = ""
|
|
try:
|
|
interaction = await client.post(
|
|
"/proxy/interaction/interactions",
|
|
headers=operator,
|
|
json={
|
|
"channel": "voice",
|
|
"subject": "UAT preflight voice interaction",
|
|
"customer_id": artifacts["customer_id"],
|
|
"queue_id": artifacts["queue_id"],
|
|
"priority": 3,
|
|
},
|
|
)
|
|
ok = interaction.status_code == 200
|
|
interaction_id = interaction.json().get("interaction_id") if ok else ""
|
|
artifacts["voice_interaction_id"] = interaction_id
|
|
checks.append(CheckResult("Voice interaction create", ok, f"status={interaction.status_code}"))
|
|
if not ok:
|
|
return checks, artifacts
|
|
|
|
assign = await client.patch(
|
|
f"/proxy/interaction/interactions/{interaction_id}/assign",
|
|
headers=supervisor,
|
|
json={"assignee": "operator_a"},
|
|
)
|
|
assign_ok = assign.status_code == 200 and assign.json().get("status") == "in_progress"
|
|
checks.append(CheckResult("Voice interaction assign", assign_ok, f"status={assign.status_code}"))
|
|
|
|
close = await client.patch(
|
|
f"/proxy/interaction/interactions/{interaction_id}/status",
|
|
headers=operator,
|
|
json={"status": "closed"},
|
|
)
|
|
close_ok = close.status_code == 200 and close.json().get("status") == "closed"
|
|
checks.append(CheckResult("Voice interaction close", close_ok, f"status={close.status_code}"))
|
|
except Exception as exc:
|
|
checks.append(CheckResult("Voice interaction lifecycle", False, f"exception={exc}"))
|
|
return checks, artifacts
|
|
|
|
try:
|
|
tg = await client.post(
|
|
"/proxy/telegram/integrations/telegram/webhook",
|
|
json={
|
|
"chat_id": f"uat_chat_{now_compact()}",
|
|
"text": "UAT preflight telegram",
|
|
"customer_external_id": None,
|
|
"payload": {"source": "uat_preflight"},
|
|
},
|
|
)
|
|
ok = tg.status_code == 200 and tg.json().get("message_id")
|
|
artifacts["telegram_message_id"] = tg.json().get("message_id") if tg.status_code == 200 else ""
|
|
checks.append(CheckResult("Telegram webhook", bool(ok), f"status={tg.status_code}"))
|
|
except Exception as exc:
|
|
checks.append(CheckResult("Telegram webhook", False, f"exception={exc}"))
|
|
return checks, artifacts
|
|
|
|
try:
|
|
cat = await client.post(
|
|
"/proxy/kb/knowledge/categories",
|
|
headers=analyst,
|
|
json={"name": f"UAT {now_compact()}", "description": "UAT preflight"},
|
|
)
|
|
cat_ok = cat.status_code == 200
|
|
category_id = cat.json().get("category_id") if cat_ok else ""
|
|
if not cat_ok:
|
|
checks.append(CheckResult("KB category", False, f"status={cat.status_code}"))
|
|
return checks, artifacts
|
|
|
|
art = await client.post(
|
|
"/proxy/kb/knowledge/articles",
|
|
headers=analyst,
|
|
json={
|
|
"category_id": category_id,
|
|
"title": "UAT preflight article",
|
|
"body": "Use this article during UAT flow.",
|
|
"tags": ["uat", "kb"],
|
|
},
|
|
)
|
|
art_ok = art.status_code == 200
|
|
checks.append(CheckResult("KB article", art_ok, f"status={art.status_code}"))
|
|
if not art_ok:
|
|
return checks, artifacts
|
|
|
|
search = await client.get("/proxy/kb/knowledge/search?q=preflight")
|
|
search_ok = search.status_code == 200 and len(search.json()) > 0
|
|
checks.append(CheckResult("KB search", search_ok, f"status={search.status_code}, count={len(search.json())}"))
|
|
except Exception as exc:
|
|
checks.append(CheckResult("KB checks", False, f"exception={exc}"))
|
|
return checks, artifacts
|
|
|
|
try:
|
|
ingest = await client.post(
|
|
"/proxy/reporting/reports/events",
|
|
json={
|
|
"queue_id": artifacts["queue_id"],
|
|
"answered": True,
|
|
"wait_seconds": 18,
|
|
"handle_seconds": 105,
|
|
"abandoned": False,
|
|
"resolved_first_contact": True,
|
|
},
|
|
)
|
|
ingest_ok = ingest.status_code == 200
|
|
checks.append(CheckResult("KPI event ingest", ingest_ok, f"status={ingest.status_code}"))
|
|
if not ingest_ok:
|
|
return checks, artifacts
|
|
|
|
kpi = await client.get(f"/proxy/reporting/reports/kpi?queue_id={artifacts['queue_id']}")
|
|
kpi_ok = kpi.status_code == 200 and "kpi" in kpi.json()
|
|
checks.append(CheckResult("KPI report", kpi_ok, f"status={kpi.status_code}"))
|
|
except Exception as exc:
|
|
checks.append(CheckResult("KPI checks", False, f"exception={exc}"))
|
|
return checks, artifacts
|
|
|
|
try:
|
|
up1 = await client.post(
|
|
"/proxy/supervisor/supervisor/agent-states",
|
|
json={"agent_id": "uat_agent_1", "state": "READY", "queue_id": artifacts["queue_id"]},
|
|
)
|
|
up2 = await client.post(
|
|
"/proxy/supervisor/supervisor/agent-states",
|
|
json={"agent_id": "uat_agent_2", "state": "BUSY", "queue_id": artifacts["queue_id"]},
|
|
)
|
|
rt = await client.get("/proxy/supervisor/supervisor/realtime")
|
|
realtime_ok = up1.status_code == 200 and up2.status_code == 200 and rt.status_code == 200
|
|
checks.append(CheckResult("Supervisor realtime", realtime_ok, f"status={rt.status_code}"))
|
|
except Exception as exc:
|
|
checks.append(CheckResult("Supervisor realtime", False, f"exception={exc}"))
|
|
return checks, artifacts
|
|
|
|
try:
|
|
audit = await client.get("/proxy/audit/audit/events?limit=5")
|
|
ok = audit.status_code == 200
|
|
checks.append(CheckResult("Audit availability", ok, f"status={audit.status_code}"))
|
|
except Exception as exc:
|
|
checks.append(CheckResult("Audit availability", False, f"exception={exc}"))
|
|
|
|
artifacts["completed_at"] = iso_now()
|
|
return checks, artifacts
|
|
|
|
|
|
def write_report(
|
|
output_path: Path,
|
|
base_url: str,
|
|
checks: list[CheckResult],
|
|
artifacts: dict[str, Any],
|
|
auto_start: bool,
|
|
) -> None:
|
|
ok_count = sum(1 for c in checks if c.ok)
|
|
total = len(checks)
|
|
overall = "PASS" if ok_count == total else "FAIL"
|
|
|
|
lines: list[str] = []
|
|
lines.append("# UAT Preflight Report")
|
|
lines.append("")
|
|
lines.append(f"- Timestamp (UTC): {iso_now()}")
|
|
lines.append(f"- Base URL: {base_url}")
|
|
lines.append(f"- Auto-start mode: {str(auto_start).lower()}")
|
|
lines.append(f"- Overall: {overall} ({ok_count}/{total})")
|
|
lines.append("")
|
|
lines.append("## Scope Baseline")
|
|
lines.append("- Pilot scope freeze: `docs/gates/mvp-pilot-baseline.md`")
|
|
lines.append("- Deferred scope backlog: `docs/roadmap/05-wave2-backlog.md`")
|
|
lines.append("")
|
|
lines.append("## Checks")
|
|
for item in checks:
|
|
marker = "PASS" if item.ok else "FAIL"
|
|
lines.append(f"- [{marker}] {item.name}: {item.details}")
|
|
lines.append("")
|
|
lines.append("## Artifacts")
|
|
if artifacts:
|
|
for key, value in artifacts.items():
|
|
lines.append(f"- {key}: {value}")
|
|
else:
|
|
lines.append("- none")
|
|
lines.append("")
|
|
lines.append("## Next Action")
|
|
lines.append("- If all checks passed, run `scripts/uat_dry_run.py` and prepare the manual UAT session.")
|
|
lines.append("")
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
output_path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
async def main() -> None:
|
|
parser = argparse.ArgumentParser(description="UAT preflight checker for pilot session")
|
|
parser.add_argument("--base-url", default="http://localhost:8080", help="Gateway base URL")
|
|
parser.add_argument("--auto-start", action="store_true", help="Auto-start local services for preflight")
|
|
parser.add_argument(
|
|
"--output",
|
|
default="",
|
|
help="Report output path (default: docs/uat/evidence/preflight_<timestamp>.md)",
|
|
)
|
|
parser.add_argument("--keep-data", action="store_true", help="Keep temporary auto-start data directory")
|
|
args = parser.parse_args()
|
|
|
|
output = Path(args.output) if args.output else DEFAULT_EVIDENCE_DIR / f"preflight_{now_compact()}.md"
|
|
output = output if output.is_absolute() else (ROOT / output).resolve()
|
|
|
|
processes: list[subprocess.Popen] = []
|
|
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
|
|
base_url = args.base_url
|
|
checks: list[CheckResult] = []
|
|
artifacts: dict[str, Any] = {}
|
|
try:
|
|
if args.auto_start:
|
|
processes = await start_services_sequential(run_data_dir)
|
|
base_url = "http://127.0.0.1:48080"
|
|
|
|
checks, artifacts = await run_checks(base_url)
|
|
write_report(output, base_url, checks, artifacts, args.auto_start)
|
|
|
|
ok = all(item.ok for item in checks) and len(checks) > 0
|
|
print(f"UAT preflight report: {output}")
|
|
print(f"Result: {'PASS' if ok else 'FAIL'}")
|
|
sys.exit(0 if ok else 1)
|
|
finally:
|
|
stop_services(processes)
|
|
if args.auto_start and not args.keep_data:
|
|
try:
|
|
if run_data_dir.exists():
|
|
shutil.rmtree(run_data_dir)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|