chore: remove dead supervisor analytics and archived tooling
This commit is contained in:
Yera All
@@ -1,283 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
GATE4_PATH = ROOT / "docs" / "gates" / "gate-04-pilot-hardening.md"
|
||||
P1P2_PATH = ROOT / "docs" / "gates" / "p1-p2-defects.md"
|
||||
ACCEPTED_RELEASE_PATH = ROOT / "docs" / "releases" / "v1.0.0-mvp-accepted.md"
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def iso_now() -> str:
|
||||
return utc_now().isoformat()
|
||||
|
||||
|
||||
def _normalized_status(value: str) -> str:
|
||||
return value.strip().lower()
|
||||
|
||||
|
||||
def _is_open_status(value: str) -> bool:
|
||||
normalized = _normalized_status(value)
|
||||
return normalized not in {"closed", "resolved", "verified"}
|
||||
|
||||
|
||||
def parse_open_defect_counts(defect_log_path: Path) -> dict[str, int]:
|
||||
counts = {"P1": 0, "P2": 0, "P3": 0, "P4": 0}
|
||||
with defect_log_path.open("r", newline="", encoding="utf-8") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
for row in reader:
|
||||
severity = (row.get("severity") or "").strip().upper()
|
||||
status = row.get("status") or ""
|
||||
if severity in counts and _is_open_status(status):
|
||||
counts[severity] += 1
|
||||
return counts
|
||||
|
||||
|
||||
def validate_session_protocol(session_protocol_path: Path) -> list[str]:
|
||||
content = session_protocol_path.read_text(encoding="utf-8")
|
||||
errors: list[str] = []
|
||||
blank_markers = [
|
||||
"- Session ID:",
|
||||
"- Environment URL:",
|
||||
"- Build/version:",
|
||||
"- Deployment date:",
|
||||
"- Cluster/namespace:",
|
||||
"- Business owner:",
|
||||
"- IT owner:",
|
||||
]
|
||||
for marker in blank_markers:
|
||||
if f"{marker}\n" in content or f"{marker}\r\n" in content:
|
||||
errors.append(f"Session protocol still contains blank field: {marker}")
|
||||
|
||||
if "## Decision" not in content or "[x]" not in content:
|
||||
errors.append("Session protocol does not contain a selected decision checkbox")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate_signoff_sheet(signoff_sheet_path: Path) -> list[str]:
|
||||
content = signoff_sheet_path.read_text(encoding="utf-8")
|
||||
errors: list[str] = []
|
||||
placeholder_lines = [
|
||||
" - Name:",
|
||||
" - Signature:",
|
||||
" - Date:",
|
||||
]
|
||||
for placeholder in placeholder_lines:
|
||||
if content.count(placeholder) > 0:
|
||||
errors.append(f"Sign-off sheet still contains placeholder lines matching: {placeholder.strip()}")
|
||||
break
|
||||
|
||||
decision_lines = [line for line in content.splitlines() if line.startswith("- [")]
|
||||
if sum(1 for line in decision_lines if "[x]" in line.lower()) != 1:
|
||||
errors.append("Sign-off sheet must have exactly one selected decision checkbox")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def apply_gate4_closure(
|
||||
content: str,
|
||||
*,
|
||||
session_id: str,
|
||||
session_dir: Path,
|
||||
acceptance: str,
|
||||
open_counts: dict[str, int],
|
||||
) -> str:
|
||||
updated = content
|
||||
updated = updated.replace(
|
||||
"- [ ] UAT signed with real operators/supervisors",
|
||||
"- [x] UAT signed with real operators/supervisors",
|
||||
)
|
||||
updated = updated.replace(
|
||||
"- [ ] P1/P2 defects closed",
|
||||
"- [x] P1/P2 defects closed",
|
||||
)
|
||||
|
||||
marker = "Manual closure update:"
|
||||
if marker in updated:
|
||||
updated = updated.split(marker, 1)[0].rstrip()
|
||||
|
||||
lines = [
|
||||
"",
|
||||
"Manual closure update:",
|
||||
f"- Date (UTC): {iso_now()}",
|
||||
f"- Session ID: {session_id}",
|
||||
f"- Session directory: `{session_dir.as_posix()}`",
|
||||
f"- Acceptance: {acceptance}",
|
||||
(
|
||||
"- Remaining open defects: "
|
||||
f"P1={open_counts['P1']}, P2={open_counts['P2']}, "
|
||||
f"P3={open_counts['P3']}, P4={open_counts['P4']}"
|
||||
),
|
||||
]
|
||||
return updated.rstrip() + "\n" + "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def build_p1p2_register(
|
||||
*,
|
||||
session_id: str,
|
||||
session_dir: Path,
|
||||
open_counts: dict[str, int],
|
||||
acceptance: str,
|
||||
) -> str:
|
||||
lines = [
|
||||
"# P1/P2 Defect Register (Pilot)",
|
||||
"",
|
||||
f"Last update: {utc_now().date().isoformat()} ({session_id})",
|
||||
"",
|
||||
"## Current Status",
|
||||
f"- Open P1: {open_counts['P1']}",
|
||||
f"- Open P2: {open_counts['P2']}",
|
||||
"- Source: manual UAT close-out",
|
||||
f"- Manual session evidence: `{session_dir.as_posix()}`",
|
||||
"",
|
||||
"## Triage Policy",
|
||||
"- Only `P1` and `P2` issues belong to MVP remediation.",
|
||||
"- `P3` and `P4` issues move to `docs/roadmap/05-wave2-backlog.md` unless they block sign-off.",
|
||||
"",
|
||||
"## Manual Closure",
|
||||
f"- Acceptance: {acceptance}",
|
||||
"- P1/P2 closure confirmed by signed manual UAT package.",
|
||||
]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def build_acceptance_release_note(
|
||||
*,
|
||||
session_id: str,
|
||||
session_dir: Path,
|
||||
acceptance: str,
|
||||
open_counts: dict[str, int],
|
||||
) -> str:
|
||||
lines = [
|
||||
"# Release Notes - v1.0.0-mvp Accepted",
|
||||
"",
|
||||
f"Date: {utc_now().date().isoformat()}",
|
||||
"",
|
||||
"## Acceptance Result",
|
||||
f"- Session ID: {session_id}",
|
||||
f"- Acceptance: {acceptance}",
|
||||
f"- Manual session evidence: `{session_dir.as_posix()}`",
|
||||
"",
|
||||
"## Closed Gate 4 Conditions",
|
||||
"- UAT signed with real operators/supervisors",
|
||||
"- P1/P2 defects closed",
|
||||
"",
|
||||
"## Remaining Deferred Items",
|
||||
f"- Open P3: {open_counts['P3']}",
|
||||
f"- Open P4: {open_counts['P4']}",
|
||||
"- Deferred scope remains tracked in `docs/roadmap/05-wave2-backlog.md`.",
|
||||
"",
|
||||
"## Baseline",
|
||||
"- Accepted baseline remains the current MVP v1 scope without API expansion.",
|
||||
"- Stage 1-3 public routes remain unchanged.",
|
||||
]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def finalize_manual_session(
|
||||
*,
|
||||
session_dir: Path,
|
||||
acceptance: str,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, int]:
|
||||
required_files = {
|
||||
"session_protocol": session_dir / "session-protocol.md",
|
||||
"scenario_checklist": session_dir / "scenario-checklist.md",
|
||||
"defect_log": session_dir / "defect-log.csv",
|
||||
"signoff_sheet": session_dir / "signoff-sheet.md",
|
||||
"manifest": session_dir / "manifest.json",
|
||||
}
|
||||
missing = [name for name, path in required_files.items() if not path.exists()]
|
||||
attachments_dir = session_dir / "attachments"
|
||||
if not attachments_dir.exists():
|
||||
missing.append("attachments")
|
||||
if missing:
|
||||
raise FileNotFoundError(f"Manual session bundle is incomplete: {missing}")
|
||||
|
||||
validation_errors: list[str] = []
|
||||
validation_errors.extend(validate_session_protocol(required_files["session_protocol"]))
|
||||
validation_errors.extend(validate_signoff_sheet(required_files["signoff_sheet"]))
|
||||
open_counts = parse_open_defect_counts(required_files["defect_log"])
|
||||
if open_counts["P1"] != 0 or open_counts["P2"] != 0:
|
||||
validation_errors.append(
|
||||
f"Cannot close MVP pilot with open P1/P2 defects: P1={open_counts['P1']}, P2={open_counts['P2']}"
|
||||
)
|
||||
|
||||
if validation_errors:
|
||||
details = "\n".join(f"- {error}" for error in validation_errors)
|
||||
raise ValueError(f"Manual UAT package validation failed:\n{details}")
|
||||
|
||||
if dry_run:
|
||||
return open_counts
|
||||
|
||||
session_id = session_dir.name.replace("manual_", "", 1)
|
||||
gate4_content = GATE4_PATH.read_text(encoding="utf-8")
|
||||
updated_gate4 = apply_gate4_closure(
|
||||
gate4_content,
|
||||
session_id=session_id,
|
||||
session_dir=session_dir,
|
||||
acceptance=acceptance,
|
||||
open_counts=open_counts,
|
||||
)
|
||||
GATE4_PATH.write_text(updated_gate4, encoding="utf-8")
|
||||
P1P2_PATH.write_text(
|
||||
build_p1p2_register(
|
||||
session_id=session_id,
|
||||
session_dir=session_dir,
|
||||
open_counts=open_counts,
|
||||
acceptance=acceptance,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
ACCEPTED_RELEASE_PATH.write_text(
|
||||
build_acceptance_release_note(
|
||||
session_id=session_id,
|
||||
session_dir=session_dir,
|
||||
acceptance=acceptance,
|
||||
open_counts=open_counts,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return open_counts
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Validate a manual UAT bundle and close MVP Gate 4")
|
||||
parser.add_argument("--session-dir", required=True, help="Path to the prepared manual session directory")
|
||||
parser.add_argument(
|
||||
"--acceptance",
|
||||
choices=["accepted_for_pilot_completion", "accepted_with_conditions"],
|
||||
default="accepted_for_pilot_completion",
|
||||
help="Acceptance mode recorded in gate closure artifacts",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Validate the manual session bundle without updating gate/release documents",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
session_dir = Path(args.session_dir)
|
||||
if not session_dir.is_absolute():
|
||||
session_dir = (ROOT / session_dir).resolve()
|
||||
|
||||
counts = finalize_manual_session(
|
||||
session_dir=session_dir,
|
||||
acceptance=args.acceptance,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
prefix = "MVP pilot finalization dry-run passed:" if args.dry_run else "MVP pilot finalized:"
|
||||
print(f"{prefix} P1={counts['P1']} P2={counts['P2']} P3={counts['P3']} P4={counts['P4']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,64 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
REQUIRED_SERVICES = [
|
||||
"auth",
|
||||
"audit",
|
||||
"customer",
|
||||
"interaction",
|
||||
"routing",
|
||||
"voice",
|
||||
"telegram",
|
||||
"kb",
|
||||
"reporting",
|
||||
"supervisor",
|
||||
]
|
||||
|
||||
|
||||
async def check_gateway(base_url: str) -> bool:
|
||||
async with httpx.AsyncClient(base_url=base_url, timeout=5) as client:
|
||||
try:
|
||||
health = await client.get("/health")
|
||||
if health.status_code != 200:
|
||||
print(f"[FAIL] gateway /health status={health.status_code}")
|
||||
return False
|
||||
|
||||
registry = await client.get("/registry")
|
||||
if registry.status_code != 200:
|
||||
print(f"[FAIL] gateway /registry status={registry.status_code}")
|
||||
return False
|
||||
data = registry.json()
|
||||
services = data.get("services", {})
|
||||
missing = [s for s in REQUIRED_SERVICES if s not in services]
|
||||
if missing:
|
||||
print(f"[FAIL] registry missing services: {missing}")
|
||||
return False
|
||||
|
||||
login = await client.post(
|
||||
"/proxy/auth/auth/login",
|
||||
json={"username": "admin", "password": "admin123"},
|
||||
)
|
||||
if login.status_code != 200:
|
||||
print(f"[FAIL] auth login status={login.status_code}")
|
||||
return False
|
||||
|
||||
print("[PASS] Stage 1 gateway/auth health checks passed")
|
||||
return True
|
||||
except Exception as exc:
|
||||
print(f"[FAIL] exception: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
base_url = "http://localhost:8080"
|
||||
ok = await check_gateway(base_url)
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,303 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DATA_ROOT = ROOT / ".data_gate3"
|
||||
|
||||
SERVICE_SPECS = [
|
||||
{"name": "auth", "module": "services.auth_service.app:app", "port": 28001},
|
||||
{"name": "audit", "module": "services.audit_service.app:app", "port": 28002},
|
||||
{"name": "customer", "module": "services.customer_service.app:app", "port": 28003},
|
||||
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 28004},
|
||||
{"name": "routing", "module": "services.routing_service.app:app", "port": 28005},
|
||||
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 28006},
|
||||
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 28007},
|
||||
{"name": "kb", "module": "services.kb_service.app:app", "port": 28008},
|
||||
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 28009},
|
||||
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 28010},
|
||||
{"name": "gateway", "module": "gateway.app:app", "port": 28080},
|
||||
]
|
||||
|
||||
REQUIRED_STAGE3_SERVICES = ["kb", "reporting", "supervisor"]
|
||||
|
||||
|
||||
async def wait_for_health(base_url: str, retries: int = 80, delay: float = 0.25) -> None:
|
||||
async with httpx.AsyncClient(timeout=2) as client:
|
||||
for _ in range(retries):
|
||||
try:
|
||||
response = await client.get(f"{base_url}/health")
|
||||
if response.status_code == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(delay)
|
||||
raise RuntimeError(f"Service not healthy: {base_url}/health")
|
||||
|
||||
|
||||
def start_one_service(spec: dict[str, Any], database_url: str | None, data_dir: Path) -> subprocess.Popen:
|
||||
env = os.environ.copy()
|
||||
env["CC_DATA_DIR"] = str(data_dir)
|
||||
if database_url:
|
||||
env["DATABASE_URL"] = database_url
|
||||
|
||||
if spec["name"] == "gateway":
|
||||
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:28001"
|
||||
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:28002"
|
||||
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:28003"
|
||||
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:28004"
|
||||
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:28005"
|
||||
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:28006"
|
||||
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:28007"
|
||||
env["KB_SERVICE_URL"] = "http://127.0.0.1:28008"
|
||||
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:28009"
|
||||
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:28010"
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uvicorn",
|
||||
spec["module"],
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(spec["port"]),
|
||||
]
|
||||
return subprocess.Popen(
|
||||
cmd,
|
||||
cwd=str(ROOT),
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
async def start_services_sequential(database_url: str | None, data_dir: Path) -> list[subprocess.Popen]:
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
processes: list[subprocess.Popen] = []
|
||||
for spec in SERVICE_SPECS:
|
||||
proc = start_one_service(spec, database_url, data_dir)
|
||||
processes.append(proc)
|
||||
await wait_for_health(f"http://127.0.0.1:{spec['port']}", retries=60, delay=0.2)
|
||||
return processes
|
||||
|
||||
|
||||
def stop_services(processes: list[subprocess.Popen]) -> None:
|
||||
for proc in processes:
|
||||
if proc.poll() is not None:
|
||||
continue
|
||||
try:
|
||||
proc.terminate()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for proc in processes:
|
||||
try:
|
||||
proc.wait(timeout=2)
|
||||
except Exception:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def run_gate3_checks(base_url: str, reporting_url: str | None = None) -> bool:
|
||||
contract_path = ROOT / "contracts" / "openapi" / "stage3-kb-reporting-supervisor.yaml"
|
||||
if not contract_path.exists():
|
||||
print(f"[FAIL] Missing Stage 3 OpenAPI: {contract_path}")
|
||||
return False
|
||||
print(f"[PASS] Stage 3 OpenAPI present: {contract_path.name}")
|
||||
|
||||
operator = {"X-User": "operator", "X-Role": "operator"}
|
||||
analyst = {"X-User": "analyst", "X-Role": "analyst"}
|
||||
|
||||
async with httpx.AsyncClient(base_url=base_url, timeout=10) as client:
|
||||
health = await client.get("/health")
|
||||
if health.status_code != 200:
|
||||
print(f"[FAIL] gateway /health status={health.status_code}")
|
||||
return False
|
||||
|
||||
registry = await client.get("/registry")
|
||||
if registry.status_code != 200:
|
||||
print(f"[FAIL] gateway /registry status={registry.status_code}")
|
||||
return False
|
||||
services = registry.json().get("services", {})
|
||||
missing = [name for name in REQUIRED_STAGE3_SERVICES if name not in services]
|
||||
if missing:
|
||||
print(f"[FAIL] registry missing Stage 3 services: {missing}")
|
||||
return False
|
||||
print("[PASS] Gateway and registry checks")
|
||||
|
||||
denied = await client.post(
|
||||
"/proxy/kb/knowledge/categories",
|
||||
headers=operator,
|
||||
json={"name": "Forbidden", "description": "Role check"},
|
||||
)
|
||||
if denied.status_code != 403:
|
||||
print(f"[FAIL] KB role check expected 403, got {denied.status_code}")
|
||||
return False
|
||||
|
||||
category = await client.post(
|
||||
"/proxy/kb/knowledge/categories",
|
||||
headers=analyst,
|
||||
json={"name": "Stage 3 KB", "description": "Gate 3"},
|
||||
)
|
||||
if category.status_code != 200:
|
||||
print(f"[FAIL] KB category create status={category.status_code}")
|
||||
return False
|
||||
category_id = category.json()["category_id"]
|
||||
|
||||
article = await client.post(
|
||||
"/proxy/kb/knowledge/articles",
|
||||
headers=analyst,
|
||||
json={
|
||||
"category_id": category_id,
|
||||
"title": "Install helper",
|
||||
"body": "Step-by-step instruction for operator",
|
||||
"tags": ["stage3", "kb"],
|
||||
},
|
||||
)
|
||||
if article.status_code != 200:
|
||||
print(f"[FAIL] KB article create status={article.status_code}")
|
||||
return False
|
||||
|
||||
search = await client.get("/proxy/kb/knowledge/search?q=helper")
|
||||
if search.status_code != 200 or len(search.json()) < 1:
|
||||
print(
|
||||
f"[FAIL] KB search failed status={search.status_code} count={len(search.json()) if search.status_code == 200 else 0}"
|
||||
)
|
||||
return False
|
||||
print("[PASS] KB checks")
|
||||
|
||||
rows = [
|
||||
{
|
||||
"queue_id": "q_gate3",
|
||||
"answered": True,
|
||||
"wait_seconds": 15,
|
||||
"handle_seconds": 95,
|
||||
"abandoned": False,
|
||||
"resolved_first_contact": True,
|
||||
},
|
||||
{
|
||||
"queue_id": "q_gate3",
|
||||
"answered": False,
|
||||
"wait_seconds": 20,
|
||||
"handle_seconds": 0,
|
||||
"abandoned": True,
|
||||
"resolved_first_contact": False,
|
||||
},
|
||||
]
|
||||
for row in rows:
|
||||
ingested = await client.post("/proxy/reporting/reports/events", json=row)
|
||||
if ingested.status_code != 200:
|
||||
print(f"[FAIL] reporting ingest status={ingested.status_code}")
|
||||
return False
|
||||
|
||||
kpi = await client.get("/proxy/reporting/reports/kpi?queue_id=q_gate3&sl_threshold_seconds=30")
|
||||
if kpi.status_code != 200:
|
||||
print(f"[FAIL] reporting KPI status={kpi.status_code}")
|
||||
return False
|
||||
kpi_body = kpi.json()
|
||||
if "kpi" not in kpi_body or not {"SL", "ASA", "AHT", "Abandon", "FCR"}.issubset(set(kpi_body["kpi"])):
|
||||
print("[FAIL] KPI payload missing required keys")
|
||||
return False
|
||||
print("[PASS] Reporting KPI checks")
|
||||
|
||||
report_service_url = reporting_url or services.get("reporting")
|
||||
if not report_service_url:
|
||||
print("[FAIL] reporting service URL not found")
|
||||
return False
|
||||
|
||||
async with httpx.AsyncClient(base_url=report_service_url, timeout=10) as reporting_client:
|
||||
exported = await reporting_client.get("/reports/export")
|
||||
if exported.status_code != 200:
|
||||
print(f"[FAIL] reporting export status={exported.status_code}")
|
||||
return False
|
||||
if "queue_id,answered,wait_seconds,handle_seconds,abandoned,resolved_first_contact,created_at" not in exported.text:
|
||||
print("[FAIL] reporting export missing CSV header")
|
||||
return False
|
||||
if "q_gate3" not in exported.text:
|
||||
print("[FAIL] reporting export missing inserted data")
|
||||
return False
|
||||
print("[PASS] Reporting CSV export checks")
|
||||
|
||||
up1 = await client.post(
|
||||
"/proxy/supervisor/supervisor/agent-states",
|
||||
json={"agent_id": "a_gate3_1", "state": "READY", "queue_id": "q_gate3"},
|
||||
)
|
||||
up2 = await client.post(
|
||||
"/proxy/supervisor/supervisor/agent-states",
|
||||
json={"agent_id": "a_gate3_2", "state": "BUSY", "queue_id": "q_gate3"},
|
||||
)
|
||||
queue = await client.post(
|
||||
"/proxy/supervisor/supervisor/queue-metrics?queue_id=q_gate3&in_queue=3&avg_wait_seconds=21"
|
||||
)
|
||||
if up1.status_code != 200 or up2.status_code != 200 or queue.status_code != 200:
|
||||
print(
|
||||
f"[FAIL] supervisor updates status: agent1={up1.status_code}, agent2={up2.status_code}, queue={queue.status_code}"
|
||||
)
|
||||
return False
|
||||
|
||||
realtime = await client.get("/proxy/supervisor/supervisor/realtime")
|
||||
if realtime.status_code != 200:
|
||||
print(f"[FAIL] supervisor realtime status={realtime.status_code}")
|
||||
return False
|
||||
body: dict[str, Any] = realtime.json()
|
||||
if body.get("agents", {}).get("total", 0) < 2:
|
||||
print("[FAIL] supervisor realtime has less than 2 agents")
|
||||
return False
|
||||
queues = body.get("queues", [])
|
||||
if not any(item.get("queue_id") == "q_gate3" for item in queues):
|
||||
print("[FAIL] supervisor realtime missing q_gate3 queue snapshot")
|
||||
return False
|
||||
print("[PASS] Supervisor realtime checks")
|
||||
|
||||
print("[PASS] Gate 3 checks completed")
|
||||
return True
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Gate 3 checker")
|
||||
parser.add_argument("--base-url", default="http://localhost:8080", help="Gateway base URL")
|
||||
parser.add_argument("--auto-start", action="store_true", help="Auto-start local services for the check")
|
||||
parser.add_argument(
|
||||
"--database-url",
|
||||
default=os.getenv("DATABASE_URL", ""),
|
||||
help="Optional DB URL for auto-start mode",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.auto_start:
|
||||
ok = await run_gate3_checks(args.base_url)
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
db_url = args.database_url.strip() or None
|
||||
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
|
||||
processes: list[subprocess.Popen] = []
|
||||
try:
|
||||
processes = await start_services_sequential(db_url, run_data_dir)
|
||||
ok = await run_gate3_checks("http://127.0.0.1:28080", reporting_url="http://127.0.0.1:28009")
|
||||
if db_url:
|
||||
print(f"[INFO] DB mode: {db_url}")
|
||||
sys.exit(0 if ok else 1)
|
||||
finally:
|
||||
stop_services(processes)
|
||||
try:
|
||||
shutil.rmtree(run_data_dir)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,350 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DATA_ROOT = ROOT / ".data_gate4"
|
||||
|
||||
SERVICE_SPECS = [
|
||||
{"name": "auth", "module": "services.auth_service.app:app", "port": 38001},
|
||||
{"name": "audit", "module": "services.audit_service.app:app", "port": 38002},
|
||||
{"name": "customer", "module": "services.customer_service.app:app", "port": 38003},
|
||||
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 38004},
|
||||
{"name": "routing", "module": "services.routing_service.app:app", "port": 38005},
|
||||
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 38006},
|
||||
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 38007},
|
||||
{"name": "kb", "module": "services.kb_service.app:app", "port": 38008},
|
||||
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 38009},
|
||||
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 38010},
|
||||
{"name": "gateway", "module": "gateway.app:app", "port": 38080},
|
||||
]
|
||||
|
||||
|
||||
async def wait_for_health(base_url: str, retries: int = 60, delay: float = 0.2) -> None:
|
||||
async with httpx.AsyncClient(timeout=2) as client:
|
||||
for _ in range(retries):
|
||||
try:
|
||||
response = await client.get(f"{base_url}/health")
|
||||
if response.status_code == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(delay)
|
||||
raise RuntimeError(f"Service not healthy: {base_url}/health")
|
||||
|
||||
|
||||
def start_one_service(spec: dict[str, Any], data_dir: Path) -> subprocess.Popen:
|
||||
env = os.environ.copy()
|
||||
env["CC_DATA_DIR"] = str(data_dir)
|
||||
|
||||
if spec["name"] == "gateway":
|
||||
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:38001"
|
||||
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:38002"
|
||||
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:38003"
|
||||
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:38004"
|
||||
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:38005"
|
||||
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:38006"
|
||||
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:38007"
|
||||
env["KB_SERVICE_URL"] = "http://127.0.0.1:38008"
|
||||
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:38009"
|
||||
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:38010"
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uvicorn",
|
||||
spec["module"],
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(spec["port"]),
|
||||
]
|
||||
return subprocess.Popen(
|
||||
cmd,
|
||||
cwd=str(ROOT),
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
async def start_services_sequential(data_dir: Path) -> list[subprocess.Popen]:
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
processes: list[subprocess.Popen] = []
|
||||
for spec in SERVICE_SPECS:
|
||||
proc = start_one_service(spec, data_dir)
|
||||
processes.append(proc)
|
||||
await wait_for_health(f"http://127.0.0.1:{spec['port']}")
|
||||
return processes
|
||||
|
||||
|
||||
def stop_services(processes: list[subprocess.Popen]) -> None:
|
||||
for proc in processes:
|
||||
if proc.poll() is not None:
|
||||
continue
|
||||
try:
|
||||
proc.terminate()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for proc in processes:
|
||||
try:
|
||||
proc.wait(timeout=2)
|
||||
except Exception:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _create_interaction(client: httpx.AsyncClient, channel: str, idx: int) -> tuple[bool, float]:
|
||||
headers = {"X-User": "operator", "X-Role": "operator"}
|
||||
payload = {
|
||||
"channel": channel,
|
||||
"subject": f"Gate4 load {channel} #{idx}",
|
||||
"customer_id": None,
|
||||
"queue_id": "q_gate4",
|
||||
"priority": 3,
|
||||
}
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
resp = await client.post("/proxy/interaction/interactions", headers=headers, json=payload, timeout=10)
|
||||
elapsed = time.perf_counter() - started
|
||||
return resp.status_code == 200, elapsed
|
||||
except Exception:
|
||||
elapsed = time.perf_counter() - started
|
||||
return False, elapsed
|
||||
|
||||
|
||||
async def run_load_wave(base_url: str, voice: int, digital: int) -> dict[str, float | int]:
|
||||
async with httpx.AsyncClient(base_url=base_url) as client:
|
||||
tasks = []
|
||||
for i in range(voice):
|
||||
tasks.append(_create_interaction(client, "voice", i))
|
||||
for i in range(digital):
|
||||
tasks.append(_create_interaction(client, "telegram", i))
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
ok_times = [t for ok, t in results if ok]
|
||||
failed = len(results) - len(ok_times)
|
||||
if ok_times:
|
||||
p95 = sorted(ok_times)[max(0, int(len(ok_times) * 0.95) - 1)]
|
||||
avg = statistics.mean(ok_times)
|
||||
else:
|
||||
p95 = 0.0
|
||||
avg = 0.0
|
||||
|
||||
return {
|
||||
"total": len(results),
|
||||
"success": len(ok_times),
|
||||
"failed": failed,
|
||||
"avg_seconds": round(avg, 4),
|
||||
"p95_seconds": round(p95, 4),
|
||||
}
|
||||
|
||||
|
||||
async def get_interaction_count(base_url: str, limit: int = 1000) -> int:
|
||||
async with httpx.AsyncClient(base_url=base_url, timeout=10) as client:
|
||||
response = await client.get(f"/proxy/interaction/interactions?limit={limit}")
|
||||
response.raise_for_status()
|
||||
items = response.json()
|
||||
return len(items)
|
||||
|
||||
|
||||
async def run_security_smoke(base_url: str) -> bool:
|
||||
operator = {"X-User": "operator", "X-Role": "operator"}
|
||||
|
||||
async with httpx.AsyncClient(base_url=base_url, timeout=10) as client:
|
||||
create_user = await client.post(
|
||||
"/proxy/auth/users",
|
||||
headers=operator,
|
||||
json={
|
||||
"username": "sec_denied_user",
|
||||
"password": "secret123",
|
||||
"full_name": "Security Denied",
|
||||
"role": "operator",
|
||||
},
|
||||
)
|
||||
if create_user.status_code != 403:
|
||||
print(f"[FAIL] Security RBAC auth/users expected 403, got {create_user.status_code}")
|
||||
return False
|
||||
|
||||
create_queue = await client.post(
|
||||
"/proxy/routing/queues",
|
||||
json={
|
||||
"name": "Denied queue",
|
||||
"description": "Role check",
|
||||
"rules": [{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 30}],
|
||||
},
|
||||
)
|
||||
if create_queue.status_code != 403:
|
||||
print(f"[FAIL] Security RBAC routing/queues expected 403, got {create_queue.status_code}")
|
||||
return False
|
||||
|
||||
kb_create = await client.post(
|
||||
"/proxy/kb/knowledge/categories",
|
||||
headers=operator,
|
||||
json={"name": "Denied KB", "description": "Role check"},
|
||||
)
|
||||
if kb_create.status_code != 403:
|
||||
print(f"[FAIL] Security RBAC kb category expected 403, got {kb_create.status_code}")
|
||||
return False
|
||||
|
||||
print("[PASS] Security smoke checks")
|
||||
return True
|
||||
|
||||
|
||||
def run_backup_restore(source_dir: Path) -> tuple[Path, Path]:
|
||||
backup_script = ROOT / "scripts" / "backup_data.ps1"
|
||||
restore_script = ROOT / "scripts" / "restore_data.ps1"
|
||||
backup_output = DATA_ROOT / f"backups_{int(time.time() * 1000)}"
|
||||
backup_output.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
backup_cmd = [
|
||||
"powershell",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-File",
|
||||
str(backup_script),
|
||||
"-SourceDir",
|
||||
str(source_dir),
|
||||
"-OutputDir",
|
||||
str(backup_output),
|
||||
]
|
||||
result = subprocess.run(backup_cmd, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"Backup failed: {result.stdout}\n{result.stderr}")
|
||||
|
||||
archives = sorted(backup_output.glob("*.zip"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
if not archives:
|
||||
raise RuntimeError("Backup archive not created")
|
||||
archive = archives[0]
|
||||
|
||||
if source_dir.exists():
|
||||
shutil.rmtree(source_dir)
|
||||
|
||||
restore_cmd = [
|
||||
"powershell",
|
||||
"-ExecutionPolicy",
|
||||
"Bypass",
|
||||
"-File",
|
||||
str(restore_script),
|
||||
"-BackupZip",
|
||||
str(archive),
|
||||
"-TargetDir",
|
||||
str(source_dir),
|
||||
]
|
||||
restore = subprocess.run(restore_cmd, cwd=str(ROOT), capture_output=True, text=True, check=False)
|
||||
if restore.returncode != 0:
|
||||
raise RuntimeError(f"Restore failed: {restore.stdout}\n{restore.stderr}")
|
||||
|
||||
db_file = source_dir / "mvp_cc.db"
|
||||
if not db_file.exists():
|
||||
raise RuntimeError(f"Restored DB file not found: {db_file}")
|
||||
|
||||
return backup_output, archive
|
||||
|
||||
|
||||
def check_docs_bundle() -> bool:
|
||||
required = [
|
||||
ROOT / "docs" / "security" / "checklist.md",
|
||||
ROOT / "docs" / "releases" / "v1.0.0-mvp.md",
|
||||
ROOT / "docs" / "gates" / "mvp-pilot-baseline.md",
|
||||
ROOT / "docs" / "gates" / "p1-p2-defects.md",
|
||||
ROOT / "docs" / "roadmap" / "05-wave2-backlog.md",
|
||||
ROOT / "docs" / "runbooks" / "backup-restore.md",
|
||||
ROOT / "docs" / "runbooks" / "load-test-plan.md",
|
||||
ROOT / "docs" / "runbooks" / "pilot-uat.md",
|
||||
ROOT / "docs" / "uat" / "README.md",
|
||||
ROOT / "docs" / "uat" / "scenario-checklist.md",
|
||||
ROOT / "docs" / "uat" / "session-template.md",
|
||||
ROOT / "docs" / "uat" / "signoff-template.md",
|
||||
ROOT / "docs" / "uat" / "defect-log-template.csv",
|
||||
]
|
||||
missing = [str(path) for path in required if not path.exists()]
|
||||
if missing:
|
||||
print(f"[FAIL] Missing required Stage 4 docs: {missing}")
|
||||
return False
|
||||
print("[PASS] Stage 4 docs bundle present")
|
||||
return True
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Gate 4 checker: load, backup/restore, security smoke")
|
||||
parser.add_argument("--voice", type=int, default=100, help="Concurrent voice requests")
|
||||
parser.add_argument("--digital", type=int, default=100, help="Concurrent digital requests")
|
||||
parser.add_argument("--keep-artifacts", action="store_true", help="Keep .data_gate4 run and backup artifacts")
|
||||
args = parser.parse_args()
|
||||
|
||||
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
|
||||
backup_output: Path | None = None
|
||||
processes: list[subprocess.Popen] = []
|
||||
try:
|
||||
processes = await start_services_sequential(run_data_dir)
|
||||
base_url = "http://127.0.0.1:38080"
|
||||
|
||||
load_result = await run_load_wave(base_url, args.voice, args.digital)
|
||||
print("[INFO] Load result:", load_result)
|
||||
if load_result["failed"] != 0 or load_result["success"] != load_result["total"]:
|
||||
print("[FAIL] Load target not reached with zero errors")
|
||||
sys.exit(1)
|
||||
print("[PASS] Load target reached")
|
||||
|
||||
count_before = await get_interaction_count(base_url, limit=max(500, args.voice + args.digital + 100))
|
||||
if count_before < (args.voice + args.digital):
|
||||
print(
|
||||
f"[FAIL] Interaction count before backup too low: {count_before} < {args.voice + args.digital}"
|
||||
)
|
||||
sys.exit(1)
|
||||
print(f"[PASS] Interaction count before backup: {count_before}")
|
||||
|
||||
security_ok = await run_security_smoke(base_url)
|
||||
if not security_ok:
|
||||
sys.exit(1)
|
||||
|
||||
stop_services(processes)
|
||||
processes = []
|
||||
|
||||
backup_output, archive = run_backup_restore(run_data_dir)
|
||||
print(f"[PASS] Backup/restore scripts completed, archive: {archive}")
|
||||
|
||||
processes = await start_services_sequential(run_data_dir)
|
||||
count_after = await get_interaction_count(base_url, limit=max(500, args.voice + args.digital + 100))
|
||||
if count_after != count_before:
|
||||
print(f"[FAIL] Restored interaction count mismatch: before={count_before}, after={count_after}")
|
||||
sys.exit(1)
|
||||
print(f"[PASS] Restore data integrity validated: {count_after} interactions")
|
||||
|
||||
if not check_docs_bundle():
|
||||
sys.exit(1)
|
||||
|
||||
print("[PASS] Gate 4 automated checks completed")
|
||||
finally:
|
||||
stop_services(processes)
|
||||
if not args.keep_artifacts:
|
||||
try:
|
||||
if run_data_dir.exists():
|
||||
shutil.rmtree(run_data_dir)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if backup_output and backup_output.exists():
|
||||
shutil.rmtree(backup_output)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,251 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DATA_ROOT = ROOT / ".data_smoke"
|
||||
|
||||
SERVICE_SPECS = [
|
||||
{"name": "auth", "module": "services.auth_service.app:app", "port": 18001},
|
||||
{"name": "audit", "module": "services.audit_service.app:app", "port": 18002},
|
||||
{"name": "customer", "module": "services.customer_service.app:app", "port": 18003},
|
||||
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 18004},
|
||||
{"name": "routing", "module": "services.routing_service.app:app", "port": 18005},
|
||||
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 18006},
|
||||
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 18007},
|
||||
{"name": "kb", "module": "services.kb_service.app:app", "port": 18008},
|
||||
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 18009},
|
||||
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 18010},
|
||||
{"name": "gateway", "module": "gateway.app:app", "port": 18080},
|
||||
]
|
||||
|
||||
|
||||
async def wait_for_health(base_url: str, retries: int = 80, delay: float = 0.25) -> None:
|
||||
async with httpx.AsyncClient(timeout=2) as client:
|
||||
for _ in range(retries):
|
||||
try:
|
||||
response = await client.get(f"{base_url}/health")
|
||||
if response.status_code == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(delay)
|
||||
raise RuntimeError(f"Service not healthy: {base_url}/health")
|
||||
|
||||
|
||||
def start_services(database_url: str | None, data_dir: Path) -> list[subprocess.Popen]:
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
processes: list[subprocess.Popen] = []
|
||||
for spec in SERVICE_SPECS:
|
||||
env = os.environ.copy()
|
||||
env["CC_DATA_DIR"] = str(data_dir)
|
||||
if database_url:
|
||||
env["DATABASE_URL"] = database_url
|
||||
|
||||
if spec["name"] == "gateway":
|
||||
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:18001"
|
||||
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:18002"
|
||||
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:18003"
|
||||
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:18004"
|
||||
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:18005"
|
||||
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:18006"
|
||||
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:18007"
|
||||
env["KB_SERVICE_URL"] = "http://127.0.0.1:18008"
|
||||
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:18009"
|
||||
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:18010"
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uvicorn",
|
||||
spec["module"],
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(spec["port"]),
|
||||
]
|
||||
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=str(ROOT),
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
processes.append(proc)
|
||||
|
||||
return processes
|
||||
|
||||
|
||||
def stop_services(processes: list[subprocess.Popen]) -> None:
|
||||
for proc in processes:
|
||||
if proc.poll() is not None:
|
||||
continue
|
||||
try:
|
||||
proc.terminate()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for proc in processes:
|
||||
try:
|
||||
proc.wait(timeout=2)
|
||||
except Exception:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def run_gate_checks() -> dict[str, Any]:
|
||||
for spec in SERVICE_SPECS:
|
||||
await wait_for_health(f"http://127.0.0.1:{spec['port']}")
|
||||
|
||||
base = "http://127.0.0.1:18080"
|
||||
admin = {"X-User": "admin", "X-Role": "admin"}
|
||||
operator = {"X-User": "operator", "X-Role": "operator"}
|
||||
supervisor = {"X-User": "supervisor", "X-Role": "supervisor"}
|
||||
|
||||
async with httpx.AsyncClient(base_url=base, timeout=10) as client:
|
||||
health = await client.get("/health")
|
||||
health.raise_for_status()
|
||||
|
||||
login = await client.post("/proxy/auth/auth/login", json={"username": "admin", "password": "admin123"})
|
||||
login.raise_for_status()
|
||||
|
||||
queue = await client.post(
|
||||
"/proxy/routing/queues",
|
||||
headers=admin,
|
||||
json={
|
||||
"name": "Main Queue",
|
||||
"description": "Smoke queue",
|
||||
"rules": [
|
||||
{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 30},
|
||||
{"channel": "telegram", "priority": 3, "strategy": "round_robin", "sla_seconds": 45},
|
||||
],
|
||||
},
|
||||
)
|
||||
queue.raise_for_status()
|
||||
queue_id = queue.json()["queue_id"]
|
||||
|
||||
customer = await client.post(
|
||||
"/proxy/customer/customers",
|
||||
json={
|
||||
"display_name": "Gate User",
|
||||
"phones": ["+77010000011"],
|
||||
"preferred_phone": "+77010000011",
|
||||
"tags": ["smoke"],
|
||||
},
|
||||
)
|
||||
customer.raise_for_status()
|
||||
customer_id = customer.json()["customer_id"]
|
||||
|
||||
interaction = await client.post(
|
||||
"/proxy/interaction/interactions",
|
||||
headers=operator,
|
||||
json={
|
||||
"channel": "voice",
|
||||
"subject": "Gate voice check",
|
||||
"customer_id": customer_id,
|
||||
"queue_id": queue_id,
|
||||
"priority": 3,
|
||||
},
|
||||
)
|
||||
interaction.raise_for_status()
|
||||
interaction_id = interaction.json()["interaction_id"]
|
||||
|
||||
assign = await client.patch(
|
||||
f"/proxy/interaction/interactions/{interaction_id}/assign",
|
||||
headers=supervisor,
|
||||
json={"assignee": "operator_a"},
|
||||
)
|
||||
assign.raise_for_status()
|
||||
|
||||
escalate = await client.post(
|
||||
f"/proxy/interaction/interactions/{interaction_id}/escalate",
|
||||
headers=operator,
|
||||
json={"target_queue_id": "line2"},
|
||||
)
|
||||
escalate.raise_for_status()
|
||||
|
||||
close = await client.patch(
|
||||
f"/proxy/interaction/interactions/{interaction_id}/status",
|
||||
headers=operator,
|
||||
json={"status": "closed"},
|
||||
)
|
||||
close.raise_for_status()
|
||||
|
||||
voice_event = await client.post(
|
||||
"/proxy/voice/integrations/voice/events",
|
||||
headers=operator,
|
||||
json={
|
||||
"event_type": "call.started",
|
||||
"call_id": "smoke_call_1",
|
||||
"interaction_id": interaction_id,
|
||||
"payload": {"source": "live_smoke"},
|
||||
},
|
||||
)
|
||||
voice_event.raise_for_status()
|
||||
|
||||
tg_event = await client.post(
|
||||
"/proxy/telegram/integrations/telegram/webhook",
|
||||
json={
|
||||
"chat_id": "smoke_chat",
|
||||
"text": "Smoke message",
|
||||
"customer_external_id": None,
|
||||
"payload": {"source": "live_smoke"},
|
||||
},
|
||||
)
|
||||
tg_event.raise_for_status()
|
||||
|
||||
interactions = await client.get("/proxy/interaction/interactions")
|
||||
interactions.raise_for_status()
|
||||
rows = interactions.json()
|
||||
|
||||
return {
|
||||
"gateway": health.json(),
|
||||
"queue_id": queue_id,
|
||||
"customer_id": customer_id,
|
||||
"interaction_id": interaction_id,
|
||||
"interactions_total": len(rows),
|
||||
}
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Live Gate 1/2 smoke")
|
||||
parser.add_argument(
|
||||
"--database-url",
|
||||
default=os.getenv("DATABASE_URL", ""),
|
||||
help="Optional DB URL override for started services",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
db_url = args.database_url.strip() or None
|
||||
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
|
||||
processes = start_services(db_url, run_data_dir)
|
||||
try:
|
||||
result = await run_gate_checks()
|
||||
print("Gate 1/2 live smoke passed")
|
||||
print(result)
|
||||
if db_url:
|
||||
print(f"DB mode: {db_url}")
|
||||
finally:
|
||||
stop_services(processes)
|
||||
try:
|
||||
shutil.rmtree(run_data_dir)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,295 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from statistics import median
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
|
||||
def _parse_iso(value: str | None) -> datetime | None:
|
||||
raw = str(value or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=timezone.utc)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _percentile(values: list[float], q: float) -> float | None:
|
||||
if not values:
|
||||
return None
|
||||
if len(values) == 1:
|
||||
return values[0]
|
||||
sorted_values = sorted(values)
|
||||
idx = (len(sorted_values) - 1) * q
|
||||
lo = int(idx)
|
||||
hi = min(lo + 1, len(sorted_values) - 1)
|
||||
if lo == hi:
|
||||
return sorted_values[lo]
|
||||
frac = idx - lo
|
||||
return sorted_values[lo] + (sorted_values[hi] - sorted_values[lo]) * frac
|
||||
|
||||
|
||||
@dataclass
|
||||
class CallTimeline:
|
||||
call_id: str
|
||||
started_at: datetime | None = None
|
||||
ended_at: datetime | None = None
|
||||
ended_reconciled: bool | None = None
|
||||
recording_ready_at: datetime | None = None
|
||||
recording_ready_reconciled: bool | None = None
|
||||
recording_uploaded_at: datetime | None = None
|
||||
interaction_id: str | None = None
|
||||
|
||||
def start_to_end_seconds(self) -> float | None:
|
||||
if self.started_at and self.ended_at:
|
||||
return max((self.ended_at - self.started_at).total_seconds(), 0.0)
|
||||
return None
|
||||
|
||||
def end_to_recording_ready_seconds(self) -> float | None:
|
||||
if self.ended_at and self.recording_ready_at:
|
||||
return max((self.recording_ready_at - self.ended_at).total_seconds(), 0.0)
|
||||
return None
|
||||
|
||||
def end_to_recording_upload_seconds(self) -> float | None:
|
||||
if self.ended_at and self.recording_uploaded_at:
|
||||
return max((self.recording_uploaded_at - self.ended_at).total_seconds(), 0.0)
|
||||
return None
|
||||
|
||||
def status(self) -> str:
|
||||
if self.started_at and not self.ended_at:
|
||||
return "active_no_end"
|
||||
if self.ended_at and not self.recording_uploaded_at:
|
||||
return "ended_no_recording_upload"
|
||||
if self.ended_at and self.recording_uploaded_at:
|
||||
return "complete"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Track 10: report Asterisk voice lifecycle latency."
|
||||
)
|
||||
parser.add_argument("--database-url", required=True)
|
||||
parser.add_argument("--since-hours", type=int, default=6)
|
||||
parser.add_argument("--since-minutes", type=int, default=0)
|
||||
parser.add_argument("--limit-events", type=int, default=5000)
|
||||
parser.add_argument("--max-started-to-ended-seconds", type=float, default=30.0)
|
||||
parser.add_argument("--max-ended-to-recording-seconds", type=float, default=45.0)
|
||||
parser.add_argument("--print-limit", type=int, default=15)
|
||||
parser.add_argument("--json-out", default="")
|
||||
parser.add_argument("--breach-mode", choices=["all", "direct"], default="direct")
|
||||
parser.add_argument("--fail-on-breach", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.since_minutes and args.since_minutes > 0:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=args.since_minutes)
|
||||
else:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=max(args.since_hours, 1))
|
||||
cutoff_iso = cutoff.isoformat()
|
||||
|
||||
engine = create_engine(args.database_url, future=True)
|
||||
calls: dict[str, CallTimeline] = {}
|
||||
started_from_asterisk: set[str] = set()
|
||||
|
||||
with engine.begin() as conn:
|
||||
rows = conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT call_id, interaction_id, event_type, payload_json, created_at
|
||||
FROM voice_events
|
||||
WHERE event_type IN ('call.started', 'call.ended', 'recording.ready')
|
||||
AND created_at >= :cutoff
|
||||
ORDER BY id DESC
|
||||
LIMIT :limit
|
||||
"""
|
||||
),
|
||||
{"cutoff": cutoff_iso, "limit": max(args.limit_events, 100)},
|
||||
).mappings().all()
|
||||
|
||||
for row in rows:
|
||||
call_id = str(row["call_id"] or "").strip()
|
||||
if not call_id:
|
||||
continue
|
||||
payload_raw = str(row["payload_json"] or "{}")
|
||||
try:
|
||||
payload = json.loads(payload_raw)
|
||||
except json.JSONDecodeError:
|
||||
payload = {}
|
||||
event_type = str(row["event_type"] or "")
|
||||
created_at = _parse_iso(str(row["created_at"] or ""))
|
||||
if created_at is None:
|
||||
continue
|
||||
|
||||
timeline = calls.get(call_id) or CallTimeline(call_id=call_id)
|
||||
if event_type == "call.started":
|
||||
if payload.get("source") != "asterisk":
|
||||
continue
|
||||
started_from_asterisk.add(call_id)
|
||||
timeline.started_at = max(filter(None, [timeline.started_at, created_at]))
|
||||
elif event_type == "call.ended":
|
||||
if timeline.ended_at is None or created_at > timeline.ended_at:
|
||||
timeline.ended_at = created_at
|
||||
timeline.ended_reconciled = bool(payload.get("reconciled"))
|
||||
elif event_type == "recording.ready":
|
||||
if timeline.recording_ready_at is None or created_at > timeline.recording_ready_at:
|
||||
timeline.recording_ready_at = created_at
|
||||
timeline.recording_ready_reconciled = bool(payload.get("reconciled"))
|
||||
timeline.interaction_id = timeline.interaction_id or str(row["interaction_id"] or "").strip() or None
|
||||
calls[call_id] = timeline
|
||||
|
||||
rec_rows = conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT call_id, interaction_id, created_at
|
||||
FROM call_recordings
|
||||
WHERE created_at >= :cutoff
|
||||
ORDER BY id DESC
|
||||
LIMIT :limit
|
||||
"""
|
||||
),
|
||||
{"cutoff": cutoff_iso, "limit": max(args.limit_events, 100)},
|
||||
).mappings().all()
|
||||
for row in rec_rows:
|
||||
call_id = str(row["call_id"] or "").strip()
|
||||
if call_id not in calls:
|
||||
continue
|
||||
created_at = _parse_iso(str(row["created_at"] or ""))
|
||||
if created_at is None:
|
||||
continue
|
||||
timeline = calls[call_id]
|
||||
if timeline.recording_uploaded_at is None or created_at > timeline.recording_uploaded_at:
|
||||
timeline.recording_uploaded_at = created_at
|
||||
timeline.interaction_id = timeline.interaction_id or str(row["interaction_id"] or "").strip() or None
|
||||
|
||||
items = [calls[call_id] for call_id in started_from_asterisk if call_id in calls]
|
||||
items.sort(key=lambda x: x.started_at or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
|
||||
|
||||
start_end = [v for v in (item.start_to_end_seconds() for item in items) if v is not None]
|
||||
end_rec = [v for v in (item.end_to_recording_upload_seconds() for item in items) if v is not None]
|
||||
direct_items = [item for item in items if item.ended_at is not None and item.ended_reconciled is False]
|
||||
direct_start_end = [v for v in (item.start_to_end_seconds() for item in direct_items) if v is not None]
|
||||
direct_end_rec = [
|
||||
v for v in (item.end_to_recording_upload_seconds() for item in direct_items) if v is not None
|
||||
]
|
||||
ended_no_recording = [item for item in items if item.status() == "ended_no_recording_upload"]
|
||||
active_no_end = [item for item in items if item.status() == "active_no_end"]
|
||||
|
||||
p95_start_end = _percentile(start_end, 0.95)
|
||||
p95_end_rec = _percentile(end_rec, 0.95)
|
||||
p95_direct_start_end = _percentile(direct_start_end, 0.95)
|
||||
p95_direct_end_rec = _percentile(direct_end_rec, 0.95)
|
||||
|
||||
breach = False
|
||||
if args.breach_mode == "direct":
|
||||
left = p95_direct_start_end if p95_direct_start_end is not None else p95_start_end
|
||||
right = p95_direct_end_rec if p95_direct_end_rec is not None else p95_end_rec
|
||||
else:
|
||||
left = p95_start_end
|
||||
right = p95_end_rec
|
||||
|
||||
if left is not None and left > args.max_started_to_ended_seconds:
|
||||
breach = True
|
||||
if right is not None and right > args.max_ended_to_recording_seconds:
|
||||
breach = True
|
||||
if ended_no_recording:
|
||||
breach = True
|
||||
|
||||
report = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"since_hours": args.since_hours,
|
||||
"since_minutes": args.since_minutes,
|
||||
"thresholds": {
|
||||
"max_started_to_ended_seconds": args.max_started_to_ended_seconds,
|
||||
"max_ended_to_recording_seconds": args.max_ended_to_recording_seconds,
|
||||
},
|
||||
"summary": {
|
||||
"total_calls": len(items),
|
||||
"complete_calls": sum(1 for item in items if item.status() == "complete"),
|
||||
"active_no_end": len(active_no_end),
|
||||
"ended_no_recording_upload": len(ended_no_recording),
|
||||
"ended_reconciled_calls": sum(1 for item in items if item.ended_reconciled is True),
|
||||
"ended_direct_calls": sum(1 for item in items if item.ended_reconciled is False),
|
||||
"start_to_end": {
|
||||
"count": len(start_end),
|
||||
"median": median(start_end) if start_end else None,
|
||||
"p95": p95_start_end,
|
||||
},
|
||||
"end_to_recording_upload": {
|
||||
"count": len(end_rec),
|
||||
"median": median(end_rec) if end_rec else None,
|
||||
"p95": p95_end_rec,
|
||||
},
|
||||
"direct_start_to_end": {
|
||||
"count": len(direct_start_end),
|
||||
"median": median(direct_start_end) if direct_start_end else None,
|
||||
"p95": p95_direct_start_end,
|
||||
},
|
||||
"direct_end_to_recording_upload": {
|
||||
"count": len(direct_end_rec),
|
||||
"median": median(direct_end_rec) if direct_end_rec else None,
|
||||
"p95": p95_direct_end_rec,
|
||||
},
|
||||
"breach_mode": args.breach_mode,
|
||||
},
|
||||
"worst_calls": [
|
||||
{
|
||||
"call_id": item.call_id,
|
||||
"interaction_id": item.interaction_id,
|
||||
"status": item.status(),
|
||||
"started_at": item.started_at.isoformat() if item.started_at else None,
|
||||
"ended_at": item.ended_at.isoformat() if item.ended_at else None,
|
||||
"ended_reconciled": item.ended_reconciled,
|
||||
"recording_ready_at": item.recording_ready_at.isoformat() if item.recording_ready_at else None,
|
||||
"recording_ready_reconciled": item.recording_ready_reconciled,
|
||||
"recording_uploaded_at": item.recording_uploaded_at.isoformat() if item.recording_uploaded_at else None,
|
||||
"start_to_end_seconds": item.start_to_end_seconds(),
|
||||
"end_to_recording_upload_seconds": item.end_to_recording_upload_seconds(),
|
||||
}
|
||||
for item in items[: max(args.print_limit, 1)]
|
||||
],
|
||||
}
|
||||
|
||||
print(
|
||||
"[INFO] Track10 voice latency report: "
|
||||
f"calls={report['summary']['total_calls']} "
|
||||
f"complete={report['summary']['complete_calls']} "
|
||||
f"active_no_end={report['summary']['active_no_end']} "
|
||||
f"ended_no_recording_upload={report['summary']['ended_no_recording_upload']}"
|
||||
)
|
||||
print(
|
||||
"[INFO] p95 started->ended="
|
||||
f"{report['summary']['start_to_end']['p95']}s; "
|
||||
"p95 ended->recording_upload="
|
||||
f"{report['summary']['end_to_recording_upload']['p95']}s"
|
||||
)
|
||||
print(
|
||||
"[INFO] direct p95 started->ended="
|
||||
f"{report['summary']['direct_start_to_end']['p95']}s; "
|
||||
"direct p95 ended->recording_upload="
|
||||
f"{report['summary']['direct_end_to_recording_upload']['p95']}s"
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
if args.json_out.strip():
|
||||
out_path = Path(args.json_out).expanduser().resolve()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"[INFO] report written: {out_path}")
|
||||
|
||||
if args.fail_on_breach and breach:
|
||||
print("[FAIL] SLO breach detected for Track 10 criteria")
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,163 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.error import URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DB_PATH = ROOT / ".data_local" / "mvp_cc.db"
|
||||
OUT_DIR = ROOT / ".artifacts" / "track12-monitor"
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
OUT_PATH = OUT_DIR / "latest.log"
|
||||
|
||||
BASE_URL = "http://127.0.0.1:8080"
|
||||
AUTH_BASE_URL = "http://127.0.0.1:8001"
|
||||
USERNAME = "operator"
|
||||
PASSWORD = "op12345"
|
||||
ROLE = "operator"
|
||||
WINDOW_SECONDS = 180
|
||||
POLL_SECONDS = 1.0
|
||||
|
||||
|
||||
def iso_now() -> str:
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def write_line(handle, label: str, payload) -> None:
|
||||
handle.write(f"[{iso_now()}] {label} {json.dumps(payload, ensure_ascii=False, sort_keys=True)}\n")
|
||||
handle.flush()
|
||||
|
||||
|
||||
def login_headers() -> dict[str, str]:
|
||||
body = json.dumps(
|
||||
{
|
||||
"username": USERNAME,
|
||||
"password": PASSWORD,
|
||||
"role": ROLE,
|
||||
}
|
||||
).encode("utf-8")
|
||||
req = Request(
|
||||
f"{AUTH_BASE_URL}/auth/login",
|
||||
headers={"Content-Type": "application/json"},
|
||||
data=body,
|
||||
method="POST",
|
||||
)
|
||||
with urlopen(req, timeout=10) as response:
|
||||
payload = json.loads(response.read().decode("utf-8", errors="replace"))
|
||||
token = str(payload.get("access_token") or "").strip()
|
||||
if not token:
|
||||
raise RuntimeError("auth/login did not return access_token")
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
def fetch_json(path: str, headers: dict[str, str]):
|
||||
req = Request(
|
||||
f"{BASE_URL}{path}",
|
||||
headers=headers,
|
||||
method="GET",
|
||||
)
|
||||
with urlopen(req, timeout=10) as response:
|
||||
body = response.read().decode("utf-8", errors="replace")
|
||||
return json.loads(body)
|
||||
|
||||
|
||||
def latest_events(conn: sqlite3.Connection, limit: int = 6) -> list[dict]:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
select event_id, event_type, call_id, interaction_id, payload_json, created_at
|
||||
from voice_events
|
||||
order by id desc
|
||||
limit ?
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
result = []
|
||||
for row in rows:
|
||||
payload = row["payload_json"]
|
||||
try:
|
||||
payload = json.loads(payload)
|
||||
except Exception:
|
||||
pass
|
||||
result.append(
|
||||
{
|
||||
"event_id": row["event_id"],
|
||||
"event_type": row["event_type"],
|
||||
"call_id": row["call_id"],
|
||||
"interaction_id": row["interaction_id"],
|
||||
"payload": payload,
|
||||
"created_at": row["created_at"],
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def latest_links(conn: sqlite3.Connection, limit: int = 4) -> list[dict]:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
select call_id, interaction_id, status, telephony_status, claimed_by_user,
|
||||
operator_extension, started_at, connected_at, ended_at
|
||||
from asterisk_call_links
|
||||
order by id desc
|
||||
limit ?
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
with OUT_PATH.open("w", encoding="utf-8") as handle:
|
||||
write_line(handle, "monitor", {"status": "started", "window_seconds": WINDOW_SECONDS})
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
headers = login_headers()
|
||||
write_line(handle, "auth", {"status": "ok", "user": USERNAME, "role": ROLE})
|
||||
end_at = time.time() + WINDOW_SECONDS
|
||||
seen = {
|
||||
"live": None,
|
||||
"recent": None,
|
||||
"events": None,
|
||||
"links": None,
|
||||
}
|
||||
try:
|
||||
while time.time() < end_at:
|
||||
try:
|
||||
live = fetch_json("/proxy/asterisk-bridge/asterisk/live-calls", headers)
|
||||
if live != seen["live"]:
|
||||
seen["live"] = live
|
||||
write_line(handle, "live-calls", live)
|
||||
except URLError as exc:
|
||||
write_line(handle, "live-calls-error", {"error": str(exc)})
|
||||
|
||||
try:
|
||||
recent = fetch_json("/proxy/asterisk-bridge/asterisk/recent-calls", headers)
|
||||
if recent != seen["recent"]:
|
||||
seen["recent"] = recent
|
||||
write_line(handle, "recent-calls", recent)
|
||||
except URLError as exc:
|
||||
write_line(handle, "recent-calls-error", {"error": str(exc)})
|
||||
|
||||
events = latest_events(conn)
|
||||
if events != seen["events"]:
|
||||
seen["events"] = events
|
||||
write_line(handle, "voice-events", events)
|
||||
|
||||
links = latest_links(conn)
|
||||
if links != seen["links"]:
|
||||
seen["links"] = links
|
||||
write_line(handle, "call-links", links)
|
||||
|
||||
time.sleep(POLL_SECONDS)
|
||||
finally:
|
||||
conn.close()
|
||||
write_line(handle, "monitor", {"status": "finished"})
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,154 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def load_summary(report_dir: Path) -> dict[str, Any]:
|
||||
summary_path = report_dir / "summary.json"
|
||||
if not summary_path.exists():
|
||||
raise FileNotFoundError(f"Missing summary.json: {summary_path}")
|
||||
return json.loads(summary_path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def evaluate_summary(
|
||||
summary: dict[str, Any],
|
||||
*,
|
||||
require_success_rate: float,
|
||||
require_p95_seconds: float,
|
||||
require_p99_seconds: float,
|
||||
) -> list[str]:
|
||||
issues: list[str] = []
|
||||
results = summary.get("results", {})
|
||||
success_rate = float(results.get("success_rate", 0.0))
|
||||
p95 = float(results.get("p95_seconds", 0.0))
|
||||
p99 = float(results.get("p99_seconds", 0.0))
|
||||
five_xx_rate = float(results.get("five_xx_rate", 0.0))
|
||||
|
||||
if success_rate < require_success_rate:
|
||||
issues.append(f"Success rate {success_rate:.2f}% is below threshold {require_success_rate:.2f}%")
|
||||
if p95 > require_p95_seconds:
|
||||
issues.append(f"P95 {p95:.2f}s exceeds threshold {require_p95_seconds:.2f}s")
|
||||
if p99 > require_p99_seconds:
|
||||
issues.append(f"P99 {p99:.2f}s exceeds threshold {require_p99_seconds:.2f}s")
|
||||
if five_xx_rate > 0.5:
|
||||
issues.append(f"5xx/transport error rate {five_xx_rate:.2f}% exceeds 0.50%")
|
||||
return issues
|
||||
|
||||
|
||||
def _run_kubectl(args: list[str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(args, capture_output=True, text=True, check=False)
|
||||
|
||||
|
||||
def check_pod_health(namespace: str, kubectl_context: str | None = None) -> list[str]:
|
||||
cmd = ["kubectl"]
|
||||
if kubectl_context:
|
||||
cmd.extend(["--context", kubectl_context])
|
||||
cmd.extend(["get", "pods", "-n", namespace, "-o", "json"])
|
||||
result = _run_kubectl(cmd)
|
||||
if result.returncode != 0:
|
||||
return [f"kubectl get pods failed: {result.stderr.strip() or result.stdout.strip()}"]
|
||||
|
||||
payload = json.loads(result.stdout)
|
||||
issues: list[str] = []
|
||||
for item in payload.get("items", []):
|
||||
name = item.get("metadata", {}).get("name", "<unknown>")
|
||||
status = item.get("status", {})
|
||||
phase = status.get("phase")
|
||||
if phase not in {"Running", "Succeeded"}:
|
||||
issues.append(f"Pod {name} is in unexpected phase {phase}")
|
||||
for container in status.get("containerStatuses", []):
|
||||
if container.get("restartCount", 0) > 0:
|
||||
issues.append(f"Pod {name} restartCount={container.get('restartCount')}")
|
||||
state = container.get("state", {})
|
||||
waiting = state.get("waiting") or {}
|
||||
terminated = state.get("terminated") or {}
|
||||
if waiting.get("reason") == "CrashLoopBackOff":
|
||||
issues.append(f"Pod {name} is in CrashLoopBackOff")
|
||||
if terminated.get("reason") == "OOMKilled":
|
||||
issues.append(f"Pod {name} had OOMKilled termination")
|
||||
return issues
|
||||
|
||||
|
||||
def check_kubectl_top(namespace: str, kubectl_context: str | None = None) -> list[str]:
|
||||
cmd = ["kubectl"]
|
||||
if kubectl_context:
|
||||
cmd.extend(["--context", kubectl_context])
|
||||
cmd.extend(["top", "pods", "-n", namespace, "--no-headers"])
|
||||
result = _run_kubectl(cmd)
|
||||
if result.returncode != 0:
|
||||
return [f"kubectl top pods failed: {result.stderr.strip() or result.stdout.strip()}"]
|
||||
if not result.stdout.strip():
|
||||
return ["kubectl top pods returned no rows"]
|
||||
return []
|
||||
|
||||
|
||||
def check_hpa(namespace: str, kubectl_context: str | None = None) -> list[str]:
|
||||
cmd = ["kubectl"]
|
||||
if kubectl_context:
|
||||
cmd.extend(["--context", kubectl_context])
|
||||
cmd.extend(["get", "hpa", "-n", namespace, "-o", "json"])
|
||||
result = _run_kubectl(cmd)
|
||||
if result.returncode != 0:
|
||||
return [f"kubectl get hpa failed: {result.stderr.strip() or result.stdout.strip()}"]
|
||||
|
||||
payload = json.loads(result.stdout)
|
||||
items = payload.get("items", [])
|
||||
if not items:
|
||||
return ["No HPA objects found in namespace"]
|
||||
|
||||
issues: list[str] = []
|
||||
for item in items:
|
||||
name = item.get("metadata", {}).get("name", "<unknown>")
|
||||
status = item.get("status", {})
|
||||
current_metrics = status.get("currentMetrics") or []
|
||||
if not current_metrics and status.get("currentCPUUtilizationPercentage") is None:
|
||||
issues.append(f"HPA {name} has no current utilization metrics")
|
||||
return issues
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Validate Track 7 scale/hardening evidence.")
|
||||
parser.add_argument("--namespace", required=True, help="Kubernetes namespace")
|
||||
parser.add_argument("--report-dir", required=True, help="Directory produced by scripts/load_test.py")
|
||||
parser.add_argument("--require-success-rate", type=float, default=99.0)
|
||||
parser.add_argument("--require-p95-seconds", type=float, default=2.0)
|
||||
parser.add_argument("--require-p99-seconds", type=float, default=3.5)
|
||||
parser.add_argument("--kubectl-context", default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
report_dir = Path(args.report_dir)
|
||||
issues: list[str] = []
|
||||
try:
|
||||
summary = load_summary(report_dir)
|
||||
except Exception as exc:
|
||||
raise SystemExit(f"[FAIL] {exc}") from exc
|
||||
|
||||
issues.extend(
|
||||
evaluate_summary(
|
||||
summary,
|
||||
require_success_rate=args.require_success_rate,
|
||||
require_p95_seconds=args.require_p95_seconds,
|
||||
require_p99_seconds=args.require_p99_seconds,
|
||||
)
|
||||
)
|
||||
issues.extend(check_pod_health(args.namespace, kubectl_context=args.kubectl_context))
|
||||
issues.extend(check_kubectl_top(args.namespace, kubectl_context=args.kubectl_context))
|
||||
issues.extend(check_hpa(args.namespace, kubectl_context=args.kubectl_context))
|
||||
|
||||
if issues:
|
||||
print("[FAIL] Track 7 validation failed:")
|
||||
for issue in issues:
|
||||
print(f"- {issue}")
|
||||
raise SystemExit(1)
|
||||
|
||||
print("[PASS] Track 7 validation checks passed")
|
||||
print(f"- namespace: {args.namespace}")
|
||||
print(f"- report_dir: {report_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,220 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
if __package__ in {None, ""}:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from scripts import load_test, track7_check
|
||||
else:
|
||||
from scripts import load_test, track7_check
|
||||
|
||||
|
||||
PROFILE_THRESHOLDS = {
|
||||
"step_250_250": {
|
||||
"require_success_rate": 99.0,
|
||||
"require_p95_seconds": 1.5,
|
||||
"require_p99_seconds": 3.0,
|
||||
},
|
||||
"target_500_500": {
|
||||
"require_success_rate": 99.0,
|
||||
"require_p95_seconds": 2.0,
|
||||
"require_p99_seconds": 3.5,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_stage_root(report_root: str | None = None) -> Path:
|
||||
if report_root:
|
||||
target = Path(report_root)
|
||||
else:
|
||||
timestamp = time.strftime("%Y%m%d_%H%M%S")
|
||||
target = Path(".artifacts") / "track7" / f"staged_{timestamp}"
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
return target
|
||||
|
||||
|
||||
def resolve_profiles(profiles_raw: str) -> list[str]:
|
||||
requested = [item.strip() for item in profiles_raw.split(",") if item.strip()]
|
||||
if not requested:
|
||||
raise ValueError("At least one profile is required")
|
||||
unsupported = [name for name in requested if name not in PROFILE_THRESHOLDS]
|
||||
if unsupported:
|
||||
raise ValueError(f"Unsupported staged profiles: {', '.join(unsupported)}")
|
||||
return requested
|
||||
|
||||
|
||||
def evaluate_stage(
|
||||
*,
|
||||
namespace: str,
|
||||
report_dir: Path,
|
||||
kubectl_context: str | None,
|
||||
thresholds: dict[str, float],
|
||||
) -> list[str]:
|
||||
summary = track7_check.load_summary(report_dir)
|
||||
issues: list[str] = []
|
||||
issues.extend(
|
||||
track7_check.evaluate_summary(
|
||||
summary,
|
||||
require_success_rate=thresholds["require_success_rate"],
|
||||
require_p95_seconds=thresholds["require_p95_seconds"],
|
||||
require_p99_seconds=thresholds["require_p99_seconds"],
|
||||
)
|
||||
)
|
||||
issues.extend(track7_check.check_pod_health(namespace, kubectl_context=kubectl_context))
|
||||
issues.extend(track7_check.check_kubectl_top(namespace, kubectl_context=kubectl_context))
|
||||
issues.extend(track7_check.check_hpa(namespace, kubectl_context=kubectl_context))
|
||||
return issues
|
||||
|
||||
|
||||
def write_acceptance_pack(stage_root: Path, payload: dict[str, Any]) -> None:
|
||||
(stage_root / "acceptance_summary.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
|
||||
lines = [
|
||||
"# Track 7 Staged Validation",
|
||||
"",
|
||||
f"- Namespace: `{payload['namespace']}`",
|
||||
f"- Base URL: `{payload['base_url']}`",
|
||||
f"- Auth mode: `{payload['auth_mode']}`",
|
||||
f"- Include read traffic: `{int(payload['include_read_traffic'])}`",
|
||||
f"- Started at: `{payload['started_at']}`",
|
||||
f"- Completed at: `{payload['completed_at']}`",
|
||||
f"- Passed: `{payload['passed']}`",
|
||||
"",
|
||||
"## Stages",
|
||||
"",
|
||||
]
|
||||
for stage in payload["stages"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {stage['profile']}",
|
||||
"",
|
||||
f"- Report dir: `{stage['report_dir']}`",
|
||||
f"- Passed: `{stage['passed']}`",
|
||||
f"- Success rate: `{stage['results']['success_rate']:.2f}%`",
|
||||
f"- P95: `{stage['results']['p95_seconds']:.4f}s`",
|
||||
f"- P99: `{stage['results']['p99_seconds']:.4f}s`",
|
||||
]
|
||||
)
|
||||
if stage["issues"]:
|
||||
lines.append("- Issues:")
|
||||
for issue in stage["issues"]:
|
||||
lines.append(f" - {issue}")
|
||||
lines.append("")
|
||||
(stage_root / "acceptance_summary.md").write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
async def run_staged_validation(
|
||||
*,
|
||||
base_url: str,
|
||||
namespace: str,
|
||||
profiles: list[str],
|
||||
auth_mode: str,
|
||||
include_read_traffic: bool,
|
||||
kubectl_context: str | None,
|
||||
report_root: str | None,
|
||||
) -> tuple[dict[str, Any], Path]:
|
||||
stage_root = build_stage_root(report_root)
|
||||
started_at = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
stages: list[dict[str, Any]] = []
|
||||
|
||||
for profile_name in profiles:
|
||||
thresholds = PROFILE_THRESHOLDS[profile_name]
|
||||
report_dir = stage_root / profile_name
|
||||
summary, out_dir = await load_test.run_profile(
|
||||
base_url=base_url,
|
||||
profile_name=profile_name,
|
||||
ramp_seconds=None,
|
||||
hold_seconds=None,
|
||||
voice=None,
|
||||
digital=None,
|
||||
include_read_traffic=include_read_traffic,
|
||||
auth_mode=auth_mode,
|
||||
report_dir=str(report_dir),
|
||||
require_success_rate=thresholds["require_success_rate"],
|
||||
require_p95_seconds=thresholds["require_p95_seconds"],
|
||||
require_p99_seconds=thresholds["require_p99_seconds"],
|
||||
)
|
||||
issues = evaluate_stage(
|
||||
namespace=namespace,
|
||||
report_dir=out_dir,
|
||||
kubectl_context=kubectl_context,
|
||||
thresholds=thresholds,
|
||||
)
|
||||
stage_payload = {
|
||||
"profile": profile_name,
|
||||
"report_dir": str(out_dir),
|
||||
"thresholds": thresholds,
|
||||
"results": summary["results"],
|
||||
"passed": not issues,
|
||||
"issues": issues,
|
||||
}
|
||||
stages.append(stage_payload)
|
||||
if issues:
|
||||
break
|
||||
|
||||
completed_at = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
payload = {
|
||||
"namespace": namespace,
|
||||
"base_url": base_url,
|
||||
"auth_mode": auth_mode,
|
||||
"include_read_traffic": include_read_traffic,
|
||||
"started_at": started_at,
|
||||
"completed_at": completed_at,
|
||||
"stages": stages,
|
||||
"passed": all(stage["passed"] for stage in stages) and len(stages) == len(profiles),
|
||||
}
|
||||
write_acceptance_pack(stage_root, payload)
|
||||
return payload, stage_root
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Run staged Track 7 validation and write an acceptance pack.")
|
||||
parser.add_argument("--base-url", required=True, help="Gateway base URL")
|
||||
parser.add_argument("--namespace", required=True, help="Kubernetes namespace to validate")
|
||||
parser.add_argument(
|
||||
"--profiles",
|
||||
default="step_250_250,target_500_500",
|
||||
help="Comma-separated staged profiles. Supported: step_250_250,target_500_500",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--auth-mode",
|
||||
choices=["legacy_headers", "bearer"],
|
||||
default="bearer",
|
||||
help="Authentication mode for load traffic",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-read-traffic",
|
||||
type=int,
|
||||
choices=[0, 1],
|
||||
default=1,
|
||||
help="Include background supervisor/reporting reads during staged runs",
|
||||
)
|
||||
parser.add_argument("--kubectl-context", default=None, help="Optional kubectl context override")
|
||||
parser.add_argument("--report-root", default=None, help="Root directory for stage artifacts")
|
||||
args = parser.parse_args()
|
||||
|
||||
profiles = resolve_profiles(args.profiles)
|
||||
payload, stage_root = await run_staged_validation(
|
||||
base_url=args.base_url,
|
||||
namespace=args.namespace,
|
||||
profiles=profiles,
|
||||
auth_mode=args.auth_mode,
|
||||
include_read_traffic=bool(args.include_read_traffic),
|
||||
kubectl_context=args.kubectl_context,
|
||||
report_root=args.report_root,
|
||||
)
|
||||
print("Track 7 staged validation:")
|
||||
print(json.dumps(payload, indent=2))
|
||||
print(f"Acceptance pack: {stage_root}")
|
||||
if not payload["passed"]:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,104 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine, inspect, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from scripts.event_bus_smoke import run_smoke_check
|
||||
from services.shared.db import DATABASE_URL, _normalize_database_url
|
||||
from services.shared.sql_models import EventOutboxRow
|
||||
|
||||
|
||||
def _engine_for(database_url: str | None):
|
||||
return create_engine(_normalize_database_url(database_url or DATABASE_URL), future=True)
|
||||
|
||||
|
||||
def _parse_iso(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def evaluate_track8(
|
||||
*,
|
||||
database_url: str | None = None,
|
||||
require_bus_enabled: bool = False,
|
||||
base_url: str | None = None,
|
||||
failed_age_seconds: int = 300,
|
||||
) -> list[str]:
|
||||
issues: list[str] = []
|
||||
engine = _engine_for(database_url)
|
||||
inspector = inspect(engine)
|
||||
tables = set(inspector.get_table_names())
|
||||
required_tables = {"event_outbox", "event_inbox", "reporting_event_log"}
|
||||
missing = sorted(required_tables - tables)
|
||||
if missing:
|
||||
issues.append(f"Missing required tables: {', '.join(missing)}")
|
||||
|
||||
if require_bus_enabled and os.getenv("EVENT_BUS_ENABLED", "0").strip() not in {"1", "true", "yes", "on"}:
|
||||
issues.append("EVENT_BUS_ENABLED is not enabled in the current environment")
|
||||
|
||||
with Session(engine) as session:
|
||||
failed_rows = session.execute(
|
||||
select(EventOutboxRow).where(EventOutboxRow.status == "failed")
|
||||
).scalars().all()
|
||||
stale_cutoff = datetime.now(timezone.utc) - timedelta(seconds=failed_age_seconds)
|
||||
stale_failed = [
|
||||
row for row in failed_rows if (_parse_iso(row.updated_at) or _parse_iso(row.created_at) or stale_cutoff) < stale_cutoff
|
||||
]
|
||||
if stale_failed:
|
||||
issues.append(f"Found {len(stale_failed)} failed outbox events older than {failed_age_seconds}s")
|
||||
|
||||
pending_rows = session.execute(
|
||||
select(EventOutboxRow).where(EventOutboxRow.status == "pending")
|
||||
).scalars().all()
|
||||
if len(pending_rows) > 500:
|
||||
issues.append(f"Outbox backlog too high: {len(pending_rows)} pending events")
|
||||
|
||||
if not (ROOT / "contracts" / "events" / "ivr.completed.json").exists():
|
||||
issues.append("Missing contracts/events/ivr.completed.json")
|
||||
|
||||
if base_url:
|
||||
smoke = run_smoke_check(base_url=base_url, database_url=database_url)
|
||||
if not smoke["passed"]:
|
||||
issues.append("Event bus smoke check failed")
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Validate Track 8 event-bus readiness.")
|
||||
parser.add_argument("--database-url", default=None)
|
||||
parser.add_argument("--require-bus-enabled", action="store_true")
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--failed-age-seconds", type=int, default=300)
|
||||
args = parser.parse_args()
|
||||
|
||||
issues = evaluate_track8(
|
||||
database_url=args.database_url,
|
||||
require_bus_enabled=args.require_bus_enabled,
|
||||
base_url=args.base_url,
|
||||
failed_age_seconds=args.failed_age_seconds,
|
||||
)
|
||||
if issues:
|
||||
print("[FAIL] Track 8 validation failed:")
|
||||
for issue in issues:
|
||||
print(f"- {issue}")
|
||||
raise SystemExit(1)
|
||||
print("[PASS] Track 8 validation checks passed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,967 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import csv
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DATA_ROOT = ROOT / ".data_uat_dry_run"
|
||||
DEFAULT_OUTPUT_ROOT = ROOT / "docs" / "uat" / "evidence"
|
||||
|
||||
SCENARIO_SERVICE_MAP = {
|
||||
"SYS": "platform",
|
||||
"S0": "auth-service/gateway",
|
||||
"S1": "customer-service",
|
||||
"S2": "interaction-service",
|
||||
"S3": "interaction-service/routing-service",
|
||||
"S4": "routing-service/interaction-service",
|
||||
"S5": "voice-adapter-service",
|
||||
"S6": "telegram-adapter-service",
|
||||
"S7": "kb-service",
|
||||
"S8": "supervisor-service",
|
||||
"S9": "reporting-service",
|
||||
}
|
||||
|
||||
SEVERITY_RISK_MAP = {
|
||||
"P1": "High",
|
||||
"P2": "Medium",
|
||||
"P3": "Medium",
|
||||
"P4": "Low",
|
||||
}
|
||||
|
||||
SERVICE_SPECS = [
|
||||
{"name": "auth", "module": "services.auth_service.app:app", "port": 58001},
|
||||
{"name": "audit", "module": "services.audit_service.app:app", "port": 58002},
|
||||
{"name": "customer", "module": "services.customer_service.app:app", "port": 58003},
|
||||
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 58004},
|
||||
{"name": "routing", "module": "services.routing_service.app:app", "port": 58005},
|
||||
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 58006},
|
||||
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 58007},
|
||||
{"name": "kb", "module": "services.kb_service.app:app", "port": 58008},
|
||||
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 58009},
|
||||
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 58010},
|
||||
{"name": "gateway", "module": "gateway.app:app", "port": 58080},
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScenarioResult:
|
||||
scenario_id: str
|
||||
title: str
|
||||
passed: bool
|
||||
severity_on_fail: str
|
||||
details: str
|
||||
evidence: dict[str, Any]
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def now_compact() -> str:
|
||||
return utc_now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
|
||||
def iso_now() -> str:
|
||||
return utc_now().isoformat()
|
||||
|
||||
|
||||
async def wait_for_health(base_url: str, retries: int = 80, delay: float = 0.2) -> None:
|
||||
async with httpx.AsyncClient(timeout=2) as client:
|
||||
for _ in range(retries):
|
||||
try:
|
||||
response = await client.get(f"{base_url}/health")
|
||||
if response.status_code == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(delay)
|
||||
raise RuntimeError(f"Service not healthy: {base_url}/health")
|
||||
|
||||
|
||||
def start_one_service(spec: dict[str, Any], data_dir: Path) -> subprocess.Popen:
|
||||
env = os.environ.copy()
|
||||
env["CC_DATA_DIR"] = str(data_dir)
|
||||
|
||||
if spec["name"] == "gateway":
|
||||
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:58001"
|
||||
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:58002"
|
||||
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:58003"
|
||||
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:58004"
|
||||
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:58005"
|
||||
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:58006"
|
||||
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:58007"
|
||||
env["KB_SERVICE_URL"] = "http://127.0.0.1:58008"
|
||||
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:58009"
|
||||
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:58010"
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uvicorn",
|
||||
spec["module"],
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(spec["port"]),
|
||||
]
|
||||
return subprocess.Popen(
|
||||
cmd,
|
||||
cwd=str(ROOT),
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
async def start_services_sequential(data_dir: Path) -> list[subprocess.Popen]:
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
processes: list[subprocess.Popen] = []
|
||||
for spec in SERVICE_SPECS:
|
||||
proc = start_one_service(spec, data_dir)
|
||||
processes.append(proc)
|
||||
await wait_for_health(f"http://127.0.0.1:{spec['port']}")
|
||||
return processes
|
||||
|
||||
|
||||
def stop_services(processes: list[subprocess.Popen]) -> None:
|
||||
for proc in processes:
|
||||
if proc.poll() is not None:
|
||||
continue
|
||||
try:
|
||||
proc.terminate()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for proc in processes:
|
||||
try:
|
||||
proc.wait(timeout=2)
|
||||
except Exception:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def scenario_result(
|
||||
scenario_id: str,
|
||||
title: str,
|
||||
passed: bool,
|
||||
severity_on_fail: str,
|
||||
details: str,
|
||||
evidence: dict[str, Any] | None = None,
|
||||
) -> ScenarioResult:
|
||||
return ScenarioResult(
|
||||
scenario_id=scenario_id,
|
||||
title=title,
|
||||
passed=passed,
|
||||
severity_on_fail=severity_on_fail,
|
||||
details=details,
|
||||
evidence=evidence or {},
|
||||
)
|
||||
|
||||
|
||||
async def run_scenarios(base_url: str) -> tuple[list[ScenarioResult], dict[str, Any]]:
|
||||
admin = {"X-User": "admin", "X-Role": "admin"}
|
||||
supervisor = {"X-User": "supervisor", "X-Role": "supervisor"}
|
||||
operator = {"X-User": "operator", "X-Role": "operator"}
|
||||
analyst = {"X-User": "analyst", "X-Role": "analyst"}
|
||||
results: list[ScenarioResult] = []
|
||||
context: dict[str, Any] = {
|
||||
"started_at": iso_now(),
|
||||
"baseline_doc": "docs/gates/mvp-pilot-baseline.md",
|
||||
"wave2_backlog": "docs/roadmap/05-wave2-backlog.md",
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(base_url=base_url, timeout=12) as client:
|
||||
health = await client.get("/health")
|
||||
registry = await client.get("/registry")
|
||||
if health.status_code != 200 or registry.status_code != 200:
|
||||
details = f"health={health.status_code}, registry={registry.status_code}"
|
||||
return [scenario_result("SYS", "System precheck", False, "P1", details)], context
|
||||
services = registry.json().get("services", {})
|
||||
context["reporting_url"] = services.get("reporting", "")
|
||||
|
||||
login = await client.post("/proxy/auth/auth/login", json={"username": "admin", "password": "admin123"})
|
||||
login_ok = False
|
||||
if login.status_code == 200:
|
||||
login_ok = bool(login.json().get("access_token"))
|
||||
denied = await client.post(
|
||||
"/proxy/auth/users",
|
||||
headers=operator,
|
||||
json={
|
||||
"username": f"uat_denied_{now_compact()}",
|
||||
"password": "secret123",
|
||||
"full_name": "UAT Denied",
|
||||
"role": "operator",
|
||||
},
|
||||
)
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S0",
|
||||
"Login and RBAC deny",
|
||||
login_ok and denied.status_code == 403,
|
||||
"P1",
|
||||
f"login={login.status_code}, denied={denied.status_code}",
|
||||
{"user": "admin", "denied_status": denied.status_code},
|
||||
)
|
||||
)
|
||||
|
||||
queue = await client.post(
|
||||
"/proxy/routing/queues",
|
||||
headers=admin,
|
||||
json={
|
||||
"name": f"UAT Dry Queue {now_compact()}",
|
||||
"description": "UAT dry-run queue",
|
||||
"rules": [
|
||||
{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 30},
|
||||
{"channel": "telegram", "priority": 3, "strategy": "round_robin", "sla_seconds": 45},
|
||||
],
|
||||
},
|
||||
)
|
||||
if queue.status_code != 200:
|
||||
results.append(
|
||||
scenario_result(
|
||||
"SYS",
|
||||
"System precheck",
|
||||
False,
|
||||
"P1",
|
||||
f"queue create failed status={queue.status_code}",
|
||||
)
|
||||
)
|
||||
context["completed_at"] = iso_now()
|
||||
return results, context
|
||||
queue_id = queue.json()["queue_id"]
|
||||
context["queue_id"] = queue_id
|
||||
|
||||
customer = await client.post(
|
||||
"/proxy/customer/customers",
|
||||
json={
|
||||
"display_name": "UAT Dry Customer",
|
||||
"phones": ["+77017770000"],
|
||||
"preferred_phone": "+77017770000",
|
||||
"tags": ["uat", "dry-run"],
|
||||
},
|
||||
)
|
||||
customer_id = ""
|
||||
search_ok = False
|
||||
if customer.status_code == 200:
|
||||
customer_id = customer.json()["customer_id"]
|
||||
search = await client.get("/proxy/customer/customers?query=dry-run")
|
||||
if search.status_code == 200:
|
||||
search_ok = any(item.get("customer_id") == customer_id for item in search.json())
|
||||
context["customer_id"] = customer_id
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S1",
|
||||
"Customer create and search",
|
||||
customer.status_code == 200 and search_ok,
|
||||
"P1",
|
||||
f"create={customer.status_code}, search={search_ok}",
|
||||
{"customer_id": customer_id, "query": "dry-run"},
|
||||
)
|
||||
)
|
||||
|
||||
voice_interaction_id = ""
|
||||
interaction = await client.post(
|
||||
"/proxy/interaction/interactions",
|
||||
headers=operator,
|
||||
json={
|
||||
"channel": "voice",
|
||||
"subject": "UAT dry-run S2 voice",
|
||||
"customer_id": customer_id or None,
|
||||
"queue_id": queue_id,
|
||||
"priority": 3,
|
||||
},
|
||||
)
|
||||
if interaction.status_code != 200:
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S2",
|
||||
"Voice interaction lifecycle",
|
||||
False,
|
||||
"P1",
|
||||
f"create status={interaction.status_code}",
|
||||
)
|
||||
)
|
||||
else:
|
||||
voice_interaction_id = interaction.json()["interaction_id"]
|
||||
context["voice_interaction_id"] = voice_interaction_id
|
||||
assign = await client.patch(
|
||||
f"/proxy/interaction/interactions/{voice_interaction_id}/assign",
|
||||
headers=supervisor,
|
||||
json={"assignee": "operator_a"},
|
||||
)
|
||||
close = await client.patch(
|
||||
f"/proxy/interaction/interactions/{voice_interaction_id}/status",
|
||||
headers=operator,
|
||||
json={"status": "closed"},
|
||||
)
|
||||
timeline = await client.get(f"/proxy/interaction/interactions/{voice_interaction_id}/timeline")
|
||||
actions: set[str] = set()
|
||||
if timeline.status_code == 200:
|
||||
actions = {event.get("action", "") for event in timeline.json().get("events", [])}
|
||||
expected = {"interaction.created", "interaction.assigned", "interaction.status_changed"}
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S2",
|
||||
"Voice interaction lifecycle",
|
||||
assign.status_code == 200 and close.status_code == 200 and expected.issubset(actions),
|
||||
"P1",
|
||||
f"assign={assign.status_code}, close={close.status_code}, actions={sorted(actions)}",
|
||||
{"interaction_id": voice_interaction_id},
|
||||
)
|
||||
)
|
||||
|
||||
esc_create = await client.post(
|
||||
"/proxy/interaction/interactions",
|
||||
headers=operator,
|
||||
json={
|
||||
"channel": "voice",
|
||||
"subject": "UAT dry-run S3 escalation",
|
||||
"customer_id": customer_id or None,
|
||||
"queue_id": queue_id,
|
||||
"priority": 3,
|
||||
},
|
||||
)
|
||||
if esc_create.status_code != 200:
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S3",
|
||||
"Assignment and escalation to second line",
|
||||
False,
|
||||
"P1",
|
||||
f"create status={esc_create.status_code}",
|
||||
)
|
||||
)
|
||||
else:
|
||||
esc_interaction_id = esc_create.json()["interaction_id"]
|
||||
assign = await client.patch(
|
||||
f"/proxy/interaction/interactions/{esc_interaction_id}/assign",
|
||||
headers=supervisor,
|
||||
json={"assignee": "operator_b"},
|
||||
)
|
||||
escalated = await client.post(
|
||||
f"/proxy/interaction/interactions/{esc_interaction_id}/escalate",
|
||||
headers=operator,
|
||||
json={"target_queue_id": "line2"},
|
||||
)
|
||||
timeline = await client.get(f"/proxy/interaction/interactions/{esc_interaction_id}/timeline")
|
||||
actions = []
|
||||
if timeline.status_code == 200:
|
||||
actions = [event.get("action", "") for event in timeline.json().get("events", [])]
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S3",
|
||||
"Assignment and escalation to second line",
|
||||
assign.status_code == 200
|
||||
and escalated.status_code == 200
|
||||
and escalated.json().get("status") == "escalated"
|
||||
and escalated.json().get("queue_id") == "line2"
|
||||
and "interaction.escalated" in actions,
|
||||
"P1",
|
||||
f"assign={assign.status_code}, escalate={escalated.status_code}, queue={escalated.json().get('queue_id') if escalated.status_code == 200 else ''}",
|
||||
{"interaction_id": esc_interaction_id, "target_queue": "line2"},
|
||||
)
|
||||
)
|
||||
|
||||
route = await client.post(f"/proxy/routing/queues/{queue_id}/route?channel=voice&priority=3")
|
||||
timeline = (
|
||||
await client.get(f"/proxy/interaction/interactions/{voice_interaction_id}/timeline")
|
||||
if voice_interaction_id
|
||||
else None
|
||||
)
|
||||
timeline_ok = False
|
||||
timeline_count = 0
|
||||
if timeline and timeline.status_code == 200:
|
||||
timeline_count = len(timeline.json().get("events", []))
|
||||
timeline_ok = timeline_count >= 3
|
||||
route_ok = False
|
||||
if route.status_code == 200:
|
||||
route_ok = bool(route.json().get("assignee"))
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S4",
|
||||
"Routing and timeline verification",
|
||||
route_ok and timeline_ok,
|
||||
"P2",
|
||||
f"route={route.status_code}, timeline_ok={timeline_ok}, timeline_events={timeline_count}",
|
||||
{
|
||||
"queue_id": queue_id,
|
||||
"routed_assignee": route.json().get("assignee") if route.status_code == 200 else "",
|
||||
"interaction_id": voice_interaction_id,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
voice_event = await client.post(
|
||||
"/proxy/voice/integrations/voice/events",
|
||||
headers=operator,
|
||||
json={
|
||||
"event_type": "call.started",
|
||||
"call_id": f"uat_call_{now_compact()}",
|
||||
"interaction_id": voice_interaction_id or None,
|
||||
"payload": {"source": "uat_dry_run"},
|
||||
},
|
||||
)
|
||||
voice_event_id = voice_event.json().get("event_id") if voice_event.status_code == 200 else ""
|
||||
voice_list_ok = False
|
||||
if voice_event.status_code == 200:
|
||||
voice_events = await client.get(
|
||||
"/proxy/voice/integrations/voice/events?limit=10",
|
||||
headers=supervisor,
|
||||
)
|
||||
if voice_events.status_code == 200:
|
||||
voice_list_ok = any(item.get("event_id") == voice_event_id for item in voice_events.json())
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S5",
|
||||
"Voice event intake",
|
||||
voice_event.status_code == 200 and voice_list_ok,
|
||||
"P2",
|
||||
f"create={voice_event.status_code}, listed={voice_list_ok}",
|
||||
{"event_id": voice_event_id},
|
||||
)
|
||||
)
|
||||
|
||||
telegram = await client.post(
|
||||
"/proxy/telegram/integrations/telegram/webhook",
|
||||
json={
|
||||
"chat_id": f"uat_dry_chat_{now_compact()}",
|
||||
"text": "UAT dry-run telegram",
|
||||
"payload": {"source": "uat_dry_run"},
|
||||
},
|
||||
)
|
||||
telegram_id = telegram.json().get("message_id") if telegram.status_code == 200 else ""
|
||||
telegram_list_ok = False
|
||||
if telegram.status_code == 200:
|
||||
messages = await client.get("/proxy/telegram/integrations/telegram/messages?limit=10")
|
||||
if messages.status_code == 200:
|
||||
telegram_list_ok = any(item.get("message_id") == telegram_id for item in messages.json())
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S6",
|
||||
"Telegram interaction lifecycle",
|
||||
telegram.status_code == 200 and telegram_list_ok,
|
||||
"P1",
|
||||
f"create={telegram.status_code}, listed={telegram_list_ok}",
|
||||
{"message_id": telegram_id},
|
||||
)
|
||||
)
|
||||
|
||||
category = await client.post(
|
||||
"/proxy/kb/knowledge/categories",
|
||||
headers=analyst,
|
||||
json={"name": f"UAT Dry {now_compact()}", "description": "dry-run"},
|
||||
)
|
||||
if category.status_code != 200:
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S7",
|
||||
"KB usage in active handling",
|
||||
False,
|
||||
"P2",
|
||||
f"category create status={category.status_code}",
|
||||
)
|
||||
)
|
||||
else:
|
||||
category_id = category.json()["category_id"]
|
||||
article = await client.post(
|
||||
"/proxy/kb/knowledge/articles",
|
||||
headers=analyst,
|
||||
json={
|
||||
"category_id": category_id,
|
||||
"title": "UAT dry KB article",
|
||||
"body": "KB article body for dry run",
|
||||
"tags": ["uat", "kb", "dry"],
|
||||
},
|
||||
)
|
||||
search = await client.get("/proxy/kb/knowledge/search?q=dry")
|
||||
article_id = article.json().get("article_id") if article.status_code == 200 else ""
|
||||
found = False
|
||||
if search.status_code == 200:
|
||||
found = any(item.get("article_id") == article_id for item in search.json())
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S7",
|
||||
"KB usage in active handling",
|
||||
article.status_code == 200 and search.status_code == 200 and found,
|
||||
"P2",
|
||||
f"article={article.status_code}, search={search.status_code}, found={found}",
|
||||
{"article_id": article_id, "keyword": "dry"},
|
||||
)
|
||||
)
|
||||
|
||||
up1 = await client.post(
|
||||
"/proxy/supervisor/supervisor/agent-states",
|
||||
json={"agent_id": "uat_dry_a1", "state": "READY", "queue_id": queue_id},
|
||||
)
|
||||
up2 = await client.post(
|
||||
"/proxy/supervisor/supervisor/agent-states",
|
||||
json={"agent_id": "uat_dry_a2", "state": "BUSY", "queue_id": queue_id},
|
||||
)
|
||||
metrics = await client.post(
|
||||
f"/proxy/supervisor/supervisor/queue-metrics?queue_id={queue_id}&in_queue=2&avg_wait_seconds=19"
|
||||
)
|
||||
realtime = await client.get("/proxy/supervisor/supervisor/realtime")
|
||||
queue_seen = False
|
||||
agent_total = 0
|
||||
if realtime.status_code == 200:
|
||||
body = realtime.json()
|
||||
agent_total = int(body.get("agents", {}).get("total", 0))
|
||||
queue_seen = any(row.get("queue_id") == queue_id for row in body.get("queues", []))
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S8",
|
||||
"Supervisor realtime",
|
||||
up1.status_code == 200
|
||||
and up2.status_code == 200
|
||||
and metrics.status_code == 200
|
||||
and realtime.status_code == 200
|
||||
and agent_total >= 2
|
||||
and queue_seen,
|
||||
"P2",
|
||||
f"state_updates=({up1.status_code},{up2.status_code}), metrics={metrics.status_code}, realtime={realtime.status_code}",
|
||||
{"queue_id": queue_id, "agent_total": agent_total},
|
||||
)
|
||||
)
|
||||
|
||||
kpi_rows = [
|
||||
{
|
||||
"queue_id": queue_id,
|
||||
"answered": True,
|
||||
"wait_seconds": 15,
|
||||
"handle_seconds": 90,
|
||||
"abandoned": False,
|
||||
"resolved_first_contact": True,
|
||||
},
|
||||
{
|
||||
"queue_id": queue_id,
|
||||
"answered": True,
|
||||
"wait_seconds": 33,
|
||||
"handle_seconds": 110,
|
||||
"abandoned": False,
|
||||
"resolved_first_contact": False,
|
||||
},
|
||||
{
|
||||
"queue_id": queue_id,
|
||||
"answered": False,
|
||||
"wait_seconds": 10,
|
||||
"handle_seconds": 0,
|
||||
"abandoned": True,
|
||||
"resolved_first_contact": False,
|
||||
},
|
||||
]
|
||||
ingest_statuses: list[int] = []
|
||||
for row in kpi_rows:
|
||||
ingested = await client.post("/proxy/reporting/reports/events", json=row)
|
||||
ingest_statuses.append(ingested.status_code)
|
||||
kpi = await client.get(f"/proxy/reporting/reports/kpi?queue_id={queue_id}&sl_threshold_seconds=30")
|
||||
has_kpi = False
|
||||
if kpi.status_code == 200:
|
||||
payload = kpi.json().get("kpi", {})
|
||||
has_kpi = {"SL", "ASA", "AHT", "Abandon", "FCR"}.issubset(set(payload.keys()))
|
||||
exported = await client.get("/proxy/reporting/reports/export")
|
||||
export_raw = exported.json().get("raw", "") if exported.status_code == 200 else ""
|
||||
csv_ok = (
|
||||
exported.status_code == 200
|
||||
and "queue_id,answered,wait_seconds,handle_seconds,abandoned,resolved_first_contact,created_at" in export_raw
|
||||
and queue_id in export_raw
|
||||
)
|
||||
results.append(
|
||||
scenario_result(
|
||||
"S9",
|
||||
"KPI report and export validation",
|
||||
all(code == 200 for code in ingest_statuses) and kpi.status_code == 200 and has_kpi and csv_ok,
|
||||
"P2",
|
||||
f"ingest={ingest_statuses}, kpi={kpi.status_code}, csv={csv_ok}",
|
||||
{"queue_id": queue_id},
|
||||
)
|
||||
)
|
||||
|
||||
context["completed_at"] = iso_now()
|
||||
return results, context
|
||||
|
||||
|
||||
def build_defects(results: list[ScenarioResult]) -> list[dict[str, str]]:
|
||||
defects: list[dict[str, str]] = []
|
||||
stamp = iso_now()
|
||||
counter = 1
|
||||
for row in results:
|
||||
if row.passed:
|
||||
continue
|
||||
defect_id = f"UAT-DRY-{counter:03d}"
|
||||
counter += 1
|
||||
severity = row.severity_on_fail
|
||||
defects.append(
|
||||
{
|
||||
"defect_id": defect_id,
|
||||
"severity": severity,
|
||||
"status": "Open",
|
||||
"scenario": row.scenario_id,
|
||||
"service": SCENARIO_SERVICE_MAP.get(row.scenario_id, "platform"),
|
||||
"risk_level": SEVERITY_RISK_MAP.get(severity, "Medium"),
|
||||
"summary": row.title,
|
||||
"repro_steps": "See scenario checklist and automated dry-run logs",
|
||||
"actual_result": row.details,
|
||||
"expected_result": "Scenario should pass",
|
||||
"verification_step": f"Re-run scenario {row.scenario_id} after fix",
|
||||
"owner": "TBD",
|
||||
"opened_at": stamp,
|
||||
"closed_at": "",
|
||||
"backlog_bucket": "MVP" if severity in {"P1", "P2"} else "Wave2",
|
||||
"notes": "Generated by scripts/uat_dry_run.py",
|
||||
}
|
||||
)
|
||||
return defects
|
||||
|
||||
|
||||
def write_scenario_json(output_dir: Path, results: list[ScenarioResult], context: dict[str, Any]) -> Path:
|
||||
payload = {
|
||||
"generated_at": iso_now(),
|
||||
"context": context,
|
||||
"scenarios": [
|
||||
{
|
||||
"scenario_id": row.scenario_id,
|
||||
"title": row.title,
|
||||
"passed": row.passed,
|
||||
"severity_on_fail": row.severity_on_fail,
|
||||
"details": row.details,
|
||||
"evidence": row.evidence,
|
||||
}
|
||||
for row in results
|
||||
],
|
||||
}
|
||||
path = output_dir / "scenario-results.json"
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def write_defect_csv(output_dir: Path, defects: list[dict[str, str]]) -> Path:
|
||||
path = output_dir / "defect-log.csv"
|
||||
fieldnames = [
|
||||
"defect_id",
|
||||
"severity",
|
||||
"status",
|
||||
"scenario",
|
||||
"service",
|
||||
"risk_level",
|
||||
"summary",
|
||||
"repro_steps",
|
||||
"actual_result",
|
||||
"expected_result",
|
||||
"verification_step",
|
||||
"owner",
|
||||
"opened_at",
|
||||
"closed_at",
|
||||
"backlog_bucket",
|
||||
"notes",
|
||||
]
|
||||
with path.open("w", newline="", encoding="utf-8") as fp:
|
||||
writer = csv.DictWriter(fp, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for row in defects:
|
||||
writer.writerow(row)
|
||||
return path
|
||||
|
||||
|
||||
def write_session_protocol(
|
||||
output_dir: Path,
|
||||
session_id: str,
|
||||
base_url: str,
|
||||
results: list[ScenarioResult],
|
||||
defects: list[dict[str, str]],
|
||||
build_version: str,
|
||||
deployment_date: str,
|
||||
cluster_namespace: str,
|
||||
) -> Path:
|
||||
passed = sum(1 for row in results if row.passed)
|
||||
failed = sum(1 for row in results if not row.passed)
|
||||
p1 = sum(1 for row in defects if row["severity"] == "P1")
|
||||
p2 = sum(1 for row in defects if row["severity"] == "P2")
|
||||
p3 = sum(1 for row in defects if row["severity"] == "P3")
|
||||
p4 = sum(1 for row in defects if row["severity"] == "P4")
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append("# UAT Session Protocol (Automated Dry Run)")
|
||||
lines.append("")
|
||||
lines.append("## Session Metadata")
|
||||
lines.append("")
|
||||
lines.append(f"- Session ID: {session_id}")
|
||||
lines.append(f"- Date: {utc_now().date().isoformat()}")
|
||||
lines.append(f"- Start time (UTC): {iso_now()}")
|
||||
lines.append(f"- End time (UTC): {iso_now()}")
|
||||
lines.append(f"- Environment URL: {base_url}")
|
||||
lines.append(f"- Build/version: {build_version}")
|
||||
lines.append(f"- Deployment date: {deployment_date}")
|
||||
lines.append(f"- Cluster/namespace: {cluster_namespace}")
|
||||
lines.append("")
|
||||
lines.append("## Scope Confirmation")
|
||||
lines.append("")
|
||||
lines.append("- [x] MVP pilot scope reviewed against `docs/gates/mvp-pilot-baseline.md`")
|
||||
lines.append("- [x] Out-of-scope requests remain deferred to `docs/roadmap/05-wave2-backlog.md`")
|
||||
lines.append("- [x] API stabilization rules remain in effect (backward-compatible fixes only)")
|
||||
lines.append("")
|
||||
lines.append("## Participants")
|
||||
lines.append("")
|
||||
lines.append("- Operators: automated dry-run actor")
|
||||
lines.append("- Supervisor: automated dry-run actor")
|
||||
lines.append("- Analyst: automated dry-run actor")
|
||||
lines.append("- Admin: automated dry-run actor")
|
||||
lines.append("- Business owner: pending manual UAT")
|
||||
lines.append("- IT owner: pending manual UAT")
|
||||
lines.append("")
|
||||
lines.append("## Preconditions")
|
||||
lines.append("")
|
||||
lines.append("- [x] Environment is reachable")
|
||||
lines.append("- [x] Test accounts are active")
|
||||
lines.append("- [x] Test queue is configured")
|
||||
lines.append("- [x] Audit and reporting endpoints are reachable")
|
||||
lines.append("")
|
||||
lines.append("## Execution Summary")
|
||||
lines.append("")
|
||||
lines.append(f"- Mandatory scenarios executed: {len(results)}")
|
||||
lines.append(f"- Passed: {passed}")
|
||||
lines.append(f"- Failed: {failed}")
|
||||
lines.append("- Blocked: 0")
|
||||
lines.append("")
|
||||
lines.append("## Scenario Results")
|
||||
lines.append("")
|
||||
for row in results:
|
||||
mark = "PASS" if row.passed else "FAIL"
|
||||
lines.append(f"- [{mark}] {row.scenario_id} {row.title}: {row.details}")
|
||||
lines.append("")
|
||||
lines.append("## Defect Summary")
|
||||
lines.append("")
|
||||
lines.append(f"- P1: {p1}")
|
||||
lines.append(f"- P2: {p2}")
|
||||
lines.append(f"- P3: {p3}")
|
||||
lines.append(f"- P4: {p4}")
|
||||
lines.append("- Defect log attachment: defect-log.csv")
|
||||
lines.append("")
|
||||
lines.append("## Decision")
|
||||
lines.append("")
|
||||
if failed == 0:
|
||||
lines.append("- [x] Accepted with conditions")
|
||||
lines.append("- [ ] Accepted for pilot completion")
|
||||
lines.append("- [ ] Not accepted")
|
||||
else:
|
||||
lines.append("- [ ] Accepted with conditions")
|
||||
lines.append("- [ ] Accepted for pilot completion")
|
||||
lines.append("- [x] Not accepted")
|
||||
lines.append("")
|
||||
lines.append("## Comments")
|
||||
lines.append("")
|
||||
lines.append("- Automated dry-run completed.")
|
||||
lines.append("- Manual UAT with real operators/supervisor is still required for final sign-off.")
|
||||
lines.append("- Any future scope additions must be recorded in `docs/roadmap/05-wave2-backlog.md`.")
|
||||
lines.append("")
|
||||
lines.append("## Signatures")
|
||||
lines.append("")
|
||||
lines.append("- Business owner: pending")
|
||||
lines.append("- IT owner: pending")
|
||||
lines.append("- Supervisor representative: pending")
|
||||
lines.append(f"- Date: {utc_now().date().isoformat()}")
|
||||
lines.append("")
|
||||
path = output_dir / "session-protocol.md"
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def write_signoff_draft(output_dir: Path, session_id: str, failed: int) -> Path:
|
||||
lines: list[str] = []
|
||||
lines.append("# UAT Sign-off Sheet (Draft)")
|
||||
lines.append("")
|
||||
lines.append(f"- Session ID: {session_id}")
|
||||
lines.append(f"- Date: {utc_now().date().isoformat()}")
|
||||
lines.append(f"- Scenario failures: {failed}")
|
||||
lines.append("")
|
||||
lines.append("## Scope Confirmation")
|
||||
lines.append("")
|
||||
lines.append("- MVP pilot scope confirmed against `docs/gates/mvp-pilot-baseline.md`")
|
||||
lines.append("- Out-of-scope requests remain deferred to `docs/roadmap/05-wave2-backlog.md`")
|
||||
lines.append("")
|
||||
lines.append("## Acceptance Statement")
|
||||
lines.append("")
|
||||
lines.append("This draft is generated by automated dry-run and cannot replace real UAT signatures.")
|
||||
lines.append("")
|
||||
lines.append("## Required Manual Signatures")
|
||||
lines.append("")
|
||||
lines.append("- Business owner: pending")
|
||||
lines.append("- IT owner: pending")
|
||||
lines.append("- Supervisor representative: pending")
|
||||
lines.append("")
|
||||
path = output_dir / "signoff-draft.md"
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def write_summary(
|
||||
output_dir: Path,
|
||||
session_id: str,
|
||||
base_url: str,
|
||||
scenario_file: Path,
|
||||
defect_file: Path,
|
||||
protocol_file: Path,
|
||||
signoff_file: Path,
|
||||
results: list[ScenarioResult],
|
||||
defects: list[dict[str, str]],
|
||||
) -> Path:
|
||||
passed = sum(1 for row in results if row.passed)
|
||||
failed = sum(1 for row in results if not row.passed)
|
||||
p1 = sum(1 for row in defects if row["severity"] == "P1")
|
||||
p2 = sum(1 for row in defects if row["severity"] == "P2")
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append("# UAT Dry-Run Summary")
|
||||
lines.append("")
|
||||
lines.append(f"- Session ID: {session_id}")
|
||||
lines.append(f"- Generated at (UTC): {iso_now()}")
|
||||
lines.append(f"- Environment URL: {base_url}")
|
||||
lines.append(f"- Scenarios: {len(results)} total, {passed} passed, {failed} failed")
|
||||
lines.append(f"- Defects: P1={p1}, P2={p2}")
|
||||
lines.append("")
|
||||
lines.append("## Files")
|
||||
lines.append("")
|
||||
lines.append(f"- Scenario results: `{scenario_file.name}`")
|
||||
lines.append(f"- Session protocol: `{protocol_file.name}`")
|
||||
lines.append(f"- Defect log: `{defect_file.name}`")
|
||||
lines.append(f"- Sign-off draft: `{signoff_file.name}`")
|
||||
lines.append("")
|
||||
lines.append("## Next Step")
|
||||
lines.append("")
|
||||
lines.append("Run manual UAT with real participants, fix only P1/P2 items, and collect signatures.")
|
||||
lines.append("Route feature requests and deferred scope to `docs/roadmap/05-wave2-backlog.md`.")
|
||||
lines.append("")
|
||||
path = output_dir / "summary.md"
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def update_p1p2_register(defects: list[dict[str, str]], session_id: str) -> None:
|
||||
p1 = sum(1 for row in defects if row["severity"] == "P1" and row["status"] == "Open")
|
||||
p2 = sum(1 for row in defects if row["severity"] == "P2" and row["status"] == "Open")
|
||||
path = ROOT / "docs" / "gates" / "p1-p2-defects.md"
|
||||
lines: list[str] = []
|
||||
lines.append("# P1/P2 Defect Register (Pilot)")
|
||||
lines.append("")
|
||||
lines.append(f"Last update: {utc_now().date().isoformat()} ({session_id})")
|
||||
lines.append("")
|
||||
lines.append("## Current Status")
|
||||
lines.append(f"- Open P1: {p1}")
|
||||
lines.append(f"- Open P2: {p2}")
|
||||
lines.append("- Source: `scripts/uat_dry_run.py` and automated checks")
|
||||
lines.append(f"- Latest dry-run evidence: `docs/uat/evidence/dry_run_{session_id}/`")
|
||||
lines.append("")
|
||||
lines.append("## Triage Policy")
|
||||
lines.append("- Only `P1` and `P2` issues belong to MVP remediation.")
|
||||
lines.append("- `P3` and `P4` issues move to `docs/roadmap/05-wave2-backlog.md` unless they block sign-off.")
|
||||
lines.append("")
|
||||
lines.append("## Pilot Note")
|
||||
lines.append("- Final P1/P2 closure is confirmed only after the real-operator UAT cycle and sign-off.")
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Run automated UAT dry-run and generate session artifacts")
|
||||
parser.add_argument("--base-url", default="http://localhost:8080", help="Gateway base URL")
|
||||
parser.add_argument("--auto-start", action="store_true", help="Auto-start local services for dry-run")
|
||||
parser.add_argument("--keep-data", action="store_true", help="Keep temporary auto-start data directory")
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default="",
|
||||
help="Output root directory (default: docs/uat/evidence)",
|
||||
)
|
||||
parser.add_argument("--session-id", default="", help="Optional session ID override")
|
||||
parser.add_argument("--build-version", default="v1.0.0-mvp", help="Build/version for protocol")
|
||||
parser.add_argument(
|
||||
"--deployment-date",
|
||||
default=utc_now().date().isoformat(),
|
||||
help="Deployment date for protocol",
|
||||
)
|
||||
parser.add_argument("--cluster-namespace", default="local/standalone", help="Cluster/namespace for protocol")
|
||||
parser.add_argument(
|
||||
"--update-defect-register",
|
||||
action="store_true",
|
||||
help="Update docs/gates/p1-p2-defects.md using dry-run result",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
output_root = Path(args.output_dir) if args.output_dir else DEFAULT_OUTPUT_ROOT
|
||||
output_root = output_root if output_root.is_absolute() else (ROOT / output_root).resolve()
|
||||
session_id = args.session_id.strip() or f"UAT-DRY-{now_compact()}"
|
||||
session_dir = output_root / f"dry_run_{session_id}"
|
||||
session_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
|
||||
processes: list[subprocess.Popen] = []
|
||||
base_url = args.base_url
|
||||
|
||||
try:
|
||||
if args.auto_start:
|
||||
processes = await start_services_sequential(run_data_dir)
|
||||
base_url = "http://127.0.0.1:58080"
|
||||
|
||||
results, context = await run_scenarios(base_url)
|
||||
defects = build_defects(results)
|
||||
|
||||
scenario_file = write_scenario_json(session_dir, results, context)
|
||||
defect_file = write_defect_csv(session_dir, defects)
|
||||
protocol_file = write_session_protocol(
|
||||
session_dir,
|
||||
session_id,
|
||||
base_url,
|
||||
results,
|
||||
defects,
|
||||
args.build_version,
|
||||
args.deployment_date,
|
||||
args.cluster_namespace,
|
||||
)
|
||||
signoff_file = write_signoff_draft(session_dir, session_id, sum(1 for row in results if not row.passed))
|
||||
summary_file = write_summary(
|
||||
session_dir,
|
||||
session_id,
|
||||
base_url,
|
||||
scenario_file,
|
||||
defect_file,
|
||||
protocol_file,
|
||||
signoff_file,
|
||||
results,
|
||||
defects,
|
||||
)
|
||||
|
||||
if args.update_defect_register:
|
||||
update_p1p2_register(defects, session_id)
|
||||
|
||||
passed = sum(1 for row in results if row.passed)
|
||||
failed = sum(1 for row in results if not row.passed)
|
||||
print(f"UAT dry-run session: {session_id}")
|
||||
print(f"Output directory: {session_dir}")
|
||||
print(f"Scenarios: total={len(results)} passed={passed} failed={failed}")
|
||||
print(f"Summary: {summary_file}")
|
||||
sys.exit(0 if failed == 0 else 1)
|
||||
finally:
|
||||
stop_services(processes)
|
||||
if args.auto_start and not args.keep_data:
|
||||
try:
|
||||
if run_data_dir.exists():
|
||||
shutil.rmtree(run_data_dir)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,329 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_EVIDENCE_ROOT = ROOT / "docs" / "uat" / "evidence"
|
||||
DEFAULT_TEMPLATE_ROOT = ROOT / "docs" / "uat"
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def now_compact() -> str:
|
||||
return utc_now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
|
||||
def iso_now() -> str:
|
||||
return utc_now().isoformat()
|
||||
|
||||
|
||||
def find_latest_preflight(evidence_root: Path) -> Path:
|
||||
candidates = sorted(
|
||||
evidence_root.glob("preflight_*.md"),
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
if not candidates:
|
||||
raise FileNotFoundError("No preflight report found in docs/uat/evidence")
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def find_latest_dry_run_dir(evidence_root: Path) -> Path:
|
||||
candidates = sorted(
|
||||
[path for path in evidence_root.glob("dry_run_*") if path.is_dir()],
|
||||
key=lambda path: path.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
if not candidates:
|
||||
raise FileNotFoundError("No dry-run directory found in docs/uat/evidence")
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def _replace_line(content: str, prefix: str, value: str) -> str:
|
||||
replacement = f"{prefix} {value}".rstrip()
|
||||
return content.replace(prefix, replacement, 1)
|
||||
|
||||
|
||||
def render_session_protocol(
|
||||
template_text: str,
|
||||
*,
|
||||
session_id: str,
|
||||
environment_url: str,
|
||||
build_version: str,
|
||||
deployment_date: str,
|
||||
cluster_namespace: str,
|
||||
participants: dict[str, str],
|
||||
preflight_name: str,
|
||||
defect_log_name: str,
|
||||
) -> str:
|
||||
content = template_text
|
||||
today = utc_now().date().isoformat()
|
||||
content = _replace_line(content, "- Session ID:", session_id)
|
||||
content = _replace_line(content, "- Date:", today)
|
||||
content = _replace_line(content, "- Start time:", "")
|
||||
content = _replace_line(content, "- End time:", "")
|
||||
content = _replace_line(content, "- Environment URL:", environment_url)
|
||||
content = _replace_line(content, "- Build/version:", build_version)
|
||||
content = _replace_line(content, "- Deployment date:", deployment_date)
|
||||
content = _replace_line(content, "- Cluster/namespace:", cluster_namespace)
|
||||
|
||||
field_map = {
|
||||
"- Operators:": participants["operators"],
|
||||
"- Supervisor:": participants["supervisor"],
|
||||
"- Analyst:": participants["analyst"],
|
||||
"- Admin:": participants["admin"],
|
||||
"- Business owner:": participants["business_owner"],
|
||||
"- IT owner:": participants["it_owner"],
|
||||
}
|
||||
for prefix, value in field_map.items():
|
||||
content = _replace_line(content, prefix, value)
|
||||
|
||||
content = content.replace(
|
||||
"- [ ] Preflight report attached (`docs/uat/evidence/preflight_*.md`)",
|
||||
f"- [x] Preflight report attached (`attachments/{preflight_name}`)",
|
||||
1,
|
||||
)
|
||||
content = content.replace(
|
||||
"- [ ] Defect log prepared from `docs/uat/defect-log-template.csv`",
|
||||
f"- [x] Defect log prepared from `{defect_log_name}`",
|
||||
1,
|
||||
)
|
||||
return content
|
||||
|
||||
|
||||
def render_signoff_sheet(
|
||||
template_text: str,
|
||||
*,
|
||||
session_id: str,
|
||||
environment_url: str,
|
||||
build_version: str,
|
||||
) -> str:
|
||||
lines = template_text.splitlines()
|
||||
if not lines:
|
||||
return template_text
|
||||
|
||||
header = [
|
||||
lines[0],
|
||||
"",
|
||||
f"- Session ID: {session_id}",
|
||||
f"- Environment URL: {environment_url}",
|
||||
f"- Build/version: {build_version}",
|
||||
f"- Prepared at (UTC): {iso_now()}",
|
||||
"",
|
||||
]
|
||||
return "\n".join(header + lines[1:])
|
||||
|
||||
|
||||
def build_manual_readme(
|
||||
*,
|
||||
session_id: str,
|
||||
environment_url: str,
|
||||
preflight_name: str,
|
||||
dry_run_name: str,
|
||||
) -> str:
|
||||
lines = [
|
||||
"# Manual UAT Session Bundle",
|
||||
"",
|
||||
f"- Session ID: {session_id}",
|
||||
f"- Environment URL: {environment_url}",
|
||||
f"- Prepared at (UTC): {iso_now()}",
|
||||
"",
|
||||
"## Included Files",
|
||||
"",
|
||||
"- `session-protocol.md`",
|
||||
"- `scenario-checklist.md`",
|
||||
"- `defect-log.csv`",
|
||||
"- `signoff-sheet.md`",
|
||||
"- `manifest.json`",
|
||||
"",
|
||||
"## Attached Evidence",
|
||||
"",
|
||||
f"- `attachments/{preflight_name}`",
|
||||
f"- `attachments/{dry_run_name}/`",
|
||||
"",
|
||||
"## Next Step",
|
||||
"",
|
||||
"Run the live manual UAT on the pilot gateway, record any defects, and use the",
|
||||
"`scripts/finalize_mvp_pilot.py` command only after signatures are captured and",
|
||||
"all open P1/P2 defects are closed.",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def prepare_manual_bundle(
|
||||
*,
|
||||
session_id: str,
|
||||
environment_url: str,
|
||||
build_version: str,
|
||||
deployment_date: str,
|
||||
cluster_namespace: str,
|
||||
participants: dict[str, str],
|
||||
output_root: Path,
|
||||
template_root: Path,
|
||||
preflight_report: Path,
|
||||
dry_run_dir: Path,
|
||||
overwrite: bool = False,
|
||||
) -> Path:
|
||||
session_dir = output_root / f"manual_{session_id}"
|
||||
if session_dir.exists():
|
||||
if not overwrite:
|
||||
raise FileExistsError(f"Session directory already exists: {session_dir}")
|
||||
shutil.rmtree(session_dir)
|
||||
|
||||
attachments_dir = session_dir / "attachments"
|
||||
attachments_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
copied_preflight = attachments_dir / preflight_report.name
|
||||
shutil.copy2(preflight_report, copied_preflight)
|
||||
|
||||
copied_dry_run = attachments_dir / dry_run_dir.name
|
||||
shutil.copytree(dry_run_dir, copied_dry_run)
|
||||
|
||||
session_template = (template_root / "session-template.md").read_text(encoding="utf-8")
|
||||
signoff_template = (template_root / "signoff-template.md").read_text(encoding="utf-8")
|
||||
checklist_template = (template_root / "scenario-checklist.md").read_text(encoding="utf-8")
|
||||
defect_template = (template_root / "defect-log-template.csv").read_text(encoding="utf-8")
|
||||
|
||||
rendered_session = render_session_protocol(
|
||||
session_template,
|
||||
session_id=session_id,
|
||||
environment_url=environment_url,
|
||||
build_version=build_version,
|
||||
deployment_date=deployment_date,
|
||||
cluster_namespace=cluster_namespace,
|
||||
participants=participants,
|
||||
preflight_name=preflight_report.name,
|
||||
defect_log_name="defect-log.csv",
|
||||
)
|
||||
rendered_signoff = render_signoff_sheet(
|
||||
signoff_template,
|
||||
session_id=session_id,
|
||||
environment_url=environment_url,
|
||||
build_version=build_version,
|
||||
)
|
||||
|
||||
(session_dir / "session-protocol.md").write_text(rendered_session, encoding="utf-8")
|
||||
(session_dir / "signoff-sheet.md").write_text(rendered_signoff, encoding="utf-8")
|
||||
(session_dir / "scenario-checklist.md").write_text(checklist_template, encoding="utf-8")
|
||||
(session_dir / "defect-log.csv").write_text(defect_template, encoding="utf-8")
|
||||
|
||||
manifest: dict[str, Any] = {
|
||||
"prepared_at": iso_now(),
|
||||
"session_id": session_id,
|
||||
"environment_url": environment_url,
|
||||
"build_version": build_version,
|
||||
"deployment_date": deployment_date,
|
||||
"cluster_namespace": cluster_namespace,
|
||||
"participants": participants,
|
||||
"artifacts": {
|
||||
"preflight_report": f"attachments/{preflight_report.name}",
|
||||
"dry_run_bundle": f"attachments/{dry_run_dir.name}",
|
||||
"session_protocol": "session-protocol.md",
|
||||
"scenario_checklist": "scenario-checklist.md",
|
||||
"defect_log": "defect-log.csv",
|
||||
"signoff_sheet": "signoff-sheet.md",
|
||||
},
|
||||
}
|
||||
(session_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
(session_dir / "README.md").write_text(
|
||||
build_manual_readme(
|
||||
session_id=session_id,
|
||||
environment_url=environment_url,
|
||||
preflight_name=preflight_report.name,
|
||||
dry_run_name=dry_run_dir.name,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
return session_dir
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Prepare a manual UAT bundle from the latest preflight and dry-run")
|
||||
parser.add_argument("--session-id", default=f"UAT-MANUAL-{now_compact()}", help="Manual UAT session id")
|
||||
parser.add_argument(
|
||||
"--environment-url",
|
||||
default="http://<pilot-gateway>:8080",
|
||||
help="Pilot gateway URL recorded in the session pack",
|
||||
)
|
||||
parser.add_argument("--build-version", default="v1.0.0-mvp", help="Build/version under test")
|
||||
parser.add_argument(
|
||||
"--deployment-date",
|
||||
default=utc_now().date().isoformat(),
|
||||
help="Deployment date recorded in the session pack",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cluster-namespace",
|
||||
default="pilot/production-like",
|
||||
help="Cluster or namespace label recorded in the session pack",
|
||||
)
|
||||
parser.add_argument("--operators", default="TBD", help="Operator participants")
|
||||
parser.add_argument("--supervisor", default="TBD", help="Supervisor participant")
|
||||
parser.add_argument("--analyst", default="TBD", help="Analyst participant")
|
||||
parser.add_argument("--admin", default="TBD", help="Admin participant")
|
||||
parser.add_argument("--business-owner", default="TBD", help="Business owner")
|
||||
parser.add_argument("--it-owner", default="TBD", help="IT owner")
|
||||
parser.add_argument(
|
||||
"--preflight-report",
|
||||
default="",
|
||||
help="Optional explicit preflight report path (default: latest preflight report)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run-dir",
|
||||
default="",
|
||||
help="Optional explicit dry-run directory path (default: latest dry-run evidence folder)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
default="",
|
||||
help="Output root directory (default: docs/uat/evidence)",
|
||||
)
|
||||
parser.add_argument("--overwrite", action="store_true", help="Overwrite the session folder if it already exists")
|
||||
args = parser.parse_args()
|
||||
|
||||
output_root = Path(args.output_root) if args.output_root else DEFAULT_EVIDENCE_ROOT
|
||||
output_root = output_root if output_root.is_absolute() else (ROOT / output_root).resolve()
|
||||
|
||||
preflight_report = Path(args.preflight_report) if args.preflight_report else find_latest_preflight(DEFAULT_EVIDENCE_ROOT)
|
||||
if not preflight_report.is_absolute():
|
||||
preflight_report = (ROOT / preflight_report).resolve()
|
||||
|
||||
dry_run_dir = Path(args.dry_run_dir) if args.dry_run_dir else find_latest_dry_run_dir(DEFAULT_EVIDENCE_ROOT)
|
||||
if not dry_run_dir.is_absolute():
|
||||
dry_run_dir = (ROOT / dry_run_dir).resolve()
|
||||
|
||||
participants = {
|
||||
"operators": args.operators,
|
||||
"supervisor": args.supervisor,
|
||||
"analyst": args.analyst,
|
||||
"admin": args.admin,
|
||||
"business_owner": args.business_owner,
|
||||
"it_owner": args.it_owner,
|
||||
}
|
||||
|
||||
session_dir = prepare_manual_bundle(
|
||||
session_id=args.session_id,
|
||||
environment_url=args.environment_url,
|
||||
build_version=args.build_version,
|
||||
deployment_date=args.deployment_date,
|
||||
cluster_namespace=args.cluster_namespace,
|
||||
participants=participants,
|
||||
output_root=output_root,
|
||||
template_root=DEFAULT_TEMPLATE_ROOT,
|
||||
preflight_report=preflight_report,
|
||||
dry_run_dir=dry_run_dir,
|
||||
overwrite=args.overwrite,
|
||||
)
|
||||
print(f"Manual UAT bundle prepared: {session_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,457 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DATA_ROOT = ROOT / ".data_uat_preflight"
|
||||
DEFAULT_EVIDENCE_DIR = ROOT / "docs" / "uat" / "evidence"
|
||||
|
||||
SERVICE_SPECS = [
|
||||
{"name": "auth", "module": "services.auth_service.app:app", "port": 48001},
|
||||
{"name": "audit", "module": "services.audit_service.app:app", "port": 48002},
|
||||
{"name": "customer", "module": "services.customer_service.app:app", "port": 48003},
|
||||
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 48004},
|
||||
{"name": "routing", "module": "services.routing_service.app:app", "port": 48005},
|
||||
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 48006},
|
||||
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 48007},
|
||||
{"name": "kb", "module": "services.kb_service.app:app", "port": 48008},
|
||||
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 48009},
|
||||
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 48010},
|
||||
{"name": "gateway", "module": "gateway.app:app", "port": 48080},
|
||||
]
|
||||
|
||||
REQUIRED_SERVICES = [
|
||||
"auth",
|
||||
"audit",
|
||||
"customer",
|
||||
"interaction",
|
||||
"routing",
|
||||
"voice",
|
||||
"telegram",
|
||||
"kb",
|
||||
"reporting",
|
||||
"supervisor",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckResult:
|
||||
name: str
|
||||
ok: bool
|
||||
details: str
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def now_compact() -> str:
|
||||
return utc_now().strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
|
||||
def iso_now() -> str:
|
||||
return utc_now().isoformat()
|
||||
|
||||
|
||||
async def wait_for_health(base_url: str, retries: int = 80, delay: float = 0.2) -> None:
|
||||
async with httpx.AsyncClient(timeout=2) as client:
|
||||
for _ in range(retries):
|
||||
try:
|
||||
response = await client.get(f"{base_url}/health")
|
||||
if response.status_code == 200:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(delay)
|
||||
raise RuntimeError(f"Service not healthy: {base_url}/health")
|
||||
|
||||
|
||||
def start_one_service(spec: dict[str, Any], data_dir: Path) -> subprocess.Popen:
|
||||
env = os.environ.copy()
|
||||
env["CC_DATA_DIR"] = str(data_dir)
|
||||
|
||||
if spec["name"] == "gateway":
|
||||
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:48001"
|
||||
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:48002"
|
||||
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:48003"
|
||||
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:48004"
|
||||
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:48005"
|
||||
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:48006"
|
||||
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:48007"
|
||||
env["KB_SERVICE_URL"] = "http://127.0.0.1:48008"
|
||||
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:48009"
|
||||
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:48010"
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"uvicorn",
|
||||
spec["module"],
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
str(spec["port"]),
|
||||
]
|
||||
return subprocess.Popen(
|
||||
cmd,
|
||||
cwd=str(ROOT),
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
async def start_services_sequential(data_dir: Path) -> list[subprocess.Popen]:
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
processes: list[subprocess.Popen] = []
|
||||
for spec in SERVICE_SPECS:
|
||||
proc = start_one_service(spec, data_dir)
|
||||
processes.append(proc)
|
||||
await wait_for_health(f"http://127.0.0.1:{spec['port']}")
|
||||
return processes
|
||||
|
||||
|
||||
def stop_services(processes: list[subprocess.Popen]) -> None:
|
||||
for proc in processes:
|
||||
if proc.poll() is not None:
|
||||
continue
|
||||
try:
|
||||
proc.terminate()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for proc in processes:
|
||||
try:
|
||||
proc.wait(timeout=2)
|
||||
except Exception:
|
||||
try:
|
||||
proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def run_checks(base_url: str) -> tuple[list[CheckResult], dict[str, Any]]:
|
||||
checks: list[CheckResult] = []
|
||||
artifacts: dict[str, Any] = {}
|
||||
admin = {"X-User": "admin", "X-Role": "admin"}
|
||||
supervisor = {"X-User": "supervisor", "X-Role": "supervisor"}
|
||||
operator = {"X-User": "operator", "X-Role": "operator"}
|
||||
analyst = {"X-User": "analyst", "X-Role": "analyst"}
|
||||
|
||||
async with httpx.AsyncClient(base_url=base_url, timeout=10) as client:
|
||||
try:
|
||||
health = await client.get("/health")
|
||||
ok = health.status_code == 200
|
||||
checks.append(CheckResult("Gateway health", ok, f"status={health.status_code}"))
|
||||
if not ok:
|
||||
return checks, artifacts
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("Gateway health", False, f"exception={exc}"))
|
||||
return checks, artifacts
|
||||
|
||||
try:
|
||||
registry = await client.get("/registry")
|
||||
data = registry.json() if registry.status_code == 200 else {}
|
||||
services = data.get("services", {})
|
||||
missing = [name for name in REQUIRED_SERVICES if name not in services]
|
||||
ok = registry.status_code == 200 and not missing
|
||||
details = f"status={registry.status_code}"
|
||||
if missing:
|
||||
details += f", missing={missing}"
|
||||
checks.append(CheckResult("Service registry", ok, details))
|
||||
if not ok:
|
||||
return checks, artifacts
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("Service registry", False, f"exception={exc}"))
|
||||
return checks, artifacts
|
||||
|
||||
try:
|
||||
login = await client.post("/proxy/auth/auth/login", json={"username": "admin", "password": "admin123"})
|
||||
ok = login.status_code == 200 and "access_token" in login.json()
|
||||
checks.append(CheckResult("Auth login", ok, f"status={login.status_code}"))
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("Auth login", False, f"exception={exc}"))
|
||||
return checks, artifacts
|
||||
|
||||
try:
|
||||
queue = await client.post(
|
||||
"/proxy/routing/queues",
|
||||
headers=admin,
|
||||
json={
|
||||
"name": f"UAT Queue {now_compact()}",
|
||||
"description": "UAT preflight queue",
|
||||
"rules": [
|
||||
{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 30},
|
||||
{"channel": "telegram", "priority": 3, "strategy": "round_robin", "sla_seconds": 45},
|
||||
],
|
||||
},
|
||||
)
|
||||
queue_ok = queue.status_code == 200
|
||||
queue_id = queue.json().get("queue_id") if queue_ok else ""
|
||||
artifacts["queue_id"] = queue_id
|
||||
checks.append(CheckResult("Queue create", queue_ok, f"status={queue.status_code}, queue_id={queue_id}"))
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("Queue create", False, f"exception={exc}"))
|
||||
return checks, artifacts
|
||||
|
||||
try:
|
||||
customer = await client.post(
|
||||
"/proxy/customer/customers",
|
||||
json={
|
||||
"display_name": "UAT Preflight Customer",
|
||||
"phones": ["+77019990000"],
|
||||
"preferred_phone": "+77019990000",
|
||||
"tags": ["uat", "preflight"],
|
||||
},
|
||||
)
|
||||
customer_ok = customer.status_code == 200
|
||||
customer_id = customer.json().get("customer_id") if customer_ok else ""
|
||||
artifacts["customer_id"] = customer_id
|
||||
checks.append(
|
||||
CheckResult("Customer create", customer_ok, f"status={customer.status_code}, customer_id={customer_id}")
|
||||
)
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("Customer create", False, f"exception={exc}"))
|
||||
return checks, artifacts
|
||||
|
||||
interaction_id = ""
|
||||
try:
|
||||
interaction = await client.post(
|
||||
"/proxy/interaction/interactions",
|
||||
headers=operator,
|
||||
json={
|
||||
"channel": "voice",
|
||||
"subject": "UAT preflight voice interaction",
|
||||
"customer_id": artifacts["customer_id"],
|
||||
"queue_id": artifacts["queue_id"],
|
||||
"priority": 3,
|
||||
},
|
||||
)
|
||||
ok = interaction.status_code == 200
|
||||
interaction_id = interaction.json().get("interaction_id") if ok else ""
|
||||
artifacts["voice_interaction_id"] = interaction_id
|
||||
checks.append(CheckResult("Voice interaction create", ok, f"status={interaction.status_code}"))
|
||||
if not ok:
|
||||
return checks, artifacts
|
||||
|
||||
assign = await client.patch(
|
||||
f"/proxy/interaction/interactions/{interaction_id}/assign",
|
||||
headers=supervisor,
|
||||
json={"assignee": "operator_a"},
|
||||
)
|
||||
assign_ok = assign.status_code == 200 and assign.json().get("status") == "in_progress"
|
||||
checks.append(CheckResult("Voice interaction assign", assign_ok, f"status={assign.status_code}"))
|
||||
|
||||
close = await client.patch(
|
||||
f"/proxy/interaction/interactions/{interaction_id}/status",
|
||||
headers=operator,
|
||||
json={"status": "closed"},
|
||||
)
|
||||
close_ok = close.status_code == 200 and close.json().get("status") == "closed"
|
||||
checks.append(CheckResult("Voice interaction close", close_ok, f"status={close.status_code}"))
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("Voice interaction lifecycle", False, f"exception={exc}"))
|
||||
return checks, artifacts
|
||||
|
||||
try:
|
||||
tg = await client.post(
|
||||
"/proxy/telegram/integrations/telegram/webhook",
|
||||
json={
|
||||
"chat_id": f"uat_chat_{now_compact()}",
|
||||
"text": "UAT preflight telegram",
|
||||
"customer_external_id": None,
|
||||
"payload": {"source": "uat_preflight"},
|
||||
},
|
||||
)
|
||||
ok = tg.status_code == 200 and tg.json().get("message_id")
|
||||
artifacts["telegram_message_id"] = tg.json().get("message_id") if tg.status_code == 200 else ""
|
||||
checks.append(CheckResult("Telegram webhook", bool(ok), f"status={tg.status_code}"))
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("Telegram webhook", False, f"exception={exc}"))
|
||||
return checks, artifacts
|
||||
|
||||
try:
|
||||
cat = await client.post(
|
||||
"/proxy/kb/knowledge/categories",
|
||||
headers=analyst,
|
||||
json={"name": f"UAT {now_compact()}", "description": "UAT preflight"},
|
||||
)
|
||||
cat_ok = cat.status_code == 200
|
||||
category_id = cat.json().get("category_id") if cat_ok else ""
|
||||
if not cat_ok:
|
||||
checks.append(CheckResult("KB category", False, f"status={cat.status_code}"))
|
||||
return checks, artifacts
|
||||
|
||||
art = await client.post(
|
||||
"/proxy/kb/knowledge/articles",
|
||||
headers=analyst,
|
||||
json={
|
||||
"category_id": category_id,
|
||||
"title": "UAT preflight article",
|
||||
"body": "Use this article during UAT flow.",
|
||||
"tags": ["uat", "kb"],
|
||||
},
|
||||
)
|
||||
art_ok = art.status_code == 200
|
||||
checks.append(CheckResult("KB article", art_ok, f"status={art.status_code}"))
|
||||
if not art_ok:
|
||||
return checks, artifacts
|
||||
|
||||
search = await client.get("/proxy/kb/knowledge/search?q=preflight")
|
||||
search_ok = search.status_code == 200 and len(search.json()) > 0
|
||||
checks.append(CheckResult("KB search", search_ok, f"status={search.status_code}, count={len(search.json())}"))
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("KB checks", False, f"exception={exc}"))
|
||||
return checks, artifacts
|
||||
|
||||
try:
|
||||
ingest = await client.post(
|
||||
"/proxy/reporting/reports/events",
|
||||
json={
|
||||
"queue_id": artifacts["queue_id"],
|
||||
"answered": True,
|
||||
"wait_seconds": 18,
|
||||
"handle_seconds": 105,
|
||||
"abandoned": False,
|
||||
"resolved_first_contact": True,
|
||||
},
|
||||
)
|
||||
ingest_ok = ingest.status_code == 200
|
||||
checks.append(CheckResult("KPI event ingest", ingest_ok, f"status={ingest.status_code}"))
|
||||
if not ingest_ok:
|
||||
return checks, artifacts
|
||||
|
||||
kpi = await client.get(f"/proxy/reporting/reports/kpi?queue_id={artifacts['queue_id']}")
|
||||
kpi_ok = kpi.status_code == 200 and "kpi" in kpi.json()
|
||||
checks.append(CheckResult("KPI report", kpi_ok, f"status={kpi.status_code}"))
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("KPI checks", False, f"exception={exc}"))
|
||||
return checks, artifacts
|
||||
|
||||
try:
|
||||
up1 = await client.post(
|
||||
"/proxy/supervisor/supervisor/agent-states",
|
||||
json={"agent_id": "uat_agent_1", "state": "READY", "queue_id": artifacts["queue_id"]},
|
||||
)
|
||||
up2 = await client.post(
|
||||
"/proxy/supervisor/supervisor/agent-states",
|
||||
json={"agent_id": "uat_agent_2", "state": "BUSY", "queue_id": artifacts["queue_id"]},
|
||||
)
|
||||
rt = await client.get("/proxy/supervisor/supervisor/realtime")
|
||||
realtime_ok = up1.status_code == 200 and up2.status_code == 200 and rt.status_code == 200
|
||||
checks.append(CheckResult("Supervisor realtime", realtime_ok, f"status={rt.status_code}"))
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("Supervisor realtime", False, f"exception={exc}"))
|
||||
return checks, artifacts
|
||||
|
||||
try:
|
||||
audit = await client.get("/proxy/audit/audit/events?limit=5")
|
||||
ok = audit.status_code == 200
|
||||
checks.append(CheckResult("Audit availability", ok, f"status={audit.status_code}"))
|
||||
except Exception as exc:
|
||||
checks.append(CheckResult("Audit availability", False, f"exception={exc}"))
|
||||
|
||||
artifacts["completed_at"] = iso_now()
|
||||
return checks, artifacts
|
||||
|
||||
|
||||
def write_report(
|
||||
output_path: Path,
|
||||
base_url: str,
|
||||
checks: list[CheckResult],
|
||||
artifacts: dict[str, Any],
|
||||
auto_start: bool,
|
||||
) -> None:
|
||||
ok_count = sum(1 for c in checks if c.ok)
|
||||
total = len(checks)
|
||||
overall = "PASS" if ok_count == total else "FAIL"
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append("# UAT Preflight Report")
|
||||
lines.append("")
|
||||
lines.append(f"- Timestamp (UTC): {iso_now()}")
|
||||
lines.append(f"- Base URL: {base_url}")
|
||||
lines.append(f"- Auto-start mode: {str(auto_start).lower()}")
|
||||
lines.append(f"- Overall: {overall} ({ok_count}/{total})")
|
||||
lines.append("")
|
||||
lines.append("## Scope Baseline")
|
||||
lines.append("- Pilot scope freeze: `docs/gates/mvp-pilot-baseline.md`")
|
||||
lines.append("- Deferred scope backlog: `docs/roadmap/05-wave2-backlog.md`")
|
||||
lines.append("")
|
||||
lines.append("## Checks")
|
||||
for item in checks:
|
||||
marker = "PASS" if item.ok else "FAIL"
|
||||
lines.append(f"- [{marker}] {item.name}: {item.details}")
|
||||
lines.append("")
|
||||
lines.append("## Artifacts")
|
||||
if artifacts:
|
||||
for key, value in artifacts.items():
|
||||
lines.append(f"- {key}: {value}")
|
||||
else:
|
||||
lines.append("- none")
|
||||
lines.append("")
|
||||
lines.append("## Next Action")
|
||||
lines.append("- If all checks passed, run `scripts/uat_dry_run.py` and prepare the manual UAT session.")
|
||||
lines.append("")
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="UAT preflight checker for pilot session")
|
||||
parser.add_argument("--base-url", default="http://localhost:8080", help="Gateway base URL")
|
||||
parser.add_argument("--auto-start", action="store_true", help="Auto-start local services for preflight")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="",
|
||||
help="Report output path (default: docs/uat/evidence/preflight_<timestamp>.md)",
|
||||
)
|
||||
parser.add_argument("--keep-data", action="store_true", help="Keep temporary auto-start data directory")
|
||||
args = parser.parse_args()
|
||||
|
||||
output = Path(args.output) if args.output else DEFAULT_EVIDENCE_DIR / f"preflight_{now_compact()}.md"
|
||||
output = output if output.is_absolute() else (ROOT / output).resolve()
|
||||
|
||||
processes: list[subprocess.Popen] = []
|
||||
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
|
||||
base_url = args.base_url
|
||||
checks: list[CheckResult] = []
|
||||
artifacts: dict[str, Any] = {}
|
||||
try:
|
||||
if args.auto_start:
|
||||
processes = await start_services_sequential(run_data_dir)
|
||||
base_url = "http://127.0.0.1:48080"
|
||||
|
||||
checks, artifacts = await run_checks(base_url)
|
||||
write_report(output, base_url, checks, artifacts, args.auto_start)
|
||||
|
||||
ok = all(item.ok for item in checks) and len(checks) > 0
|
||||
print(f"UAT preflight report: {output}")
|
||||
print(f"Result: {'PASS' if ok else 'FAIL'}")
|
||||
sys.exit(0 if ok else 1)
|
||||
finally:
|
||||
stop_services(processes)
|
||||
if args.auto_start and not args.keep_data:
|
||||
try:
|
||||
if run_data_dir.exists():
|
||||
shutil.rmtree(run_data_dir)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user