Initial import with GitLab CI/CD and registry deploy flow

This commit is contained in:
Yera All
2026-04-02 17:05:45 +05:00
commit 5374c202d9
338 changed files with 81297 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
# Script helpers package marker for tests.
+99
View File
@@ -0,0 +1,99 @@
from __future__ import annotations
import argparse
import select
import socket
import sys
import threading
import time
import paramiko
def _bridge_channels(chan: paramiko.Channel, upstream: socket.socket) -> None:
try:
while True:
readers, _, _ = select.select([chan, upstream], [], [], 1.0)
if chan in readers:
data = chan.recv(4096)
if not data:
break
upstream.sendall(data)
if upstream in readers:
data = upstream.recv(4096)
if not data:
break
chan.sendall(data)
finally:
try:
chan.close()
except Exception:
pass
try:
upstream.close()
except Exception:
pass
def _accept_channels(transport: paramiko.Transport, local_host: str, local_port: int) -> None:
while transport.is_active():
chan = transport.accept(1000)
if chan is None:
continue
try:
upstream = socket.create_connection((local_host, local_port), timeout=10)
except OSError:
chan.close()
continue
threading.Thread(target=_bridge_channels, args=(chan, upstream), daemon=True).start()
def main() -> int:
parser = argparse.ArgumentParser(description="Maintain reverse SSH tunnel for Asterisk IVR FastAGI.")
parser.add_argument("--ssh-host", required=True)
parser.add_argument("--ssh-port", type=int, default=22)
parser.add_argument("--ssh-user", required=True)
parser.add_argument("--ssh-password", required=True)
parser.add_argument("--remote-bind-host", default="127.0.0.1")
parser.add_argument("--remote-port", type=int, default=4573)
parser.add_argument("--local-host", default="127.0.0.1")
parser.add_argument("--local-port", type=int, default=4573)
args = parser.parse_args()
while True:
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
client.connect(
args.ssh_host,
port=args.ssh_port,
username=args.ssh_user,
password=args.ssh_password,
timeout=15,
banner_timeout=15,
auth_timeout=15,
)
transport = client.get_transport()
if transport is None:
raise RuntimeError("SSH transport is unavailable")
transport.request_port_forward(args.remote_bind_host, args.remote_port)
print(
f"Reverse tunnel active: {args.remote_bind_host}:{args.remote_port} -> "
f"{args.local_host}:{args.local_port}",
flush=True,
)
_accept_channels(transport, args.local_host, args.local_port)
except KeyboardInterrupt:
return 0
except Exception as exc:
print(f"Reverse tunnel error: {exc}", file=sys.stderr, flush=True)
time.sleep(3)
finally:
try:
client.close()
except Exception:
pass
if __name__ == "__main__":
raise SystemExit(main())
+139
View File
@@ -0,0 +1,139 @@
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
import sys
import time
import httpx
from sqlalchemy import create_engine, text
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from services.shared.security import issue_app_token
def _load_env_file(path: str) -> None:
env_path = Path(path).expanduser().resolve()
if not env_path.exists() or not env_path.is_file():
return
for line in env_path.read_text(encoding="utf-8").splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("#") or "=" not in stripped:
continue
key, value = stripped.split("=", 1)
key = key.strip()
if key and key not in os.environ:
os.environ[key] = value.strip()
def _bool_env(name: str, default: bool) -> bool:
raw = os.getenv(name)
if raw is None:
return default
return raw.strip().lower() in {"1", "true", "yes", "on"}
def _ops_headers() -> dict[str, str]:
if _bool_env("ALLOW_LEGACY_HEADER_AUTH", True):
return {"X-User": "admin", "X-Role": "admin"}
token = issue_app_token(
subject="ops:track9-smoke",
username="track9-smoke",
role="admin",
auth_source="service",
provider="track9-script",
ttl_seconds=300,
)
return {"Authorization": f"Bearer {token}"}
def _get_json(url: str) -> dict:
with httpx.Client(timeout=10, trust_env=False) as client:
response = client.get(url, headers=_ops_headers())
response.raise_for_status()
return response.json()
def _wait_for_ami_connected(base_url: str, retries: int = 8, delay_seconds: float = 1.0) -> dict:
last_status: dict = {}
for _ in range(retries):
status = _get_json(f"{base_url.rstrip('/')}/proxy/asterisk-bridge/asterisk/status")
last_status = status
if status.get("ami_connected"):
return status
time.sleep(delay_seconds)
return last_status
def _scalar(conn, sql: str) -> int:
value = conn.execute(text(sql)).scalar()
return int(value or 0)
def main() -> int:
parser = argparse.ArgumentParser(description="Track 9 Asterisk bridge smoke check")
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--env-file", default=".env.production")
parser.add_argument("--database-url", required=True)
parser.add_argument("--require-recording", action="store_true")
args = parser.parse_args()
_load_env_file(args.env_file)
health = _get_json(f"{args.base_url.rstrip('/')}/proxy/asterisk-bridge/health")
status = _wait_for_ami_connected(args.base_url)
failures: list[str] = []
if health.get("status") != "ok":
failures.append("asterisk-bridge health is not ok")
if not status.get("ami_connected"):
failures.append("AMI is not connected")
if not status.get("queue_codes_loaded"):
failures.append("Queue mapping is empty")
engine = create_engine(args.database_url, future=True)
with engine.begin() as conn:
started = _scalar(
conn,
"SELECT COUNT(*) FROM asterisk_event_log WHERE ami_event_name = 'MVPCCCallStarted' AND forward_status = 'forwarded'",
)
ended = _scalar(
conn,
"SELECT COUNT(*) FROM asterisk_event_log WHERE ami_event_name = 'MVPCCCallEnded' AND forward_status = 'forwarded'",
)
links = _scalar(conn, "SELECT COUNT(*) FROM asterisk_call_links")
recordings = _scalar(
conn,
"SELECT COUNT(*) FROM asterisk_event_log WHERE ami_event_name = 'MVPCCRecordingReady' AND recording_id IS NOT NULL",
)
if started < 1:
failures.append("No forwarded MVPCCCallStarted events")
if ended < 1:
failures.append("No forwarded MVPCCCallEnded events")
if links < 1:
failures.append("No asterisk_call_links rows")
if args.require_recording and recordings < 1:
failures.append("No uploaded recording linked to an Asterisk event")
if failures:
print("[FAIL] Asterisk lab smoke failed")
for item in failures:
print(f"- {item}")
return 1
print("[PASS] Asterisk lab smoke passed")
print(f"- started_events: {started}")
print(f"- ended_events: {ended}")
print(f"- call_links: {links}")
print(f"- recording_events_with_upload: {recordings}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+18
View File
@@ -0,0 +1,18 @@
param(
[string]$SourceDir = "e:\Zhan\.data",
[string]$OutputDir = "e:\Zhan\backups"
)
if (!(Test-Path $SourceDir)) {
Write-Error "Source data directory not found: $SourceDir"
exit 1
}
New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
$archivePath = Join-Path $OutputDir ("mvp_cc_data_" + $timestamp + ".zip")
Compress-Archive -Path (Join-Path $SourceDir "*") -DestinationPath $archivePath -Force
Write-Host "Backup created:" $archivePath
+61
View File
@@ -0,0 +1,61 @@
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
function Remove-DirectoryIfExists {
param(
[Parameter(Mandatory = $true)]
[string]$Path
)
if (Test-Path -LiteralPath $Path) {
Write-Host "Removing directory:" $Path
Remove-Item -LiteralPath $Path -Recurse -Force
}
}
function Clear-DirectoryContents {
param(
[Parameter(Mandatory = $true)]
[string]$Path
)
if (!(Test-Path -LiteralPath $Path)) {
return
}
Write-Host "Clearing directory contents:" $Path
Get-ChildItem -LiteralPath $Path -Force -ErrorAction SilentlyContinue | ForEach-Object {
Remove-Item -LiteralPath $_.FullName -Recurse -Force
}
}
$cacheDirs = Get-ChildItem -Path $root -Directory -Recurse -Force -ErrorAction SilentlyContinue |
Where-Object { $_.Name -eq "__pycache__" }
foreach ($dir in $cacheDirs) {
Write-Host "Removing cache directory:" $dir.FullName
Remove-Item -LiteralPath $dir.FullName -Recurse -Force
}
Remove-DirectoryIfExists -Path (Join-Path $root ".pytest_cache")
Remove-DirectoryIfExists -Path (Join-Path $root ".codex_tmp")
Remove-DirectoryIfExists -Path (Join-Path $root ".local_stack")
Remove-DirectoryIfExists -Path (Join-Path $root ".artifacts")
$dataDirs = Get-ChildItem -Path $root -Directory -Force -ErrorAction SilentlyContinue |
Where-Object {
$_.Name -eq ".data" -or
$_.Name -eq ".data_local" -or
$_.Name -eq ".data_smoke" -or
$_.Name -like ".data_gate*" -or
$_.Name -like ".data_uat_*"
}
foreach ($dir in $dataDirs) {
Clear-DirectoryContents -Path $dir.FullName
}
Write-Host ""
Write-Host "Workspace cleanup completed."
Write-Host "Preserved source directories, docs, templates, and backups."
+605
View File
@@ -0,0 +1,605 @@
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import json
import math
import os
from pathlib import Path
import struct
from typing import Any
import wave
import httpx
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_OUTPUT = ROOT / ".local_stack" / "demo-seed-summary.json"
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def now_stamp() -> str:
return utc_now().strftime("%Y%m%d_%H%M%S")
def build_seed_plan(tag: str) -> dict[str, Any]:
return {
"queue_name": "Demo Queue",
"customer_name": "Demo Customer",
"customer_phone": "+77010000001",
"voice_subject": "Demo voice escalation",
"chat_subject": "Demo Telegram follow-up",
"webchat_text": "Demo webchat request from the website",
"email_subject": "Demo email request about account access",
"email_body": "Please restore access to the portal. This message is preloaded for the management demo.",
"kb_category": "Demo Knowledge",
"kb_title": "Demo Knowledge Article",
"kb_keyword": "demo-showcase",
"telegram_text": "Demo inbound Telegram message",
"voice_event_type": "call.started",
"recording_file_name": "demo-call.wav",
"ivr_root_prompt_kz": "\u0421\u0430\u043b\u0430\u043c\u0430\u0442\u0441\u044b\u0437 \u0431\u0430! \u049a\u0430\u0437\u0430\u049b \u0442\u0456\u043b\u0456\u043d \u0442\u0430\u04a3\u0434\u0430\u0443 \u04af\u0448\u0456\u043d \u0431\u0456\u0440 \u0446\u0438\u0444\u0440\u044b\u043d \u0442\u0435\u0440\u0456\u04a3\u0456\u0437.",
"ivr_root_prompt_ru": "\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435! \u0414\u043e\u0431\u0440\u043e \u043f\u043e\u0436\u0430\u043b\u043e\u0432\u0430\u0442\u044c \u0432 \u043a\u043e\u043d\u0442\u0430\u043a\u0442-\u0446\u0435\u043d\u0442\u0440. \u0414\u043b\u044f \u0440\u0443\u0441\u0441\u043a\u043e\u0433\u043e \u044f\u0437\u044b\u043a\u0430 \u043d\u0430\u0436\u043c\u0438\u0442\u0435 \u0446\u0438\u0444\u0440\u0443 \u0434\u0432\u0430.",
"ivr_root_prompt": "Саламатсыз ба! Қазақ тілін таңдау үшін бір цифрын теріңіз. Здравствуйте! Добро пожаловать в контакт-центр. Для русского языка нажмите цифру два.",
"ivr_ru_menu_prompt": "Для отдела продаж нажмите 1. Для службы поддержки нажмите 2.",
"ivr_kz_menu_prompt": "Сату бөлімі үшін 1 басыңыз. Қолдау қызметі үшін 2 басыңыз.",
}
def _demo_ivr_flow_document(plan: dict[str, Any], *, sales_queue_id: str, support_queue_id: str) -> dict[str, Any]:
return {
"nodes": [
{
"node_id": "root",
"prompt_text": plan["ivr_root_prompt"],
"prompt_sequence": [
{
"prompt_audio_key": "ivr/demo-language-kz",
"prompt_text": plan["ivr_root_prompt_kz"],
"language": "kz",
},
{
"prompt_audio_key": "ivr/demo-language-ru",
"prompt_text": plan["ivr_root_prompt_ru"],
"language": "ru",
},
],
"is_terminal": False,
"invalid_target_node_id": "root",
"no_input_target_node_id": "root",
"options": [
{"digit": "1", "target_node_id": "menu_kz"},
{"digit": "2", "target_node_id": "menu_ru"},
],
},
{
"node_id": "menu_ru",
"prompt_text": plan["ivr_ru_menu_prompt"],
"prompt_audio_key": "ivr/demo-menu-ru",
"is_terminal": False,
"invalid_target_node_id": "menu_ru",
"no_input_target_node_id": "menu_ru",
"options": [
{"digit": "1", "target_node_id": "sales_ru"},
{"digit": "2", "target_node_id": "support_ru"},
],
},
{
"node_id": "menu_kz",
"prompt_text": plan["ivr_kz_menu_prompt"],
"prompt_audio_key": "ivr/demo-menu-kz",
"is_terminal": False,
"invalid_target_node_id": "menu_kz",
"no_input_target_node_id": "menu_kz",
"options": [
{"digit": "1", "target_node_id": "sales_kz"},
{"digit": "2", "target_node_id": "support_kz"},
],
},
{
"node_id": "sales_ru",
"prompt_text": "Переводим в отдел продаж.",
"prompt_audio_key": "ivr/demo-sales-ru",
"is_terminal": True,
"outcome_code": "sales_route_ru",
"resolved_queue_id": sales_queue_id,
"resolved_queue_code": "ivr_sales_ai_ru",
"options": [],
},
{
"node_id": "support_ru",
"prompt_text": "Переводим в службу поддержки.",
"prompt_audio_key": "ivr/demo-support-ru",
"is_terminal": True,
"outcome_code": "support_route_ru",
"resolved_queue_id": support_queue_id,
"resolved_queue_code": "ivr_support_ai_ru",
"options": [],
},
{
"node_id": "sales_kz",
"prompt_text": "Сату бөліміне қосып жатырмыз.",
"prompt_audio_key": "ivr/demo-sales-kz",
"is_terminal": True,
"outcome_code": "sales_route_kz",
"resolved_queue_id": sales_queue_id,
"resolved_queue_code": "ivr_sales_ai_kz",
"options": [],
},
{
"node_id": "support_kz",
"prompt_text": "Қолдау қызметіне қосып жатырмыз.",
"prompt_audio_key": "ivr/demo-support-kz",
"is_terminal": True,
"outcome_code": "support_route_kz",
"resolved_queue_id": support_queue_id,
"resolved_queue_code": "ivr_support_ai_kz",
"options": [],
},
]
}
def _raise_for_status(response: httpx.Response, action: str) -> None:
try:
response.raise_for_status()
except httpx.HTTPStatusError as exc:
detail = response.text.strip()
raise RuntimeError(f"{action} failed: {exc.response.status_code} {detail}") from exc
def _build_demo_wav(target_path: Path, duration_seconds: int = 1) -> Path:
target_path.parent.mkdir(parents=True, exist_ok=True)
sample_rate = 8000
amplitude = 12000
frequency = 440.0
total_frames = sample_rate * duration_seconds
with wave.open(str(target_path), "wb") as handle:
handle.setnchannels(1)
handle.setsampwidth(2)
handle.setframerate(sample_rate)
frames = bytearray()
for index in range(total_frames):
sample = int(amplitude * math.sin(2 * math.pi * frequency * (index / sample_rate)))
frames.extend(struct.pack("<h", sample))
handle.writeframes(bytes(frames))
return target_path
def _seed_data_dir() -> Path:
raw = os.getenv("CC_DATA_DIR")
if raw:
return Path(raw).resolve()
local_dir = (ROOT / ".data_local").resolve()
if local_dir.exists():
return local_dir
return (ROOT / ".data").resolve()
def seed_demo(base_url: str) -> dict[str, Any]:
admin = {"X-User": "admin", "X-Role": "admin"}
supervisor = {"X-User": "supervisor", "X-Role": "supervisor"}
operator = {"X-User": "operator", "X-Role": "operator"}
analyst = {"X-User": "analyst", "X-Role": "analyst"}
tag = now_stamp()
plan = build_seed_plan(tag)
sample_audio_path = _build_demo_wav((_seed_data_dir() / "demo_assets" / plan["recording_file_name"]).resolve())
with httpx.Client(base_url=base_url, timeout=10) as client:
health = client.get("/health")
_raise_for_status(health, "gateway health check")
queue = client.post(
"/proxy/routing/queues",
headers=admin,
json={
"name": plan["queue_name"],
"description": "Prepared automatically for a management demo",
"rules": [
{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 30},
{"channel": "telegram", "priority": 3, "strategy": "round_robin", "sla_seconds": 45},
{"channel": "webchat", "priority": 3, "strategy": "round_robin", "sla_seconds": 40},
{"channel": "email", "priority": 3, "strategy": "round_robin", "sla_seconds": 120},
],
},
)
_raise_for_status(queue, "queue create")
queue_id = queue.json()["queue_id"]
sales_queue = client.post(
"/proxy/routing/queues",
headers=admin,
json={
"name": "IVR Sales Queue",
"description": "Prepared automatically for IVR routing demo",
"rules": [
{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 25},
],
},
)
_raise_for_status(sales_queue, "sales queue create")
sales_queue_id = sales_queue.json()["queue_id"]
support_queue = client.post(
"/proxy/routing/queues",
headers=admin,
json={
"name": "IVR Support Queue",
"description": "Prepared automatically for IVR routing demo",
"rules": [
{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 35},
],
},
)
_raise_for_status(support_queue, "support queue create")
support_queue_id = support_queue.json()["queue_id"]
customer = client.post(
"/proxy/customer/customers",
json={
"display_name": plan["customer_name"],
"phones": [plan["customer_phone"]],
"preferred_phone": plan["customer_phone"],
"tags": ["demo", "showcase"],
},
)
_raise_for_status(customer, "customer create")
customer_id = customer.json()["customer_id"]
voice_interaction = client.post(
"/proxy/interaction/interactions",
headers=operator,
json={
"channel": "voice",
"subject": plan["voice_subject"],
"customer_id": customer_id,
"queue_id": queue_id,
"priority": 3,
},
)
_raise_for_status(voice_interaction, "voice interaction create")
voice_interaction_id = voice_interaction.json()["interaction_id"]
assign = client.patch(
f"/proxy/interaction/interactions/{voice_interaction_id}/assign",
headers=supervisor,
json={"assignee": "operator_a"},
)
_raise_for_status(assign, "voice interaction assign")
escalate = client.post(
f"/proxy/interaction/interactions/{voice_interaction_id}/escalate",
headers=operator,
json={"target_queue_id": "line2"},
)
_raise_for_status(escalate, "voice interaction escalate")
close = client.patch(
f"/proxy/interaction/interactions/{voice_interaction_id}/status",
headers=operator,
json={"status": "closed"},
)
_raise_for_status(close, "voice interaction close")
active_interaction = client.post(
"/proxy/interaction/interactions",
headers=operator,
json={
"channel": "telegram",
"subject": plan["chat_subject"],
"customer_id": customer_id,
"queue_id": "line2",
"priority": 3,
},
)
_raise_for_status(active_interaction, "active interaction create")
active_interaction_id = active_interaction.json()["interaction_id"]
voice_event = client.post(
"/proxy/voice/integrations/voice/events",
headers=operator,
json={
"event_type": plan["voice_event_type"],
"call_id": "demo_call",
"interaction_id": voice_interaction_id,
"payload": {"source": "demo-seed"},
},
)
_raise_for_status(voice_event, "voice event send")
voice_event_id = voice_event.json()["event_id"]
recording_ready_event = client.post(
"/proxy/voice/integrations/voice/events",
headers=operator,
json={
"event_type": "recording.ready",
"call_id": "demo_call",
"interaction_id": voice_interaction_id,
"payload": {
"source": "demo-seed",
"source_path": str(sample_audio_path),
"file_name": plan["recording_file_name"],
"mime_type": "audio/wav",
"duration_seconds": 1,
"recorded_at": utc_now().replace(microsecond=0).isoformat(),
},
},
)
_raise_for_status(recording_ready_event, "recording ready event send")
recording_event_id = recording_ready_event.json()["event_id"]
recording = client.post(
f"/proxy/recording/recordings/import-from-voice-event/{recording_event_id}",
headers=supervisor,
)
_raise_for_status(recording, "recording import")
recording_id = recording.json()["recording_id"]
ivr_flow = client.post(
"/proxy/ivr/ivr/flows",
headers=admin,
json={
"name": "Demo IVR Flow",
"description": "Prepared automatically for a management demo",
"queue_id": queue_id,
"entry_node_id": "root",
"flow_json": _demo_ivr_flow_document(
plan,
sales_queue_id=sales_queue_id,
support_queue_id=support_queue_id,
),
"is_active": True,
},
)
_raise_for_status(ivr_flow, "ivr flow create")
ivr_flow_id = ivr_flow.json()["flow_id"]
ivr_session_start = client.post(
"/proxy/ivr/ivr/sessions/start",
headers=admin,
json={
"call_id": "demo_call_ivr",
"queue_id": queue_id,
"interaction_id": voice_interaction_id,
},
)
_raise_for_status(ivr_session_start, "ivr session start")
ivr_session_id = ivr_session_start.json()["session"]["session_id"]
ivr_session_step = client.post(
f"/proxy/ivr/ivr/sessions/{ivr_session_id}/dtmf",
headers=admin,
json={"digit": "1"},
)
_raise_for_status(ivr_session_step, "ivr language step")
ivr_session_step = client.post(
f"/proxy/ivr/ivr/sessions/{ivr_session_id}/dtmf",
headers=admin,
json={"digit": "2"},
)
_raise_for_status(ivr_session_step, "ivr dtmf step")
ivr_session_payload = ivr_session_step.json()
ivr_route_preview = client.post(
f"/proxy/routing/queues/{queue_id}/route?channel=voice&priority=3&ivr_session_id={ivr_session_id}",
headers=admin,
)
_raise_for_status(ivr_route_preview, "ivr route preview")
telegram = client.post(
"/proxy/telegram/integrations/telegram/webhook",
json={
"chat_id": "demo_chat",
"text": plan["telegram_text"],
"customer_external_id": customer_id,
"payload": {"source": "demo-seed"},
},
)
_raise_for_status(telegram, "telegram webhook send")
telegram_id = telegram.json()["message_id"]
webchat = client.post(
"/proxy/webchat/integrations/webchat/messages",
json={
"session_id": "demo_webchat_session",
"text": plan["webchat_text"],
"visitor_name": "Demo Visitor",
"customer_external_id": customer_id,
"queue_id": "line2",
"priority": 3,
"payload": {"source": "demo-seed", "subject": "Demo webchat intake"},
},
)
_raise_for_status(webchat, "webchat message send")
webchat_id = webchat.json()["message_id"]
webchat_interaction_id = webchat.json()["interaction_id"]
email = client.post(
"/proxy/email/integrations/email/messages",
json={
"from_email": "demo.user@example.com",
"subject": plan["email_subject"],
"body": plan["email_body"],
"customer_external_id": customer_id,
"queue_id": "line2",
"priority": 3,
"payload": {"source": "demo-seed", "mailbox": "support@example.com"},
},
)
_raise_for_status(email, "email message send")
email_id = email.json()["message_id"]
email_interaction_id = email.json()["interaction_id"]
category = client.post(
"/proxy/kb/knowledge/categories",
headers=analyst,
json={"name": plan["kb_category"], "description": "Demo search content"},
)
_raise_for_status(category, "kb category create")
category_id = category.json()["category_id"]
article = client.post(
"/proxy/kb/knowledge/articles",
headers=analyst,
json={
"category_id": category_id,
"title": plan["kb_title"],
"body": "Use the demo-showcase article when management asks about answer prompts.",
"tags": ["demo", plan["kb_keyword"]],
},
)
_raise_for_status(article, "kb article create")
article_id = article.json()["article_id"]
search = client.get(f"/proxy/kb/knowledge/search?q={plan['kb_keyword']}")
_raise_for_status(search, "kb search")
agent1 = client.post(
"/proxy/supervisor/supervisor/agent-states",
headers=supervisor,
json={"agent_id": "demo_agent_a", "state": "READY", "queue_id": "line2"},
)
_raise_for_status(agent1, "supervisor agent update 1")
agent2 = client.post(
"/proxy/supervisor/supervisor/agent-states",
headers=supervisor,
json={"agent_id": "demo_agent_b", "state": "BUSY", "queue_id": "line2"},
)
_raise_for_status(agent2, "supervisor agent update 2")
queue_metrics = client.post(
"/proxy/supervisor/supervisor/queue-metrics?queue_id=line2&in_queue=1&avg_wait_seconds=14",
headers=supervisor,
)
_raise_for_status(queue_metrics, "supervisor queue metrics")
realtime = client.get("/proxy/supervisor/supervisor/realtime", headers=supervisor)
_raise_for_status(realtime, "supervisor realtime")
kpi_rows = [
{
"queue_id": "line2",
"channel": "voice",
"agent_id": "demo_agent_b",
"answered": True,
"wait_seconds": 15,
"handle_seconds": 90,
"abandoned": False,
"resolved_first_contact": True,
},
{
"queue_id": "line2",
"channel": "webchat",
"agent_id": "demo_agent_a",
"answered": True,
"wait_seconds": 25,
"handle_seconds": 110,
"abandoned": False,
"resolved_first_contact": False,
},
{
"queue_id": "line2",
"channel": "email",
"agent_id": None,
"answered": False,
"wait_seconds": 12,
"handle_seconds": 0,
"abandoned": True,
"resolved_first_contact": False,
},
]
for idx, row in enumerate(kpi_rows, start=1):
created = client.post("/proxy/reporting/reports/events", json=row)
_raise_for_status(created, f"kpi event ingest {idx}")
kpi = client.get("/proxy/reporting/reports/kpi?queue_id=line2&sl_threshold_seconds=30")
_raise_for_status(kpi, "kpi query")
event_bus_enabled = os.getenv("EVENT_BUS_ENABLED", "0").strip().lower() in {"1", "true", "yes", "on"}
event_bus_smoke_passed = False
seeded_event_id = None
if event_bus_enabled:
try:
from scripts.event_bus_smoke import run_smoke_check
smoke = run_smoke_check(base_url=base_url, auth_mode="legacy_headers", timeout_seconds=10)
event_bus_smoke_passed = bool(smoke.get("passed"))
seeded_event_id = smoke.get("event_id")
except Exception:
event_bus_smoke_passed = False
return {
"generated_at": utc_now().isoformat(),
"base_url": base_url,
"demo_login": {"username": "admin", "password": "admin123"},
"queue_id": queue_id,
"customer_id": customer_id,
"voice_interaction_id": voice_interaction_id,
"active_interaction_id": active_interaction_id,
"voice_event_id": voice_event_id,
"recording_event_id": recording_event_id,
"recording_id": recording_id,
"recording_call_id": "demo_call",
"recording_file_name": plan["recording_file_name"],
"event_bus_enabled": event_bus_enabled,
"event_bus_smoke_passed": event_bus_smoke_passed,
"seeded_event_id": seeded_event_id,
"ivr_flow_id": ivr_flow_id,
"ivr_session_id": ivr_session_id,
"ivr_outcome_code": ivr_session_payload["session"]["outcome_code"],
"ivr_resolved_queue_id": ivr_session_payload["session"]["resolved_queue_id"],
"ivr_route_preview": ivr_route_preview.json(),
"telegram_message_id": telegram_id,
"webchat_message_id": webchat_id,
"webchat_interaction_id": webchat_interaction_id,
"email_message_id": email_id,
"email_interaction_id": email_interaction_id,
"kb_category_id": category_id,
"kb_article_id": article_id,
"kb_keyword": plan["kb_keyword"],
"sample_recording_path": str(sample_audio_path),
"kpi_snapshot": kpi.json(),
"supervisor_snapshot": realtime.json(),
}
def main() -> int:
parser = argparse.ArgumentParser(description="Seed demo-ready data into the local MVP stack")
parser.add_argument("--base-url", default="http://localhost:8080", help="Gateway base URL")
parser.add_argument(
"--output",
default=str(DEFAULT_OUTPUT),
help="Where to store the seed summary JSON",
)
args = parser.parse_args()
output_path = Path(args.output)
if not output_path.is_absolute():
output_path = (ROOT / output_path).resolve()
output_path.parent.mkdir(parents=True, exist_ok=True)
summary = seed_demo(args.base_url)
output_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
print("Demo data is ready.")
print(f"UI: {args.base_url}/operator")
print(f"Summary: {output_path}")
print(f"Customer: {summary['customer_id']}")
print(f"Closed interaction: {summary['voice_interaction_id']}")
print(f"Active interaction: {summary['active_interaction_id']}")
print(f"Recording: {summary['recording_id']}")
print(f"IVR flow: {summary['ivr_flow_id']}")
print(f"IVR session: {summary['ivr_session_id']}")
print(f"Webchat interaction: {summary['webchat_interaction_id']}")
print(f"Email interaction: {summary['email_interaction_id']}")
print(f"KB keyword: {summary['kb_keyword']}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+130
View File
@@ -0,0 +1,130 @@
from __future__ import annotations
import argparse
import sys
import time
from pathlib import Path
import httpx
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from services.shared.core import utc_now_iso
from services.shared.db import DATABASE_URL, _normalize_database_url
from services.shared.sql_models import EventInboxRow, EventOutboxRow
def _engine_for(database_url: str | None):
url = _normalize_database_url(database_url or DATABASE_URL)
return create_engine(url, future=True)
def _auth_headers(client: httpx.Client, auth_mode: str) -> dict[str, str]:
if auth_mode == "legacy_headers":
return {"X-User": "admin", "X-Role": "admin"}
response = client.post(
"/proxy/auth/auth/login",
json={"username": "admin", "password": "admin123"},
timeout=10,
)
response.raise_for_status()
token = response.json()["access_token"]
return {"Authorization": f"Bearer {token}"}
def run_smoke_check(
*,
base_url: str,
database_url: str | None = None,
auth_mode: str = "legacy_headers",
timeout_seconds: int = 15,
) -> dict:
engine = _engine_for(database_url)
with httpx.Client(base_url=base_url, timeout=10) as client:
headers = _auth_headers(client, auth_mode)
event_bus_health = client.get("/proxy/event-bus/health", headers=headers)
if event_bus_health.status_code != 200:
raise RuntimeError("event-bus-service health check failed")
created = client.post(
"/proxy/interaction/interactions",
headers=headers,
json={
"channel": "voice",
"subject": f"Track8 smoke {utc_now_iso()}",
"customer_id": None,
"queue_id": "track8_smoke_queue",
"priority": 3,
},
)
created.raise_for_status()
interaction_id = created.json()["interaction_id"]
deadline = time.time() + timeout_seconds
found: dict[str, str] = {}
with Session(engine) as session:
while time.time() < deadline:
outbox_row = session.execute(
select(EventOutboxRow).where(
EventOutboxRow.entity_id == interaction_id,
EventOutboxRow.event_type == "interaction.created",
)
).scalar_one_or_none()
if outbox_row:
found["event_id"] = outbox_row.event_id
found["outbox_status"] = outbox_row.status
audit_seen = session.execute(
select(EventInboxRow).where(
EventInboxRow.consumer_name == "audit-service",
EventInboxRow.event_id == outbox_row.event_id,
)
).scalar_one_or_none()
reporting_seen = session.execute(
select(EventInboxRow).where(
EventInboxRow.consumer_name == "reporting-service",
EventInboxRow.event_id == outbox_row.event_id,
)
).scalar_one_or_none()
if outbox_row.status == "published" and audit_seen and reporting_seen:
return {
"passed": True,
"interaction_id": interaction_id,
"event_id": outbox_row.event_id,
"outbox_status": outbox_row.status,
}
session.expire_all()
time.sleep(0.5)
return {
"passed": False,
"interaction_id": interaction_id,
"event_id": found.get("event_id"),
"outbox_status": found.get("outbox_status"),
}
def main() -> None:
parser = argparse.ArgumentParser(description="Run a lightweight Track 8 event-bus smoke check.")
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--database-url", default=None)
parser.add_argument("--auth-mode", choices=["legacy_headers", "bearer"], default="legacy_headers")
parser.add_argument("--timeout-seconds", type=int, default=15)
args = parser.parse_args()
result = run_smoke_check(
base_url=args.base_url,
database_url=args.database_url,
auth_mode=args.auth_mode,
timeout_seconds=args.timeout_seconds,
)
print(result)
if not result["passed"]:
raise SystemExit(1)
if __name__ == "__main__":
main()
+283
View File
@@ -0,0 +1,283 @@
from __future__ import annotations
import argparse
import csv
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
GATE4_PATH = ROOT / "docs" / "gates" / "gate-04-pilot-hardening.md"
P1P2_PATH = ROOT / "docs" / "gates" / "p1-p2-defects.md"
ACCEPTED_RELEASE_PATH = ROOT / "docs" / "releases" / "v1.0.0-mvp-accepted.md"
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def iso_now() -> str:
return utc_now().isoformat()
def _normalized_status(value: str) -> str:
return value.strip().lower()
def _is_open_status(value: str) -> bool:
normalized = _normalized_status(value)
return normalized not in {"closed", "resolved", "verified"}
def parse_open_defect_counts(defect_log_path: Path) -> dict[str, int]:
counts = {"P1": 0, "P2": 0, "P3": 0, "P4": 0}
with defect_log_path.open("r", newline="", encoding="utf-8") as handle:
reader = csv.DictReader(handle)
for row in reader:
severity = (row.get("severity") or "").strip().upper()
status = row.get("status") or ""
if severity in counts and _is_open_status(status):
counts[severity] += 1
return counts
def validate_session_protocol(session_protocol_path: Path) -> list[str]:
content = session_protocol_path.read_text(encoding="utf-8")
errors: list[str] = []
blank_markers = [
"- Session ID:",
"- Environment URL:",
"- Build/version:",
"- Deployment date:",
"- Cluster/namespace:",
"- Business owner:",
"- IT owner:",
]
for marker in blank_markers:
if f"{marker}\n" in content or f"{marker}\r\n" in content:
errors.append(f"Session protocol still contains blank field: {marker}")
if "## Decision" not in content or "[x]" not in content:
errors.append("Session protocol does not contain a selected decision checkbox")
return errors
def validate_signoff_sheet(signoff_sheet_path: Path) -> list[str]:
content = signoff_sheet_path.read_text(encoding="utf-8")
errors: list[str] = []
placeholder_lines = [
" - Name:",
" - Signature:",
" - Date:",
]
for placeholder in placeholder_lines:
if content.count(placeholder) > 0:
errors.append(f"Sign-off sheet still contains placeholder lines matching: {placeholder.strip()}")
break
decision_lines = [line for line in content.splitlines() if line.startswith("- [")]
if sum(1 for line in decision_lines if "[x]" in line.lower()) != 1:
errors.append("Sign-off sheet must have exactly one selected decision checkbox")
return errors
def apply_gate4_closure(
content: str,
*,
session_id: str,
session_dir: Path,
acceptance: str,
open_counts: dict[str, int],
) -> str:
updated = content
updated = updated.replace(
"- [ ] UAT signed with real operators/supervisors",
"- [x] UAT signed with real operators/supervisors",
)
updated = updated.replace(
"- [ ] P1/P2 defects closed",
"- [x] P1/P2 defects closed",
)
marker = "Manual closure update:"
if marker in updated:
updated = updated.split(marker, 1)[0].rstrip()
lines = [
"",
"Manual closure update:",
f"- Date (UTC): {iso_now()}",
f"- Session ID: {session_id}",
f"- Session directory: `{session_dir.as_posix()}`",
f"- Acceptance: {acceptance}",
(
"- Remaining open defects: "
f"P1={open_counts['P1']}, P2={open_counts['P2']}, "
f"P3={open_counts['P3']}, P4={open_counts['P4']}"
),
]
return updated.rstrip() + "\n" + "\n".join(lines) + "\n"
def build_p1p2_register(
*,
session_id: str,
session_dir: Path,
open_counts: dict[str, int],
acceptance: str,
) -> str:
lines = [
"# P1/P2 Defect Register (Pilot)",
"",
f"Last update: {utc_now().date().isoformat()} ({session_id})",
"",
"## Current Status",
f"- Open P1: {open_counts['P1']}",
f"- Open P2: {open_counts['P2']}",
"- Source: manual UAT close-out",
f"- Manual session evidence: `{session_dir.as_posix()}`",
"",
"## Triage Policy",
"- Only `P1` and `P2` issues belong to MVP remediation.",
"- `P3` and `P4` issues move to `docs/roadmap/05-wave2-backlog.md` unless they block sign-off.",
"",
"## Manual Closure",
f"- Acceptance: {acceptance}",
"- P1/P2 closure confirmed by signed manual UAT package.",
]
return "\n".join(lines) + "\n"
def build_acceptance_release_note(
*,
session_id: str,
session_dir: Path,
acceptance: str,
open_counts: dict[str, int],
) -> str:
lines = [
"# Release Notes - v1.0.0-mvp Accepted",
"",
f"Date: {utc_now().date().isoformat()}",
"",
"## Acceptance Result",
f"- Session ID: {session_id}",
f"- Acceptance: {acceptance}",
f"- Manual session evidence: `{session_dir.as_posix()}`",
"",
"## Closed Gate 4 Conditions",
"- UAT signed with real operators/supervisors",
"- P1/P2 defects closed",
"",
"## Remaining Deferred Items",
f"- Open P3: {open_counts['P3']}",
f"- Open P4: {open_counts['P4']}",
"- Deferred scope remains tracked in `docs/roadmap/05-wave2-backlog.md`.",
"",
"## Baseline",
"- Accepted baseline remains the current MVP v1 scope without API expansion.",
"- Stage 1-3 public routes remain unchanged.",
]
return "\n".join(lines) + "\n"
def finalize_manual_session(
*,
session_dir: Path,
acceptance: str,
dry_run: bool = False,
) -> dict[str, int]:
required_files = {
"session_protocol": session_dir / "session-protocol.md",
"scenario_checklist": session_dir / "scenario-checklist.md",
"defect_log": session_dir / "defect-log.csv",
"signoff_sheet": session_dir / "signoff-sheet.md",
"manifest": session_dir / "manifest.json",
}
missing = [name for name, path in required_files.items() if not path.exists()]
attachments_dir = session_dir / "attachments"
if not attachments_dir.exists():
missing.append("attachments")
if missing:
raise FileNotFoundError(f"Manual session bundle is incomplete: {missing}")
validation_errors: list[str] = []
validation_errors.extend(validate_session_protocol(required_files["session_protocol"]))
validation_errors.extend(validate_signoff_sheet(required_files["signoff_sheet"]))
open_counts = parse_open_defect_counts(required_files["defect_log"])
if open_counts["P1"] != 0 or open_counts["P2"] != 0:
validation_errors.append(
f"Cannot close MVP pilot with open P1/P2 defects: P1={open_counts['P1']}, P2={open_counts['P2']}"
)
if validation_errors:
details = "\n".join(f"- {error}" for error in validation_errors)
raise ValueError(f"Manual UAT package validation failed:\n{details}")
if dry_run:
return open_counts
session_id = session_dir.name.replace("manual_", "", 1)
gate4_content = GATE4_PATH.read_text(encoding="utf-8")
updated_gate4 = apply_gate4_closure(
gate4_content,
session_id=session_id,
session_dir=session_dir,
acceptance=acceptance,
open_counts=open_counts,
)
GATE4_PATH.write_text(updated_gate4, encoding="utf-8")
P1P2_PATH.write_text(
build_p1p2_register(
session_id=session_id,
session_dir=session_dir,
open_counts=open_counts,
acceptance=acceptance,
),
encoding="utf-8",
)
ACCEPTED_RELEASE_PATH.write_text(
build_acceptance_release_note(
session_id=session_id,
session_dir=session_dir,
acceptance=acceptance,
open_counts=open_counts,
),
encoding="utf-8",
)
return open_counts
def main() -> None:
parser = argparse.ArgumentParser(description="Validate a manual UAT bundle and close MVP Gate 4")
parser.add_argument("--session-dir", required=True, help="Path to the prepared manual session directory")
parser.add_argument(
"--acceptance",
choices=["accepted_for_pilot_completion", "accepted_with_conditions"],
default="accepted_for_pilot_completion",
help="Acceptance mode recorded in gate closure artifacts",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Validate the manual session bundle without updating gate/release documents",
)
args = parser.parse_args()
session_dir = Path(args.session_dir)
if not session_dir.is_absolute():
session_dir = (ROOT / session_dir).resolve()
counts = finalize_manual_session(
session_dir=session_dir,
acceptance=args.acceptance,
dry_run=args.dry_run,
)
prefix = "MVP pilot finalization dry-run passed:" if args.dry_run else "MVP pilot finalized:"
print(f"{prefix} P1={counts['P1']} P2={counts['P2']} P3={counts['P3']} P4={counts['P4']}")
if __name__ == "__main__":
main()
+64
View File
@@ -0,0 +1,64 @@
from __future__ import annotations
import asyncio
import sys
import httpx
REQUIRED_SERVICES = [
"auth",
"audit",
"customer",
"interaction",
"routing",
"voice",
"telegram",
"kb",
"reporting",
"supervisor",
]
async def check_gateway(base_url: str) -> bool:
async with httpx.AsyncClient(base_url=base_url, timeout=5) as client:
try:
health = await client.get("/health")
if health.status_code != 200:
print(f"[FAIL] gateway /health status={health.status_code}")
return False
registry = await client.get("/registry")
if registry.status_code != 200:
print(f"[FAIL] gateway /registry status={registry.status_code}")
return False
data = registry.json()
services = data.get("services", {})
missing = [s for s in REQUIRED_SERVICES if s not in services]
if missing:
print(f"[FAIL] registry missing services: {missing}")
return False
login = await client.post(
"/proxy/auth/auth/login",
json={"username": "admin", "password": "admin123"},
)
if login.status_code != 200:
print(f"[FAIL] auth login status={login.status_code}")
return False
print("[PASS] Stage 1 gateway/auth health checks passed")
return True
except Exception as exc:
print(f"[FAIL] exception: {exc}")
return False
async def main() -> None:
base_url = "http://localhost:8080"
ok = await check_gateway(base_url)
sys.exit(0 if ok else 1)
if __name__ == "__main__":
asyncio.run(main())
+303
View File
@@ -0,0 +1,303 @@
from __future__ import annotations
import argparse
import asyncio
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
from typing import Any
import httpx
ROOT = Path(__file__).resolve().parents[1]
DATA_ROOT = ROOT / ".data_gate3"
SERVICE_SPECS = [
{"name": "auth", "module": "services.auth_service.app:app", "port": 28001},
{"name": "audit", "module": "services.audit_service.app:app", "port": 28002},
{"name": "customer", "module": "services.customer_service.app:app", "port": 28003},
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 28004},
{"name": "routing", "module": "services.routing_service.app:app", "port": 28005},
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 28006},
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 28007},
{"name": "kb", "module": "services.kb_service.app:app", "port": 28008},
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 28009},
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 28010},
{"name": "gateway", "module": "gateway.app:app", "port": 28080},
]
REQUIRED_STAGE3_SERVICES = ["kb", "reporting", "supervisor"]
async def wait_for_health(base_url: str, retries: int = 80, delay: float = 0.25) -> None:
async with httpx.AsyncClient(timeout=2) as client:
for _ in range(retries):
try:
response = await client.get(f"{base_url}/health")
if response.status_code == 200:
return
except Exception:
pass
await asyncio.sleep(delay)
raise RuntimeError(f"Service not healthy: {base_url}/health")
def start_one_service(spec: dict[str, Any], database_url: str | None, data_dir: Path) -> subprocess.Popen:
env = os.environ.copy()
env["CC_DATA_DIR"] = str(data_dir)
if database_url:
env["DATABASE_URL"] = database_url
if spec["name"] == "gateway":
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:28001"
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:28002"
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:28003"
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:28004"
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:28005"
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:28006"
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:28007"
env["KB_SERVICE_URL"] = "http://127.0.0.1:28008"
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:28009"
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:28010"
cmd = [
sys.executable,
"-m",
"uvicorn",
spec["module"],
"--host",
"127.0.0.1",
"--port",
str(spec["port"]),
]
return subprocess.Popen(
cmd,
cwd=str(ROOT),
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
async def start_services_sequential(database_url: str | None, data_dir: Path) -> list[subprocess.Popen]:
data_dir.mkdir(parents=True, exist_ok=True)
processes: list[subprocess.Popen] = []
for spec in SERVICE_SPECS:
proc = start_one_service(spec, database_url, data_dir)
processes.append(proc)
await wait_for_health(f"http://127.0.0.1:{spec['port']}", retries=60, delay=0.2)
return processes
def stop_services(processes: list[subprocess.Popen]) -> None:
for proc in processes:
if proc.poll() is not None:
continue
try:
proc.terminate()
except Exception:
pass
for proc in processes:
try:
proc.wait(timeout=2)
except Exception:
try:
proc.kill()
except Exception:
pass
async def run_gate3_checks(base_url: str, reporting_url: str | None = None) -> bool:
contract_path = ROOT / "contracts" / "openapi" / "stage3-kb-reporting-supervisor.yaml"
if not contract_path.exists():
print(f"[FAIL] Missing Stage 3 OpenAPI: {contract_path}")
return False
print(f"[PASS] Stage 3 OpenAPI present: {contract_path.name}")
operator = {"X-User": "operator", "X-Role": "operator"}
analyst = {"X-User": "analyst", "X-Role": "analyst"}
async with httpx.AsyncClient(base_url=base_url, timeout=10) as client:
health = await client.get("/health")
if health.status_code != 200:
print(f"[FAIL] gateway /health status={health.status_code}")
return False
registry = await client.get("/registry")
if registry.status_code != 200:
print(f"[FAIL] gateway /registry status={registry.status_code}")
return False
services = registry.json().get("services", {})
missing = [name for name in REQUIRED_STAGE3_SERVICES if name not in services]
if missing:
print(f"[FAIL] registry missing Stage 3 services: {missing}")
return False
print("[PASS] Gateway and registry checks")
denied = await client.post(
"/proxy/kb/knowledge/categories",
headers=operator,
json={"name": "Forbidden", "description": "Role check"},
)
if denied.status_code != 403:
print(f"[FAIL] KB role check expected 403, got {denied.status_code}")
return False
category = await client.post(
"/proxy/kb/knowledge/categories",
headers=analyst,
json={"name": "Stage 3 KB", "description": "Gate 3"},
)
if category.status_code != 200:
print(f"[FAIL] KB category create status={category.status_code}")
return False
category_id = category.json()["category_id"]
article = await client.post(
"/proxy/kb/knowledge/articles",
headers=analyst,
json={
"category_id": category_id,
"title": "Install helper",
"body": "Step-by-step instruction for operator",
"tags": ["stage3", "kb"],
},
)
if article.status_code != 200:
print(f"[FAIL] KB article create status={article.status_code}")
return False
search = await client.get("/proxy/kb/knowledge/search?q=helper")
if search.status_code != 200 or len(search.json()) < 1:
print(
f"[FAIL] KB search failed status={search.status_code} count={len(search.json()) if search.status_code == 200 else 0}"
)
return False
print("[PASS] KB checks")
rows = [
{
"queue_id": "q_gate3",
"answered": True,
"wait_seconds": 15,
"handle_seconds": 95,
"abandoned": False,
"resolved_first_contact": True,
},
{
"queue_id": "q_gate3",
"answered": False,
"wait_seconds": 20,
"handle_seconds": 0,
"abandoned": True,
"resolved_first_contact": False,
},
]
for row in rows:
ingested = await client.post("/proxy/reporting/reports/events", json=row)
if ingested.status_code != 200:
print(f"[FAIL] reporting ingest status={ingested.status_code}")
return False
kpi = await client.get("/proxy/reporting/reports/kpi?queue_id=q_gate3&sl_threshold_seconds=30")
if kpi.status_code != 200:
print(f"[FAIL] reporting KPI status={kpi.status_code}")
return False
kpi_body = kpi.json()
if "kpi" not in kpi_body or not {"SL", "ASA", "AHT", "Abandon", "FCR"}.issubset(set(kpi_body["kpi"])):
print("[FAIL] KPI payload missing required keys")
return False
print("[PASS] Reporting KPI checks")
report_service_url = reporting_url or services.get("reporting")
if not report_service_url:
print("[FAIL] reporting service URL not found")
return False
async with httpx.AsyncClient(base_url=report_service_url, timeout=10) as reporting_client:
exported = await reporting_client.get("/reports/export")
if exported.status_code != 200:
print(f"[FAIL] reporting export status={exported.status_code}")
return False
if "queue_id,answered,wait_seconds,handle_seconds,abandoned,resolved_first_contact,created_at" not in exported.text:
print("[FAIL] reporting export missing CSV header")
return False
if "q_gate3" not in exported.text:
print("[FAIL] reporting export missing inserted data")
return False
print("[PASS] Reporting CSV export checks")
up1 = await client.post(
"/proxy/supervisor/supervisor/agent-states",
json={"agent_id": "a_gate3_1", "state": "READY", "queue_id": "q_gate3"},
)
up2 = await client.post(
"/proxy/supervisor/supervisor/agent-states",
json={"agent_id": "a_gate3_2", "state": "BUSY", "queue_id": "q_gate3"},
)
queue = await client.post(
"/proxy/supervisor/supervisor/queue-metrics?queue_id=q_gate3&in_queue=3&avg_wait_seconds=21"
)
if up1.status_code != 200 or up2.status_code != 200 or queue.status_code != 200:
print(
f"[FAIL] supervisor updates status: agent1={up1.status_code}, agent2={up2.status_code}, queue={queue.status_code}"
)
return False
realtime = await client.get("/proxy/supervisor/supervisor/realtime")
if realtime.status_code != 200:
print(f"[FAIL] supervisor realtime status={realtime.status_code}")
return False
body: dict[str, Any] = realtime.json()
if body.get("agents", {}).get("total", 0) < 2:
print("[FAIL] supervisor realtime has less than 2 agents")
return False
queues = body.get("queues", [])
if not any(item.get("queue_id") == "q_gate3" for item in queues):
print("[FAIL] supervisor realtime missing q_gate3 queue snapshot")
return False
print("[PASS] Supervisor realtime checks")
print("[PASS] Gate 3 checks completed")
return True
async def main() -> None:
parser = argparse.ArgumentParser(description="Gate 3 checker")
parser.add_argument("--base-url", default="http://localhost:8080", help="Gateway base URL")
parser.add_argument("--auto-start", action="store_true", help="Auto-start local services for the check")
parser.add_argument(
"--database-url",
default=os.getenv("DATABASE_URL", ""),
help="Optional DB URL for auto-start mode",
)
args = parser.parse_args()
if not args.auto_start:
ok = await run_gate3_checks(args.base_url)
sys.exit(0 if ok else 1)
db_url = args.database_url.strip() or None
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
processes: list[subprocess.Popen] = []
try:
processes = await start_services_sequential(db_url, run_data_dir)
ok = await run_gate3_checks("http://127.0.0.1:28080", reporting_url="http://127.0.0.1:28009")
if db_url:
print(f"[INFO] DB mode: {db_url}")
sys.exit(0 if ok else 1)
finally:
stop_services(processes)
try:
shutil.rmtree(run_data_dir)
except Exception:
pass
if __name__ == "__main__":
asyncio.run(main())
+350
View File
@@ -0,0 +1,350 @@
from __future__ import annotations
import argparse
import asyncio
import os
from pathlib import Path
import shutil
import statistics
import subprocess
import sys
import time
from typing import Any
import httpx
ROOT = Path(__file__).resolve().parents[1]
DATA_ROOT = ROOT / ".data_gate4"
SERVICE_SPECS = [
{"name": "auth", "module": "services.auth_service.app:app", "port": 38001},
{"name": "audit", "module": "services.audit_service.app:app", "port": 38002},
{"name": "customer", "module": "services.customer_service.app:app", "port": 38003},
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 38004},
{"name": "routing", "module": "services.routing_service.app:app", "port": 38005},
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 38006},
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 38007},
{"name": "kb", "module": "services.kb_service.app:app", "port": 38008},
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 38009},
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 38010},
{"name": "gateway", "module": "gateway.app:app", "port": 38080},
]
async def wait_for_health(base_url: str, retries: int = 60, delay: float = 0.2) -> None:
async with httpx.AsyncClient(timeout=2) as client:
for _ in range(retries):
try:
response = await client.get(f"{base_url}/health")
if response.status_code == 200:
return
except Exception:
pass
await asyncio.sleep(delay)
raise RuntimeError(f"Service not healthy: {base_url}/health")
def start_one_service(spec: dict[str, Any], data_dir: Path) -> subprocess.Popen:
env = os.environ.copy()
env["CC_DATA_DIR"] = str(data_dir)
if spec["name"] == "gateway":
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:38001"
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:38002"
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:38003"
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:38004"
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:38005"
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:38006"
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:38007"
env["KB_SERVICE_URL"] = "http://127.0.0.1:38008"
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:38009"
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:38010"
cmd = [
sys.executable,
"-m",
"uvicorn",
spec["module"],
"--host",
"127.0.0.1",
"--port",
str(spec["port"]),
]
return subprocess.Popen(
cmd,
cwd=str(ROOT),
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
async def start_services_sequential(data_dir: Path) -> list[subprocess.Popen]:
data_dir.mkdir(parents=True, exist_ok=True)
processes: list[subprocess.Popen] = []
for spec in SERVICE_SPECS:
proc = start_one_service(spec, data_dir)
processes.append(proc)
await wait_for_health(f"http://127.0.0.1:{spec['port']}")
return processes
def stop_services(processes: list[subprocess.Popen]) -> None:
for proc in processes:
if proc.poll() is not None:
continue
try:
proc.terminate()
except Exception:
pass
for proc in processes:
try:
proc.wait(timeout=2)
except Exception:
try:
proc.kill()
except Exception:
pass
async def _create_interaction(client: httpx.AsyncClient, channel: str, idx: int) -> tuple[bool, float]:
headers = {"X-User": "operator", "X-Role": "operator"}
payload = {
"channel": channel,
"subject": f"Gate4 load {channel} #{idx}",
"customer_id": None,
"queue_id": "q_gate4",
"priority": 3,
}
started = time.perf_counter()
try:
resp = await client.post("/proxy/interaction/interactions", headers=headers, json=payload, timeout=10)
elapsed = time.perf_counter() - started
return resp.status_code == 200, elapsed
except Exception:
elapsed = time.perf_counter() - started
return False, elapsed
async def run_load_wave(base_url: str, voice: int, digital: int) -> dict[str, float | int]:
async with httpx.AsyncClient(base_url=base_url) as client:
tasks = []
for i in range(voice):
tasks.append(_create_interaction(client, "voice", i))
for i in range(digital):
tasks.append(_create_interaction(client, "telegram", i))
results = await asyncio.gather(*tasks)
ok_times = [t for ok, t in results if ok]
failed = len(results) - len(ok_times)
if ok_times:
p95 = sorted(ok_times)[max(0, int(len(ok_times) * 0.95) - 1)]
avg = statistics.mean(ok_times)
else:
p95 = 0.0
avg = 0.0
return {
"total": len(results),
"success": len(ok_times),
"failed": failed,
"avg_seconds": round(avg, 4),
"p95_seconds": round(p95, 4),
}
async def get_interaction_count(base_url: str, limit: int = 1000) -> int:
async with httpx.AsyncClient(base_url=base_url, timeout=10) as client:
response = await client.get(f"/proxy/interaction/interactions?limit={limit}")
response.raise_for_status()
items = response.json()
return len(items)
async def run_security_smoke(base_url: str) -> bool:
operator = {"X-User": "operator", "X-Role": "operator"}
async with httpx.AsyncClient(base_url=base_url, timeout=10) as client:
create_user = await client.post(
"/proxy/auth/users",
headers=operator,
json={
"username": "sec_denied_user",
"password": "secret123",
"full_name": "Security Denied",
"role": "operator",
},
)
if create_user.status_code != 403:
print(f"[FAIL] Security RBAC auth/users expected 403, got {create_user.status_code}")
return False
create_queue = await client.post(
"/proxy/routing/queues",
json={
"name": "Denied queue",
"description": "Role check",
"rules": [{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 30}],
},
)
if create_queue.status_code != 403:
print(f"[FAIL] Security RBAC routing/queues expected 403, got {create_queue.status_code}")
return False
kb_create = await client.post(
"/proxy/kb/knowledge/categories",
headers=operator,
json={"name": "Denied KB", "description": "Role check"},
)
if kb_create.status_code != 403:
print(f"[FAIL] Security RBAC kb category expected 403, got {kb_create.status_code}")
return False
print("[PASS] Security smoke checks")
return True
def run_backup_restore(source_dir: Path) -> tuple[Path, Path]:
backup_script = ROOT / "scripts" / "backup_data.ps1"
restore_script = ROOT / "scripts" / "restore_data.ps1"
backup_output = DATA_ROOT / f"backups_{int(time.time() * 1000)}"
backup_output.mkdir(parents=True, exist_ok=True)
backup_cmd = [
"powershell",
"-ExecutionPolicy",
"Bypass",
"-File",
str(backup_script),
"-SourceDir",
str(source_dir),
"-OutputDir",
str(backup_output),
]
result = subprocess.run(backup_cmd, cwd=str(ROOT), capture_output=True, text=True, check=False)
if result.returncode != 0:
raise RuntimeError(f"Backup failed: {result.stdout}\n{result.stderr}")
archives = sorted(backup_output.glob("*.zip"), key=lambda p: p.stat().st_mtime, reverse=True)
if not archives:
raise RuntimeError("Backup archive not created")
archive = archives[0]
if source_dir.exists():
shutil.rmtree(source_dir)
restore_cmd = [
"powershell",
"-ExecutionPolicy",
"Bypass",
"-File",
str(restore_script),
"-BackupZip",
str(archive),
"-TargetDir",
str(source_dir),
]
restore = subprocess.run(restore_cmd, cwd=str(ROOT), capture_output=True, text=True, check=False)
if restore.returncode != 0:
raise RuntimeError(f"Restore failed: {restore.stdout}\n{restore.stderr}")
db_file = source_dir / "mvp_cc.db"
if not db_file.exists():
raise RuntimeError(f"Restored DB file not found: {db_file}")
return backup_output, archive
def check_docs_bundle() -> bool:
required = [
ROOT / "docs" / "security" / "checklist.md",
ROOT / "docs" / "releases" / "v1.0.0-mvp.md",
ROOT / "docs" / "gates" / "mvp-pilot-baseline.md",
ROOT / "docs" / "gates" / "p1-p2-defects.md",
ROOT / "docs" / "roadmap" / "05-wave2-backlog.md",
ROOT / "docs" / "runbooks" / "backup-restore.md",
ROOT / "docs" / "runbooks" / "load-test-plan.md",
ROOT / "docs" / "runbooks" / "pilot-uat.md",
ROOT / "docs" / "uat" / "README.md",
ROOT / "docs" / "uat" / "scenario-checklist.md",
ROOT / "docs" / "uat" / "session-template.md",
ROOT / "docs" / "uat" / "signoff-template.md",
ROOT / "docs" / "uat" / "defect-log-template.csv",
]
missing = [str(path) for path in required if not path.exists()]
if missing:
print(f"[FAIL] Missing required Stage 4 docs: {missing}")
return False
print("[PASS] Stage 4 docs bundle present")
return True
async def main() -> None:
parser = argparse.ArgumentParser(description="Gate 4 checker: load, backup/restore, security smoke")
parser.add_argument("--voice", type=int, default=100, help="Concurrent voice requests")
parser.add_argument("--digital", type=int, default=100, help="Concurrent digital requests")
parser.add_argument("--keep-artifacts", action="store_true", help="Keep .data_gate4 run and backup artifacts")
args = parser.parse_args()
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
backup_output: Path | None = None
processes: list[subprocess.Popen] = []
try:
processes = await start_services_sequential(run_data_dir)
base_url = "http://127.0.0.1:38080"
load_result = await run_load_wave(base_url, args.voice, args.digital)
print("[INFO] Load result:", load_result)
if load_result["failed"] != 0 or load_result["success"] != load_result["total"]:
print("[FAIL] Load target not reached with zero errors")
sys.exit(1)
print("[PASS] Load target reached")
count_before = await get_interaction_count(base_url, limit=max(500, args.voice + args.digital + 100))
if count_before < (args.voice + args.digital):
print(
f"[FAIL] Interaction count before backup too low: {count_before} < {args.voice + args.digital}"
)
sys.exit(1)
print(f"[PASS] Interaction count before backup: {count_before}")
security_ok = await run_security_smoke(base_url)
if not security_ok:
sys.exit(1)
stop_services(processes)
processes = []
backup_output, archive = run_backup_restore(run_data_dir)
print(f"[PASS] Backup/restore scripts completed, archive: {archive}")
processes = await start_services_sequential(run_data_dir)
count_after = await get_interaction_count(base_url, limit=max(500, args.voice + args.digital + 100))
if count_after != count_before:
print(f"[FAIL] Restored interaction count mismatch: before={count_before}, after={count_after}")
sys.exit(1)
print(f"[PASS] Restore data integrity validated: {count_after} interactions")
if not check_docs_bundle():
sys.exit(1)
print("[PASS] Gate 4 automated checks completed")
finally:
stop_services(processes)
if not args.keep_artifacts:
try:
if run_data_dir.exists():
shutil.rmtree(run_data_dir)
except Exception:
pass
try:
if backup_output and backup_output.exists():
shutil.rmtree(backup_output)
except Exception:
pass
if __name__ == "__main__":
asyncio.run(main())
@@ -0,0 +1,99 @@
{
"category_name": "Базовые ответы Voice AI",
"source_language": "ru",
"target_language": "kz",
"localizations": [
{
"source_title": "График работы",
"title": "Жұмыс кестесі",
"body": "Нақты жұмыс кестесін айту үшін қаланы, филиалды немесе мекенжайды көрсетіңіз. Егер филиалды білмесеңіз, кемінде қаланы айтыңыз, мен ақпаратты нақтылауға көмектесемін немесе қоңырауды операторға бағыттаймын.",
"tags": [
"жұмыс кестесі",
"жұмыс режимі",
"жұмыс уақыты",
"қашан жұмыс істейсіз",
"қандай жұмыс кестесі",
"жұмыс кестесін білгім келеді",
"жұмыс кестесін білу",
"филиалдың жұмыс уақыты",
"кесте"
]
},
{
"source_title": "Статус заявки",
"title": "Өтінім мәртебесі",
"body": "Өтінім мәртебесін тексеру үшін өтінім нөмірін немесе рәсімделген телефон нөмірін атаңыз. Нөмір болмаса, қолмен тексеру үшін қоңырауды операторға бағыттаймын.",
"tags": [
"өтінім мәртебесі",
"мәртебе",
"өтінім",
"өтінім нөмірі",
"тапсырыс мәртебесі",
"өтінім мәртебесін тексеру",
"өтінім мәртебесін білу",
"менің тапсырысым"
]
},
{
"source_title": "Адрес и филиал",
"title": "Мекенжай және филиал",
"body": "Мекенжайды немесе филиалды айту үшін қаланы және мүмкін болса ауданды, көшені немесе бағдарды атаңыз. Егер қолайлы филиалды анықтай алмасам, қоңырауды операторға бағыттаймын.",
"tags": [
"мекенжай",
"филиал",
"офис",
"бөлімше",
"қайда орналасқансыз",
"сіздерді қалай табамын",
"офистің мекенжайы",
"филиалдың мекенжайы",
"қала"
]
},
{
"source_title": "Тарифы и стоимость",
"title": "Тарифтер мен құны",
"body": "Тарифті, бағаны немесе құнын айту үшін сізді қызықтыратын қызметті немесе өнімді атаңыз. Дәл есептеу немесе жеке ұсыныс үшін қоңырау операторға бағытталуы мүмкін.",
"tags": [
"тариф",
"тарифтер",
"құны",
"баға",
"қанша тұрады",
"қандай баға",
"құнын білу",
"қызмет ақысы",
"қызмет құны"
]
},
{
"source_title": "Проблема с услугой",
"title": "Қызметке қатысты мәселе",
"body": "Егер сервис жұмыс істемесе немесе қате шықса, мәселені қысқаша сипаттап, телефон нөмірін, жеке шотты немесе өтінім нөмірін көрсетіңіз. Диагностика үшін қоңырау операторға бағытталуы мүмкін.",
"tags": [
"жұмыс істемейді",
"қате",
"мәселе",
"ақау",
"болмай тұр",
"интернет жұмыс істемейді",
"қате бар",
"қызмет жұмыс істемейді"
]
},
{
"source_title": "Соединение с оператором",
"title": "Оператормен қосу",
"body": "Егер тірі маман керек болса, былай айтыңыз: оператормен қосыңыз, операторға ауыстырыңыз немесе адам керек. Осыдан кейін қоңырау операторға бағытталады.",
"tags": [
"оператор",
"тірі оператор",
"оператормен қосыңыз",
"операторға ауыстырыңыз",
"оператор керек",
"адам керек",
"оператормен байланыстырыңыз"
]
}
]
}
+94
View File
@@ -0,0 +1,94 @@
{
"category": {
"name": "Базовые ответы Voice AI",
"description": "Стартовые customer-facing статьи для голосового AI и ручного наполнения KB."
},
"articles": [
{
"title": "График работы",
"body": "Чтобы назвать точный график работы, укажите город, филиал или адрес. Если вы не знаете филиал, скажите хотя бы город, и я помогу уточнить информацию или передам звонок оператору.",
"tags": [
"график работы",
"режим работы",
"часы работы",
"время работы",
"когда вы работаете",
"какой график работы",
"хочу узнать график работы",
"узнать график работы",
"время работы филиала"
]
},
{
"title": "Статус заявки",
"body": "Чтобы проверить статус заявки, назовите номер заявки или телефон, по которому она оформлялась. Если номера нет, я передам звонок оператору для ручной проверки.",
"tags": [
"статус заявки",
"статус",
"заявка",
"номер заявки",
"статус заказа",
"проверить статус заявки",
"узнать статус заявки",
"мой заказ"
]
},
{
"title": "Адрес и филиал",
"body": "Чтобы подсказать адрес или филиал, назовите город и по возможности район, улицу или ориентир. Если подходящий филиал не удастся определить, я передам звонок оператору.",
"tags": [
"адрес",
"филиал",
"офис",
"отделение",
"где вы находитесь",
"как вас найти",
"адрес офиса",
"адрес филиала",
"город"
]
},
{
"title": "Тарифы и стоимость",
"body": "Чтобы подсказать тариф, цену или стоимость, назовите услугу или продукт, который вас интересует. Для точного расчета или персонального предложения звонок может быть передан оператору.",
"tags": [
"тариф",
"тарифы",
"стоимость",
"цена",
"сколько стоит",
"какая цена",
"узнать стоимость",
"оплата услуг",
"стоимость услуги"
]
},
{
"title": "Проблема с услугой",
"body": "Если сервис не работает или появилась ошибка, кратко опишите проблему и укажите номер телефона, лицевого счета или заявки. Для диагностики звонок может быть передан оператору.",
"tags": [
"не работает",
"ошибка",
"проблема",
"сбой",
"не получается",
"интернет не работает",
"есть ошибка",
"не работает услуга"
]
},
{
"title": "Соединение с оператором",
"body": "Если нужен живой специалист, скажите: соедините с оператором, переведите на оператора или нужен человек. После этого звонок будет передан оператору.",
"tags": [
"оператор",
"живой оператор",
"соедините с оператором",
"переведите на оператора",
"нужен оператор",
"нужен человек",
"свяжите с оператором"
]
}
]
}
+251
View File
@@ -0,0 +1,251 @@
from __future__ import annotations
import argparse
import asyncio
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
from typing import Any
import httpx
ROOT = Path(__file__).resolve().parents[1]
DATA_ROOT = ROOT / ".data_smoke"
SERVICE_SPECS = [
{"name": "auth", "module": "services.auth_service.app:app", "port": 18001},
{"name": "audit", "module": "services.audit_service.app:app", "port": 18002},
{"name": "customer", "module": "services.customer_service.app:app", "port": 18003},
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 18004},
{"name": "routing", "module": "services.routing_service.app:app", "port": 18005},
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 18006},
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 18007},
{"name": "kb", "module": "services.kb_service.app:app", "port": 18008},
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 18009},
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 18010},
{"name": "gateway", "module": "gateway.app:app", "port": 18080},
]
async def wait_for_health(base_url: str, retries: int = 80, delay: float = 0.25) -> None:
async with httpx.AsyncClient(timeout=2) as client:
for _ in range(retries):
try:
response = await client.get(f"{base_url}/health")
if response.status_code == 200:
return
except Exception:
pass
await asyncio.sleep(delay)
raise RuntimeError(f"Service not healthy: {base_url}/health")
def start_services(database_url: str | None, data_dir: Path) -> list[subprocess.Popen]:
data_dir.mkdir(parents=True, exist_ok=True)
processes: list[subprocess.Popen] = []
for spec in SERVICE_SPECS:
env = os.environ.copy()
env["CC_DATA_DIR"] = str(data_dir)
if database_url:
env["DATABASE_URL"] = database_url
if spec["name"] == "gateway":
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:18001"
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:18002"
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:18003"
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:18004"
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:18005"
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:18006"
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:18007"
env["KB_SERVICE_URL"] = "http://127.0.0.1:18008"
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:18009"
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:18010"
cmd = [
sys.executable,
"-m",
"uvicorn",
spec["module"],
"--host",
"127.0.0.1",
"--port",
str(spec["port"]),
]
proc = subprocess.Popen(
cmd,
cwd=str(ROOT),
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
processes.append(proc)
return processes
def stop_services(processes: list[subprocess.Popen]) -> None:
for proc in processes:
if proc.poll() is not None:
continue
try:
proc.terminate()
except Exception:
pass
for proc in processes:
try:
proc.wait(timeout=2)
except Exception:
try:
proc.kill()
except Exception:
pass
async def run_gate_checks() -> dict[str, Any]:
for spec in SERVICE_SPECS:
await wait_for_health(f"http://127.0.0.1:{spec['port']}")
base = "http://127.0.0.1:18080"
admin = {"X-User": "admin", "X-Role": "admin"}
operator = {"X-User": "operator", "X-Role": "operator"}
supervisor = {"X-User": "supervisor", "X-Role": "supervisor"}
async with httpx.AsyncClient(base_url=base, timeout=10) as client:
health = await client.get("/health")
health.raise_for_status()
login = await client.post("/proxy/auth/auth/login", json={"username": "admin", "password": "admin123"})
login.raise_for_status()
queue = await client.post(
"/proxy/routing/queues",
headers=admin,
json={
"name": "Main Queue",
"description": "Smoke queue",
"rules": [
{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 30},
{"channel": "telegram", "priority": 3, "strategy": "round_robin", "sla_seconds": 45},
],
},
)
queue.raise_for_status()
queue_id = queue.json()["queue_id"]
customer = await client.post(
"/proxy/customer/customers",
json={
"display_name": "Gate User",
"phones": ["+77010000011"],
"preferred_phone": "+77010000011",
"tags": ["smoke"],
},
)
customer.raise_for_status()
customer_id = customer.json()["customer_id"]
interaction = await client.post(
"/proxy/interaction/interactions",
headers=operator,
json={
"channel": "voice",
"subject": "Gate voice check",
"customer_id": customer_id,
"queue_id": queue_id,
"priority": 3,
},
)
interaction.raise_for_status()
interaction_id = interaction.json()["interaction_id"]
assign = await client.patch(
f"/proxy/interaction/interactions/{interaction_id}/assign",
headers=supervisor,
json={"assignee": "operator_a"},
)
assign.raise_for_status()
escalate = await client.post(
f"/proxy/interaction/interactions/{interaction_id}/escalate",
headers=operator,
json={"target_queue_id": "line2"},
)
escalate.raise_for_status()
close = await client.patch(
f"/proxy/interaction/interactions/{interaction_id}/status",
headers=operator,
json={"status": "closed"},
)
close.raise_for_status()
voice_event = await client.post(
"/proxy/voice/integrations/voice/events",
headers=operator,
json={
"event_type": "call.started",
"call_id": "smoke_call_1",
"interaction_id": interaction_id,
"payload": {"source": "live_smoke"},
},
)
voice_event.raise_for_status()
tg_event = await client.post(
"/proxy/telegram/integrations/telegram/webhook",
json={
"chat_id": "smoke_chat",
"text": "Smoke message",
"customer_external_id": None,
"payload": {"source": "live_smoke"},
},
)
tg_event.raise_for_status()
interactions = await client.get("/proxy/interaction/interactions")
interactions.raise_for_status()
rows = interactions.json()
return {
"gateway": health.json(),
"queue_id": queue_id,
"customer_id": customer_id,
"interaction_id": interaction_id,
"interactions_total": len(rows),
}
async def main() -> None:
parser = argparse.ArgumentParser(description="Live Gate 1/2 smoke")
parser.add_argument(
"--database-url",
default=os.getenv("DATABASE_URL", ""),
help="Optional DB URL override for started services",
)
args = parser.parse_args()
db_url = args.database_url.strip() or None
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
processes = start_services(db_url, run_data_dir)
try:
result = await run_gate_checks()
print("Gate 1/2 live smoke passed")
print(result)
if db_url:
print(f"DB mode: {db_url}")
finally:
stop_services(processes)
try:
shutil.rmtree(run_data_dir)
except Exception:
pass
if __name__ == "__main__":
asyncio.run(main())
+452
View File
@@ -0,0 +1,452 @@
from __future__ import annotations
import argparse
import asyncio
import csv
import json
import math
import statistics
import time
from collections import Counter
from pathlib import Path
from typing import Any
import httpx
PROFILE_DEFAULTS = {
"baseline_100_100": {"voice": 100, "digital": 100, "ramp_seconds": 15, "hold_seconds": 60},
"step_250_250": {"voice": 250, "digital": 250, "ramp_seconds": 30, "hold_seconds": 120},
"target_500_500": {"voice": 500, "digital": 500, "ramp_seconds": 60, "hold_seconds": 180},
}
DIGITAL_DISTRIBUTION = (("telegram", 0.4), ("webchat", 0.3), ("email", 0.3))
READ_TRAFFIC_INTERVAL_SECONDS = 1.0
MAX_ERROR_SAMPLES = 200
WORKER_PAUSE_SECONDS = 1.0
def resolve_profile(
name: str,
ramp_seconds: int | None = None,
hold_seconds: int | None = None,
voice: int | None = None,
digital: int | None = None,
) -> dict[str, int | str]:
if name not in PROFILE_DEFAULTS:
raise ValueError(f"Unsupported profile: {name}")
profile = dict(PROFILE_DEFAULTS[name])
profile["profile"] = name
if ramp_seconds is not None:
profile["ramp_seconds"] = ramp_seconds
if hold_seconds is not None:
profile["hold_seconds"] = hold_seconds
if voice is not None:
profile["voice"] = voice
if digital is not None:
profile["digital"] = digital
return profile
def split_digital_mix(total: int) -> dict[str, int]:
remaining = total
result: dict[str, int] = {}
for idx, (name, ratio) in enumerate(DIGITAL_DISTRIBUTION):
if idx == len(DIGITAL_DISTRIBUTION) - 1:
count = remaining
else:
count = int(round(total * ratio))
remaining -= count
result[name] = count
return result
def build_mix_profile(profile: dict[str, int | str], include_read_traffic: bool) -> dict[str, Any]:
voice = int(profile["voice"])
digital = int(profile["digital"])
digital_mix = split_digital_mix(digital)
payload = {
"profile": profile["profile"],
"voice_workers": voice,
"digital_workers": digital,
"ramp_seconds": int(profile["ramp_seconds"]),
"hold_seconds": int(profile["hold_seconds"]),
"channels": {"voice": voice, **digital_mix},
"include_read_traffic": include_read_traffic,
"read_endpoints": [
"/proxy/supervisor/supervisor/realtime",
"/proxy/reporting/reports/kpi",
]
if include_read_traffic
else [],
}
return payload
def _build_voice_payload(idx: int) -> tuple[str, str, dict[str, Any]]:
return (
"POST",
"/proxy/interaction/interactions",
{
"channel": "voice",
"subject": f"Track7 voice load #{idx}",
"customer_id": None,
"queue_id": "q_load_voice",
"priority": 3,
},
)
def _build_telegram_payload(idx: int) -> tuple[str, str, dict[str, Any]]:
return (
"POST",
"/proxy/telegram/integrations/telegram/webhook",
{
"chat_id": f"load_chat_{idx}",
"text": f"Track7 telegram load #{idx}",
"customer_external_id": None,
"payload": {"source": "track7-load"},
},
)
def _build_webchat_payload(idx: int) -> tuple[str, str, dict[str, Any]]:
return (
"POST",
"/proxy/webchat/integrations/webchat/messages",
{
"session_id": f"load_session_{idx}",
"text": f"Track7 webchat load #{idx}",
"visitor_name": "Load Test Visitor",
"customer_external_id": None,
"queue_id": "q_load_webchat",
"priority": 3,
"payload": {"source": "track7-load"},
},
)
def _build_email_payload(idx: int) -> tuple[str, str, dict[str, Any]]:
return (
"POST",
"/proxy/email/integrations/email/messages",
{
"from_email": f"load{idx}@example.test",
"subject": f"Track7 email load #{idx}",
"body": "Synthetic email for scale validation.",
"customer_external_id": None,
"queue_id": "q_load_email",
"priority": 3,
"payload": {"source": "track7-load"},
},
)
def build_request(target: str, idx: int) -> tuple[str, str, dict[str, Any]]:
if target == "voice":
return _build_voice_payload(idx)
if target == "telegram":
return _build_telegram_payload(idx)
if target == "webchat":
return _build_webchat_payload(idx)
if target == "email":
return _build_email_payload(idx)
raise ValueError(f"Unsupported target: {target}")
def build_worker_targets(profile: dict[str, int | str]) -> list[str]:
targets = ["voice"] * int(profile["voice"])
digital_mix = split_digital_mix(int(profile["digital"]))
for name, count in digital_mix.items():
targets.extend([name] * count)
return targets
def percentile(samples: list[float], ratio: float) -> float:
if not samples:
return 0.0
sorted_samples = sorted(samples)
index = max(0, math.ceil(len(sorted_samples) * ratio) - 1)
return sorted_samples[index]
def ensure_report_dir(report_dir: str | None = None) -> Path:
if report_dir:
target = Path(report_dir)
else:
timestamp = time.strftime("%Y%m%d_%H%M%S")
target = Path(".artifacts") / "track7" / timestamp
target.mkdir(parents=True, exist_ok=True)
return target
async def build_auth_headers(client: httpx.AsyncClient, auth_mode: str) -> dict[str, str]:
if auth_mode == "legacy_headers":
return {"X-User": "admin", "X-Role": "admin"}
if auth_mode != "bearer":
raise ValueError(f"Unsupported auth mode: {auth_mode}")
response = await client.post(
"/proxy/auth/auth/login",
json={"username": "admin", "password": "admin123"},
timeout=10,
)
response.raise_for_status()
token = response.json()["access_token"]
return {"Authorization": f"Bearer {token}"}
def _record_result(
results: list[dict[str, Any]],
errors: list[dict[str, Any]],
target: str,
started_at: float,
elapsed: float,
status_code: int,
ok: bool,
error_message: str | None = None,
) -> None:
results.append(
{
"ts": round(started_at, 6),
"target": target,
"status_code": status_code,
"latency_seconds": round(elapsed, 6),
"ok": ok,
}
)
if (not ok or status_code >= 500 or error_message) and len(errors) < MAX_ERROR_SAMPLES:
errors.append(
{
"ts": round(started_at, 6),
"target": target,
"status_code": status_code,
"latency_seconds": round(elapsed, 6),
"error": error_message,
}
)
async def _run_worker(
client: httpx.AsyncClient,
headers: dict[str, str],
target: str,
idx: int,
stop_event: asyncio.Event,
results: list[dict[str, Any]],
errors: list[dict[str, Any]],
) -> None:
method, path, payload = build_request(target, idx)
while not stop_event.is_set():
started = time.perf_counter()
try:
response = await client.request(method, path, headers=headers, json=payload, timeout=20)
elapsed = time.perf_counter() - started
ok = response.status_code < 500 and response.status_code < 400
_record_result(results, errors, target, started, elapsed, response.status_code, ok)
except Exception as exc:
elapsed = time.perf_counter() - started
_record_result(results, errors, target, started, elapsed, 0, False, str(exc))
await asyncio.sleep(WORKER_PAUSE_SECONDS)
async def _run_read_probe(
client: httpx.AsyncClient,
headers: dict[str, str],
path: str,
stop_event: asyncio.Event,
results: list[dict[str, Any]],
errors: list[dict[str, Any]],
) -> None:
target = f"read:{path.rsplit('/', 1)[-1]}"
while not stop_event.is_set():
started = time.perf_counter()
try:
response = await client.get(path, headers=headers, timeout=20)
elapsed = time.perf_counter() - started
ok = response.status_code < 500 and response.status_code < 400
_record_result(results, errors, target, started, elapsed, response.status_code, ok)
except Exception as exc:
elapsed = time.perf_counter() - started
_record_result(results, errors, target, started, elapsed, 0, False, str(exc))
await asyncio.sleep(READ_TRAFFIC_INTERVAL_SECONDS)
def build_summary(
results: list[dict[str, Any]],
mix_profile: dict[str, Any],
thresholds: dict[str, float | None],
started_at: str,
completed_at: str,
) -> dict[str, Any]:
latencies = [item["latency_seconds"] for item in results]
total = len(results)
success = sum(1 for item in results if item["ok"])
failures = total - success
five_xx = sum(1 for item in results if int(item["status_code"]) >= 500 or int(item["status_code"]) == 0)
by_target = Counter(item["target"] for item in results)
summary = {
"profile": mix_profile["profile"],
"started_at": started_at,
"completed_at": completed_at,
"traffic": mix_profile,
"results": {
"total_requests": total,
"success": success,
"failed": failures,
"five_xx_or_transport": five_xx,
"success_rate": round((success / total) * 100, 2) if total else 0.0,
"five_xx_rate": round((five_xx / total) * 100, 2) if total else 0.0,
"avg_seconds": round(statistics.mean(latencies), 4) if latencies else 0.0,
"p95_seconds": round(percentile(latencies, 0.95), 4) if latencies else 0.0,
"p99_seconds": round(percentile(latencies, 0.99), 4) if latencies else 0.0,
"by_target": dict(by_target),
},
"thresholds": thresholds,
}
summary["passed"] = evaluate_thresholds(summary)
return summary
def evaluate_thresholds(summary: dict[str, Any]) -> bool:
results = summary["results"]
thresholds = summary["thresholds"]
if thresholds.get("require_success_rate") is not None and results["success_rate"] < thresholds["require_success_rate"]:
return False
if thresholds.get("require_p95_seconds") is not None and results["p95_seconds"] > thresholds["require_p95_seconds"]:
return False
if thresholds.get("require_p99_seconds") is not None and results["p99_seconds"] > thresholds["require_p99_seconds"]:
return False
return True
def write_report(report_dir: Path, summary: dict[str, Any], results: list[dict[str, Any]], errors: list[dict[str, Any]]) -> None:
(report_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
(report_dir / "mix_profile.json").write_text(json.dumps(summary["traffic"], indent=2), encoding="utf-8")
(report_dir / "error_samples.json").write_text(json.dumps(errors, indent=2), encoding="utf-8")
with (report_dir / "latency_samples.csv").open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=["ts", "target", "status_code", "latency_seconds", "ok"])
writer.writeheader()
writer.writerows(results)
async def run_profile(
*,
base_url: str,
profile_name: str,
ramp_seconds: int | None,
hold_seconds: int | None,
voice: int | None,
digital: int | None,
include_read_traffic: bool,
auth_mode: str,
report_dir: str | None,
require_success_rate: float | None,
require_p95_seconds: float | None,
require_p99_seconds: float | None,
) -> tuple[dict[str, Any], Path]:
profile = resolve_profile(
profile_name,
ramp_seconds=ramp_seconds,
hold_seconds=hold_seconds,
voice=voice,
digital=digital,
)
mix_profile = build_mix_profile(profile, include_read_traffic)
out_dir = ensure_report_dir(report_dir)
started_at_iso = time.strftime("%Y-%m-%dT%H:%M:%S")
results: list[dict[str, Any]] = []
errors: list[dict[str, Any]] = []
stop_event = asyncio.Event()
worker_targets = build_worker_targets(profile)
async with httpx.AsyncClient(base_url=base_url) as client:
headers = await build_auth_headers(client, auth_mode)
workers: list[asyncio.Task[None]] = []
ramp_window = max(1, int(profile["ramp_seconds"]))
spacing = ramp_window / max(1, len(worker_targets))
for idx, target in enumerate(worker_targets):
workers.append(asyncio.create_task(_run_worker(client, headers, target, idx, stop_event, results, errors)))
if spacing > 0:
await asyncio.sleep(spacing)
read_tasks: list[asyncio.Task[None]] = []
if include_read_traffic:
read_tasks = [
asyncio.create_task(
_run_read_probe(client, headers, "/proxy/supervisor/supervisor/realtime", stop_event, results, errors)
),
asyncio.create_task(
_run_read_probe(client, headers, "/proxy/reporting/reports/kpi", stop_event, results, errors)
),
]
await asyncio.sleep(int(profile["hold_seconds"]))
stop_event.set()
await asyncio.gather(*workers, *read_tasks, return_exceptions=True)
completed_at_iso = time.strftime("%Y-%m-%dT%H:%M:%S")
thresholds = {
"require_success_rate": require_success_rate,
"require_p95_seconds": require_p95_seconds,
"require_p99_seconds": require_p99_seconds,
}
summary = build_summary(results, mix_profile, thresholds, started_at_iso, completed_at_iso)
write_report(out_dir, summary, results, errors)
return summary, out_dir
async def main() -> None:
parser = argparse.ArgumentParser(description="Track 7 mixed workload load harness through the gateway.")
parser.add_argument("--base-url", default="http://localhost:8080", help="Gateway URL")
parser.add_argument(
"--profile",
default="baseline_100_100",
choices=sorted(PROFILE_DEFAULTS),
help="Named load profile",
)
parser.add_argument("--ramp-seconds", type=int, default=None, help="Override ramp duration")
parser.add_argument("--hold-seconds", type=int, default=None, help="Override hold duration")
parser.add_argument("--voice", type=int, default=None, help="Backward-compatible voice worker override")
parser.add_argument("--digital", type=int, default=None, help="Backward-compatible digital worker override")
parser.add_argument("--report-dir", default=None, help="Directory for JSON/CSV reports")
parser.add_argument("--require-success-rate", type=float, default=None, help="Optional success-rate threshold")
parser.add_argument("--require-p95-seconds", type=float, default=None, help="Optional p95 threshold")
parser.add_argument("--require-p99-seconds", type=float, default=None, help="Optional p99 threshold")
parser.add_argument(
"--include-read-traffic",
type=int,
choices=[0, 1],
default=1,
help="Include background supervisor/reporting reads during the hold window",
)
parser.add_argument(
"--auth-mode",
choices=["legacy_headers", "bearer"],
default="legacy_headers",
help="Authentication mode for load traffic",
)
args = parser.parse_args()
summary, out_dir = await run_profile(
base_url=args.base_url,
profile_name=args.profile,
ramp_seconds=args.ramp_seconds,
hold_seconds=args.hold_seconds,
voice=args.voice,
digital=args.digital,
include_read_traffic=bool(args.include_read_traffic),
auth_mode=args.auth_mode,
report_dir=args.report_dir,
require_success_rate=args.require_success_rate,
require_p95_seconds=args.require_p95_seconds,
require_p99_seconds=args.require_p99_seconds,
)
print("Track 7 load test result:")
print(json.dumps(summary, indent=2))
print(f"Report dir: {out_dir}")
if not summary["passed"]:
raise SystemExit(1)
if __name__ == "__main__":
asyncio.run(main())
+567
View File
@@ -0,0 +1,567 @@
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
from typing import Any
from urllib.error import URLError
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_RUNTIME_DIR = ROOT / ".local_stack"
DEFAULT_DATA_DIR = ROOT / ".data_local"
DEFAULT_ENV_FILES = [
ROOT / ".env.production",
ROOT / ".env.local",
ROOT / ".env",
]
def _env_int(name: str, default: int, env: dict[str, str] | None = None) -> int:
source = env or os.environ
raw = str(source.get(name, "")).strip()
if not raw:
return default
try:
value = int(raw)
except ValueError as exc:
raise RuntimeError(f"{name} must be an integer") from exc
if value <= 0:
raise RuntimeError(f"{name} must be positive")
return value
def utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def load_env_file(path: Path) -> dict[str, str]:
if not path.exists():
return {}
result: dict[str, str] = {}
for raw_line in path.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
key = key.strip()
value = value.strip()
if not key:
continue
if value and value[0] == value[-1] and value[0] in {'"', "'"}:
value = value[1:-1]
result[key] = value
return result
def load_stack_env(extra_env_files: list[Path] | None = None) -> dict[str, str]:
env = os.environ.copy()
for path in DEFAULT_ENV_FILES:
for key, value in load_env_file(path).items():
env.setdefault(key, value)
for path in extra_env_files or []:
for key, value in load_env_file(path).items():
env[key] = value
return env
def build_service_specs(env: dict[str, str] | None = None) -> list[dict[str, Any]]:
gateway_port = _env_int("LOCAL_STACK_GATEWAY_PORT", 8080, env)
return [
{"name": "auth", "port": 8001, "module": "services.auth_service.app:app"},
{"name": "audit", "port": 8002, "module": "services.audit_service.app:app"},
{"name": "customer", "port": 8003, "module": "services.customer_service.app:app"},
{"name": "interaction", "port": 8004, "module": "services.interaction_service.app:app"},
{"name": "routing", "port": 8005, "module": "services.routing_service.app:app"},
{"name": "voice", "port": 8006, "module": "services.voice_adapter_service.app:app"},
{"name": "recording", "port": 8013, "module": "services.recording_service.app:app"},
{"name": "ivr", "port": 8014, "module": "services.ivr_service.app:app"},
{"name": "event-bus", "port": 8015, "module": "services.event_bus_service.app:app"},
{"name": "asterisk-bridge", "port": 8016, "module": "services.asterisk_bridge_service.app:app"},
{"name": "telegram", "port": 8007, "module": "services.telegram_adapter_service.app:app"},
{"name": "whatsapp", "port": 8019, "module": "services.whatsapp_adapter_service.app:app"},
{"name": "ai", "port": 8017, "module": "services.ai_orchestrator_service.app:app"},
{"name": "ai-voice-runtime", "port": 8018, "module": "services.ai_voice_runtime_service.app:app"},
{"name": "webchat", "port": 8011, "module": "services.webchat_adapter_service.app:app"},
{"name": "email", "port": 8012, "module": "services.email_adapter_service.app:app"},
{"name": "kb", "port": 8008, "module": "services.kb_service.app:app"},
{"name": "reporting", "port": 8009, "module": "services.reporting_service.app:app"},
{"name": "supervisor", "port": 8010, "module": "services.supervisor_service.app:app"},
{"name": "gateway", "port": gateway_port, "module": "gateway.app:app"},
]
def build_gateway_env(env: dict[str, str] | None = None) -> dict[str, str]:
ports = {spec["name"]: int(spec["port"]) for spec in build_service_specs(env)}
base = "http://127.0.0.1"
return {
"AUTH_SERVICE_URL": f"{base}:{ports['auth']}",
"AUDIT_SERVICE_URL": f"{base}:{ports['audit']}",
"CUSTOMER_SERVICE_URL": f"{base}:{ports['customer']}",
"INTERACTION_SERVICE_URL": f"{base}:{ports['interaction']}",
"ROUTING_SERVICE_URL": f"{base}:{ports['routing']}",
"VOICE_ADAPTER_SERVICE_URL": f"{base}:{ports['voice']}",
"RECORDING_SERVICE_URL": f"{base}:{ports['recording']}",
"IVR_SERVICE_URL": f"{base}:{ports['ivr']}",
"EVENT_BUS_SERVICE_URL": f"{base}:{ports['event-bus']}",
"ASTERISK_BRIDGE_SERVICE_URL": f"{base}:{ports['asterisk-bridge']}",
"TELEGRAM_ADAPTER_SERVICE_URL": f"{base}:{ports['telegram']}",
"WHATSAPP_ADAPTER_SERVICE_URL": f"{base}:{ports['whatsapp']}",
"AI_ORCHESTRATOR_SERVICE_URL": f"{base}:{ports['ai']}",
"AI_VOICE_RUNTIME_SERVICE_URL": f"{base}:{ports['ai-voice-runtime']}",
"WEBCHAT_ADAPTER_SERVICE_URL": f"{base}:{ports['webchat']}",
"EMAIL_ADAPTER_SERVICE_URL": f"{base}:{ports['email']}",
"KB_SERVICE_URL": f"{base}:{ports['kb']}",
"REPORTING_SERVICE_URL": f"{base}:{ports['reporting']}",
"SUPERVISOR_SERVICE_URL": f"{base}:{ports['supervisor']}",
}
def service_bind_host(spec_name: str, env: dict[str, str]) -> str:
if spec_name == "gateway":
return env.get("LOCAL_STACK_GATEWAY_HOST", "127.0.0.1").strip() or "127.0.0.1"
return env.get("LOCAL_STACK_SERVICE_HOST", "127.0.0.1").strip() or "127.0.0.1"
def service_health_host(spec_name: str, env: dict[str, str]) -> str:
if spec_name == "gateway":
configured = env.get("LOCAL_STACK_GATEWAY_HEALTH_HOST", "").strip()
if configured:
return configured
bind_host = service_bind_host(spec_name, env)
if bind_host in {"0.0.0.0", "::"}:
return "127.0.0.1"
return bind_host
def public_base_url(env: dict[str, str] | None = None) -> str:
current_env = env or {}
default_url = "http://127.0.0.1:8080"
return current_env.get("LOCAL_STACK_PUBLIC_BASE_URL", default_url).strip() or default_url
def build_manifest_payload(
runtime_dir: Path,
data_dir: Path,
services: list[dict[str, Any]],
env: dict[str, str] | None = None,
) -> dict[str, Any]:
return {
"generated_at": utc_now_iso(),
"root": str(ROOT),
"runtime_dir": str(runtime_dir),
"data_dir": str(data_dir),
"base_url": public_base_url(env),
"services": services,
}
def manifest_path(runtime_dir: Path) -> Path:
return runtime_dir / "manifest.json"
def load_manifest(runtime_dir: Path) -> dict[str, Any] | None:
path = manifest_path(runtime_dir)
if not path.exists():
return None
return json.loads(path.read_text(encoding="utf-8"))
def save_manifest(runtime_dir: Path, payload: dict[str, Any]) -> None:
runtime_dir.mkdir(parents=True, exist_ok=True)
manifest_path(runtime_dir).write_text(json.dumps(payload, indent=2), encoding="utf-8")
def remove_manifest(runtime_dir: Path) -> None:
path = manifest_path(runtime_dir)
if path.exists():
path.unlink()
def is_pid_running(pid: int) -> bool:
if pid <= 0:
return False
if os.name == "nt":
result = subprocess.run(
["tasklist", "/FI", f"PID eq {pid}"],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
check=False,
)
if result.returncode != 0:
return False
return str(pid) in (result.stdout or "")
try:
os.kill(pid, 0)
except (OSError, SystemError):
return False
return True
def resolve_listener_pid(port: int) -> int | None:
if os.name == "nt":
result = subprocess.run(
["netstat", "-ano", "-p", "tcp"],
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
check=False,
)
if result.returncode != 0:
return None
markers = {f"127.0.0.1:{port}", f"0.0.0.0:{port}", f"[::]:{port}"}
for raw_line in (result.stdout or "").splitlines():
line = raw_line.strip()
if "LISTENING" not in line:
continue
parts = line.split()
if len(parts) < 5:
continue
local_address = parts[1]
state = parts[3]
pid_raw = parts[4]
if local_address not in markers or state != "LISTENING":
continue
try:
pid = int(pid_raw)
except ValueError:
continue
if pid > 0:
return pid
return None
return None
def wait_for_health(host: str, port: int, retries: int = 80, delay: float = 0.25) -> None:
url = f"http://{host}:{port}/health"
for _ in range(retries):
try:
with urlopen(url, timeout=1.5) as response: # noqa: S310 - local health probe
if response.status == 200:
return
except (OSError, URLError):
pass
time.sleep(delay)
raise RuntimeError(f"Service did not become healthy: {url}")
def create_log_paths(runtime_dir: Path, name: str) -> tuple[Path, Path]:
log_dir = runtime_dir / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
return log_dir / f"{name}.stdout.log", log_dir / f"{name}.stderr.log"
def spawn_service(spec: dict[str, Any], runtime_dir: Path, data_dir: Path, base_env: dict[str, str]) -> dict[str, Any]:
env = base_env.copy()
env["CC_DATA_DIR"] = str(data_dir)
env["EVENT_BUS_ENABLED"] = env.get("EVENT_BUS_ENABLED", "0")
env["EVENT_BUS_URL"] = env.get("EVENT_BUS_URL", "amqp://guest:guest@127.0.0.1:5672/")
env["EVENT_BUS_EXCHANGE"] = env.get("EVENT_BUS_EXCHANGE", "mvpcc.domain.events")
env["EVENT_BUS_DISPATCH_BATCH_SIZE"] = env.get("EVENT_BUS_DISPATCH_BATCH_SIZE", "50")
env["EVENT_BUS_MAX_ATTEMPTS"] = env.get("EVENT_BUS_MAX_ATTEMPTS", "5")
env["EVENT_BUS_POLL_INTERVAL_SECONDS"] = env.get("EVENT_BUS_POLL_INTERVAL_SECONDS", "2")
env["EVENT_BUS_CONSUMER_ENABLED"] = env.get("EVENT_BUS_CONSUMER_ENABLED", "1")
env["EVENT_BUS_AUDIT_QUEUE"] = env.get("EVENT_BUS_AUDIT_QUEUE", "mvpcc.audit.events")
env["EVENT_BUS_REPORTING_QUEUE"] = env.get("EVENT_BUS_REPORTING_QUEUE", "mvpcc.reporting.events")
env["ASTERISK_BRIDGE_ENABLED"] = env.get("ASTERISK_BRIDGE_ENABLED", "0")
env["ASTERISK_AMI_EVENT_PREFIX"] = env.get("ASTERISK_AMI_EVENT_PREFIX", "MVPCC")
env["ASTERISK_QUEUE_MAP_JSON"] = env.get("ASTERISK_QUEUE_MAP_JSON", "{}")
env["ASTERISK_CALLCONTROL_ENABLED"] = env.get("ASTERISK_CALLCONTROL_ENABLED", "0")
env["ASTERISK_CALLCONTROL_ACTION_TIMEOUT_SECONDS"] = env.get(
"ASTERISK_CALLCONTROL_ACTION_TIMEOUT_SECONDS",
"10",
)
env["ASTERISK_WEBRTC_ENABLED"] = env.get("ASTERISK_WEBRTC_ENABLED", "0")
env["ASTERISK_WEBRTC_WS_URL"] = env.get("ASTERISK_WEBRTC_WS_URL", "")
env["ASTERISK_WEBRTC_ICE_SERVERS_JSON"] = env.get("ASTERISK_WEBRTC_ICE_SERVERS_JSON", "[]")
env["ASTERISK_OPERATOR_EXTENSION_MAP_JSON"] = env.get("ASTERISK_OPERATOR_EXTENSION_MAP_JSON", "{}")
env["ASTERISK_BROWSER_SIP_MAP_JSON"] = env.get("ASTERISK_BROWSER_SIP_MAP_JSON", "{}")
env["ASTERISK_TRANSFER_TARGET_MAP_JSON"] = env.get("ASTERISK_TRANSFER_TARGET_MAP_JSON", "{}")
env["AI_PROVIDER"] = env.get("AI_PROVIDER", "stub")
env["AI_API_BASE"] = env.get("AI_API_BASE", "")
env["AI_API_KEY"] = env.get("AI_API_KEY", "")
env["AI_MODEL"] = env.get("AI_MODEL", "stub-telegram-assistant")
env["AI_TIMEOUT_SECONDS"] = env.get("AI_TIMEOUT_SECONDS", "20")
env["AI_TELEGRAM_ENABLED"] = env.get("AI_TELEGRAM_ENABLED", "0")
env["AI_TELEGRAM_ALWAYS_REPLY"] = env.get("AI_TELEGRAM_ALWAYS_REPLY", "0")
env["AI_TELEGRAM_MAX_CONTEXT_MESSAGES"] = env.get("AI_TELEGRAM_MAX_CONTEXT_MESSAGES", "20")
env["AI_TELEGRAM_MAX_KB_RESULTS"] = env.get("AI_TELEGRAM_MAX_KB_RESULTS", "3")
env["AI_TELEGRAM_CONFIDENCE_HANDOFF_THRESHOLD"] = env.get(
"AI_TELEGRAM_CONFIDENCE_HANDOFF_THRESHOLD",
"0.65",
)
env["AI_WHATSAPP_ENABLED"] = env.get("AI_WHATSAPP_ENABLED", "0")
env["AI_WHATSAPP_ALWAYS_REPLY"] = env.get("AI_WHATSAPP_ALWAYS_REPLY", "0")
env["AI_WHATSAPP_MAX_CONTEXT_MESSAGES"] = env.get("AI_WHATSAPP_MAX_CONTEXT_MESSAGES", "20")
env["AI_WHATSAPP_MAX_KB_RESULTS"] = env.get("AI_WHATSAPP_MAX_KB_RESULTS", "3")
env["AI_WHATSAPP_CONFIDENCE_HANDOFF_THRESHOLD"] = env.get(
"AI_WHATSAPP_CONFIDENCE_HANDOFF_THRESHOLD",
"0.65",
)
env["AI_VOICE_RUNTIME_SERVICE_URL"] = env.get("AI_VOICE_RUNTIME_SERVICE_URL", "http://127.0.0.1:8018")
env["AI_VOICE_ENABLED"] = env.get("AI_VOICE_ENABLED", "0")
env["AI_VOICE_QUEUE_CONFIG_JSON"] = env.get("AI_VOICE_QUEUE_CONFIG_JSON", "{}")
env["AI_VOICE_ASR_PROVIDER"] = env.get("AI_VOICE_ASR_PROVIDER", "openai")
env["AI_VOICE_TTS_PROVIDER"] = env.get("AI_VOICE_TTS_PROVIDER", "openai")
env["AI_VOICE_MAX_CONTEXT_SEGMENTS"] = env.get("AI_VOICE_MAX_CONTEXT_SEGMENTS", "8")
env["AI_VOICE_HANDOFF_TIMEOUT_SECONDS"] = env.get("AI_VOICE_HANDOFF_TIMEOUT_SECONDS", "8")
env["AI_VOICE_RUNTIME_TRUSTED_SERVICE_SUBJECTS"] = env.get(
"AI_VOICE_RUNTIME_TRUSTED_SERVICE_SUBJECTS",
"svc:ai-voice-runtime",
)
env["ASTERISK_CALLCONTROL_CLAIM_CONTEXT"] = env.get("ASTERISK_CALLCONTROL_CLAIM_CONTEXT", "mvpcc-claim")
env["ASTERISK_CALLCONTROL_TRANSFER_CONTEXT"] = env.get(
"ASTERISK_CALLCONTROL_TRANSFER_CONTEXT",
"mvpcc-transfer",
)
if spec["name"] == "recording":
env["CC_RECORDINGS_DIR"] = str((data_dir / "recordings").resolve())
env["RECORDING_MAX_BYTES"] = env.get("RECORDING_MAX_BYTES", "26214400")
if spec["name"] == "gateway":
env.update(build_gateway_env(env))
stdout_path, stderr_path = create_log_paths(runtime_dir, str(spec["name"]))
creationflags = 0
if os.name == "nt":
creationflags = getattr(subprocess, "CREATE_NO_WINDOW", 0)
cmd = [
sys.executable,
"-m",
"uvicorn",
str(spec["module"]),
"--host",
service_bind_host(str(spec["name"]), env),
"--port",
str(spec["port"]),
]
with stdout_path.open("ab") as stdout_file, stderr_path.open("ab") as stderr_file:
proc = subprocess.Popen(
cmd,
cwd=str(ROOT),
env=env,
stdin=subprocess.DEVNULL,
stdout=stdout_file,
stderr=stderr_file,
creationflags=creationflags,
)
time.sleep(0.15)
if proc.poll() is not None:
raise RuntimeError(f"{spec['name']} failed to start; check {stderr_path}")
return {
"name": spec["name"],
"module": spec["module"],
"port": spec["port"],
"pid": proc.pid,
"stdout_log": str(stdout_path),
"stderr_log": str(stderr_path),
}
def terminate_pid(pid: int) -> None:
if not is_pid_running(pid):
return
try:
os.kill(pid, signal.SIGTERM)
except OSError:
return
for _ in range(20):
if not is_pid_running(pid):
return
time.sleep(0.2)
if os.name == "nt":
subprocess.run(
["taskkill", "/PID", str(pid), "/T", "/F"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
return
try:
os.kill(pid, signal.SIGKILL)
except OSError:
pass
def running_services(manifest: dict[str, Any] | None) -> list[dict[str, Any]]:
if not manifest or not isinstance(manifest, dict):
return []
active: list[dict[str, Any]] = []
for svc in manifest.get("services", []):
if not isinstance(svc, dict):
continue
try:
pid = int(svc.get("pid", 0))
except Exception: # noqa: BLE001
continue
if is_pid_running(pid):
active.append(svc)
return active
def start_stack(runtime_dir: Path, data_dir: Path, force_restart: bool, env: dict[str, str]) -> int:
service_specs = build_service_specs(env)
current = load_manifest(runtime_dir)
active = running_services(current)
if active and not force_restart:
print("Local stack is already running.")
print(f"Base URL: {current.get('base_url', public_base_url(env))}")
print(f"Logs: {runtime_dir / 'logs'}")
print("Use `python scripts\\local_stack.py stop` first or rerun with --force-restart.")
return 0
if active and force_restart:
stop_stack(runtime_dir)
if current and not active:
remove_manifest(runtime_dir)
runtime_dir.mkdir(parents=True, exist_ok=True)
data_dir.mkdir(parents=True, exist_ok=True)
launched: list[dict[str, Any]] = []
try:
for spec in service_specs:
meta = spawn_service(spec, runtime_dir, data_dir, env)
wait_for_health(service_health_host(str(spec["name"]), env), int(meta["port"]))
if not is_pid_running(int(meta["pid"])):
listener_pid = resolve_listener_pid(int(meta["port"]))
if listener_pid and is_pid_running(listener_pid):
meta["pid"] = listener_pid
else:
raise RuntimeError(f"{meta['name']} exited before becoming ready; check {meta['stderr_log']}")
launched.append(meta)
print(f"[ok] {meta['name']} on {meta['port']} (pid {meta['pid']})")
except Exception as exc:
print(f"[fail] {exc}")
print(f"Check logs: {runtime_dir / 'logs'}")
print("If ports are already busy, stop old local processes and rerun.")
for meta in reversed(launched):
terminate_pid(int(meta["pid"]))
return 1
save_manifest(runtime_dir, build_manifest_payload(runtime_dir, data_dir, launched, env))
print("")
print("Local stack is ready.")
print(f"UI: {public_base_url(env)}/")
print(f"Logs: {runtime_dir / 'logs'}")
print("Stop: python scripts\\local_stack.py stop")
return 0
def stop_stack(runtime_dir: Path) -> int:
current = load_manifest(runtime_dir)
if not current:
print("Local stack is not running (no manifest found).")
return 0
services = current.get("services", []) if isinstance(current, dict) else []
for meta in reversed(services):
if not isinstance(meta, dict):
continue
name = str(meta.get("name", "service"))
try:
pid = int(meta.get("pid", 0))
except Exception: # noqa: BLE001
print(f"[skip] {name} (pid invalid)")
continue
if is_pid_running(pid):
terminate_pid(pid)
print(f"[stopped] {name} (pid {pid})")
else:
print(f"[skip] {name} (pid {pid}) already stopped")
remove_manifest(runtime_dir)
print("Local stack is stopped.")
return 0
def status_stack(runtime_dir: Path, env: dict[str, str]) -> int:
current = load_manifest(runtime_dir)
if not current:
print("Local stack is not running.")
return 0
print(f"Base URL: {current.get('base_url', public_base_url(env))}")
print(f"Data dir: {current.get('data_dir', str(DEFAULT_DATA_DIR))}")
print(f"Logs: {runtime_dir / 'logs'}")
print("")
services = current.get("services", []) if isinstance(current, dict) else []
for meta in services:
if not isinstance(meta, dict):
continue
try:
pid = int(meta.get("pid", 0))
except Exception: # noqa: BLE001
pid = 0
label = "running" if is_pid_running(pid) else "stopped"
print(f"- {meta.get('name')}: {label} on {meta.get('port')} (pid {pid})")
return 0
def main() -> int:
parser = argparse.ArgumentParser(description="Manage the local MVP stack without opening extra consoles")
parser.add_argument("command", choices=["start", "stop", "status"], help="Action to perform")
parser.add_argument(
"--runtime-dir",
default=str(DEFAULT_RUNTIME_DIR),
help="Directory for manifest and logs",
)
parser.add_argument(
"--data-dir",
default=None,
help="Directory for local runtime data and recordings",
)
parser.add_argument(
"--env-file",
action="append",
default=[],
help="Additional env file to apply after the default stack env files",
)
parser.add_argument(
"--force-restart",
action="store_true",
help="Stop the existing managed stack before starting a new one",
)
args = parser.parse_args()
extra_env_files: list[Path] = []
for raw_path in args.env_file:
path = Path(raw_path)
if not path.is_absolute():
path = (ROOT / path).resolve()
extra_env_files.append(path)
env = load_stack_env(extra_env_files)
runtime_dir = Path(args.runtime_dir)
if not runtime_dir.is_absolute():
runtime_dir = (ROOT / runtime_dir).resolve()
raw_data_dir = args.data_dir or env.get("CC_DATA_DIR") or str(DEFAULT_DATA_DIR)
data_dir = Path(raw_data_dir)
if not data_dir.is_absolute():
data_dir = (ROOT / data_dir).resolve()
if args.command == "start":
return start_stack(runtime_dir, data_dir, args.force_restart, env)
if args.command == "stop":
return stop_stack(runtime_dir)
return status_stack(runtime_dir, env)
if __name__ == "__main__":
raise SystemExit(main())
+252
View File
@@ -0,0 +1,252 @@
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
import shutil
import sqlite3
import sys
import tempfile
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8")
from services.ai_voice_runtime_service.audiosocket import pcm16le_to_wav_bytes, resample_pcm16le
from services.ai_voice_runtime_service.providers.tts import build_tts_provider
def _default_provider_name() -> str:
return os.getenv("AI_VOICE_TTS_PROVIDER", "openai").strip() or "openai"
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Bulk-materialize IVR prompt WAV files from a flow JSON document or IVR SQLite DB.",
)
source = parser.add_mutually_exclusive_group(required=True)
source.add_argument("--flow-json-file", help="Path to a flow JSON document.")
source.add_argument("--sqlite-db", help="Path to SQLite DB containing ivr_flows.")
parser.add_argument("--flow-id", default=None, help="Optional specific flow_id when reading from SQLite.")
parser.add_argument("--output-dir", required=True, help="Directory where <prompt_audio_key>.wav files will be written.")
parser.add_argument("--preview-dir", default=None, help="Optional directory to mirror generated WAV files into.")
parser.add_argument(
"--provider",
default=_default_provider_name(),
help="TTS provider name. Defaults to AI_VOICE_TTS_PROVIDER or openai.",
)
parser.add_argument("--sample-rate", type=int, default=8000, help="Target WAV sample rate. Defaults to 8000.")
parser.add_argument("--default-language", default="ru", help="Fallback language for ambiguous prompts. Defaults to ru.")
parser.add_argument(
"--language-override",
action="append",
default=[],
help="Explicit language override in prompt_audio_key=language form. May be repeated.",
)
parser.add_argument("--dry-run", action="store_true", help="Print planned prompt files without generating audio.")
return parser.parse_args()
def _parse_language_overrides(raw_overrides: list[str]) -> dict[str, str]:
overrides: dict[str, str] = {}
for raw in raw_overrides:
item = str(raw or "").strip()
if not item:
continue
key, sep, value = item.partition("=")
if not sep or not key.strip() or not value.strip():
raise SystemExit(f"Invalid --language-override value: {raw!r}")
overrides[key.strip()] = value.strip()
return overrides
def _load_flow_document(args: argparse.Namespace) -> dict[str, Any]:
if args.flow_json_file:
return json.loads(Path(args.flow_json_file).read_text(encoding="utf-8"))
conn = sqlite3.connect(str(args.sqlite_db))
try:
if str(args.flow_id or "").strip():
row = conn.execute(
"select flow_json from ivr_flows where flow_id = ? order by id desc limit 1",
(args.flow_id.strip(),),
).fetchone()
else:
row = conn.execute(
"select flow_json from ivr_flows where is_active = 1 order by id desc limit 1"
).fetchone()
finally:
conn.close()
if row is None or not str(row[0] or "").strip():
raise SystemExit("Unable to load IVR flow JSON")
return json.loads(row[0])
def _node_identity(node: dict[str, Any]) -> str:
return str(node.get("node_id") or node.get("id") or "").strip()
def _infer_language(node: dict[str, Any], *, prompt_key: str, default_language: str) -> str:
candidates = [
prompt_key,
_node_identity(node),
str(node.get("resolved_queue_code") or "").strip(),
str(node.get("outcome_code") or "").strip(),
]
for candidate in candidates:
lowered = candidate.lower()
if any(token in lowered for token in ("_kz", "-kz", "_kk", "-kk", "kz_", "kk_")):
return "kz"
if any(token in lowered for token in ("_ru", "-ru", "ru_")):
return "ru"
text = str(node.get("prompt_text") or "").strip().lower()
has_kazakh_letters = any(char in text for char in "әіңғүұқөһ")
has_russian_wording = any(token in text for token in ("здравствуйте", "добро пожаловать", "русского", "службы поддержки", "отдела продаж"))
if has_kazakh_letters and not has_russian_wording:
return "kz"
return str(default_language or "ru").strip() or "ru"
def _collect_prompts(
flow_document: dict[str, Any],
*,
default_language: str,
language_overrides: dict[str, str],
) -> list[dict[str, str]]:
prompts: list[dict[str, str]] = []
seen: dict[str, str] = {}
def _append_prompt(*, node: dict[str, Any], prompt_key: str, prompt_text: str, language: str | None = None) -> None:
normalized_key = str(prompt_key or "").strip()
normalized_text = str(prompt_text or "").strip()
if not normalized_key or not normalized_text:
return
previous_text = seen.get(normalized_key)
if previous_text is not None and previous_text != normalized_text:
raise SystemExit(f"Prompt key {normalized_key!r} is reused with different texts")
seen[normalized_key] = normalized_text
prompts.append(
{
"prompt_audio_key": normalized_key,
"prompt_text": normalized_text,
"language": language_overrides.get(
normalized_key,
str(language or "").strip()
or _infer_language(node, prompt_key=normalized_key, default_language=default_language),
),
"node_id": _node_identity(node),
}
)
for raw_node in flow_document.get("nodes", []):
node = raw_node if isinstance(raw_node, dict) else None
if node is None:
continue
prompt_sequence = node.get("prompt_sequence")
if isinstance(prompt_sequence, list):
for prompt in prompt_sequence:
if not isinstance(prompt, dict):
continue
_append_prompt(
node=node,
prompt_key=str(prompt.get("prompt_audio_key") or ""),
prompt_text=str(prompt.get("prompt_text") or ""),
language=str(prompt.get("language") or "").strip() or None,
)
_append_prompt(
node=node,
prompt_key=str(node.get("prompt_audio_key") or ""),
prompt_text=str(node.get("prompt_text") or ""),
)
return prompts
def _write_atomic(target: Path, payload: bytes) -> None:
target.parent.mkdir(parents=True, exist_ok=True)
temp_path: str | None = None
try:
with tempfile.NamedTemporaryFile(dir=target.parent, delete=False, suffix=".tmp") as handle:
handle.write(payload)
temp_path = handle.name
Path(temp_path).replace(target)
target.chmod(0o644)
finally:
if temp_path:
try:
Path(temp_path).unlink(missing_ok=True)
except OSError:
pass
def _materialize_prompt(
*,
provider_name: str,
prompt_text: str,
language: str,
sample_rate: int,
output_path: Path,
) -> None:
provider = build_tts_provider(provider_name)
synthesis = provider.synthesize(prompt_text, language=language)
pcm_bytes = resample_pcm16le(
synthesis.audio_bytes,
input_rate_hz=synthesis.sample_rate_hz,
output_rate_hz=max(sample_rate, 1),
)
wav_bytes = pcm16le_to_wav_bytes(
pcm_bytes,
sample_rate_hz=max(sample_rate, 1),
)
_write_atomic(output_path, wav_bytes)
def main() -> int:
args = _parse_args()
flow_document = _load_flow_document(args)
language_overrides = _parse_language_overrides(args.language_override)
prompts = _collect_prompts(
flow_document,
default_language=args.default_language,
language_overrides=language_overrides,
)
output_dir = Path(args.output_dir).expanduser()
preview_dir = Path(args.preview_dir).expanduser() if args.preview_dir else None
output_dir.mkdir(parents=True, exist_ok=True)
if preview_dir is not None:
preview_dir.mkdir(parents=True, exist_ok=True)
generated: list[dict[str, str]] = []
for prompt in prompts:
output_path = output_dir / f"{prompt['prompt_audio_key']}.wav"
if not args.dry_run:
_materialize_prompt(
provider_name=args.provider,
prompt_text=prompt["prompt_text"],
language=prompt["language"],
sample_rate=args.sample_rate,
output_path=output_path,
)
if preview_dir is not None:
preview_path = preview_dir / output_path.name
shutil.copyfile(output_path, preview_path)
preview_path.chmod(0o644)
generated.append(
{
**prompt,
"output_path": str(output_path),
}
)
print(json.dumps({"provider": args.provider, "count": len(generated), "prompts": generated}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+72
View File
@@ -0,0 +1,72 @@
from __future__ import annotations
import argparse
import os
from pathlib import Path
import sys
import tempfile
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from services.ai_voice_runtime_service.audiosocket import pcm16le_to_wav_bytes, resample_pcm16le
from services.ai_voice_runtime_service.providers.tts import build_tts_provider
def _parse_args() -> argparse.Namespace:
default_provider = os.getenv("AI_VOICE_TTS_PROVIDER", "openai").strip() or "openai"
parser = argparse.ArgumentParser(
description="Generate TTS once, cache it on the server, and materialize a WAV prompt file.",
)
parser.add_argument("--text", required=True, help="Text to synthesize.")
parser.add_argument("--output", required=True, help="Target WAV path.")
parser.add_argument("--language", default=None, help="Optional language hint.")
parser.add_argument(
"--provider",
default=default_provider,
help="TTS provider name. Defaults to AI_VOICE_TTS_PROVIDER or openai.",
)
parser.add_argument("--sample-rate", type=int, default=8000, help="Target WAV sample rate. Defaults to 8000.")
return parser.parse_args()
def _write_atomic(target: Path, payload: bytes) -> None:
target.parent.mkdir(parents=True, exist_ok=True)
temp_path: str | None = None
try:
with tempfile.NamedTemporaryFile(dir=target.parent, delete=False, suffix=".tmp") as handle:
handle.write(payload)
temp_path = handle.name
Path(temp_path).replace(target)
target.chmod(0o644)
finally:
if temp_path:
try:
Path(temp_path).unlink(missing_ok=True)
except OSError:
pass
def main() -> int:
args = _parse_args()
provider = build_tts_provider(args.provider)
synthesis = provider.synthesize(args.text, language=args.language)
pcm_bytes = resample_pcm16le(
synthesis.audio_bytes,
input_rate_hz=synthesis.sample_rate_hz,
output_rate_hz=max(args.sample_rate, 1),
)
wav_bytes = pcm16le_to_wav_bytes(
pcm_bytes,
sample_rate_hz=max(args.sample_rate, 1),
)
output_path = Path(args.output).expanduser()
_write_atomic(output_path, wav_bytes)
print(output_path)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+81
View File
@@ -0,0 +1,81 @@
from __future__ import annotations
from pathlib import Path
import sys
from sqlalchemy.exc import OperationalError, ProgrammingError
from sqlalchemy import text
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from services.shared.db import engine
from services.shared.schema_migrations import (
MIGRATIONS_DIR,
applied_migration_versions,
database_backend_name,
ensure_schema_migrations_table,
list_migration_files,
)
def _split_statements(sql: str) -> list[str]:
chunks = []
for part in sql.split(";"):
stmt = part.strip()
if stmt:
chunks.append(stmt)
return chunks
def _apply_file(path: Path) -> None:
# Some checked-in SQL files may contain a UTF-8 BOM; accept them for both
# local SQLite runs and Postgres migration jobs.
statements = _split_statements(path.read_text(encoding="utf-8-sig"))
with engine.begin() as conn:
for stmt in statements:
try:
conn.execute(text(stmt))
except (OperationalError, ProgrammingError) as exc:
message = str(exc).lower()
duplicate_markers = [
"duplicate column name",
"already exists",
"duplicate key value violates unique constraint",
]
if any(marker in message for marker in duplicate_markers):
continue
raise
conn.execute(
text("INSERT INTO schema_migrations(version, applied_at) VALUES (:v, CURRENT_TIMESTAMP)"),
{"v": path.name},
)
def main() -> None:
files = list_migration_files()
if not files:
raise RuntimeError(f"No migration files found for dialect in {MIGRATIONS_DIR}")
ensure_schema_migrations_table()
applied = applied_migration_versions()
applied_now = []
for file_path in files:
if file_path.name in applied:
continue
_apply_file(file_path)
applied_now.append(file_path.name)
print(f"Dialect: {database_backend_name()}")
if applied_now:
print("Applied migrations:")
for m in applied_now:
print(f"- {m}")
else:
print("No new migrations to apply")
if __name__ == "__main__":
main()
+100
View File
@@ -0,0 +1,100 @@
from __future__ import annotations
import argparse
import json
import sys
from typing import Any
import httpx
def _normalize_base_url(base_url: str) -> str:
return base_url.rstrip("/")
def check_oidc(
base_url: str,
*,
require_enabled: bool = False,
force_health: bool = False,
timeout: float = 5.0,
client: httpx.Client | None = None,
) -> dict[str, Any]:
base_url = _normalize_base_url(base_url)
own_client = client is None
client = client or httpx.Client(timeout=timeout, follow_redirects=False)
try:
config_resp = client.get(f"{base_url}/proxy/auth/auth/oidc/config")
config_resp.raise_for_status()
config = config_resp.json()
enabled = bool(config.get("enabled"))
if require_enabled and not enabled:
raise RuntimeError("OIDC is disabled but --require-enabled was specified")
summary: dict[str, Any] = {
"base_url": base_url,
"config": config,
"health": None,
}
if enabled or force_health:
health_resp = client.get(f"{base_url}/proxy/auth/auth/oidc/health")
health_resp.raise_for_status()
summary["health"] = health_resp.json()
return summary
finally:
if own_client:
client.close()
def _build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Smoke-check OIDC config and provider health via the gateway")
parser.add_argument("--base-url", default="http://localhost:8080", help="Gateway base URL")
parser.add_argument(
"--require-enabled",
action="store_true",
help="Fail if OIDC is disabled",
)
parser.add_argument(
"--force-health",
action="store_true",
help="Call the OIDC health endpoint even if OIDC is disabled",
)
parser.add_argument("--timeout", type=float, default=5.0, help="HTTP timeout in seconds")
return parser
def main(argv: list[str] | None = None) -> int:
parser = _build_parser()
args = parser.parse_args(argv)
try:
summary = check_oidc(
args.base_url,
require_enabled=args.require_enabled,
force_health=args.force_health,
timeout=args.timeout,
)
except (httpx.HTTPError, RuntimeError) as exc:
print(f"OIDC smoke check failed: {exc}", file=sys.stderr)
return 1
config = summary["config"]
health = summary["health"]
print("OIDC config:")
print(json.dumps(config, ensure_ascii=False, indent=2))
if health is not None:
print("OIDC health:")
print(json.dumps(health, ensure_ascii=False, indent=2))
else:
print("OIDC health skipped because OIDC is disabled")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+316
View File
@@ -0,0 +1,316 @@
from __future__ import annotations
import argparse
from dataclasses import asdict, dataclass
import json
import os
from pathlib import Path
import socket
import sys
from urllib.parse import urlparse
import httpx
from sqlalchemy import create_engine, text
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from services.shared.schema_migrations import applied_migration_versions, list_migration_files, validate_schema_migrations_applied
from services.shared.security import issue_app_token
REQUIRED_PROXY_HEALTH_SERVICES = [
"auth",
"customer",
"interaction",
"routing",
"voice",
"telegram",
"whatsapp",
"recording",
"ivr",
"ai",
"ai-voice-runtime",
"kb",
"reporting",
"supervisor",
]
@dataclass
class CheckResult:
name: str
ok: bool
details: str
def _load_env_file(path: Path) -> None:
if not path.exists() or not path.is_file():
return
for line in path.read_text(encoding="utf-8").splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("#") or "=" not in stripped:
continue
key, value = stripped.split("=", 1)
key = key.strip()
if key and key not in os.environ:
os.environ[key] = value.strip()
def _bool_env(name: str, default: bool) -> bool:
raw = os.getenv(name)
if raw is None:
return default
return raw.strip().lower() in {"1", "true", "yes", "on"}
def _normalize_database_url(url: str) -> str:
if url.startswith("postgres://"):
return "postgresql+psycopg://" + url[len("postgres://") :]
if url.startswith("postgresql://") and "+psycopg" not in url:
return "postgresql+psycopg://" + url[len("postgresql://") :]
return url
def _ops_headers() -> dict[str, str]:
if _bool_env("ALLOW_LEGACY_HEADER_AUTH", True):
return {"X-User": "admin", "X-Role": "admin"}
token = issue_app_token(
subject="ops:postgres-dev-preflight",
username="postgres-dev-preflight",
role="admin",
auth_source="service",
provider="postgres-preflight",
ttl_seconds=300,
)
return {"Authorization": f"Bearer {token}"}
def _check_tcp_endpoint(name: str, url: str, default_port: int) -> CheckResult:
parsed = urlparse(url)
host = parsed.hostname
port = parsed.port or default_port
if not host:
return CheckResult(name, False, "host is missing")
try:
with socket.create_connection((host, port), timeout=3.0):
pass
except OSError as exc:
return CheckResult(name, False, f"{host}:{port} unreachable: {exc}")
return CheckResult(name, True, f"{host}:{port} reachable")
def _check_database_connection(database_url: str) -> CheckResult:
engine = create_engine(
_normalize_database_url(database_url),
future=True,
pool_pre_ping=True,
connect_args={"connect_timeout": 5},
)
try:
with engine.connect() as conn:
conn.execute(text("SELECT 1"))
except Exception as exc: # noqa: BLE001
return CheckResult("database_connection", False, str(exc))
finally:
engine.dispose()
return CheckResult("database_connection", True, "SELECT 1 succeeded")
def _check_schema_migrations(database_url: str) -> CheckResult:
engine = create_engine(
_normalize_database_url(database_url),
future=True,
pool_pre_ping=True,
connect_args={"connect_timeout": 5},
)
try:
validate_schema_migrations_applied(db_engine=engine)
expected = list_migration_files(db_engine=engine)
applied = applied_migration_versions(db_engine=engine)
except Exception as exc: # noqa: BLE001
return CheckResult("schema_migrations", False, str(exc))
finally:
engine.dispose()
return CheckResult("schema_migrations", True, f"applied={len(applied)}/{len(expected)}")
def _http_preflight_checks(base_url: str) -> list[CheckResult]:
results: list[CheckResult] = []
normalized = base_url.rstrip("/")
ops_headers = _ops_headers()
read_headers = dict(ops_headers)
with httpx.Client(base_url=normalized, timeout=10, trust_env=False) as client:
try:
response = client.get("/health")
ok = response.status_code == 200 and response.json().get("status") == "ok"
results.append(CheckResult("gateway_health", ok, f"status={response.status_code}"))
except Exception as exc: # noqa: BLE001
return [CheckResult("gateway_health", False, str(exc))]
try:
response = client.get("/registry")
payload = response.json() if response.status_code == 200 else {}
services = payload.get("services", {}) if isinstance(payload, dict) else {}
missing = [name for name in REQUIRED_PROXY_HEALTH_SERVICES if name not in services]
ok = response.status_code == 200 and not missing
details = f"status={response.status_code}"
if missing:
details += f", missing={missing}"
results.append(CheckResult("gateway_registry", ok, details))
except Exception as exc: # noqa: BLE001
results.append(CheckResult("gateway_registry", False, str(exc)))
return results
services_to_check = list(REQUIRED_PROXY_HEALTH_SERVICES)
if _bool_env("EVENT_BUS_ENABLED", False):
services_to_check.append("event-bus")
if _bool_env("ASTERISK_BRIDGE_ENABLED", False):
services_to_check.append("asterisk-bridge")
for service in services_to_check:
try:
response = client.get(f"/proxy/{service}/health", headers=ops_headers)
payload = response.json() if response.status_code == 200 else {}
status_value = str(payload.get("status") or "").strip().lower() if isinstance(payload, dict) else ""
ok = response.status_code == 200 and status_value == "ok"
results.append(CheckResult(f"{service}_health", ok, f"status={response.status_code}"))
except Exception as exc: # noqa: BLE001
results.append(CheckResult(f"{service}_health", False, str(exc)))
try:
response = client.post(
"/proxy/auth/auth/login",
json={"username": "admin", "password": "admin123"},
)
payload = response.json() if response.status_code == 200 else {}
access_token = str(payload.get("access_token") or "").strip() if isinstance(payload, dict) else ""
if access_token:
read_headers = {"Authorization": f"Bearer {access_token}"}
ok = response.status_code == 200 and bool(access_token)
results.append(CheckResult("auth_login", ok, f"status={response.status_code}"))
except Exception as exc: # noqa: BLE001
results.append(CheckResult("auth_login", False, str(exc)))
try:
response = client.get("/proxy/routing/queues", headers=read_headers)
ok = response.status_code == 200 and isinstance(response.json(), list)
results.append(CheckResult("routing_read", ok, f"status={response.status_code}"))
except Exception as exc: # noqa: BLE001
results.append(CheckResult("routing_read", False, str(exc)))
try:
response = client.get("/proxy/voice/integrations/voice/events?limit=1", headers=read_headers)
ok = response.status_code == 200 and isinstance(response.json(), list)
results.append(CheckResult("voice_read", ok, f"status={response.status_code}"))
except Exception as exc: # noqa: BLE001
results.append(CheckResult("voice_read", False, str(exc)))
try:
response = client.get("/proxy/whatsapp/integrations/whatsapp/threads", headers=read_headers)
ok = response.status_code == 200 and isinstance(response.json(), list)
results.append(CheckResult("whatsapp_read", ok, f"status={response.status_code}"))
except Exception as exc: # noqa: BLE001
results.append(CheckResult("whatsapp_read", False, str(exc)))
return results
def run_preflight(
*,
database_url: str,
base_url: str | None,
require_rabbitmq: bool,
) -> list[CheckResult]:
results: list[CheckResult] = []
raw_database_url = str(database_url or "").strip()
if not raw_database_url:
return [CheckResult("database_url", False, "DATABASE_URL is required")]
if not raw_database_url.startswith(("postgres://", "postgresql://", "postgresql+psycopg://")):
return [CheckResult("database_url", False, "DATABASE_URL must point to PostgreSQL")]
results.append(CheckResult("database_url", True, "PostgreSQL URL detected"))
mode = str(os.getenv("SCHEMA_MANAGEMENT_MODE", "")).strip().lower()
if mode != "migrations":
results.append(CheckResult("schema_management_mode", False, "SCHEMA_MANAGEMENT_MODE must be set to migrations"))
return results
results.append(CheckResult("schema_management_mode", True, "migrations"))
results.append(_check_tcp_endpoint("postgres_tcp", raw_database_url, 5432))
if not results[-1].ok:
return results
results.append(_check_database_connection(raw_database_url))
if not results[-1].ok:
return results
results.append(_check_schema_migrations(raw_database_url))
if not results[-1].ok:
return results
if require_rabbitmq or _bool_env("EVENT_BUS_ENABLED", False):
event_bus_url = str(os.getenv("EVENT_BUS_URL", "")).strip()
if not event_bus_url:
results.append(CheckResult("rabbitmq_tcp", False, "EVENT_BUS_URL is required when RabbitMQ check is enabled"))
else:
results.append(_check_tcp_endpoint("rabbitmq_tcp", event_bus_url, 5672))
if base_url:
results.extend(_http_preflight_checks(base_url))
return results
def main() -> int:
parser = argparse.ArgumentParser(description="PostgreSQL dev preflight checks")
parser.add_argument("--env-file", action="append", default=[".env.production"])
parser.add_argument("--database-url", default=os.getenv("DATABASE_URL", ""))
parser.add_argument("--base-url", default="")
parser.add_argument("--require-rabbitmq", action="store_true")
parser.add_argument("--json", action="store_true")
args = parser.parse_args()
for raw_path in args.env_file:
path = Path(raw_path)
if not path.is_absolute():
path = (ROOT / path).resolve()
_load_env_file(path)
database_url = str(args.database_url or os.getenv("DATABASE_URL", "")).strip()
base_url = str(args.base_url or "").strip() or None
results = run_preflight(
database_url=database_url,
base_url=base_url,
require_rabbitmq=bool(args.require_rabbitmq),
)
failures = [item for item in results if not item.ok]
if args.json:
print(
json.dumps(
{
"ok": not failures,
"checks": [asdict(item) for item in results],
},
ensure_ascii=False,
indent=2,
)
)
return 0 if not failures else 1
for item in results:
label = "[ok]" if item.ok else "[fail]"
print(f"{label} {item.name}: {item.details}")
if failures:
print("[FAIL] PostgreSQL dev preflight failed")
return 1
print("[PASS] PostgreSQL dev preflight passed")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+41
View File
@@ -0,0 +1,41 @@
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
python scripts\local_stack.py stop @args
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
$dataDir = Join-Path $root ".data_local"
if (Test-Path $dataDir) {
Remove-Item -Path $dataDir -Recurse -Force
}
$summaryPath = Join-Path $root ".local_stack\demo-seed-summary.json"
if (Test-Path $summaryPath) {
Remove-Item -Path $summaryPath -Force
}
python scripts\local_stack.py start @args
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
& "$PSScriptRoot\smoke_test.ps1"
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
python scripts\demo_seed.py
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
Write-Host ""
Write-Host "Demo environment is ready."
Write-Host "UI: http://localhost:8080/operator"
Write-Host "Login: admin / admin123"
Write-Host "KB keyword: demo-showcase"
Write-Host "Summary: .local_stack\demo-seed-summary.json"
Write-Host "Stop: powershell -ExecutionPolicy Bypass -File scripts\stop_all_local.ps1"
+30
View File
@@ -0,0 +1,30 @@
param(
[string]$BackupZip = "",
[string]$TargetDir = "e:\Zhan\.data"
)
if ($BackupZip -eq "") {
$latest = Get-ChildItem -Path "e:\Zhan\backups" -Filter "*.zip" -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1
if ($null -eq $latest) {
Write-Error "No backup zip found in e:\Zhan\backups and BackupZip not provided."
exit 1
}
$BackupZip = $latest.FullName
}
if (!(Test-Path $BackupZip)) {
Write-Error "Backup zip not found: $BackupZip"
exit 1
}
if (Test-Path $TargetDir) {
$bakDir = "$TargetDir.bak.$((Get-Date).ToString('yyyyMMdd_HHmmss'))"
Move-Item -Path $TargetDir -Destination $bakDir
Write-Host "Existing data moved to:" $bakDir
}
New-Item -ItemType Directory -Force -Path $TargetDir | Out-Null
Expand-Archive -Path $BackupZip -DestinationPath $TargetDir -Force
Write-Host "Data restored from:" $BackupZip
Write-Host "Target directory:" $TargetDir
+5
View File
@@ -0,0 +1,5 @@
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
python scripts\local_stack.py start @args
exit $LASTEXITCODE
+192
View File
@@ -0,0 +1,192 @@
from __future__ import annotations
from argparse import ArgumentParser
from pathlib import Path
import json
import sys
from sqlalchemy import select
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from services.shared.core import new_id, utc_now_iso
from services.shared.db import get_session
from services.shared.kb_localization import normalize_kb_language, resolve_article_group_id
from services.shared.sql_models import KBArticleRow, KBCategoryRow
def _parser() -> ArgumentParser:
parser = ArgumentParser(description="Seed KB article localizations without creating duplicates.")
parser.add_argument(
"--seed-file",
default=str(ROOT / "scripts" / "kb_voice_basic_kz_localizations.json"),
help="Path to the localization seed JSON file.",
)
parser.add_argument(
"--apply",
action="store_true",
help="Persist changes. Without this flag the script runs in dry-run mode.",
)
parser.add_argument(
"--update-existing",
action="store_true",
help="Update existing localized articles instead of leaving them untouched.",
)
return parser
def _load_seed(path: Path) -> dict:
return json.loads(path.read_text(encoding="utf-8"))
def _find_category(session, category_name: str) -> KBCategoryRow:
row = session.execute(
select(KBCategoryRow).where(KBCategoryRow.name == category_name)
).scalar_one_or_none()
if row is None:
raise RuntimeError(f"KB category not found: {category_name}")
return row
def _find_source_article(
session,
*,
category_id: str,
source_title: str,
source_language: str,
) -> KBArticleRow:
rows = session.execute(
select(KBArticleRow)
.where(KBArticleRow.category_id == category_id)
.where(KBArticleRow.title == source_title)
.where(KBArticleRow.language == source_language)
.order_by(KBArticleRow.id.asc())
).scalars().all()
if not rows:
raise RuntimeError(f"Source article not found: {source_title}")
if len(rows) > 1:
raise RuntimeError(f"Multiple source articles found for title: {source_title}")
return rows[0]
def _find_localized_article(
session,
*,
article_group_id: str,
target_language: str,
) -> KBArticleRow | None:
return session.execute(
select(KBArticleRow)
.where(KBArticleRow.article_group_id == article_group_id)
.where(KBArticleRow.language == target_language)
).scalar_one_or_none()
def main() -> None:
args = _parser().parse_args()
seed_file = Path(args.seed_file).resolve()
seed = _load_seed(seed_file)
source_language = normalize_kb_language(seed.get("source_language"))
target_language = normalize_kb_language(seed.get("target_language"))
category_name = str(seed.get("category_name") or "").strip()
if not category_name:
raise RuntimeError("Seed file must define category_name")
session = get_session()
try:
category = _find_category(session, category_name)
summary = {
"category_id": category.category_id,
"category_name": category.name,
"seed_file": str(seed_file),
"source_language": source_language,
"target_language": target_language,
"apply": bool(args.apply),
"update_existing": bool(args.update_existing),
"created": [],
"updated": [],
"skipped": [],
}
for item in seed.get("localizations", []):
source_title = str(item.get("source_title") or "").strip()
title = str(item.get("title") or "").strip()
body = str(item.get("body") or "").strip()
tags = [str(tag).strip() for tag in item.get("tags", []) if str(tag).strip()]
if not source_title or not title or not body:
raise RuntimeError(f"Incomplete localization entry: {item!r}")
source_article = _find_source_article(
session,
category_id=category.category_id,
source_title=source_title,
source_language=source_language,
)
article_group_id = resolve_article_group_id(
source_article.article_id,
source_article.article_group_id,
)
existing = _find_localized_article(
session,
article_group_id=article_group_id,
target_language=target_language,
)
if existing is None:
row = KBArticleRow(
article_id=new_id("kba"),
category_id=category.category_id,
article_group_id=article_group_id,
language=target_language,
title=title,
body=body,
tags_json=json.dumps(tags, ensure_ascii=False),
created_at=utc_now_iso(),
updated_at=utc_now_iso(),
)
session.add(row)
summary["created"].append(
{
"source_title": source_title,
"article_group_id": article_group_id,
"target_title": title,
}
)
continue
if args.update_existing:
existing.title = title
existing.body = body
existing.tags_json = json.dumps(tags, ensure_ascii=False)
existing.updated_at = utc_now_iso()
summary["updated"].append(
{
"source_title": source_title,
"article_id": existing.article_id,
"article_group_id": article_group_id,
"target_title": title,
}
)
else:
summary["skipped"].append(
{
"source_title": source_title,
"article_id": existing.article_id,
"article_group_id": article_group_id,
"target_title": existing.title,
}
)
if args.apply:
session.commit()
else:
session.rollback()
print(json.dumps(summary, ensure_ascii=True, indent=2))
finally:
session.close()
if __name__ == "__main__":
main()
+11
View File
@@ -0,0 +1,11 @@
Write-Host 'Checking gateway health...'
Invoke-RestMethod -Uri 'http://localhost:8080/health' | ConvertTo-Json -Depth 5
Write-Host 'Checking auth health through proxy...'
Invoke-RestMethod -Uri 'http://localhost:8080/proxy/auth/health' | ConvertTo-Json -Depth 5
Write-Host 'Logging in to auth...'
$loginBody = @{ username='admin'; password='admin123' } | ConvertTo-Json
Invoke-RestMethod -Method Post -Uri 'http://localhost:8080/proxy/auth/auth/login' -Body $loginBody -ContentType 'application/json' | ConvertTo-Json -Depth 5
Write-Host 'Smoke test completed.'
+62
View File
@@ -0,0 +1,62 @@
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
$envFile = Join-Path $root ".env.production"
if (-not (Test-Path $envFile)) {
Write-Error ".env.production not found. Create it from .env.production.template first."
}
Get-Content $envFile | ForEach-Object {
$line = $_.Trim()
if (-not $line -or $line.StartsWith("#")) {
return
}
$parts = $line -split "=", 2
if ($parts.Count -ne 2) {
return
}
$name = $parts[0].Trim()
$value = $parts[1].Trim()
if ($name) {
Set-Item -Path "Env:$name" -Value $value
}
}
python scripts\local_stack.py start --force-restart
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
$amiHost = $env:ASTERISK_AMI_HOST
$amiPortValue = $env:ASTERISK_AMI_PORT
if (-not $amiPortValue) {
$amiPortValue = "5038"
}
$amiPort = [int]$amiPortValue
$sftpHost = $env:ASTERISK_SFTP_HOST
$sftpPortValue = $env:ASTERISK_SFTP_PORT
if (-not $sftpPortValue) {
$sftpPortValue = "22"
}
$sftpPort = [int]$sftpPortValue
if ($amiHost) {
$ami = Test-NetConnection $amiHost -Port $amiPort -WarningAction SilentlyContinue
Write-Host "AMI TCP check: $($ami.TcpTestSucceeded) (${amiHost}:$amiPort)"
}
if ($sftpHost) {
$sftp = Test-NetConnection $sftpHost -Port $sftpPort -WarningAction SilentlyContinue
Write-Host "SFTP TCP check: $($sftp.TcpTestSucceeded) (${sftpHost}:$sftpPort)"
}
python scripts\track9_preflight.py --base-url http://127.0.0.1:8080
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
Write-Host ""
Write-Host "Track 9 QA stack is up."
Write-Host "Next step: place real softphone call 1001 -> 7000."
+5
View File
@@ -0,0 +1,5 @@
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
python scripts\local_stack.py stop @args
exit $LASTEXITCODE
+295
View File
@@ -0,0 +1,295 @@
from __future__ import annotations
import argparse
import json
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from statistics import median
from sqlalchemy import create_engine, text
def _parse_iso(value: str | None) -> datetime | None:
raw = str(value or "").strip()
if not raw:
return None
try:
parsed = datetime.fromisoformat(raw)
except ValueError:
return None
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
def _percentile(values: list[float], q: float) -> float | None:
if not values:
return None
if len(values) == 1:
return values[0]
sorted_values = sorted(values)
idx = (len(sorted_values) - 1) * q
lo = int(idx)
hi = min(lo + 1, len(sorted_values) - 1)
if lo == hi:
return sorted_values[lo]
frac = idx - lo
return sorted_values[lo] + (sorted_values[hi] - sorted_values[lo]) * frac
@dataclass
class CallTimeline:
call_id: str
started_at: datetime | None = None
ended_at: datetime | None = None
ended_reconciled: bool | None = None
recording_ready_at: datetime | None = None
recording_ready_reconciled: bool | None = None
recording_uploaded_at: datetime | None = None
interaction_id: str | None = None
def start_to_end_seconds(self) -> float | None:
if self.started_at and self.ended_at:
return max((self.ended_at - self.started_at).total_seconds(), 0.0)
return None
def end_to_recording_ready_seconds(self) -> float | None:
if self.ended_at and self.recording_ready_at:
return max((self.recording_ready_at - self.ended_at).total_seconds(), 0.0)
return None
def end_to_recording_upload_seconds(self) -> float | None:
if self.ended_at and self.recording_uploaded_at:
return max((self.recording_uploaded_at - self.ended_at).total_seconds(), 0.0)
return None
def status(self) -> str:
if self.started_at and not self.ended_at:
return "active_no_end"
if self.ended_at and not self.recording_uploaded_at:
return "ended_no_recording_upload"
if self.ended_at and self.recording_uploaded_at:
return "complete"
return "unknown"
def main() -> int:
parser = argparse.ArgumentParser(
description="Track 10: report Asterisk voice lifecycle latency."
)
parser.add_argument("--database-url", required=True)
parser.add_argument("--since-hours", type=int, default=6)
parser.add_argument("--since-minutes", type=int, default=0)
parser.add_argument("--limit-events", type=int, default=5000)
parser.add_argument("--max-started-to-ended-seconds", type=float, default=30.0)
parser.add_argument("--max-ended-to-recording-seconds", type=float, default=45.0)
parser.add_argument("--print-limit", type=int, default=15)
parser.add_argument("--json-out", default="")
parser.add_argument("--breach-mode", choices=["all", "direct"], default="direct")
parser.add_argument("--fail-on-breach", action="store_true")
args = parser.parse_args()
if args.since_minutes and args.since_minutes > 0:
cutoff = datetime.now(timezone.utc) - timedelta(minutes=args.since_minutes)
else:
cutoff = datetime.now(timezone.utc) - timedelta(hours=max(args.since_hours, 1))
cutoff_iso = cutoff.isoformat()
engine = create_engine(args.database_url, future=True)
calls: dict[str, CallTimeline] = {}
started_from_asterisk: set[str] = set()
with engine.begin() as conn:
rows = conn.execute(
text(
"""
SELECT call_id, interaction_id, event_type, payload_json, created_at
FROM voice_events
WHERE event_type IN ('call.started', 'call.ended', 'recording.ready')
AND created_at >= :cutoff
ORDER BY id DESC
LIMIT :limit
"""
),
{"cutoff": cutoff_iso, "limit": max(args.limit_events, 100)},
).mappings().all()
for row in rows:
call_id = str(row["call_id"] or "").strip()
if not call_id:
continue
payload_raw = str(row["payload_json"] or "{}")
try:
payload = json.loads(payload_raw)
except json.JSONDecodeError:
payload = {}
event_type = str(row["event_type"] or "")
created_at = _parse_iso(str(row["created_at"] or ""))
if created_at is None:
continue
timeline = calls.get(call_id) or CallTimeline(call_id=call_id)
if event_type == "call.started":
if payload.get("source") != "asterisk":
continue
started_from_asterisk.add(call_id)
timeline.started_at = max(filter(None, [timeline.started_at, created_at]))
elif event_type == "call.ended":
if timeline.ended_at is None or created_at > timeline.ended_at:
timeline.ended_at = created_at
timeline.ended_reconciled = bool(payload.get("reconciled"))
elif event_type == "recording.ready":
if timeline.recording_ready_at is None or created_at > timeline.recording_ready_at:
timeline.recording_ready_at = created_at
timeline.recording_ready_reconciled = bool(payload.get("reconciled"))
timeline.interaction_id = timeline.interaction_id or str(row["interaction_id"] or "").strip() or None
calls[call_id] = timeline
rec_rows = conn.execute(
text(
"""
SELECT call_id, interaction_id, created_at
FROM call_recordings
WHERE created_at >= :cutoff
ORDER BY id DESC
LIMIT :limit
"""
),
{"cutoff": cutoff_iso, "limit": max(args.limit_events, 100)},
).mappings().all()
for row in rec_rows:
call_id = str(row["call_id"] or "").strip()
if call_id not in calls:
continue
created_at = _parse_iso(str(row["created_at"] or ""))
if created_at is None:
continue
timeline = calls[call_id]
if timeline.recording_uploaded_at is None or created_at > timeline.recording_uploaded_at:
timeline.recording_uploaded_at = created_at
timeline.interaction_id = timeline.interaction_id or str(row["interaction_id"] or "").strip() or None
items = [calls[call_id] for call_id in started_from_asterisk if call_id in calls]
items.sort(key=lambda x: x.started_at or datetime.min.replace(tzinfo=timezone.utc), reverse=True)
start_end = [v for v in (item.start_to_end_seconds() for item in items) if v is not None]
end_rec = [v for v in (item.end_to_recording_upload_seconds() for item in items) if v is not None]
direct_items = [item for item in items if item.ended_at is not None and item.ended_reconciled is False]
direct_start_end = [v for v in (item.start_to_end_seconds() for item in direct_items) if v is not None]
direct_end_rec = [
v for v in (item.end_to_recording_upload_seconds() for item in direct_items) if v is not None
]
ended_no_recording = [item for item in items if item.status() == "ended_no_recording_upload"]
active_no_end = [item for item in items if item.status() == "active_no_end"]
p95_start_end = _percentile(start_end, 0.95)
p95_end_rec = _percentile(end_rec, 0.95)
p95_direct_start_end = _percentile(direct_start_end, 0.95)
p95_direct_end_rec = _percentile(direct_end_rec, 0.95)
breach = False
if args.breach_mode == "direct":
left = p95_direct_start_end if p95_direct_start_end is not None else p95_start_end
right = p95_direct_end_rec if p95_direct_end_rec is not None else p95_end_rec
else:
left = p95_start_end
right = p95_end_rec
if left is not None and left > args.max_started_to_ended_seconds:
breach = True
if right is not None and right > args.max_ended_to_recording_seconds:
breach = True
if ended_no_recording:
breach = True
report = {
"generated_at": datetime.now(timezone.utc).isoformat(),
"since_hours": args.since_hours,
"since_minutes": args.since_minutes,
"thresholds": {
"max_started_to_ended_seconds": args.max_started_to_ended_seconds,
"max_ended_to_recording_seconds": args.max_ended_to_recording_seconds,
},
"summary": {
"total_calls": len(items),
"complete_calls": sum(1 for item in items if item.status() == "complete"),
"active_no_end": len(active_no_end),
"ended_no_recording_upload": len(ended_no_recording),
"ended_reconciled_calls": sum(1 for item in items if item.ended_reconciled is True),
"ended_direct_calls": sum(1 for item in items if item.ended_reconciled is False),
"start_to_end": {
"count": len(start_end),
"median": median(start_end) if start_end else None,
"p95": p95_start_end,
},
"end_to_recording_upload": {
"count": len(end_rec),
"median": median(end_rec) if end_rec else None,
"p95": p95_end_rec,
},
"direct_start_to_end": {
"count": len(direct_start_end),
"median": median(direct_start_end) if direct_start_end else None,
"p95": p95_direct_start_end,
},
"direct_end_to_recording_upload": {
"count": len(direct_end_rec),
"median": median(direct_end_rec) if direct_end_rec else None,
"p95": p95_direct_end_rec,
},
"breach_mode": args.breach_mode,
},
"worst_calls": [
{
"call_id": item.call_id,
"interaction_id": item.interaction_id,
"status": item.status(),
"started_at": item.started_at.isoformat() if item.started_at else None,
"ended_at": item.ended_at.isoformat() if item.ended_at else None,
"ended_reconciled": item.ended_reconciled,
"recording_ready_at": item.recording_ready_at.isoformat() if item.recording_ready_at else None,
"recording_ready_reconciled": item.recording_ready_reconciled,
"recording_uploaded_at": item.recording_uploaded_at.isoformat() if item.recording_uploaded_at else None,
"start_to_end_seconds": item.start_to_end_seconds(),
"end_to_recording_upload_seconds": item.end_to_recording_upload_seconds(),
}
for item in items[: max(args.print_limit, 1)]
],
}
print(
"[INFO] Track10 voice latency report: "
f"calls={report['summary']['total_calls']} "
f"complete={report['summary']['complete_calls']} "
f"active_no_end={report['summary']['active_no_end']} "
f"ended_no_recording_upload={report['summary']['ended_no_recording_upload']}"
)
print(
"[INFO] p95 started->ended="
f"{report['summary']['start_to_end']['p95']}s; "
"p95 ended->recording_upload="
f"{report['summary']['end_to_recording_upload']['p95']}s"
)
print(
"[INFO] direct p95 started->ended="
f"{report['summary']['direct_start_to_end']['p95']}s; "
"direct p95 ended->recording_upload="
f"{report['summary']['direct_end_to_recording_upload']['p95']}s"
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if args.json_out.strip():
out_path = Path(args.json_out).expanduser().resolve()
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print(f"[INFO] report written: {out_path}")
if args.fail_on_breach and breach:
print("[FAIL] SLO breach detected for Track 10 criteria")
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
+163
View File
@@ -0,0 +1,163 @@
from __future__ import annotations
import json
import sqlite3
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.error import URLError
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
DB_PATH = ROOT / ".data_local" / "mvp_cc.db"
OUT_DIR = ROOT / ".artifacts" / "track12-monitor"
OUT_DIR.mkdir(parents=True, exist_ok=True)
OUT_PATH = OUT_DIR / "latest.log"
BASE_URL = "http://127.0.0.1:8080"
AUTH_BASE_URL = "http://127.0.0.1:8001"
USERNAME = "operator"
PASSWORD = "op12345"
ROLE = "operator"
WINDOW_SECONDS = 180
POLL_SECONDS = 1.0
def iso_now() -> str:
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
def write_line(handle, label: str, payload) -> None:
handle.write(f"[{iso_now()}] {label} {json.dumps(payload, ensure_ascii=False, sort_keys=True)}\n")
handle.flush()
def login_headers() -> dict[str, str]:
body = json.dumps(
{
"username": USERNAME,
"password": PASSWORD,
"role": ROLE,
}
).encode("utf-8")
req = Request(
f"{AUTH_BASE_URL}/auth/login",
headers={"Content-Type": "application/json"},
data=body,
method="POST",
)
with urlopen(req, timeout=10) as response:
payload = json.loads(response.read().decode("utf-8", errors="replace"))
token = str(payload.get("access_token") or "").strip()
if not token:
raise RuntimeError("auth/login did not return access_token")
return {"Authorization": f"Bearer {token}"}
def fetch_json(path: str, headers: dict[str, str]):
req = Request(
f"{BASE_URL}{path}",
headers=headers,
method="GET",
)
with urlopen(req, timeout=10) as response:
body = response.read().decode("utf-8", errors="replace")
return json.loads(body)
def latest_events(conn: sqlite3.Connection, limit: int = 6) -> list[dict]:
rows = conn.execute(
"""
select event_id, event_type, call_id, interaction_id, payload_json, created_at
from voice_events
order by id desc
limit ?
""",
(limit,),
).fetchall()
result = []
for row in rows:
payload = row["payload_json"]
try:
payload = json.loads(payload)
except Exception:
pass
result.append(
{
"event_id": row["event_id"],
"event_type": row["event_type"],
"call_id": row["call_id"],
"interaction_id": row["interaction_id"],
"payload": payload,
"created_at": row["created_at"],
}
)
return result
def latest_links(conn: sqlite3.Connection, limit: int = 4) -> list[dict]:
rows = conn.execute(
"""
select call_id, interaction_id, status, telephony_status, claimed_by_user,
operator_extension, started_at, connected_at, ended_at
from asterisk_call_links
order by id desc
limit ?
""",
(limit,),
).fetchall()
return [dict(row) for row in rows]
def main() -> int:
with OUT_PATH.open("w", encoding="utf-8") as handle:
write_line(handle, "monitor", {"status": "started", "window_seconds": WINDOW_SECONDS})
conn = sqlite3.connect(DB_PATH)
conn.row_factory = sqlite3.Row
headers = login_headers()
write_line(handle, "auth", {"status": "ok", "user": USERNAME, "role": ROLE})
end_at = time.time() + WINDOW_SECONDS
seen = {
"live": None,
"recent": None,
"events": None,
"links": None,
}
try:
while time.time() < end_at:
try:
live = fetch_json("/proxy/asterisk-bridge/asterisk/live-calls", headers)
if live != seen["live"]:
seen["live"] = live
write_line(handle, "live-calls", live)
except URLError as exc:
write_line(handle, "live-calls-error", {"error": str(exc)})
try:
recent = fetch_json("/proxy/asterisk-bridge/asterisk/recent-calls", headers)
if recent != seen["recent"]:
seen["recent"] = recent
write_line(handle, "recent-calls", recent)
except URLError as exc:
write_line(handle, "recent-calls-error", {"error": str(exc)})
events = latest_events(conn)
if events != seen["events"]:
seen["events"] = events
write_line(handle, "voice-events", events)
links = latest_links(conn)
if links != seen["links"]:
seen["links"] = links
write_line(handle, "call-links", links)
time.sleep(POLL_SECONDS)
finally:
conn.close()
write_line(handle, "monitor", {"status": "finished"})
return 0
if __name__ == "__main__":
raise SystemExit(main())
+154
View File
@@ -0,0 +1,154 @@
from __future__ import annotations
import argparse
import json
import subprocess
from pathlib import Path
from typing import Any
def load_summary(report_dir: Path) -> dict[str, Any]:
summary_path = report_dir / "summary.json"
if not summary_path.exists():
raise FileNotFoundError(f"Missing summary.json: {summary_path}")
return json.loads(summary_path.read_text(encoding="utf-8"))
def evaluate_summary(
summary: dict[str, Any],
*,
require_success_rate: float,
require_p95_seconds: float,
require_p99_seconds: float,
) -> list[str]:
issues: list[str] = []
results = summary.get("results", {})
success_rate = float(results.get("success_rate", 0.0))
p95 = float(results.get("p95_seconds", 0.0))
p99 = float(results.get("p99_seconds", 0.0))
five_xx_rate = float(results.get("five_xx_rate", 0.0))
if success_rate < require_success_rate:
issues.append(f"Success rate {success_rate:.2f}% is below threshold {require_success_rate:.2f}%")
if p95 > require_p95_seconds:
issues.append(f"P95 {p95:.2f}s exceeds threshold {require_p95_seconds:.2f}s")
if p99 > require_p99_seconds:
issues.append(f"P99 {p99:.2f}s exceeds threshold {require_p99_seconds:.2f}s")
if five_xx_rate > 0.5:
issues.append(f"5xx/transport error rate {five_xx_rate:.2f}% exceeds 0.50%")
return issues
def _run_kubectl(args: list[str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(args, capture_output=True, text=True, check=False)
def check_pod_health(namespace: str, kubectl_context: str | None = None) -> list[str]:
cmd = ["kubectl"]
if kubectl_context:
cmd.extend(["--context", kubectl_context])
cmd.extend(["get", "pods", "-n", namespace, "-o", "json"])
result = _run_kubectl(cmd)
if result.returncode != 0:
return [f"kubectl get pods failed: {result.stderr.strip() or result.stdout.strip()}"]
payload = json.loads(result.stdout)
issues: list[str] = []
for item in payload.get("items", []):
name = item.get("metadata", {}).get("name", "<unknown>")
status = item.get("status", {})
phase = status.get("phase")
if phase not in {"Running", "Succeeded"}:
issues.append(f"Pod {name} is in unexpected phase {phase}")
for container in status.get("containerStatuses", []):
if container.get("restartCount", 0) > 0:
issues.append(f"Pod {name} restartCount={container.get('restartCount')}")
state = container.get("state", {})
waiting = state.get("waiting") or {}
terminated = state.get("terminated") or {}
if waiting.get("reason") == "CrashLoopBackOff":
issues.append(f"Pod {name} is in CrashLoopBackOff")
if terminated.get("reason") == "OOMKilled":
issues.append(f"Pod {name} had OOMKilled termination")
return issues
def check_kubectl_top(namespace: str, kubectl_context: str | None = None) -> list[str]:
cmd = ["kubectl"]
if kubectl_context:
cmd.extend(["--context", kubectl_context])
cmd.extend(["top", "pods", "-n", namespace, "--no-headers"])
result = _run_kubectl(cmd)
if result.returncode != 0:
return [f"kubectl top pods failed: {result.stderr.strip() or result.stdout.strip()}"]
if not result.stdout.strip():
return ["kubectl top pods returned no rows"]
return []
def check_hpa(namespace: str, kubectl_context: str | None = None) -> list[str]:
cmd = ["kubectl"]
if kubectl_context:
cmd.extend(["--context", kubectl_context])
cmd.extend(["get", "hpa", "-n", namespace, "-o", "json"])
result = _run_kubectl(cmd)
if result.returncode != 0:
return [f"kubectl get hpa failed: {result.stderr.strip() or result.stdout.strip()}"]
payload = json.loads(result.stdout)
items = payload.get("items", [])
if not items:
return ["No HPA objects found in namespace"]
issues: list[str] = []
for item in items:
name = item.get("metadata", {}).get("name", "<unknown>")
status = item.get("status", {})
current_metrics = status.get("currentMetrics") or []
if not current_metrics and status.get("currentCPUUtilizationPercentage") is None:
issues.append(f"HPA {name} has no current utilization metrics")
return issues
def main() -> None:
parser = argparse.ArgumentParser(description="Validate Track 7 scale/hardening evidence.")
parser.add_argument("--namespace", required=True, help="Kubernetes namespace")
parser.add_argument("--report-dir", required=True, help="Directory produced by scripts/load_test.py")
parser.add_argument("--require-success-rate", type=float, default=99.0)
parser.add_argument("--require-p95-seconds", type=float, default=2.0)
parser.add_argument("--require-p99-seconds", type=float, default=3.5)
parser.add_argument("--kubectl-context", default=None)
args = parser.parse_args()
report_dir = Path(args.report_dir)
issues: list[str] = []
try:
summary = load_summary(report_dir)
except Exception as exc:
raise SystemExit(f"[FAIL] {exc}") from exc
issues.extend(
evaluate_summary(
summary,
require_success_rate=args.require_success_rate,
require_p95_seconds=args.require_p95_seconds,
require_p99_seconds=args.require_p99_seconds,
)
)
issues.extend(check_pod_health(args.namespace, kubectl_context=args.kubectl_context))
issues.extend(check_kubectl_top(args.namespace, kubectl_context=args.kubectl_context))
issues.extend(check_hpa(args.namespace, kubectl_context=args.kubectl_context))
if issues:
print("[FAIL] Track 7 validation failed:")
for issue in issues:
print(f"- {issue}")
raise SystemExit(1)
print("[PASS] Track 7 validation checks passed")
print(f"- namespace: {args.namespace}")
print(f"- report_dir: {report_dir}")
if __name__ == "__main__":
main()
+220
View File
@@ -0,0 +1,220 @@
from __future__ import annotations
import argparse
import asyncio
import json
import sys
import time
from pathlib import Path
from typing import Any
if __package__ in {None, ""}:
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from scripts import load_test, track7_check
else:
from scripts import load_test, track7_check
PROFILE_THRESHOLDS = {
"step_250_250": {
"require_success_rate": 99.0,
"require_p95_seconds": 1.5,
"require_p99_seconds": 3.0,
},
"target_500_500": {
"require_success_rate": 99.0,
"require_p95_seconds": 2.0,
"require_p99_seconds": 3.5,
},
}
def build_stage_root(report_root: str | None = None) -> Path:
if report_root:
target = Path(report_root)
else:
timestamp = time.strftime("%Y%m%d_%H%M%S")
target = Path(".artifacts") / "track7" / f"staged_{timestamp}"
target.mkdir(parents=True, exist_ok=True)
return target
def resolve_profiles(profiles_raw: str) -> list[str]:
requested = [item.strip() for item in profiles_raw.split(",") if item.strip()]
if not requested:
raise ValueError("At least one profile is required")
unsupported = [name for name in requested if name not in PROFILE_THRESHOLDS]
if unsupported:
raise ValueError(f"Unsupported staged profiles: {', '.join(unsupported)}")
return requested
def evaluate_stage(
*,
namespace: str,
report_dir: Path,
kubectl_context: str | None,
thresholds: dict[str, float],
) -> list[str]:
summary = track7_check.load_summary(report_dir)
issues: list[str] = []
issues.extend(
track7_check.evaluate_summary(
summary,
require_success_rate=thresholds["require_success_rate"],
require_p95_seconds=thresholds["require_p95_seconds"],
require_p99_seconds=thresholds["require_p99_seconds"],
)
)
issues.extend(track7_check.check_pod_health(namespace, kubectl_context=kubectl_context))
issues.extend(track7_check.check_kubectl_top(namespace, kubectl_context=kubectl_context))
issues.extend(track7_check.check_hpa(namespace, kubectl_context=kubectl_context))
return issues
def write_acceptance_pack(stage_root: Path, payload: dict[str, Any]) -> None:
(stage_root / "acceptance_summary.json").write_text(json.dumps(payload, indent=2), encoding="utf-8")
lines = [
"# Track 7 Staged Validation",
"",
f"- Namespace: `{payload['namespace']}`",
f"- Base URL: `{payload['base_url']}`",
f"- Auth mode: `{payload['auth_mode']}`",
f"- Include read traffic: `{int(payload['include_read_traffic'])}`",
f"- Started at: `{payload['started_at']}`",
f"- Completed at: `{payload['completed_at']}`",
f"- Passed: `{payload['passed']}`",
"",
"## Stages",
"",
]
for stage in payload["stages"]:
lines.extend(
[
f"### {stage['profile']}",
"",
f"- Report dir: `{stage['report_dir']}`",
f"- Passed: `{stage['passed']}`",
f"- Success rate: `{stage['results']['success_rate']:.2f}%`",
f"- P95: `{stage['results']['p95_seconds']:.4f}s`",
f"- P99: `{stage['results']['p99_seconds']:.4f}s`",
]
)
if stage["issues"]:
lines.append("- Issues:")
for issue in stage["issues"]:
lines.append(f" - {issue}")
lines.append("")
(stage_root / "acceptance_summary.md").write_text("\n".join(lines), encoding="utf-8")
async def run_staged_validation(
*,
base_url: str,
namespace: str,
profiles: list[str],
auth_mode: str,
include_read_traffic: bool,
kubectl_context: str | None,
report_root: str | None,
) -> tuple[dict[str, Any], Path]:
stage_root = build_stage_root(report_root)
started_at = time.strftime("%Y-%m-%dT%H:%M:%S")
stages: list[dict[str, Any]] = []
for profile_name in profiles:
thresholds = PROFILE_THRESHOLDS[profile_name]
report_dir = stage_root / profile_name
summary, out_dir = await load_test.run_profile(
base_url=base_url,
profile_name=profile_name,
ramp_seconds=None,
hold_seconds=None,
voice=None,
digital=None,
include_read_traffic=include_read_traffic,
auth_mode=auth_mode,
report_dir=str(report_dir),
require_success_rate=thresholds["require_success_rate"],
require_p95_seconds=thresholds["require_p95_seconds"],
require_p99_seconds=thresholds["require_p99_seconds"],
)
issues = evaluate_stage(
namespace=namespace,
report_dir=out_dir,
kubectl_context=kubectl_context,
thresholds=thresholds,
)
stage_payload = {
"profile": profile_name,
"report_dir": str(out_dir),
"thresholds": thresholds,
"results": summary["results"],
"passed": not issues,
"issues": issues,
}
stages.append(stage_payload)
if issues:
break
completed_at = time.strftime("%Y-%m-%dT%H:%M:%S")
payload = {
"namespace": namespace,
"base_url": base_url,
"auth_mode": auth_mode,
"include_read_traffic": include_read_traffic,
"started_at": started_at,
"completed_at": completed_at,
"stages": stages,
"passed": all(stage["passed"] for stage in stages) and len(stages) == len(profiles),
}
write_acceptance_pack(stage_root, payload)
return payload, stage_root
async def main() -> None:
parser = argparse.ArgumentParser(description="Run staged Track 7 validation and write an acceptance pack.")
parser.add_argument("--base-url", required=True, help="Gateway base URL")
parser.add_argument("--namespace", required=True, help="Kubernetes namespace to validate")
parser.add_argument(
"--profiles",
default="step_250_250,target_500_500",
help="Comma-separated staged profiles. Supported: step_250_250,target_500_500",
)
parser.add_argument(
"--auth-mode",
choices=["legacy_headers", "bearer"],
default="bearer",
help="Authentication mode for load traffic",
)
parser.add_argument(
"--include-read-traffic",
type=int,
choices=[0, 1],
default=1,
help="Include background supervisor/reporting reads during staged runs",
)
parser.add_argument("--kubectl-context", default=None, help="Optional kubectl context override")
parser.add_argument("--report-root", default=None, help="Root directory for stage artifacts")
args = parser.parse_args()
profiles = resolve_profiles(args.profiles)
payload, stage_root = await run_staged_validation(
base_url=args.base_url,
namespace=args.namespace,
profiles=profiles,
auth_mode=args.auth_mode,
include_read_traffic=bool(args.include_read_traffic),
kubectl_context=args.kubectl_context,
report_root=args.report_root,
)
print("Track 7 staged validation:")
print(json.dumps(payload, indent=2))
print(f"Acceptance pack: {stage_root}")
if not payload["passed"]:
raise SystemExit(1)
if __name__ == "__main__":
asyncio.run(main())
+104
View File
@@ -0,0 +1,104 @@
from __future__ import annotations
import argparse
import os
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
from sqlalchemy import create_engine, inspect, select
from sqlalchemy.orm import Session
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from scripts.event_bus_smoke import run_smoke_check
from services.shared.db import DATABASE_URL, _normalize_database_url
from services.shared.sql_models import EventOutboxRow
def _engine_for(database_url: str | None):
return create_engine(_normalize_database_url(database_url or DATABASE_URL), future=True)
def _parse_iso(value: str | None) -> datetime | None:
if not value:
return None
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
return None
def evaluate_track8(
*,
database_url: str | None = None,
require_bus_enabled: bool = False,
base_url: str | None = None,
failed_age_seconds: int = 300,
) -> list[str]:
issues: list[str] = []
engine = _engine_for(database_url)
inspector = inspect(engine)
tables = set(inspector.get_table_names())
required_tables = {"event_outbox", "event_inbox", "reporting_event_log"}
missing = sorted(required_tables - tables)
if missing:
issues.append(f"Missing required tables: {', '.join(missing)}")
if require_bus_enabled and os.getenv("EVENT_BUS_ENABLED", "0").strip() not in {"1", "true", "yes", "on"}:
issues.append("EVENT_BUS_ENABLED is not enabled in the current environment")
with Session(engine) as session:
failed_rows = session.execute(
select(EventOutboxRow).where(EventOutboxRow.status == "failed")
).scalars().all()
stale_cutoff = datetime.now(timezone.utc) - timedelta(seconds=failed_age_seconds)
stale_failed = [
row for row in failed_rows if (_parse_iso(row.updated_at) or _parse_iso(row.created_at) or stale_cutoff) < stale_cutoff
]
if stale_failed:
issues.append(f"Found {len(stale_failed)} failed outbox events older than {failed_age_seconds}s")
pending_rows = session.execute(
select(EventOutboxRow).where(EventOutboxRow.status == "pending")
).scalars().all()
if len(pending_rows) > 500:
issues.append(f"Outbox backlog too high: {len(pending_rows)} pending events")
if not (ROOT / "contracts" / "events" / "ivr.completed.json").exists():
issues.append("Missing contracts/events/ivr.completed.json")
if base_url:
smoke = run_smoke_check(base_url=base_url, database_url=database_url)
if not smoke["passed"]:
issues.append("Event bus smoke check failed")
return issues
def main() -> None:
parser = argparse.ArgumentParser(description="Validate Track 8 event-bus readiness.")
parser.add_argument("--database-url", default=None)
parser.add_argument("--require-bus-enabled", action="store_true")
parser.add_argument("--base-url", default=None)
parser.add_argument("--failed-age-seconds", type=int, default=300)
args = parser.parse_args()
issues = evaluate_track8(
database_url=args.database_url,
require_bus_enabled=args.require_bus_enabled,
base_url=args.base_url,
failed_age_seconds=args.failed_age_seconds,
)
if issues:
print("[FAIL] Track 8 validation failed:")
for issue in issues:
print(f"- {issue}")
raise SystemExit(1)
print("[PASS] Track 8 validation checks passed")
if __name__ == "__main__":
main()
+249
View File
@@ -0,0 +1,249 @@
param(
[Parameter(Mandatory = $true)]
[string]$KubeContext,
[Parameter(Mandatory = $true)]
[string]$Namespace,
[Parameter(Mandatory = $true)]
[string]$Release,
[Parameter(Mandatory = $true)]
[string]$GatewayBaseUrl,
[Parameter(Mandatory = $true)]
[string]$DatabaseUrl,
[Parameter(Mandatory = $true)]
[string]$ImageTag,
[Parameter(Mandatory = $true)]
[string]$AmiHost,
[Parameter(Mandatory = $true)]
[string]$AmiUser,
[Parameter(Mandatory = $true)]
[string]$AmiSecret,
[Parameter(Mandatory = $true)]
[string]$SftpHost,
[Parameter(Mandatory = $true)]
[string]$SftpUser,
[Parameter(Mandatory = $true)]
[string]$SftpPassword,
[Parameter(Mandatory = $true)]
[string]$QueueId,
[Parameter(Mandatory = $true)]
[string]$AppTokenSecret,
[string]$ScaleValuesPath = "deployment/helm/values.scale500.yaml",
[string]$StrictValuesPath = "deployment/helm/values.track9-strict.yaml",
[string]$EvidenceOutDir = "",
[switch]$Execute,
[switch]$ForceUpgrade,
[switch]$SkipChecks
)
$ErrorActionPreference = "Stop"
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
function Run-Checked {
param(
[Parameter(Mandatory = $true)]
[string]$Label,
[Parameter(Mandatory = $true)]
[string[]]$Args
)
Write-Host ""
Write-Host "==> $Label"
Write-Host " $($Args -join ' ')"
$command = $Args[0]
$arguments = @()
if ($Args.Length -gt 1) {
$arguments = $Args[1..($Args.Length - 1)]
}
& $command @arguments
if ($LASTEXITCODE -ne 0) {
throw "$Label failed with exit code $LASTEXITCODE"
}
}
function Ensure-File {
param([string]$PathValue)
if (-not (Test-Path $PathValue)) {
throw "Required file not found: $PathValue"
}
}
Ensure-File -PathValue $ScaleValuesPath
Ensure-File -PathValue $StrictValuesPath
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
$artifactsDir = Join-Path $root ".artifacts/track9_2/$timestamp"
New-Item -ItemType Directory -Path $artifactsDir -Force | Out-Null
$helmTemplateOut = Join-Path $artifactsDir "helm_template.yaml"
$cutoverValuesFile = Join-Path $artifactsDir "cutover.override.yaml"
$podsSnapshotOut = Join-Path $artifactsDir "pods_before.txt"
$historyBeforeOut = Join-Path $artifactsDir "helm_history_before.txt"
$historyAfterOut = Join-Path $artifactsDir "helm_history_after.txt"
$cutoverSummaryOut = Join-Path $artifactsDir "cutover-summary.txt"
function Escape-YamlSingleQuoted {
param([string]$Value)
return ($Value -replace "'", "''")
}
$queueMapJson = '{"voice_lab":"' + $QueueId + '"}'
$overrideYaml = @"
namespace: '$(Escape-YamlSingleQuoted $Namespace)'
image:
tag: '$(Escape-YamlSingleQuoted $ImageTag)'
auth:
appTokenSecret: '$(Escape-YamlSingleQuoted $AppTokenSecret)'
asteriskBridge:
enabled: "1"
amiHost: '$(Escape-YamlSingleQuoted $AmiHost)'
amiUsername: '$(Escape-YamlSingleQuoted $AmiUser)'
amiSecret: '$(Escape-YamlSingleQuoted $AmiSecret)'
queueMapJson: '$(Escape-YamlSingleQuoted $queueMapJson)'
sftpHost: '$(Escape-YamlSingleQuoted $SftpHost)'
sftpUsername: '$(Escape-YamlSingleQuoted $SftpUser)'
sftpPassword: '$(Escape-YamlSingleQuoted $SftpPassword)'
"@
$overrideYaml | Out-File -FilePath $cutoverValuesFile -Encoding utf8
Run-Checked -Label "Switch kube context" -Args @("kubectl", "config", "use-context", $KubeContext)
Run-Checked -Label "Helm lint" -Args @("helm", "lint", "deployment/helm")
Write-Host ""
Write-Host "==> Helm history (before)"
Write-Host " helm -n $Namespace history $Release"
& helm -n $Namespace history $Release | Out-File -FilePath $historyBeforeOut -Encoding utf8
if ($LASTEXITCODE -ne 0) {
"release not found (this can be valid for first install)" | Out-File -FilePath $historyBeforeOut -Encoding utf8
Write-Warning "Release '$Release' not found in namespace '$Namespace'. Continuing with first-install flow."
}
Write-Host ""
Write-Host "==> Pods snapshot (before)"
Write-Host " kubectl -n $Namespace get pods -o wide"
& kubectl -n $Namespace get pods -o wide | Out-File -FilePath $podsSnapshotOut -Encoding utf8
if ($LASTEXITCODE -ne 0) {
"namespace missing or no pods yet (pre-install state)" | Out-File -FilePath $podsSnapshotOut -Encoding utf8
Write-Warning "Could not fetch pods for namespace '$Namespace'. Continuing."
}
Write-Host ""
Write-Host "==> Helm template render"
Write-Host " helm template $Release deployment/helm -n $Namespace -f $ScaleValuesPath -f $StrictValuesPath -f $cutoverValuesFile"
& helm template $Release deployment/helm -n $Namespace `
-f $ScaleValuesPath `
-f $StrictValuesPath `
-f $cutoverValuesFile | Out-File -FilePath $helmTemplateOut -Encoding utf8
if ($LASTEXITCODE -ne 0) {
throw "Helm template render failed with exit code $LASTEXITCODE"
}
$requiredStrictKeys = @(
"ALLOW_LEGACY_HEADER_AUTH"
"ASTERISK_BRIDGE_AUTH_MODE"
"ASTERISK_BRIDGE_AUTH_FALLBACK_LEGACY"
"VOICE_ADAPTER_TRUSTED_SERVICE_SUBJECTS"
"RECORDING_IMPORT_TRUSTED_SERVICE_SUBJECTS"
"RECORDING_IMPORT_ALLOW_ADMIN"
)
foreach ($key in $requiredStrictKeys) {
if (-not (Select-String -Path $helmTemplateOut -Pattern $key -SimpleMatch)) {
throw "Rendered manifest does not contain expected strict key: $key"
}
}
if (-not $Execute) {
@(
"Track 9.2 cutover dry-run completed."
"Artifacts: $artifactsDir"
"Rendered manifest: $helmTemplateOut"
"To execute deploy, rerun with -Execute"
) | Out-File -FilePath $cutoverSummaryOut -Encoding utf8
Write-Host ""
Write-Host "Dry-run completed."
Write-Host "Artifacts: $artifactsDir"
Write-Host "Run again with -Execute to apply Helm upgrade."
exit 0
}
$helmUpgradeArgs = @(
"helm", "upgrade", "--install", $Release, "deployment/helm", "-n", $Namespace,
"-f", $ScaleValuesPath,
"-f", $StrictValuesPath,
"-f", $cutoverValuesFile
)
if ($ForceUpgrade) {
$helmUpgradeArgs += "--force"
}
Run-Checked -Label "Helm upgrade/install" -Args $helmUpgradeArgs
Run-Checked -Label "Rollout status: api-gateway" -Args @("kubectl", "-n", $Namespace, "rollout", "status", "deploy/api-gateway")
Run-Checked -Label "Rollout status: asterisk-bridge-service" -Args @("kubectl", "-n", $Namespace, "rollout", "status", "deploy/asterisk-bridge-service")
Run-Checked -Label "Rollout status: voice-adapter-service" -Args @("kubectl", "-n", $Namespace, "rollout", "status", "deploy/voice-adapter-service")
Run-Checked -Label "Rollout status: recording-service" -Args @("kubectl", "-n", $Namespace, "rollout", "status", "deploy/recording-service")
Run-Checked -Label "Helm history (after)" -Args @("helm", "-n", $Namespace, "history", $Release)
& helm -n $Namespace history $Release | Out-File -FilePath $historyAfterOut -Encoding utf8
if (-not $SkipChecks) {
Run-Checked -Label "Track9 preflight strict" -Args @(
"python", "scripts/track9_preflight.py",
"--base-url", $GatewayBaseUrl,
"--check-sftp",
"--require-strict-service-auth"
)
Run-Checked -Label "Asterisk lab smoke" -Args @(
"python", "scripts/asterisk_lab_smoke.py",
"--base-url", $GatewayBaseUrl,
"--database-url", $DatabaseUrl,
"--require-recording"
)
Run-Checked -Label "Track9 check" -Args @(
"python", "scripts/track9_check.py",
"--base-url", $GatewayBaseUrl,
"--database-url", $DatabaseUrl,
"--require-recording"
)
}
$evidenceArgs = @(
"python", "scripts/track9_collect_evidence.py",
"--base-url", $GatewayBaseUrl,
"--database-url", $DatabaseUrl,
"--run-checks",
"--require-recording"
)
if ($EvidenceOutDir.Trim()) {
$evidenceArgs += @("--out-dir", $EvidenceOutDir)
}
Run-Checked -Label "Collect Track9 evidence" -Args $evidenceArgs
$historyLines = Get-Content $historyAfterOut
$deployedLine = $historyLines | Select-String -Pattern "deployed" | Select-Object -Last 1
$rollbackHint = ""
if ($deployedLine) {
$parts = ($deployedLine.ToString() -split "\s+", [System.StringSplitOptions]::RemoveEmptyEntries)
if ($parts.Length -ge 1) {
$currentRevision = $parts[0]
$previousRevision = [int]$currentRevision - 1
if ($previousRevision -ge 1) {
$rollbackHint = "helm -n $Namespace rollback $Release $previousRevision"
}
}
}
@(
"Track 9.2 cutover execution completed."
"Artifacts: $artifactsDir"
"Gateway: $GatewayBaseUrl"
"Database: $DatabaseUrl"
if ($rollbackHint) { "Rollback command: $rollbackHint" } else { "Rollback command: check helm history manually." }
) | Out-File -FilePath $cutoverSummaryOut -Encoding utf8
Write-Host ""
Write-Host "Track 9.2 cutover completed."
Write-Host "Artifacts: $artifactsDir"
if ($rollbackHint) {
Write-Host "Rollback hint: $rollbackHint"
}
+155
View File
@@ -0,0 +1,155 @@
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
import sys
import time
import httpx
from sqlalchemy import create_engine, text
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from services.shared.security import issue_app_token
def _load_env_file(path: str) -> None:
env_path = Path(path).expanduser().resolve()
if not env_path.exists() or not env_path.is_file():
return
for line in env_path.read_text(encoding="utf-8").splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("#") or "=" not in stripped:
continue
key, value = stripped.split("=", 1)
key = key.strip()
if key and key not in os.environ:
os.environ[key] = value.strip()
def _bool_env(name: str, default: bool) -> bool:
raw = os.getenv(name)
if raw is None:
return default
return raw.strip().lower() in {"1", "true", "yes", "on"}
def _ops_headers() -> dict[str, str]:
if _bool_env("ALLOW_LEGACY_HEADER_AUTH", True):
return {"X-User": "admin", "X-Role": "admin"}
token = issue_app_token(
subject="ops:track9-check",
username="track9-check",
role="admin",
auth_source="service",
provider="track9-script",
ttl_seconds=300,
)
return {"Authorization": f"Bearer {token}"}
def _get_json(url: str) -> dict:
with httpx.Client(timeout=10, trust_env=False) as client:
response = client.get(url, headers=_ops_headers())
response.raise_for_status()
return response.json()
def _wait_for_ami_connected(base_url: str, retries: int = 8, delay_seconds: float = 1.0) -> dict:
last_status: dict = {}
for _ in range(retries):
status = _get_json(f"{base_url.rstrip('/')}/proxy/asterisk-bridge/asterisk/status")
last_status = status
if status.get("ami_connected"):
return status
time.sleep(delay_seconds)
return last_status
def _table_exists(conn, name: str) -> bool:
try:
conn.execute(text(f"SELECT 1 FROM {name} LIMIT 1"))
return True
except Exception: # noqa: BLE001
return False
def _scalar(conn, sql: str) -> int:
value = conn.execute(text(sql)).scalar()
return int(value or 0)
def main() -> int:
parser = argparse.ArgumentParser(description="Track 9 Asterisk integration acceptance validator")
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--env-file", default=".env.production")
parser.add_argument("--database-url", required=True)
parser.add_argument("--require-recording", action="store_true")
args = parser.parse_args()
_load_env_file(args.env_file)
failures: list[str] = []
health = _get_json(f"{args.base_url.rstrip('/')}/proxy/asterisk-bridge/health")
status = _wait_for_ami_connected(args.base_url)
if health.get("status") != "ok":
failures.append("Bridge health is not ok")
if not status.get("ami_connected"):
failures.append("AMI is not connected")
engine = create_engine(args.database_url, future=True)
with engine.begin() as conn:
for table in ("asterisk_event_log", "asterisk_call_links", "voice_events"):
if not _table_exists(conn, table):
failures.append(f"Missing table: {table}")
started = _scalar(
conn,
"SELECT COUNT(*) FROM asterisk_event_log WHERE ami_event_name = 'MVPCCCallStarted' AND forward_status = 'forwarded'",
)
call_started = _scalar(
conn,
"SELECT COUNT(*) FROM voice_events WHERE event_type = 'call.started' AND payload_json LIKE '%\"source\": \"asterisk\"%'",
)
links = _scalar(conn, "SELECT COUNT(*) FROM asterisk_call_links")
stale_failures = _scalar(
conn,
"SELECT COUNT(*) FROM asterisk_event_log WHERE forward_status = 'failed'",
)
recordings = _scalar(
conn,
"SELECT COUNT(*) FROM asterisk_event_log WHERE ami_event_name = 'MVPCCRecordingReady' AND recording_id IS NOT NULL",
)
if started < 1:
failures.append("No forwarded MVPCCCallStarted events")
if call_started < 1:
failures.append("No platform call.started events from Asterisk source")
if links < 1:
failures.append("No linked Asterisk calls")
if stale_failures > 0:
failures.append(f"Failed bridge events present: {stale_failures}")
if args.require_recording and recordings < 1:
failures.append("No uploaded recordings linked to Asterisk events")
if failures:
print("[FAIL] Track 9 validation failed")
for item in failures:
print(f"- {item}")
return 1
print("[PASS] Track 9 validation checks passed")
print(f"- started_events: {started}")
print(f"- platform_call_started: {call_started}")
print(f"- call_links: {links}")
print(f"- recording_events_with_upload: {recordings}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+329
View File
@@ -0,0 +1,329 @@
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
from datetime import datetime
from pathlib import Path
import httpx
from sqlalchemy import create_engine, text
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from services.shared.security import issue_app_token
def _load_env_file(path: str) -> None:
env_path = Path(path).expanduser().resolve()
if not env_path.exists() or not env_path.is_file():
return
for line in env_path.read_text(encoding="utf-8").splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("#") or "=" not in stripped:
continue
key, value = stripped.split("=", 1)
key = key.strip()
if key and key not in os.environ:
os.environ[key] = value.strip()
def _bool_env(name: str, default: bool) -> bool:
raw = os.getenv(name)
if raw is None:
return default
return raw.strip().lower() in {"1", "true", "yes", "on"}
def _ops_headers() -> dict[str, str]:
if _bool_env("ALLOW_LEGACY_HEADER_AUTH", True):
return {"X-User": "admin", "X-Role": "admin"}
token = issue_app_token(
subject="ops:track9-evidence",
username="track9-evidence",
role="admin",
auth_source="service",
provider="track9-script",
ttl_seconds=300,
)
return {"Authorization": f"Bearer {token}"}
def _get_json(url: str) -> dict | list:
with httpx.Client(timeout=15, trust_env=False) as client:
response = client.get(url, headers=_ops_headers())
response.raise_for_status()
return response.json()
def _write_json(path: Path, payload: dict | list) -> None:
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def _query_rows(conn, sql: str, limit: int = 20) -> list[dict]:
rows = conn.execute(text(sql), {"limit": limit}).mappings().all()
return [dict(item) for item in rows]
def _run_command(command: list[str], output_path: Path) -> int:
result = subprocess.run( # noqa: S603
command,
capture_output=True,
text=True,
check=False,
)
output = (result.stdout or "").strip()
errors = (result.stderr or "").strip()
rendered = "\n".join(
[
f"$ {' '.join(command)}",
f"exit_code={result.returncode}",
"",
"STDOUT:",
output if output else "<empty>",
"",
"STDERR:",
errors if errors else "<empty>",
"",
]
)
output_path.write_text(rendered, encoding="utf-8")
return int(result.returncode)
def _render_summary(
*,
base_url: str,
db_url: str,
out_dir: Path,
bridge_status: dict,
failed_events: list[dict],
started_rows: list[dict],
ended_rows: list[dict],
recording_rows: list[dict],
preflight_exit_code: int | None,
smoke_exit_code: int | None,
track9_check_exit_code: int | None,
) -> str:
now = datetime.now().isoformat(timespec="seconds")
lines = [
"# Track 9 Acceptance Summary",
"",
f"- Generated at: `{now}`",
f"- Gateway: `{base_url}`",
f"- Database: `{db_url}`",
f"- Evidence directory: `{out_dir.as_posix()}`",
"",
"## Bridge status",
"",
f"- `status`: `{bridge_status.get('status')}`",
f"- `ami_connected`: `{bridge_status.get('ami_connected')}`",
f"- `queue_codes_loaded`: `{len(bridge_status.get('queue_codes_loaded') or [])}`",
f"- `sftp_enabled`: `{bridge_status.get('sftp_enabled')}`",
"",
"## Event evidence counts",
"",
f"- `failed bridge events`: `{len(failed_events)}`",
f"- `call.started (asterisk source)`: `{len(started_rows)}`",
f"- `call.ended`: `{len(ended_rows)}`",
f"- `recording-ready with recording_id`: `{len(recording_rows)}`",
"",
"## Files",
"",
"- `bridge_health.json`",
"- `bridge_status.json`",
"- `bridge_failed_events.json`",
"- `db_voice_call_started.json`",
"- `db_voice_call_ended.json`",
"- `db_recording_ready_links.json`",
"- `playback-proof.md`",
]
if preflight_exit_code is not None:
lines.extend(
[
"- `preflight_output.txt`",
f" status: `{'PASS' if preflight_exit_code == 0 else 'FAIL'}`",
]
)
if smoke_exit_code is not None:
lines.extend(
[
"- `smoke_output.txt`",
f" status: `{'PASS' if smoke_exit_code == 0 else 'FAIL'}`",
]
)
if track9_check_exit_code is not None:
lines.extend(
[
"- `track9_check_output.txt`",
f" status: `{'PASS' if track9_check_exit_code == 0 else 'FAIL'}`",
]
)
lines.extend(
[
"",
"## Manual attachments",
"",
"- Add supervisor playback proof (screenshot or note) in `playback-proof.md`.",
"",
"## Acceptance decision",
"",
"- [ ] GO",
"- [ ] NO-GO",
"",
"## Notes",
"",
"- Fill environment details and any deviations from runbook.",
]
)
return "\n".join(lines) + "\n"
def main() -> int:
parser = argparse.ArgumentParser(description="Collect Track 9 acceptance evidence package")
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--env-file", default=".env.production")
parser.add_argument("--database-url", required=True)
parser.add_argument("--out-dir", default="")
parser.add_argument("--limit", type=int, default=20)
parser.add_argument("--run-checks", action="store_true")
parser.add_argument("--require-recording", action="store_true")
parser.add_argument("--python-exe", default=sys.executable)
args = parser.parse_args()
_load_env_file(args.env_file)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
out_dir = (
Path(args.out_dir)
if args.out_dir.strip()
else Path("docs/acceptance/track9") / timestamp
)
out_dir.mkdir(parents=True, exist_ok=True)
base_url = args.base_url.rstrip("/")
health = _get_json(f"{base_url}/proxy/asterisk-bridge/health")
status = _get_json(f"{base_url}/proxy/asterisk-bridge/asterisk/status")
failed_events = _get_json(f"{base_url}/proxy/asterisk-bridge/asterisk/events?status=failed")
_write_json(out_dir / "bridge_health.json", health) # type: ignore[arg-type]
_write_json(out_dir / "bridge_status.json", status) # type: ignore[arg-type]
_write_json(out_dir / "bridge_failed_events.json", failed_events) # type: ignore[arg-type]
engine = create_engine(args.database_url, future=True)
with engine.begin() as conn:
started_rows = _query_rows(
conn,
"""
SELECT event_id, call_id, interaction_id, payload_json, created_at
FROM voice_events
WHERE event_type = 'call.started' AND payload_json LIKE '%"source": "asterisk"%'
ORDER BY id DESC
LIMIT :limit
""",
limit=args.limit,
)
ended_rows = _query_rows(
conn,
"""
SELECT event_id, call_id, interaction_id, payload_json, created_at
FROM voice_events
WHERE event_type = 'call.ended'
ORDER BY id DESC
LIMIT :limit
""",
limit=args.limit,
)
recording_rows = _query_rows(
conn,
"""
SELECT bridge_event_id, call_id, interaction_id, recording_id, forward_status, updated_at
FROM asterisk_event_log
WHERE ami_event_name = 'MVPCCRecordingReady' AND recording_id IS NOT NULL
ORDER BY id DESC
LIMIT :limit
""",
limit=args.limit,
)
_write_json(out_dir / "db_voice_call_started.json", started_rows)
_write_json(out_dir / "db_voice_call_ended.json", ended_rows)
_write_json(out_dir / "db_recording_ready_links.json", recording_rows)
(out_dir / "playback-proof.md").write_text(
"\n".join(
[
"# Playback Proof",
"",
"- Date/time:",
"- Supervisor user:",
"- Recording ID:",
"- Playback URL:",
"- Result (played/downloaded):",
"- Evidence link/screenshot path:",
"",
]
),
encoding="utf-8",
)
preflight_exit_code: int | None = None
smoke_exit_code: int | None = None
track9_check_exit_code: int | None = None
if args.run_checks:
preflight_cmd = [
args.python_exe,
"scripts/track9_preflight.py",
"--base-url",
base_url,
"--check-sftp",
]
smoke_cmd = [
args.python_exe,
"scripts/asterisk_lab_smoke.py",
"--base-url",
base_url,
"--database-url",
args.database_url,
]
check_cmd = [
args.python_exe,
"scripts/track9_check.py",
"--base-url",
base_url,
"--database-url",
args.database_url,
]
if args.require_recording:
smoke_cmd.append("--require-recording")
check_cmd.append("--require-recording")
preflight_exit_code = _run_command(preflight_cmd, out_dir / "preflight_output.txt")
smoke_exit_code = _run_command(smoke_cmd, out_dir / "smoke_output.txt")
track9_check_exit_code = _run_command(check_cmd, out_dir / "track9_check_output.txt")
summary = _render_summary(
base_url=base_url,
db_url=args.database_url,
out_dir=out_dir,
bridge_status=status if isinstance(status, dict) else {},
failed_events=failed_events if isinstance(failed_events, list) else [],
started_rows=started_rows,
ended_rows=ended_rows,
recording_rows=recording_rows,
preflight_exit_code=preflight_exit_code,
smoke_exit_code=smoke_exit_code,
track9_check_exit_code=track9_check_exit_code,
)
(out_dir / "track9-acceptance.md").write_text(summary, encoding="utf-8")
print("[PASS] Track 9 evidence package created")
print(f"- output_dir: {out_dir.as_posix()}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+216
View File
@@ -0,0 +1,216 @@
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
import sys
import httpx
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from services.shared.security import issue_app_token
def _get_json(url: str) -> dict:
with httpx.Client(timeout=10, trust_env=False) as client:
response = client.get(url, headers=_ops_headers())
response.raise_for_status()
return response.json()
def _parse_queue_map(raw: str) -> dict[str, str]:
payload = json.loads(raw or "{}")
if not isinstance(payload, dict):
raise ValueError("ASTERISK_QUEUE_MAP_JSON must be a JSON object")
result: dict[str, str] = {}
for key, value in payload.items():
key_s = str(key).strip()
val_s = str(value).strip()
if key_s and val_s:
result[key_s] = val_s
return result
def _parse_subjects(raw: str) -> set[str]:
return {item.strip() for item in (raw or "").split(",") if item.strip()}
def _load_env_file(path: str) -> None:
env_path = Path(path).expanduser().resolve()
if not env_path.exists() or not env_path.is_file():
return
for line in env_path.read_text(encoding="utf-8").splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("#") or "=" not in stripped:
continue
key, value = stripped.split("=", 1)
key = key.strip()
if key and key not in os.environ:
os.environ[key] = value.strip()
def _bool_env(name: str, default: bool) -> bool:
raw = os.getenv(name)
if raw is None:
return default
return raw.strip().lower() in {"1", "true", "yes", "on"}
def _ops_headers() -> dict[str, str]:
if _bool_env("ALLOW_LEGACY_HEADER_AUTH", True):
return {"X-User": "admin", "X-Role": "admin"}
token = issue_app_token(
subject="ops:track9-preflight",
username="track9-preflight",
role="admin",
auth_source="service",
provider="track9-script",
ttl_seconds=300,
)
return {"Authorization": f"Bearer {token}"}
def _bridge_auth_mode() -> str:
raw = os.getenv("ASTERISK_BRIDGE_AUTH_MODE", "legacy_headers").strip().lower()
if raw in {"legacy_headers", "bearer", "bearer_first"}:
return raw
raise ValueError(
"ASTERISK_BRIDGE_AUTH_MODE must be one of: legacy_headers, bearer_first, bearer"
)
def _check_sftp() -> tuple[bool, str]:
host = os.getenv("ASTERISK_SFTP_HOST", "").strip()
username = os.getenv("ASTERISK_SFTP_USERNAME", "").strip()
password = os.getenv("ASTERISK_SFTP_PASSWORD", "").strip()
base_path = os.getenv("ASTERISK_SFTP_BASE_PATH", "/var/spool/asterisk/monitor").strip()
try:
port = int(os.getenv("ASTERISK_SFTP_PORT", "22").strip())
except ValueError:
return False, "ASTERISK_SFTP_PORT is not a valid integer"
if not (host and username and password):
return False, "SFTP env is incomplete (host/username/password)"
try:
import paramiko # type: ignore
except ImportError:
return False, "paramiko is not installed"
transport = paramiko.Transport((host, port))
try:
transport.connect(username=username, password=password)
sftp = paramiko.SFTPClient.from_transport(transport)
try:
sftp.listdir(base_path)
finally:
sftp.close()
except Exception as exc: # noqa: BLE001
return False, f"SFTP check failed: {exc}"
finally:
try:
transport.close()
except Exception: # noqa: BLE001
pass
return True, "ok"
def main() -> int:
parser = argparse.ArgumentParser(description="Track 9 preflight checks")
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--env-file", default=".env.production")
parser.add_argument("--check-sftp", action="store_true")
parser.add_argument("--require-strict-service-auth", action="store_true")
args = parser.parse_args()
_load_env_file(args.env_file)
failures: list[str] = []
base_url = args.base_url.rstrip("/")
health = _get_json(f"{base_url}/proxy/asterisk-bridge/health")
status = _get_json(f"{base_url}/proxy/asterisk-bridge/asterisk/status")
queues = _get_json(f"{base_url}/proxy/routing/queues")
if health.get("status") != "ok":
failures.append("asterisk-bridge health is not ok")
if status.get("status") not in {"ok", "disabled"}:
failures.append("asterisk-bridge status is invalid")
try:
queue_map = _parse_queue_map(os.getenv("ASTERISK_QUEUE_MAP_JSON", "{}"))
except ValueError as exc:
failures.append(str(exc))
queue_map = {}
try:
auth_mode = _bridge_auth_mode()
except ValueError as exc:
failures.append(str(exc))
auth_mode = "legacy_headers"
if auth_mode in {"bearer", "bearer_first"}:
app_secret = os.getenv("APP_TOKEN_SECRET", "").strip()
if not app_secret:
failures.append("APP_TOKEN_SECRET is required for bearer bridge auth modes")
elif app_secret == "dev-secret-change-me":
failures.append("APP_TOKEN_SECRET must not use default value in bearer bridge auth modes")
bridge_subject = os.getenv("ASTERISK_BRIDGE_AUTH_SUBJECT", "svc:asterisk-bridge").strip()
voice_subjects = _parse_subjects(os.getenv("VOICE_ADAPTER_TRUSTED_SERVICE_SUBJECTS", ""))
recording_subjects = _parse_subjects(os.getenv("RECORDING_IMPORT_TRUSTED_SERVICE_SUBJECTS", ""))
if bridge_subject and voice_subjects and bridge_subject not in voice_subjects:
failures.append("VOICE_ADAPTER_TRUSTED_SERVICE_SUBJECTS does not include ASTERISK_BRIDGE_AUTH_SUBJECT")
if bridge_subject and recording_subjects and bridge_subject not in recording_subjects:
failures.append("RECORDING_IMPORT_TRUSTED_SERVICE_SUBJECTS does not include ASTERISK_BRIDGE_AUTH_SUBJECT")
if args.require_strict_service_auth:
if auth_mode != "bearer":
failures.append("Strict mode requires ASTERISK_BRIDGE_AUTH_MODE=bearer")
if _bool_env("ALLOW_LEGACY_HEADER_AUTH", True):
failures.append("Strict mode requires ALLOW_LEGACY_HEADER_AUTH=0")
if _bool_env("ASTERISK_BRIDGE_AUTH_FALLBACK_LEGACY", False):
failures.append("Strict mode requires ASTERISK_BRIDGE_AUTH_FALLBACK_LEGACY=0")
if _bool_env("RECORDING_IMPORT_ALLOW_ADMIN", True):
failures.append("Strict mode requires RECORDING_IMPORT_ALLOW_ADMIN=0")
if not voice_subjects:
failures.append("Strict mode requires VOICE_ADAPTER_TRUSTED_SERVICE_SUBJECTS")
if not recording_subjects:
failures.append("Strict mode requires RECORDING_IMPORT_TRUSTED_SERVICE_SUBJECTS")
queue_ids = {str(item.get("queue_id")) for item in queues if isinstance(item, dict)}
missing_queue_ids = sorted({value for value in queue_map.values() if value not in queue_ids})
if not queue_map:
failures.append("ASTERISK_QUEUE_MAP_JSON is empty")
if missing_queue_ids:
failures.append(
f"Queue map contains unknown queue_id values: {', '.join(missing_queue_ids)}"
)
if args.check_sftp:
sftp_ok, sftp_message = _check_sftp()
if not sftp_ok:
failures.append(sftp_message)
if failures:
print("[FAIL] Track 9 preflight failed")
for item in failures:
print(f"- {item}")
return 1
print("[PASS] Track 9 preflight passed")
print(f"- bridge_status: {status.get('status')}")
print(f"- bridge_auth_mode: {auth_mode}")
print(f"- queue_codes_loaded: {len(queue_map)}")
print(f"- mapped_queue_ids: {len(set(queue_map.values()))}")
if args.check_sftp:
print("- sftp: ok")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+967
View File
@@ -0,0 +1,967 @@
from __future__ import annotations
import argparse
import asyncio
import csv
from dataclasses import dataclass
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
from typing import Any
import httpx
ROOT = Path(__file__).resolve().parents[1]
DATA_ROOT = ROOT / ".data_uat_dry_run"
DEFAULT_OUTPUT_ROOT = ROOT / "docs" / "uat" / "evidence"
SCENARIO_SERVICE_MAP = {
"SYS": "platform",
"S0": "auth-service/gateway",
"S1": "customer-service",
"S2": "interaction-service",
"S3": "interaction-service/routing-service",
"S4": "routing-service/interaction-service",
"S5": "voice-adapter-service",
"S6": "telegram-adapter-service",
"S7": "kb-service",
"S8": "supervisor-service",
"S9": "reporting-service",
}
SEVERITY_RISK_MAP = {
"P1": "High",
"P2": "Medium",
"P3": "Medium",
"P4": "Low",
}
SERVICE_SPECS = [
{"name": "auth", "module": "services.auth_service.app:app", "port": 58001},
{"name": "audit", "module": "services.audit_service.app:app", "port": 58002},
{"name": "customer", "module": "services.customer_service.app:app", "port": 58003},
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 58004},
{"name": "routing", "module": "services.routing_service.app:app", "port": 58005},
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 58006},
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 58007},
{"name": "kb", "module": "services.kb_service.app:app", "port": 58008},
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 58009},
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 58010},
{"name": "gateway", "module": "gateway.app:app", "port": 58080},
]
@dataclass
class ScenarioResult:
scenario_id: str
title: str
passed: bool
severity_on_fail: str
details: str
evidence: dict[str, Any]
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def now_compact() -> str:
return utc_now().strftime("%Y%m%d_%H%M%S")
def iso_now() -> str:
return utc_now().isoformat()
async def wait_for_health(base_url: str, retries: int = 80, delay: float = 0.2) -> None:
async with httpx.AsyncClient(timeout=2) as client:
for _ in range(retries):
try:
response = await client.get(f"{base_url}/health")
if response.status_code == 200:
return
except Exception:
pass
await asyncio.sleep(delay)
raise RuntimeError(f"Service not healthy: {base_url}/health")
def start_one_service(spec: dict[str, Any], data_dir: Path) -> subprocess.Popen:
env = os.environ.copy()
env["CC_DATA_DIR"] = str(data_dir)
if spec["name"] == "gateway":
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:58001"
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:58002"
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:58003"
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:58004"
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:58005"
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:58006"
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:58007"
env["KB_SERVICE_URL"] = "http://127.0.0.1:58008"
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:58009"
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:58010"
cmd = [
sys.executable,
"-m",
"uvicorn",
spec["module"],
"--host",
"127.0.0.1",
"--port",
str(spec["port"]),
]
return subprocess.Popen(
cmd,
cwd=str(ROOT),
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
async def start_services_sequential(data_dir: Path) -> list[subprocess.Popen]:
data_dir.mkdir(parents=True, exist_ok=True)
processes: list[subprocess.Popen] = []
for spec in SERVICE_SPECS:
proc = start_one_service(spec, data_dir)
processes.append(proc)
await wait_for_health(f"http://127.0.0.1:{spec['port']}")
return processes
def stop_services(processes: list[subprocess.Popen]) -> None:
for proc in processes:
if proc.poll() is not None:
continue
try:
proc.terminate()
except Exception:
pass
for proc in processes:
try:
proc.wait(timeout=2)
except Exception:
try:
proc.kill()
except Exception:
pass
def scenario_result(
scenario_id: str,
title: str,
passed: bool,
severity_on_fail: str,
details: str,
evidence: dict[str, Any] | None = None,
) -> ScenarioResult:
return ScenarioResult(
scenario_id=scenario_id,
title=title,
passed=passed,
severity_on_fail=severity_on_fail,
details=details,
evidence=evidence or {},
)
async def run_scenarios(base_url: str) -> tuple[list[ScenarioResult], dict[str, Any]]:
admin = {"X-User": "admin", "X-Role": "admin"}
supervisor = {"X-User": "supervisor", "X-Role": "supervisor"}
operator = {"X-User": "operator", "X-Role": "operator"}
analyst = {"X-User": "analyst", "X-Role": "analyst"}
results: list[ScenarioResult] = []
context: dict[str, Any] = {
"started_at": iso_now(),
"baseline_doc": "docs/gates/mvp-pilot-baseline.md",
"wave2_backlog": "docs/roadmap/05-wave2-backlog.md",
}
async with httpx.AsyncClient(base_url=base_url, timeout=12) as client:
health = await client.get("/health")
registry = await client.get("/registry")
if health.status_code != 200 or registry.status_code != 200:
details = f"health={health.status_code}, registry={registry.status_code}"
return [scenario_result("SYS", "System precheck", False, "P1", details)], context
services = registry.json().get("services", {})
context["reporting_url"] = services.get("reporting", "")
login = await client.post("/proxy/auth/auth/login", json={"username": "admin", "password": "admin123"})
login_ok = False
if login.status_code == 200:
login_ok = bool(login.json().get("access_token"))
denied = await client.post(
"/proxy/auth/users",
headers=operator,
json={
"username": f"uat_denied_{now_compact()}",
"password": "secret123",
"full_name": "UAT Denied",
"role": "operator",
},
)
results.append(
scenario_result(
"S0",
"Login and RBAC deny",
login_ok and denied.status_code == 403,
"P1",
f"login={login.status_code}, denied={denied.status_code}",
{"user": "admin", "denied_status": denied.status_code},
)
)
queue = await client.post(
"/proxy/routing/queues",
headers=admin,
json={
"name": f"UAT Dry Queue {now_compact()}",
"description": "UAT dry-run queue",
"rules": [
{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 30},
{"channel": "telegram", "priority": 3, "strategy": "round_robin", "sla_seconds": 45},
],
},
)
if queue.status_code != 200:
results.append(
scenario_result(
"SYS",
"System precheck",
False,
"P1",
f"queue create failed status={queue.status_code}",
)
)
context["completed_at"] = iso_now()
return results, context
queue_id = queue.json()["queue_id"]
context["queue_id"] = queue_id
customer = await client.post(
"/proxy/customer/customers",
json={
"display_name": "UAT Dry Customer",
"phones": ["+77017770000"],
"preferred_phone": "+77017770000",
"tags": ["uat", "dry-run"],
},
)
customer_id = ""
search_ok = False
if customer.status_code == 200:
customer_id = customer.json()["customer_id"]
search = await client.get("/proxy/customer/customers?query=dry-run")
if search.status_code == 200:
search_ok = any(item.get("customer_id") == customer_id for item in search.json())
context["customer_id"] = customer_id
results.append(
scenario_result(
"S1",
"Customer create and search",
customer.status_code == 200 and search_ok,
"P1",
f"create={customer.status_code}, search={search_ok}",
{"customer_id": customer_id, "query": "dry-run"},
)
)
voice_interaction_id = ""
interaction = await client.post(
"/proxy/interaction/interactions",
headers=operator,
json={
"channel": "voice",
"subject": "UAT dry-run S2 voice",
"customer_id": customer_id or None,
"queue_id": queue_id,
"priority": 3,
},
)
if interaction.status_code != 200:
results.append(
scenario_result(
"S2",
"Voice interaction lifecycle",
False,
"P1",
f"create status={interaction.status_code}",
)
)
else:
voice_interaction_id = interaction.json()["interaction_id"]
context["voice_interaction_id"] = voice_interaction_id
assign = await client.patch(
f"/proxy/interaction/interactions/{voice_interaction_id}/assign",
headers=supervisor,
json={"assignee": "operator_a"},
)
close = await client.patch(
f"/proxy/interaction/interactions/{voice_interaction_id}/status",
headers=operator,
json={"status": "closed"},
)
timeline = await client.get(f"/proxy/interaction/interactions/{voice_interaction_id}/timeline")
actions: set[str] = set()
if timeline.status_code == 200:
actions = {event.get("action", "") for event in timeline.json().get("events", [])}
expected = {"interaction.created", "interaction.assigned", "interaction.status_changed"}
results.append(
scenario_result(
"S2",
"Voice interaction lifecycle",
assign.status_code == 200 and close.status_code == 200 and expected.issubset(actions),
"P1",
f"assign={assign.status_code}, close={close.status_code}, actions={sorted(actions)}",
{"interaction_id": voice_interaction_id},
)
)
esc_create = await client.post(
"/proxy/interaction/interactions",
headers=operator,
json={
"channel": "voice",
"subject": "UAT dry-run S3 escalation",
"customer_id": customer_id or None,
"queue_id": queue_id,
"priority": 3,
},
)
if esc_create.status_code != 200:
results.append(
scenario_result(
"S3",
"Assignment and escalation to second line",
False,
"P1",
f"create status={esc_create.status_code}",
)
)
else:
esc_interaction_id = esc_create.json()["interaction_id"]
assign = await client.patch(
f"/proxy/interaction/interactions/{esc_interaction_id}/assign",
headers=supervisor,
json={"assignee": "operator_b"},
)
escalated = await client.post(
f"/proxy/interaction/interactions/{esc_interaction_id}/escalate",
headers=operator,
json={"target_queue_id": "line2"},
)
timeline = await client.get(f"/proxy/interaction/interactions/{esc_interaction_id}/timeline")
actions = []
if timeline.status_code == 200:
actions = [event.get("action", "") for event in timeline.json().get("events", [])]
results.append(
scenario_result(
"S3",
"Assignment and escalation to second line",
assign.status_code == 200
and escalated.status_code == 200
and escalated.json().get("status") == "escalated"
and escalated.json().get("queue_id") == "line2"
and "interaction.escalated" in actions,
"P1",
f"assign={assign.status_code}, escalate={escalated.status_code}, queue={escalated.json().get('queue_id') if escalated.status_code == 200 else ''}",
{"interaction_id": esc_interaction_id, "target_queue": "line2"},
)
)
route = await client.post(f"/proxy/routing/queues/{queue_id}/route?channel=voice&priority=3")
timeline = (
await client.get(f"/proxy/interaction/interactions/{voice_interaction_id}/timeline")
if voice_interaction_id
else None
)
timeline_ok = False
timeline_count = 0
if timeline and timeline.status_code == 200:
timeline_count = len(timeline.json().get("events", []))
timeline_ok = timeline_count >= 3
route_ok = False
if route.status_code == 200:
route_ok = bool(route.json().get("assignee"))
results.append(
scenario_result(
"S4",
"Routing and timeline verification",
route_ok and timeline_ok,
"P2",
f"route={route.status_code}, timeline_ok={timeline_ok}, timeline_events={timeline_count}",
{
"queue_id": queue_id,
"routed_assignee": route.json().get("assignee") if route.status_code == 200 else "",
"interaction_id": voice_interaction_id,
},
)
)
voice_event = await client.post(
"/proxy/voice/integrations/voice/events",
headers=operator,
json={
"event_type": "call.started",
"call_id": f"uat_call_{now_compact()}",
"interaction_id": voice_interaction_id or None,
"payload": {"source": "uat_dry_run"},
},
)
voice_event_id = voice_event.json().get("event_id") if voice_event.status_code == 200 else ""
voice_list_ok = False
if voice_event.status_code == 200:
voice_events = await client.get(
"/proxy/voice/integrations/voice/events?limit=10",
headers=supervisor,
)
if voice_events.status_code == 200:
voice_list_ok = any(item.get("event_id") == voice_event_id for item in voice_events.json())
results.append(
scenario_result(
"S5",
"Voice event intake",
voice_event.status_code == 200 and voice_list_ok,
"P2",
f"create={voice_event.status_code}, listed={voice_list_ok}",
{"event_id": voice_event_id},
)
)
telegram = await client.post(
"/proxy/telegram/integrations/telegram/webhook",
json={
"chat_id": f"uat_dry_chat_{now_compact()}",
"text": "UAT dry-run telegram",
"payload": {"source": "uat_dry_run"},
},
)
telegram_id = telegram.json().get("message_id") if telegram.status_code == 200 else ""
telegram_list_ok = False
if telegram.status_code == 200:
messages = await client.get("/proxy/telegram/integrations/telegram/messages?limit=10")
if messages.status_code == 200:
telegram_list_ok = any(item.get("message_id") == telegram_id for item in messages.json())
results.append(
scenario_result(
"S6",
"Telegram interaction lifecycle",
telegram.status_code == 200 and telegram_list_ok,
"P1",
f"create={telegram.status_code}, listed={telegram_list_ok}",
{"message_id": telegram_id},
)
)
category = await client.post(
"/proxy/kb/knowledge/categories",
headers=analyst,
json={"name": f"UAT Dry {now_compact()}", "description": "dry-run"},
)
if category.status_code != 200:
results.append(
scenario_result(
"S7",
"KB usage in active handling",
False,
"P2",
f"category create status={category.status_code}",
)
)
else:
category_id = category.json()["category_id"]
article = await client.post(
"/proxy/kb/knowledge/articles",
headers=analyst,
json={
"category_id": category_id,
"title": "UAT dry KB article",
"body": "KB article body for dry run",
"tags": ["uat", "kb", "dry"],
},
)
search = await client.get("/proxy/kb/knowledge/search?q=dry")
article_id = article.json().get("article_id") if article.status_code == 200 else ""
found = False
if search.status_code == 200:
found = any(item.get("article_id") == article_id for item in search.json())
results.append(
scenario_result(
"S7",
"KB usage in active handling",
article.status_code == 200 and search.status_code == 200 and found,
"P2",
f"article={article.status_code}, search={search.status_code}, found={found}",
{"article_id": article_id, "keyword": "dry"},
)
)
up1 = await client.post(
"/proxy/supervisor/supervisor/agent-states",
json={"agent_id": "uat_dry_a1", "state": "READY", "queue_id": queue_id},
)
up2 = await client.post(
"/proxy/supervisor/supervisor/agent-states",
json={"agent_id": "uat_dry_a2", "state": "BUSY", "queue_id": queue_id},
)
metrics = await client.post(
f"/proxy/supervisor/supervisor/queue-metrics?queue_id={queue_id}&in_queue=2&avg_wait_seconds=19"
)
realtime = await client.get("/proxy/supervisor/supervisor/realtime")
queue_seen = False
agent_total = 0
if realtime.status_code == 200:
body = realtime.json()
agent_total = int(body.get("agents", {}).get("total", 0))
queue_seen = any(row.get("queue_id") == queue_id for row in body.get("queues", []))
results.append(
scenario_result(
"S8",
"Supervisor realtime",
up1.status_code == 200
and up2.status_code == 200
and metrics.status_code == 200
and realtime.status_code == 200
and agent_total >= 2
and queue_seen,
"P2",
f"state_updates=({up1.status_code},{up2.status_code}), metrics={metrics.status_code}, realtime={realtime.status_code}",
{"queue_id": queue_id, "agent_total": agent_total},
)
)
kpi_rows = [
{
"queue_id": queue_id,
"answered": True,
"wait_seconds": 15,
"handle_seconds": 90,
"abandoned": False,
"resolved_first_contact": True,
},
{
"queue_id": queue_id,
"answered": True,
"wait_seconds": 33,
"handle_seconds": 110,
"abandoned": False,
"resolved_first_contact": False,
},
{
"queue_id": queue_id,
"answered": False,
"wait_seconds": 10,
"handle_seconds": 0,
"abandoned": True,
"resolved_first_contact": False,
},
]
ingest_statuses: list[int] = []
for row in kpi_rows:
ingested = await client.post("/proxy/reporting/reports/events", json=row)
ingest_statuses.append(ingested.status_code)
kpi = await client.get(f"/proxy/reporting/reports/kpi?queue_id={queue_id}&sl_threshold_seconds=30")
has_kpi = False
if kpi.status_code == 200:
payload = kpi.json().get("kpi", {})
has_kpi = {"SL", "ASA", "AHT", "Abandon", "FCR"}.issubset(set(payload.keys()))
exported = await client.get("/proxy/reporting/reports/export")
export_raw = exported.json().get("raw", "") if exported.status_code == 200 else ""
csv_ok = (
exported.status_code == 200
and "queue_id,answered,wait_seconds,handle_seconds,abandoned,resolved_first_contact,created_at" in export_raw
and queue_id in export_raw
)
results.append(
scenario_result(
"S9",
"KPI report and export validation",
all(code == 200 for code in ingest_statuses) and kpi.status_code == 200 and has_kpi and csv_ok,
"P2",
f"ingest={ingest_statuses}, kpi={kpi.status_code}, csv={csv_ok}",
{"queue_id": queue_id},
)
)
context["completed_at"] = iso_now()
return results, context
def build_defects(results: list[ScenarioResult]) -> list[dict[str, str]]:
defects: list[dict[str, str]] = []
stamp = iso_now()
counter = 1
for row in results:
if row.passed:
continue
defect_id = f"UAT-DRY-{counter:03d}"
counter += 1
severity = row.severity_on_fail
defects.append(
{
"defect_id": defect_id,
"severity": severity,
"status": "Open",
"scenario": row.scenario_id,
"service": SCENARIO_SERVICE_MAP.get(row.scenario_id, "platform"),
"risk_level": SEVERITY_RISK_MAP.get(severity, "Medium"),
"summary": row.title,
"repro_steps": "See scenario checklist and automated dry-run logs",
"actual_result": row.details,
"expected_result": "Scenario should pass",
"verification_step": f"Re-run scenario {row.scenario_id} after fix",
"owner": "TBD",
"opened_at": stamp,
"closed_at": "",
"backlog_bucket": "MVP" if severity in {"P1", "P2"} else "Wave2",
"notes": "Generated by scripts/uat_dry_run.py",
}
)
return defects
def write_scenario_json(output_dir: Path, results: list[ScenarioResult], context: dict[str, Any]) -> Path:
payload = {
"generated_at": iso_now(),
"context": context,
"scenarios": [
{
"scenario_id": row.scenario_id,
"title": row.title,
"passed": row.passed,
"severity_on_fail": row.severity_on_fail,
"details": row.details,
"evidence": row.evidence,
}
for row in results
],
}
path = output_dir / "scenario-results.json"
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return path
def write_defect_csv(output_dir: Path, defects: list[dict[str, str]]) -> Path:
path = output_dir / "defect-log.csv"
fieldnames = [
"defect_id",
"severity",
"status",
"scenario",
"service",
"risk_level",
"summary",
"repro_steps",
"actual_result",
"expected_result",
"verification_step",
"owner",
"opened_at",
"closed_at",
"backlog_bucket",
"notes",
]
with path.open("w", newline="", encoding="utf-8") as fp:
writer = csv.DictWriter(fp, fieldnames=fieldnames)
writer.writeheader()
for row in defects:
writer.writerow(row)
return path
def write_session_protocol(
output_dir: Path,
session_id: str,
base_url: str,
results: list[ScenarioResult],
defects: list[dict[str, str]],
build_version: str,
deployment_date: str,
cluster_namespace: str,
) -> Path:
passed = sum(1 for row in results if row.passed)
failed = sum(1 for row in results if not row.passed)
p1 = sum(1 for row in defects if row["severity"] == "P1")
p2 = sum(1 for row in defects if row["severity"] == "P2")
p3 = sum(1 for row in defects if row["severity"] == "P3")
p4 = sum(1 for row in defects if row["severity"] == "P4")
lines: list[str] = []
lines.append("# UAT Session Protocol (Automated Dry Run)")
lines.append("")
lines.append("## Session Metadata")
lines.append("")
lines.append(f"- Session ID: {session_id}")
lines.append(f"- Date: {utc_now().date().isoformat()}")
lines.append(f"- Start time (UTC): {iso_now()}")
lines.append(f"- End time (UTC): {iso_now()}")
lines.append(f"- Environment URL: {base_url}")
lines.append(f"- Build/version: {build_version}")
lines.append(f"- Deployment date: {deployment_date}")
lines.append(f"- Cluster/namespace: {cluster_namespace}")
lines.append("")
lines.append("## Scope Confirmation")
lines.append("")
lines.append("- [x] MVP pilot scope reviewed against `docs/gates/mvp-pilot-baseline.md`")
lines.append("- [x] Out-of-scope requests remain deferred to `docs/roadmap/05-wave2-backlog.md`")
lines.append("- [x] API stabilization rules remain in effect (backward-compatible fixes only)")
lines.append("")
lines.append("## Participants")
lines.append("")
lines.append("- Operators: automated dry-run actor")
lines.append("- Supervisor: automated dry-run actor")
lines.append("- Analyst: automated dry-run actor")
lines.append("- Admin: automated dry-run actor")
lines.append("- Business owner: pending manual UAT")
lines.append("- IT owner: pending manual UAT")
lines.append("")
lines.append("## Preconditions")
lines.append("")
lines.append("- [x] Environment is reachable")
lines.append("- [x] Test accounts are active")
lines.append("- [x] Test queue is configured")
lines.append("- [x] Audit and reporting endpoints are reachable")
lines.append("")
lines.append("## Execution Summary")
lines.append("")
lines.append(f"- Mandatory scenarios executed: {len(results)}")
lines.append(f"- Passed: {passed}")
lines.append(f"- Failed: {failed}")
lines.append("- Blocked: 0")
lines.append("")
lines.append("## Scenario Results")
lines.append("")
for row in results:
mark = "PASS" if row.passed else "FAIL"
lines.append(f"- [{mark}] {row.scenario_id} {row.title}: {row.details}")
lines.append("")
lines.append("## Defect Summary")
lines.append("")
lines.append(f"- P1: {p1}")
lines.append(f"- P2: {p2}")
lines.append(f"- P3: {p3}")
lines.append(f"- P4: {p4}")
lines.append("- Defect log attachment: defect-log.csv")
lines.append("")
lines.append("## Decision")
lines.append("")
if failed == 0:
lines.append("- [x] Accepted with conditions")
lines.append("- [ ] Accepted for pilot completion")
lines.append("- [ ] Not accepted")
else:
lines.append("- [ ] Accepted with conditions")
lines.append("- [ ] Accepted for pilot completion")
lines.append("- [x] Not accepted")
lines.append("")
lines.append("## Comments")
lines.append("")
lines.append("- Automated dry-run completed.")
lines.append("- Manual UAT with real operators/supervisor is still required for final sign-off.")
lines.append("- Any future scope additions must be recorded in `docs/roadmap/05-wave2-backlog.md`.")
lines.append("")
lines.append("## Signatures")
lines.append("")
lines.append("- Business owner: pending")
lines.append("- IT owner: pending")
lines.append("- Supervisor representative: pending")
lines.append(f"- Date: {utc_now().date().isoformat()}")
lines.append("")
path = output_dir / "session-protocol.md"
path.write_text("\n".join(lines), encoding="utf-8")
return path
def write_signoff_draft(output_dir: Path, session_id: str, failed: int) -> Path:
lines: list[str] = []
lines.append("# UAT Sign-off Sheet (Draft)")
lines.append("")
lines.append(f"- Session ID: {session_id}")
lines.append(f"- Date: {utc_now().date().isoformat()}")
lines.append(f"- Scenario failures: {failed}")
lines.append("")
lines.append("## Scope Confirmation")
lines.append("")
lines.append("- MVP pilot scope confirmed against `docs/gates/mvp-pilot-baseline.md`")
lines.append("- Out-of-scope requests remain deferred to `docs/roadmap/05-wave2-backlog.md`")
lines.append("")
lines.append("## Acceptance Statement")
lines.append("")
lines.append("This draft is generated by automated dry-run and cannot replace real UAT signatures.")
lines.append("")
lines.append("## Required Manual Signatures")
lines.append("")
lines.append("- Business owner: pending")
lines.append("- IT owner: pending")
lines.append("- Supervisor representative: pending")
lines.append("")
path = output_dir / "signoff-draft.md"
path.write_text("\n".join(lines), encoding="utf-8")
return path
def write_summary(
output_dir: Path,
session_id: str,
base_url: str,
scenario_file: Path,
defect_file: Path,
protocol_file: Path,
signoff_file: Path,
results: list[ScenarioResult],
defects: list[dict[str, str]],
) -> Path:
passed = sum(1 for row in results if row.passed)
failed = sum(1 for row in results if not row.passed)
p1 = sum(1 for row in defects if row["severity"] == "P1")
p2 = sum(1 for row in defects if row["severity"] == "P2")
lines: list[str] = []
lines.append("# UAT Dry-Run Summary")
lines.append("")
lines.append(f"- Session ID: {session_id}")
lines.append(f"- Generated at (UTC): {iso_now()}")
lines.append(f"- Environment URL: {base_url}")
lines.append(f"- Scenarios: {len(results)} total, {passed} passed, {failed} failed")
lines.append(f"- Defects: P1={p1}, P2={p2}")
lines.append("")
lines.append("## Files")
lines.append("")
lines.append(f"- Scenario results: `{scenario_file.name}`")
lines.append(f"- Session protocol: `{protocol_file.name}`")
lines.append(f"- Defect log: `{defect_file.name}`")
lines.append(f"- Sign-off draft: `{signoff_file.name}`")
lines.append("")
lines.append("## Next Step")
lines.append("")
lines.append("Run manual UAT with real participants, fix only P1/P2 items, and collect signatures.")
lines.append("Route feature requests and deferred scope to `docs/roadmap/05-wave2-backlog.md`.")
lines.append("")
path = output_dir / "summary.md"
path.write_text("\n".join(lines), encoding="utf-8")
return path
def update_p1p2_register(defects: list[dict[str, str]], session_id: str) -> None:
p1 = sum(1 for row in defects if row["severity"] == "P1" and row["status"] == "Open")
p2 = sum(1 for row in defects if row["severity"] == "P2" and row["status"] == "Open")
path = ROOT / "docs" / "gates" / "p1-p2-defects.md"
lines: list[str] = []
lines.append("# P1/P2 Defect Register (Pilot)")
lines.append("")
lines.append(f"Last update: {utc_now().date().isoformat()} ({session_id})")
lines.append("")
lines.append("## Current Status")
lines.append(f"- Open P1: {p1}")
lines.append(f"- Open P2: {p2}")
lines.append("- Source: `scripts/uat_dry_run.py` and automated checks")
lines.append(f"- Latest dry-run evidence: `docs/uat/evidence/dry_run_{session_id}/`")
lines.append("")
lines.append("## Triage Policy")
lines.append("- Only `P1` and `P2` issues belong to MVP remediation.")
lines.append("- `P3` and `P4` issues move to `docs/roadmap/05-wave2-backlog.md` unless they block sign-off.")
lines.append("")
lines.append("## Pilot Note")
lines.append("- Final P1/P2 closure is confirmed only after the real-operator UAT cycle and sign-off.")
path.write_text("\n".join(lines), encoding="utf-8")
async def main() -> None:
parser = argparse.ArgumentParser(description="Run automated UAT dry-run and generate session artifacts")
parser.add_argument("--base-url", default="http://localhost:8080", help="Gateway base URL")
parser.add_argument("--auto-start", action="store_true", help="Auto-start local services for dry-run")
parser.add_argument("--keep-data", action="store_true", help="Keep temporary auto-start data directory")
parser.add_argument(
"--output-dir",
default="",
help="Output root directory (default: docs/uat/evidence)",
)
parser.add_argument("--session-id", default="", help="Optional session ID override")
parser.add_argument("--build-version", default="v1.0.0-mvp", help="Build/version for protocol")
parser.add_argument(
"--deployment-date",
default=utc_now().date().isoformat(),
help="Deployment date for protocol",
)
parser.add_argument("--cluster-namespace", default="local/standalone", help="Cluster/namespace for protocol")
parser.add_argument(
"--update-defect-register",
action="store_true",
help="Update docs/gates/p1-p2-defects.md using dry-run result",
)
args = parser.parse_args()
output_root = Path(args.output_dir) if args.output_dir else DEFAULT_OUTPUT_ROOT
output_root = output_root if output_root.is_absolute() else (ROOT / output_root).resolve()
session_id = args.session_id.strip() or f"UAT-DRY-{now_compact()}"
session_dir = output_root / f"dry_run_{session_id}"
session_dir.mkdir(parents=True, exist_ok=True)
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
processes: list[subprocess.Popen] = []
base_url = args.base_url
try:
if args.auto_start:
processes = await start_services_sequential(run_data_dir)
base_url = "http://127.0.0.1:58080"
results, context = await run_scenarios(base_url)
defects = build_defects(results)
scenario_file = write_scenario_json(session_dir, results, context)
defect_file = write_defect_csv(session_dir, defects)
protocol_file = write_session_protocol(
session_dir,
session_id,
base_url,
results,
defects,
args.build_version,
args.deployment_date,
args.cluster_namespace,
)
signoff_file = write_signoff_draft(session_dir, session_id, sum(1 for row in results if not row.passed))
summary_file = write_summary(
session_dir,
session_id,
base_url,
scenario_file,
defect_file,
protocol_file,
signoff_file,
results,
defects,
)
if args.update_defect_register:
update_p1p2_register(defects, session_id)
passed = sum(1 for row in results if row.passed)
failed = sum(1 for row in results if not row.passed)
print(f"UAT dry-run session: {session_id}")
print(f"Output directory: {session_dir}")
print(f"Scenarios: total={len(results)} passed={passed} failed={failed}")
print(f"Summary: {summary_file}")
sys.exit(0 if failed == 0 else 1)
finally:
stop_services(processes)
if args.auto_start and not args.keep_data:
try:
if run_data_dir.exists():
shutil.rmtree(run_data_dir)
except Exception:
pass
if __name__ == "__main__":
asyncio.run(main())
+329
View File
@@ -0,0 +1,329 @@
from __future__ import annotations
import argparse
import json
from datetime import datetime, timezone
from pathlib import Path
import shutil
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_EVIDENCE_ROOT = ROOT / "docs" / "uat" / "evidence"
DEFAULT_TEMPLATE_ROOT = ROOT / "docs" / "uat"
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def now_compact() -> str:
return utc_now().strftime("%Y%m%d_%H%M%S")
def iso_now() -> str:
return utc_now().isoformat()
def find_latest_preflight(evidence_root: Path) -> Path:
candidates = sorted(
evidence_root.glob("preflight_*.md"),
key=lambda path: path.stat().st_mtime,
reverse=True,
)
if not candidates:
raise FileNotFoundError("No preflight report found in docs/uat/evidence")
return candidates[0]
def find_latest_dry_run_dir(evidence_root: Path) -> Path:
candidates = sorted(
[path for path in evidence_root.glob("dry_run_*") if path.is_dir()],
key=lambda path: path.stat().st_mtime,
reverse=True,
)
if not candidates:
raise FileNotFoundError("No dry-run directory found in docs/uat/evidence")
return candidates[0]
def _replace_line(content: str, prefix: str, value: str) -> str:
replacement = f"{prefix} {value}".rstrip()
return content.replace(prefix, replacement, 1)
def render_session_protocol(
template_text: str,
*,
session_id: str,
environment_url: str,
build_version: str,
deployment_date: str,
cluster_namespace: str,
participants: dict[str, str],
preflight_name: str,
defect_log_name: str,
) -> str:
content = template_text
today = utc_now().date().isoformat()
content = _replace_line(content, "- Session ID:", session_id)
content = _replace_line(content, "- Date:", today)
content = _replace_line(content, "- Start time:", "")
content = _replace_line(content, "- End time:", "")
content = _replace_line(content, "- Environment URL:", environment_url)
content = _replace_line(content, "- Build/version:", build_version)
content = _replace_line(content, "- Deployment date:", deployment_date)
content = _replace_line(content, "- Cluster/namespace:", cluster_namespace)
field_map = {
"- Operators:": participants["operators"],
"- Supervisor:": participants["supervisor"],
"- Analyst:": participants["analyst"],
"- Admin:": participants["admin"],
"- Business owner:": participants["business_owner"],
"- IT owner:": participants["it_owner"],
}
for prefix, value in field_map.items():
content = _replace_line(content, prefix, value)
content = content.replace(
"- [ ] Preflight report attached (`docs/uat/evidence/preflight_*.md`)",
f"- [x] Preflight report attached (`attachments/{preflight_name}`)",
1,
)
content = content.replace(
"- [ ] Defect log prepared from `docs/uat/defect-log-template.csv`",
f"- [x] Defect log prepared from `{defect_log_name}`",
1,
)
return content
def render_signoff_sheet(
template_text: str,
*,
session_id: str,
environment_url: str,
build_version: str,
) -> str:
lines = template_text.splitlines()
if not lines:
return template_text
header = [
lines[0],
"",
f"- Session ID: {session_id}",
f"- Environment URL: {environment_url}",
f"- Build/version: {build_version}",
f"- Prepared at (UTC): {iso_now()}",
"",
]
return "\n".join(header + lines[1:])
def build_manual_readme(
*,
session_id: str,
environment_url: str,
preflight_name: str,
dry_run_name: str,
) -> str:
lines = [
"# Manual UAT Session Bundle",
"",
f"- Session ID: {session_id}",
f"- Environment URL: {environment_url}",
f"- Prepared at (UTC): {iso_now()}",
"",
"## Included Files",
"",
"- `session-protocol.md`",
"- `scenario-checklist.md`",
"- `defect-log.csv`",
"- `signoff-sheet.md`",
"- `manifest.json`",
"",
"## Attached Evidence",
"",
f"- `attachments/{preflight_name}`",
f"- `attachments/{dry_run_name}/`",
"",
"## Next Step",
"",
"Run the live manual UAT on the pilot gateway, record any defects, and use the",
"`scripts/finalize_mvp_pilot.py` command only after signatures are captured and",
"all open P1/P2 defects are closed.",
]
return "\n".join(lines)
def prepare_manual_bundle(
*,
session_id: str,
environment_url: str,
build_version: str,
deployment_date: str,
cluster_namespace: str,
participants: dict[str, str],
output_root: Path,
template_root: Path,
preflight_report: Path,
dry_run_dir: Path,
overwrite: bool = False,
) -> Path:
session_dir = output_root / f"manual_{session_id}"
if session_dir.exists():
if not overwrite:
raise FileExistsError(f"Session directory already exists: {session_dir}")
shutil.rmtree(session_dir)
attachments_dir = session_dir / "attachments"
attachments_dir.mkdir(parents=True, exist_ok=True)
copied_preflight = attachments_dir / preflight_report.name
shutil.copy2(preflight_report, copied_preflight)
copied_dry_run = attachments_dir / dry_run_dir.name
shutil.copytree(dry_run_dir, copied_dry_run)
session_template = (template_root / "session-template.md").read_text(encoding="utf-8")
signoff_template = (template_root / "signoff-template.md").read_text(encoding="utf-8")
checklist_template = (template_root / "scenario-checklist.md").read_text(encoding="utf-8")
defect_template = (template_root / "defect-log-template.csv").read_text(encoding="utf-8")
rendered_session = render_session_protocol(
session_template,
session_id=session_id,
environment_url=environment_url,
build_version=build_version,
deployment_date=deployment_date,
cluster_namespace=cluster_namespace,
participants=participants,
preflight_name=preflight_report.name,
defect_log_name="defect-log.csv",
)
rendered_signoff = render_signoff_sheet(
signoff_template,
session_id=session_id,
environment_url=environment_url,
build_version=build_version,
)
(session_dir / "session-protocol.md").write_text(rendered_session, encoding="utf-8")
(session_dir / "signoff-sheet.md").write_text(rendered_signoff, encoding="utf-8")
(session_dir / "scenario-checklist.md").write_text(checklist_template, encoding="utf-8")
(session_dir / "defect-log.csv").write_text(defect_template, encoding="utf-8")
manifest: dict[str, Any] = {
"prepared_at": iso_now(),
"session_id": session_id,
"environment_url": environment_url,
"build_version": build_version,
"deployment_date": deployment_date,
"cluster_namespace": cluster_namespace,
"participants": participants,
"artifacts": {
"preflight_report": f"attachments/{preflight_report.name}",
"dry_run_bundle": f"attachments/{dry_run_dir.name}",
"session_protocol": "session-protocol.md",
"scenario_checklist": "scenario-checklist.md",
"defect_log": "defect-log.csv",
"signoff_sheet": "signoff-sheet.md",
},
}
(session_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
(session_dir / "README.md").write_text(
build_manual_readme(
session_id=session_id,
environment_url=environment_url,
preflight_name=preflight_report.name,
dry_run_name=dry_run_dir.name,
),
encoding="utf-8",
)
return session_dir
def main() -> None:
parser = argparse.ArgumentParser(description="Prepare a manual UAT bundle from the latest preflight and dry-run")
parser.add_argument("--session-id", default=f"UAT-MANUAL-{now_compact()}", help="Manual UAT session id")
parser.add_argument(
"--environment-url",
default="http://<pilot-gateway>:8080",
help="Pilot gateway URL recorded in the session pack",
)
parser.add_argument("--build-version", default="v1.0.0-mvp", help="Build/version under test")
parser.add_argument(
"--deployment-date",
default=utc_now().date().isoformat(),
help="Deployment date recorded in the session pack",
)
parser.add_argument(
"--cluster-namespace",
default="pilot/production-like",
help="Cluster or namespace label recorded in the session pack",
)
parser.add_argument("--operators", default="TBD", help="Operator participants")
parser.add_argument("--supervisor", default="TBD", help="Supervisor participant")
parser.add_argument("--analyst", default="TBD", help="Analyst participant")
parser.add_argument("--admin", default="TBD", help="Admin participant")
parser.add_argument("--business-owner", default="TBD", help="Business owner")
parser.add_argument("--it-owner", default="TBD", help="IT owner")
parser.add_argument(
"--preflight-report",
default="",
help="Optional explicit preflight report path (default: latest preflight report)",
)
parser.add_argument(
"--dry-run-dir",
default="",
help="Optional explicit dry-run directory path (default: latest dry-run evidence folder)",
)
parser.add_argument(
"--output-root",
default="",
help="Output root directory (default: docs/uat/evidence)",
)
parser.add_argument("--overwrite", action="store_true", help="Overwrite the session folder if it already exists")
args = parser.parse_args()
output_root = Path(args.output_root) if args.output_root else DEFAULT_EVIDENCE_ROOT
output_root = output_root if output_root.is_absolute() else (ROOT / output_root).resolve()
preflight_report = Path(args.preflight_report) if args.preflight_report else find_latest_preflight(DEFAULT_EVIDENCE_ROOT)
if not preflight_report.is_absolute():
preflight_report = (ROOT / preflight_report).resolve()
dry_run_dir = Path(args.dry_run_dir) if args.dry_run_dir else find_latest_dry_run_dir(DEFAULT_EVIDENCE_ROOT)
if not dry_run_dir.is_absolute():
dry_run_dir = (ROOT / dry_run_dir).resolve()
participants = {
"operators": args.operators,
"supervisor": args.supervisor,
"analyst": args.analyst,
"admin": args.admin,
"business_owner": args.business_owner,
"it_owner": args.it_owner,
}
session_dir = prepare_manual_bundle(
session_id=args.session_id,
environment_url=args.environment_url,
build_version=args.build_version,
deployment_date=args.deployment_date,
cluster_namespace=args.cluster_namespace,
participants=participants,
output_root=output_root,
template_root=DEFAULT_TEMPLATE_ROOT,
preflight_report=preflight_report,
dry_run_dir=dry_run_dir,
overwrite=args.overwrite,
)
print(f"Manual UAT bundle prepared: {session_dir}")
if __name__ == "__main__":
main()
+457
View File
@@ -0,0 +1,457 @@
from __future__ import annotations
import argparse
import asyncio
from dataclasses import dataclass
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
from typing import Any
import httpx
ROOT = Path(__file__).resolve().parents[1]
DATA_ROOT = ROOT / ".data_uat_preflight"
DEFAULT_EVIDENCE_DIR = ROOT / "docs" / "uat" / "evidence"
SERVICE_SPECS = [
{"name": "auth", "module": "services.auth_service.app:app", "port": 48001},
{"name": "audit", "module": "services.audit_service.app:app", "port": 48002},
{"name": "customer", "module": "services.customer_service.app:app", "port": 48003},
{"name": "interaction", "module": "services.interaction_service.app:app", "port": 48004},
{"name": "routing", "module": "services.routing_service.app:app", "port": 48005},
{"name": "voice", "module": "services.voice_adapter_service.app:app", "port": 48006},
{"name": "telegram", "module": "services.telegram_adapter_service.app:app", "port": 48007},
{"name": "kb", "module": "services.kb_service.app:app", "port": 48008},
{"name": "reporting", "module": "services.reporting_service.app:app", "port": 48009},
{"name": "supervisor", "module": "services.supervisor_service.app:app", "port": 48010},
{"name": "gateway", "module": "gateway.app:app", "port": 48080},
]
REQUIRED_SERVICES = [
"auth",
"audit",
"customer",
"interaction",
"routing",
"voice",
"telegram",
"kb",
"reporting",
"supervisor",
]
@dataclass
class CheckResult:
name: str
ok: bool
details: str
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def now_compact() -> str:
return utc_now().strftime("%Y%m%d_%H%M%S")
def iso_now() -> str:
return utc_now().isoformat()
async def wait_for_health(base_url: str, retries: int = 80, delay: float = 0.2) -> None:
async with httpx.AsyncClient(timeout=2) as client:
for _ in range(retries):
try:
response = await client.get(f"{base_url}/health")
if response.status_code == 200:
return
except Exception:
pass
await asyncio.sleep(delay)
raise RuntimeError(f"Service not healthy: {base_url}/health")
def start_one_service(spec: dict[str, Any], data_dir: Path) -> subprocess.Popen:
env = os.environ.copy()
env["CC_DATA_DIR"] = str(data_dir)
if spec["name"] == "gateway":
env["AUTH_SERVICE_URL"] = "http://127.0.0.1:48001"
env["AUDIT_SERVICE_URL"] = "http://127.0.0.1:48002"
env["CUSTOMER_SERVICE_URL"] = "http://127.0.0.1:48003"
env["INTERACTION_SERVICE_URL"] = "http://127.0.0.1:48004"
env["ROUTING_SERVICE_URL"] = "http://127.0.0.1:48005"
env["VOICE_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:48006"
env["TELEGRAM_ADAPTER_SERVICE_URL"] = "http://127.0.0.1:48007"
env["KB_SERVICE_URL"] = "http://127.0.0.1:48008"
env["REPORTING_SERVICE_URL"] = "http://127.0.0.1:48009"
env["SUPERVISOR_SERVICE_URL"] = "http://127.0.0.1:48010"
cmd = [
sys.executable,
"-m",
"uvicorn",
spec["module"],
"--host",
"127.0.0.1",
"--port",
str(spec["port"]),
]
return subprocess.Popen(
cmd,
cwd=str(ROOT),
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
async def start_services_sequential(data_dir: Path) -> list[subprocess.Popen]:
data_dir.mkdir(parents=True, exist_ok=True)
processes: list[subprocess.Popen] = []
for spec in SERVICE_SPECS:
proc = start_one_service(spec, data_dir)
processes.append(proc)
await wait_for_health(f"http://127.0.0.1:{spec['port']}")
return processes
def stop_services(processes: list[subprocess.Popen]) -> None:
for proc in processes:
if proc.poll() is not None:
continue
try:
proc.terminate()
except Exception:
pass
for proc in processes:
try:
proc.wait(timeout=2)
except Exception:
try:
proc.kill()
except Exception:
pass
async def run_checks(base_url: str) -> tuple[list[CheckResult], dict[str, Any]]:
checks: list[CheckResult] = []
artifacts: dict[str, Any] = {}
admin = {"X-User": "admin", "X-Role": "admin"}
supervisor = {"X-User": "supervisor", "X-Role": "supervisor"}
operator = {"X-User": "operator", "X-Role": "operator"}
analyst = {"X-User": "analyst", "X-Role": "analyst"}
async with httpx.AsyncClient(base_url=base_url, timeout=10) as client:
try:
health = await client.get("/health")
ok = health.status_code == 200
checks.append(CheckResult("Gateway health", ok, f"status={health.status_code}"))
if not ok:
return checks, artifacts
except Exception as exc:
checks.append(CheckResult("Gateway health", False, f"exception={exc}"))
return checks, artifacts
try:
registry = await client.get("/registry")
data = registry.json() if registry.status_code == 200 else {}
services = data.get("services", {})
missing = [name for name in REQUIRED_SERVICES if name not in services]
ok = registry.status_code == 200 and not missing
details = f"status={registry.status_code}"
if missing:
details += f", missing={missing}"
checks.append(CheckResult("Service registry", ok, details))
if not ok:
return checks, artifacts
except Exception as exc:
checks.append(CheckResult("Service registry", False, f"exception={exc}"))
return checks, artifacts
try:
login = await client.post("/proxy/auth/auth/login", json={"username": "admin", "password": "admin123"})
ok = login.status_code == 200 and "access_token" in login.json()
checks.append(CheckResult("Auth login", ok, f"status={login.status_code}"))
except Exception as exc:
checks.append(CheckResult("Auth login", False, f"exception={exc}"))
return checks, artifacts
try:
queue = await client.post(
"/proxy/routing/queues",
headers=admin,
json={
"name": f"UAT Queue {now_compact()}",
"description": "UAT preflight queue",
"rules": [
{"channel": "voice", "priority": 3, "strategy": "round_robin", "sla_seconds": 30},
{"channel": "telegram", "priority": 3, "strategy": "round_robin", "sla_seconds": 45},
],
},
)
queue_ok = queue.status_code == 200
queue_id = queue.json().get("queue_id") if queue_ok else ""
artifacts["queue_id"] = queue_id
checks.append(CheckResult("Queue create", queue_ok, f"status={queue.status_code}, queue_id={queue_id}"))
except Exception as exc:
checks.append(CheckResult("Queue create", False, f"exception={exc}"))
return checks, artifacts
try:
customer = await client.post(
"/proxy/customer/customers",
json={
"display_name": "UAT Preflight Customer",
"phones": ["+77019990000"],
"preferred_phone": "+77019990000",
"tags": ["uat", "preflight"],
},
)
customer_ok = customer.status_code == 200
customer_id = customer.json().get("customer_id") if customer_ok else ""
artifacts["customer_id"] = customer_id
checks.append(
CheckResult("Customer create", customer_ok, f"status={customer.status_code}, customer_id={customer_id}")
)
except Exception as exc:
checks.append(CheckResult("Customer create", False, f"exception={exc}"))
return checks, artifacts
interaction_id = ""
try:
interaction = await client.post(
"/proxy/interaction/interactions",
headers=operator,
json={
"channel": "voice",
"subject": "UAT preflight voice interaction",
"customer_id": artifacts["customer_id"],
"queue_id": artifacts["queue_id"],
"priority": 3,
},
)
ok = interaction.status_code == 200
interaction_id = interaction.json().get("interaction_id") if ok else ""
artifacts["voice_interaction_id"] = interaction_id
checks.append(CheckResult("Voice interaction create", ok, f"status={interaction.status_code}"))
if not ok:
return checks, artifacts
assign = await client.patch(
f"/proxy/interaction/interactions/{interaction_id}/assign",
headers=supervisor,
json={"assignee": "operator_a"},
)
assign_ok = assign.status_code == 200 and assign.json().get("status") == "in_progress"
checks.append(CheckResult("Voice interaction assign", assign_ok, f"status={assign.status_code}"))
close = await client.patch(
f"/proxy/interaction/interactions/{interaction_id}/status",
headers=operator,
json={"status": "closed"},
)
close_ok = close.status_code == 200 and close.json().get("status") == "closed"
checks.append(CheckResult("Voice interaction close", close_ok, f"status={close.status_code}"))
except Exception as exc:
checks.append(CheckResult("Voice interaction lifecycle", False, f"exception={exc}"))
return checks, artifacts
try:
tg = await client.post(
"/proxy/telegram/integrations/telegram/webhook",
json={
"chat_id": f"uat_chat_{now_compact()}",
"text": "UAT preflight telegram",
"customer_external_id": None,
"payload": {"source": "uat_preflight"},
},
)
ok = tg.status_code == 200 and tg.json().get("message_id")
artifacts["telegram_message_id"] = tg.json().get("message_id") if tg.status_code == 200 else ""
checks.append(CheckResult("Telegram webhook", bool(ok), f"status={tg.status_code}"))
except Exception as exc:
checks.append(CheckResult("Telegram webhook", False, f"exception={exc}"))
return checks, artifacts
try:
cat = await client.post(
"/proxy/kb/knowledge/categories",
headers=analyst,
json={"name": f"UAT {now_compact()}", "description": "UAT preflight"},
)
cat_ok = cat.status_code == 200
category_id = cat.json().get("category_id") if cat_ok else ""
if not cat_ok:
checks.append(CheckResult("KB category", False, f"status={cat.status_code}"))
return checks, artifacts
art = await client.post(
"/proxy/kb/knowledge/articles",
headers=analyst,
json={
"category_id": category_id,
"title": "UAT preflight article",
"body": "Use this article during UAT flow.",
"tags": ["uat", "kb"],
},
)
art_ok = art.status_code == 200
checks.append(CheckResult("KB article", art_ok, f"status={art.status_code}"))
if not art_ok:
return checks, artifacts
search = await client.get("/proxy/kb/knowledge/search?q=preflight")
search_ok = search.status_code == 200 and len(search.json()) > 0
checks.append(CheckResult("KB search", search_ok, f"status={search.status_code}, count={len(search.json())}"))
except Exception as exc:
checks.append(CheckResult("KB checks", False, f"exception={exc}"))
return checks, artifacts
try:
ingest = await client.post(
"/proxy/reporting/reports/events",
json={
"queue_id": artifacts["queue_id"],
"answered": True,
"wait_seconds": 18,
"handle_seconds": 105,
"abandoned": False,
"resolved_first_contact": True,
},
)
ingest_ok = ingest.status_code == 200
checks.append(CheckResult("KPI event ingest", ingest_ok, f"status={ingest.status_code}"))
if not ingest_ok:
return checks, artifacts
kpi = await client.get(f"/proxy/reporting/reports/kpi?queue_id={artifacts['queue_id']}")
kpi_ok = kpi.status_code == 200 and "kpi" in kpi.json()
checks.append(CheckResult("KPI report", kpi_ok, f"status={kpi.status_code}"))
except Exception as exc:
checks.append(CheckResult("KPI checks", False, f"exception={exc}"))
return checks, artifacts
try:
up1 = await client.post(
"/proxy/supervisor/supervisor/agent-states",
json={"agent_id": "uat_agent_1", "state": "READY", "queue_id": artifacts["queue_id"]},
)
up2 = await client.post(
"/proxy/supervisor/supervisor/agent-states",
json={"agent_id": "uat_agent_2", "state": "BUSY", "queue_id": artifacts["queue_id"]},
)
rt = await client.get("/proxy/supervisor/supervisor/realtime")
realtime_ok = up1.status_code == 200 and up2.status_code == 200 and rt.status_code == 200
checks.append(CheckResult("Supervisor realtime", realtime_ok, f"status={rt.status_code}"))
except Exception as exc:
checks.append(CheckResult("Supervisor realtime", False, f"exception={exc}"))
return checks, artifacts
try:
audit = await client.get("/proxy/audit/audit/events?limit=5")
ok = audit.status_code == 200
checks.append(CheckResult("Audit availability", ok, f"status={audit.status_code}"))
except Exception as exc:
checks.append(CheckResult("Audit availability", False, f"exception={exc}"))
artifacts["completed_at"] = iso_now()
return checks, artifacts
def write_report(
output_path: Path,
base_url: str,
checks: list[CheckResult],
artifacts: dict[str, Any],
auto_start: bool,
) -> None:
ok_count = sum(1 for c in checks if c.ok)
total = len(checks)
overall = "PASS" if ok_count == total else "FAIL"
lines: list[str] = []
lines.append("# UAT Preflight Report")
lines.append("")
lines.append(f"- Timestamp (UTC): {iso_now()}")
lines.append(f"- Base URL: {base_url}")
lines.append(f"- Auto-start mode: {str(auto_start).lower()}")
lines.append(f"- Overall: {overall} ({ok_count}/{total})")
lines.append("")
lines.append("## Scope Baseline")
lines.append("- Pilot scope freeze: `docs/gates/mvp-pilot-baseline.md`")
lines.append("- Deferred scope backlog: `docs/roadmap/05-wave2-backlog.md`")
lines.append("")
lines.append("## Checks")
for item in checks:
marker = "PASS" if item.ok else "FAIL"
lines.append(f"- [{marker}] {item.name}: {item.details}")
lines.append("")
lines.append("## Artifacts")
if artifacts:
for key, value in artifacts.items():
lines.append(f"- {key}: {value}")
else:
lines.append("- none")
lines.append("")
lines.append("## Next Action")
lines.append("- If all checks passed, run `scripts/uat_dry_run.py` and prepare the manual UAT session.")
lines.append("")
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text("\n".join(lines), encoding="utf-8")
async def main() -> None:
parser = argparse.ArgumentParser(description="UAT preflight checker for pilot session")
parser.add_argument("--base-url", default="http://localhost:8080", help="Gateway base URL")
parser.add_argument("--auto-start", action="store_true", help="Auto-start local services for preflight")
parser.add_argument(
"--output",
default="",
help="Report output path (default: docs/uat/evidence/preflight_<timestamp>.md)",
)
parser.add_argument("--keep-data", action="store_true", help="Keep temporary auto-start data directory")
args = parser.parse_args()
output = Path(args.output) if args.output else DEFAULT_EVIDENCE_DIR / f"preflight_{now_compact()}.md"
output = output if output.is_absolute() else (ROOT / output).resolve()
processes: list[subprocess.Popen] = []
run_data_dir = DATA_ROOT / f"run_{int(time.time() * 1000)}"
base_url = args.base_url
checks: list[CheckResult] = []
artifacts: dict[str, Any] = {}
try:
if args.auto_start:
processes = await start_services_sequential(run_data_dir)
base_url = "http://127.0.0.1:48080"
checks, artifacts = await run_checks(base_url)
write_report(output, base_url, checks, artifacts, args.auto_start)
ok = all(item.ok for item in checks) and len(checks) > 0
print(f"UAT preflight report: {output}")
print(f"Result: {'PASS' if ok else 'FAIL'}")
sys.exit(0 if ok else 1)
finally:
stop_services(processes)
if args.auto_start and not args.keep_data:
try:
if run_data_dir.exists():
shutil.rmtree(run_data_dir)
except Exception:
pass
if __name__ == "__main__":
asyncio.run(main())