digiops/call-center
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Telegram channel alerts for gateway errors

Browse Source
This commit is contained in:
Magzhan Zhumabayev
2026-05-11 13:35:52 +05:00
parent a14fdea34f
commit 000ac984b6
4 changed files with 283 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+9
View File
@@ -133,6 +133,15 @@ AI_VOICE_MAX_CONTEXT_SEGMENTS=12
AI_VOICE_HANDOFF_TIMEOUT_SECONDS=8
AI_VOICE_RUNTIME_TRUSTED_SERVICE_SUBJECTS=svc:ai-voice-runtime
# Gateway error alerts (Telegram)
ERROR_TELEGRAM_ALERTS=0
ERROR_TELEGRAM_ALERT_BOT_TOKEN=<telegram-bot-token>
ERROR_TELEGRAM_ALERT_CHANNEL_ID=<telegram-channel-id-or-username>
# Alternative: ERROR_TELEGRAM_ALERT_CHAT_IDS=<id1>,<id2>
ERROR_TELEGRAM_ALERT_MIN_STATUS=500
ERROR_TELEGRAM_ALERT_COOLDOWN_SECONDS=60
ERROR_TELEGRAM_ALERT_IGNORE_PATHS=/health
# Event bus (Track 8)
EVENT_BUS_ENABLED=0
EVENT_BUS_URL=amqp://guest:guest@localhost:5672/
.env.production.template
+9
View File
@@ -115,6 +115,15 @@ AI_VOICE_MAX_CONTEXT_SEGMENTS=12
AI_VOICE_HANDOFF_TIMEOUT_SECONDS=8
AI_VOICE_RUNTIME_TRUSTED_SERVICE_SUBJECTS=svc:ai-voice-runtime
# Gateway error alerts (Telegram)
ERROR_TELEGRAM_ALERTS=0
ERROR_TELEGRAM_ALERT_BOT_TOKEN=<telegram-bot-token>
ERROR_TELEGRAM_ALERT_CHANNEL_ID=<telegram-channel-id-or-username>
# Alternative: ERROR_TELEGRAM_ALERT_CHAT_IDS=<id1>,<id2>
ERROR_TELEGRAM_ALERT_MIN_STATUS=500
ERROR_TELEGRAM_ALERT_COOLDOWN_SECONDS=60
ERROR_TELEGRAM_ALERT_IGNORE_PATHS=/health
# Event bus compatibility (Track 8 baseline)
EVENT_BUS_ENABLED=1
EVENT_BUS_URL=amqp://guest:guest@<rabbitmq-host>:5672/
gateway/app.py
+167 -7
View File
@@ -1,8 +1,10 @@
from __future__ import annotations
import asyncio
import logging
import os
from pathlib import Path
from datetime import datetime, timezone
from typing import Any
import httpx
@@ -14,6 +16,135 @@ from services.shared.models import HealthResponse
app = FastAPI(title="api-gateway", version="1.0.0")
logger = logging.getLogger("api-gateway.proxy")
_ERROR_ALERT_SENT_AT: dict[str, float] = {}
def _env_flag(name: str, default: bool = False) -> bool:
value = os.getenv(name)
if value is None:
return default
return value.strip().lower() in {"1", "true", "yes", "on"}
def _env_int(name: str, default: int) -> int:
value = os.getenv(name)
if value is None:
return default
try:
parsed = int(value)
except ValueError:
return default
return max(1, parsed)
def _error_alert_enabled() -> bool:
return _env_flag("ERROR_TELEGRAM_ALERTS", False)
def _error_alert_bot_token() -> str:
return os.getenv("ERROR_TELEGRAM_ALERT_BOT_TOKEN", "").strip()
def _error_alert_chat_ids() -> list[str]:
raw = os.getenv("ERROR_TELEGRAM_ALERT_CHAT_IDS", "").strip()
if not raw:
raw = os.getenv("ERROR_TELEGRAM_ALERT_CHANNEL_ID", "").strip()
if not raw:
raw = os.getenv("ERROR_TELEGRAM_ALERT_CHANNEL", "").strip()
return [item.strip() for item in raw.split(",") if item.strip()]
def _error_alert_min_status() -> int:
return max(1, _env_int("ERROR_TELEGRAM_ALERT_MIN_STATUS", 500))
def _error_alert_cooldown_seconds() -> int:
return _env_int("ERROR_TELEGRAM_ALERT_COOLDOWN_SECONDS", 30)
def _error_alert_ignored_paths() -> tuple[str, ...]:
raw = os.getenv("ERROR_TELEGRAM_ALERT_IGNORE_PATHS", "/health").strip()
paths = tuple(item.strip() for item in raw.split(",") if item.strip())
return paths if paths else ("/health",)
def _error_alert_service_name() -> str:
return os.getenv("ERROR_TELEGRAM_ALERT_SERVICE", "api-gateway").strip() or "api-gateway"
def _error_alert_bot_api_base() -> str:
return os.getenv("ERROR_TELEGRAM_ALERT_BOT_API_BASE", "https://api.telegram.org").rstrip("/")
def _error_alert_should_alert_path(path: str) -> bool:
ignore = _error_alert_ignored_paths()
return not any(path == item or path.startswith(f"{item}/") for item in ignore)
def _error_alert_key(service: str, status: int, path: str, method: str) -> str:
return f"{service}:{status}:{method}:{path}"
def _error_alert_is_throttled(key: str) -> bool:
now = datetime.now(tz=timezone.utc).timestamp()
cooldown = _error_alert_cooldown_seconds()
previous = _ERROR_ALERT_SENT_AT.get(key, 0.0)
if now < previous:
return True
_ERROR_ALERT_SENT_AT[key] = now + cooldown
return False
async def _send_telegram_alert(message: str) -> None:
chat_ids = _error_alert_chat_ids()
bot_token = _error_alert_bot_token()
if not bot_token or not chat_ids:
return
url = f"{_error_alert_bot_api_base()}/bot{bot_token}/sendMessage"
payload = {"text": message}
async with httpx.AsyncClient(timeout=3.0) as client:
for chat_id in chat_ids:
payload["chat_id"] = chat_id
try:
response = await client.post(url, json=payload)
except httpx.RequestError:
logger.exception("Unable to send Telegram error notification")
return
if response.status_code >= 300:
logger.warning(
"Telegram error notification failed: status=%s body=%s",
response.status_code,
response.text,
)
def _queue_telegram_error_alert(service: str, method: str, path: str, status: int, detail: str) -> None:
if not _error_alert_enabled():
return
if not _error_alert_should_alert_path(path):
return
if not _error_alert_bot_token() or not _error_alert_chat_ids():
return
if _error_alert_is_throttled(_error_alert_key(service, status, path, method)):
return
timestamp = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
message = (
f"🚨 [{service}] {status}\n"
f"method: {method}\n"
f"path: {path}\n"
f"time: {timestamp}\n"
f"detail: {detail}"
)
try:
loop = asyncio.get_running_loop()
except RuntimeError:
logger.warning("No running loop to queue telegram error notification")
return
loop.create_task(_send_telegram_alert(message))
_LEGAL_HTML = {
"privacy": """<!doctype html>
@@ -169,13 +300,6 @@ SERVICE_PATH_PREFIX_EXEMPTIONS = {
}
def _env_flag(name: str, default: bool = False) -> bool:
value = os.getenv(name)
if value is None:
return default
return value.strip().lower() in {"1", "true", "yes", "on"}
def _resolve_service_path(service: str, path: str) -> str:
normalized = path.lstrip("/")
prefix = SERVICE_PATH_PREFIXES.get(service, "").strip("/")
@@ -188,6 +312,35 @@ def _resolve_service_path(service: str, path: str) -> str:
return f"{prefix}/{normalized}" if normalized else prefix
@app.middleware("http")
async def error_notification_middleware(request: Request, call_next):
try:
response = await call_next(request)
except Exception as exc:
status_code = 500
if isinstance(exc, HTTPException):
status_code = exc.status_code
if status_code >= _error_alert_min_status():
_queue_telegram_error_alert(
service=_error_alert_service_name(),
method=request.method,
path=str(request.url.path),
status=status_code,
detail=f"{type(exc).__name__}: {exc}",
)
raise
if response.status_code >= _error_alert_min_status():
_queue_telegram_error_alert(
service=_error_alert_service_name(),
method=request.method,
path=str(request.url.path),
status=response.status_code,
detail=f"HTTP {response.status_code} from {request.method} {request.url.path}",
)
return response
@app.get("/health", response_model=HealthResponse)
def health() -> HealthResponse:
return HealthResponse(status="ok", service="api-gateway")
@@ -414,6 +567,13 @@ async def _forward(method: str, service: str, path: str, request: Request) -> Re
headers=headers,
)
except httpx.RequestError:
_queue_telegram_error_alert(
service=_error_alert_service_name(),
method=method,
path=f"/proxy/{service}/{path}",
status=502,
detail=f"Upstream service unavailable: service={service} base_url={base}",
)
logger.warning(
"Upstream service unavailable: service=%s base_url=%s method=%s path=%s",
service,
tests/test_gateway_proxy.py
+98
View File
@@ -209,3 +209,101 @@ def test_gateway_returns_bad_gateway_when_upstream_unavailable(monkeypatch):
'detail': 'Upstream service unavailable',
'service': 'sales',
}
def test_gateway_queues_error_alert_for_proxy_unreachable(monkeypatch):
async def request(self, method, url, params=None, content=None, headers=None): # noqa: ANN001, ANN201
raise gateway_module.httpx.ConnectError("All connection attempts failed")
alerts: list[dict] = []
def queue_alert(service: str, method: str, path: str, status: int, detail: str) -> None: # noqa: ARG001
alerts.append({
'service': service,
'method': method,
'path': path,
'status': status,
'detail': detail,
})
monkeypatch.setattr(DummyAsyncClient, 'request', request)
monkeypatch.setattr(gateway_module.httpx, 'AsyncClient', DummyAsyncClient)
monkeypatch.setattr(gateway_module, '_queue_telegram_error_alert', queue_alert)
monkeypatch.setenv('ERROR_TELEGRAM_ALERTS', '1')
monkeypatch.setenv('ERROR_TELEGRAM_ALERT_BOT_TOKEN', 'token')
monkeypatch.setenv('ERROR_TELEGRAM_ALERT_CHAT_IDS', '999')
client = TestClient(gateway_module.app)
response = client.get('/proxy/sales/pipelines')
assert response.status_code == 502
assert alerts == [
{
'service': 'api-gateway',
'method': 'GET',
'path': '/proxy/sales/pipelines',
'status': 502,
'detail': 'Upstream service unavailable: service=sales base_url=http://localhost:8020',
},
]
def test_gateway_queues_error_alert_for_channel_alias(monkeypatch):
async def request(self, method, url, params=None, content=None, headers=None): # noqa: ANN001, ANN201
raise gateway_module.httpx.ConnectError("All connection attempts failed")
alerts: list[dict] = []
def queue_alert(service: str, method: str, path: str, status: int, detail: str) -> None: # noqa: ARG001
alerts.append({
'service': service,
'method': method,
'path': path,
'status': status,
'detail': detail,
})
monkeypatch.setattr(DummyAsyncClient, 'request', request)
monkeypatch.setattr(gateway_module.httpx, 'AsyncClient', DummyAsyncClient)
monkeypatch.setattr(gateway_module, '_queue_telegram_error_alert', queue_alert)
monkeypatch.setenv('ERROR_TELEGRAM_ALERTS', '1')
monkeypatch.setenv('ERROR_TELEGRAM_ALERT_BOT_TOKEN', 'token')
monkeypatch.setenv('ERROR_TELEGRAM_ALERT_CHANNEL_ID', '@my_alert_channel')
client = TestClient(gateway_module.app)
client.get('/proxy/sales/pipelines')
assert alerts
assert alerts[0]['status'] == 502
def test_gateway_queues_error_alert_for_internal_exception(monkeypatch):
async def broken_forward(method: str, service: str, path: str, request: object) -> None: # noqa: ANN001, ANN201
raise ValueError('proxy exploded')
alerts: list[dict] = []
def queue_alert(service: str, method: str, path: str, status: int, detail: str) -> None: # noqa: ARG001
alerts.append({
'service': service,
'method': method,
'path': path,
'status': status,
'detail': detail,
})
monkeypatch.setattr(gateway_module, '_forward', broken_forward)
monkeypatch.setattr(gateway_module, '_queue_telegram_error_alert', queue_alert)
monkeypatch.setenv('ERROR_TELEGRAM_ALERTS', '1')
monkeypatch.setenv('ERROR_TELEGRAM_ALERT_BOT_TOKEN', 'token')
monkeypatch.setenv('ERROR_TELEGRAM_ALERT_CHAT_IDS', '999')
client = TestClient(gateway_module.app, raise_server_exceptions=False)
response = client.get('/proxy/sales/pipelines')
assert response.status_code == 500
assert alerts
assert alerts[0]['service'] == 'api-gateway'
assert alerts[0]['status'] == 500
assert alerts[0]['path'] == '/proxy/sales/pipelines'