484 lines
16 KiB
JavaScript
484 lines
16 KiB
JavaScript
const TARGET_SAMPLE_RATE = 16000;
|
|
|
|
const startBtn = document.getElementById("startBtn");
|
|
const stopBtn = document.getElementById("stopBtn");
|
|
const resetBtn = document.getElementById("resetBtn");
|
|
const statusEl = document.getElementById("status");
|
|
const partialLine = document.getElementById("partialLine");
|
|
const messagesEl = document.getElementById("messages");
|
|
const canvas = document.getElementById("waveform");
|
|
const canvasCtx = canvas.getContext("2d");
|
|
const levelEl = document.getElementById("level");
|
|
const muteLabel = document.getElementById("muteLabel");
|
|
|
|
let socket;
|
|
let mediaStream;
|
|
let audioContext;
|
|
let sourceNode;
|
|
let processorNode;
|
|
let isRunning = false;
|
|
let isStopping = false;
|
|
let muteMic = false;
|
|
let currentAssistantLine = null;
|
|
let currentAssistantText = "";
|
|
let currentAssistantAudioChunks = 0;
|
|
let currentAssistantTtsFailed = false;
|
|
let currentAssistantTtsMessage = "";
|
|
let lastErrorMessage = "";
|
|
let playbackCursor = 0;
|
|
let pendingUnmuteTimer = null;
|
|
let encodedPlaybackChain = Promise.resolve();
|
|
let pendingEncodedAudioCount = 0;
|
|
const activeEncodedAudios = new Set();
|
|
|
|
drawIdleWave();
|
|
|
|
startBtn.addEventListener("click", startCall);
|
|
stopBtn.addEventListener("click", stopCall);
|
|
resetBtn.addEventListener("click", resetSession);
|
|
|
|
async function startCall() {
|
|
lastErrorMessage = "";
|
|
isStopping = false;
|
|
startBtn.disabled = true;
|
|
setStatus("connecting", "busy");
|
|
|
|
try {
|
|
socket = new WebSocket(`${location.protocol === "https:" ? "wss" : "ws"}://${location.host}/ws`);
|
|
socket.binaryType = "arraybuffer";
|
|
socket.addEventListener("message", handleServerMessage);
|
|
socket.addEventListener("close", () => {
|
|
if (isRunning) stopCall(lastErrorMessage || "Звонок остановлен.");
|
|
if (!statusEl.classList.contains("error")) setStatus("offline", "");
|
|
});
|
|
socket.addEventListener("error", () => {
|
|
lastErrorMessage = "Ошибка WebSocket.";
|
|
setStatus("socket error", "error");
|
|
partialLine.textContent = lastErrorMessage;
|
|
addMessage("system", lastErrorMessage);
|
|
});
|
|
|
|
await waitForSocketOpen(socket);
|
|
await waitForServerReady(socket);
|
|
|
|
mediaStream = await navigator.mediaDevices.getUserMedia({
|
|
audio: {
|
|
echoCancellation: true,
|
|
noiseSuppression: true,
|
|
autoGainControl: true,
|
|
channelCount: 1,
|
|
},
|
|
});
|
|
|
|
audioContext = new AudioContext();
|
|
await audioContext.resume();
|
|
|
|
sourceNode = audioContext.createMediaStreamSource(mediaStream);
|
|
processorNode = audioContext.createScriptProcessor(4096, 1, 1);
|
|
processorNode.onaudioprocess = handleAudioProcess;
|
|
sourceNode.connect(processorNode);
|
|
processorNode.connect(audioContext.destination);
|
|
|
|
isRunning = true;
|
|
stopBtn.disabled = false;
|
|
resetBtn.disabled = false;
|
|
setStatus("live", "live");
|
|
partialLine.textContent = "Слушаю...";
|
|
socket.send(JSON.stringify({ type: "start_greeting" }));
|
|
} catch (error) {
|
|
lastErrorMessage = error.message || "Не удалось начать звонок.";
|
|
await cleanupAfterFailedStart();
|
|
startBtn.disabled = false;
|
|
stopBtn.disabled = true;
|
|
resetBtn.disabled = true;
|
|
setStatus("error", "error");
|
|
partialLine.textContent = lastErrorMessage;
|
|
addMessage("system", lastErrorMessage);
|
|
}
|
|
}
|
|
|
|
async function stopCall(message = "Звонок остановлен.") {
|
|
if (isStopping) return;
|
|
isStopping = true;
|
|
isRunning = false;
|
|
muteMic = false;
|
|
updateMuteLabel();
|
|
|
|
if (processorNode) {
|
|
processorNode.disconnect();
|
|
processorNode.onaudioprocess = null;
|
|
}
|
|
if (sourceNode) sourceNode.disconnect();
|
|
if (mediaStream) mediaStream.getTracks().forEach((track) => track.stop());
|
|
if (socket && socket.readyState === WebSocket.OPEN) socket.close();
|
|
if (audioContext && audioContext.state !== "closed") await audioContext.close();
|
|
|
|
processorNode = null;
|
|
sourceNode = null;
|
|
mediaStream = null;
|
|
audioContext = null;
|
|
socket = null;
|
|
currentAssistantLine = null;
|
|
currentAssistantText = "";
|
|
currentAssistantAudioChunks = 0;
|
|
currentAssistantTtsFailed = false;
|
|
currentAssistantTtsMessage = "";
|
|
playbackCursor = 0;
|
|
pendingEncodedAudioCount = 0;
|
|
activeEncodedAudios.forEach((audio) => {
|
|
audio.pause();
|
|
audio.src = "";
|
|
});
|
|
activeEncodedAudios.clear();
|
|
encodedPlaybackChain = Promise.resolve();
|
|
|
|
startBtn.disabled = false;
|
|
stopBtn.disabled = true;
|
|
resetBtn.disabled = true;
|
|
partialLine.textContent = message;
|
|
levelEl.style.width = "0%";
|
|
drawIdleWave();
|
|
isStopping = false;
|
|
}
|
|
|
|
function resetSession() {
|
|
messagesEl.innerHTML = "";
|
|
currentAssistantLine = null;
|
|
currentAssistantText = "";
|
|
currentAssistantAudioChunks = 0;
|
|
currentAssistantTtsFailed = false;
|
|
currentAssistantTtsMessage = "";
|
|
partialLine.textContent = "Контекст очищен.";
|
|
if (socket && socket.readyState === WebSocket.OPEN) {
|
|
socket.send(JSON.stringify({ type: "reset" }));
|
|
}
|
|
}
|
|
|
|
function handleAudioProcess(event) {
|
|
const input = event.inputBuffer.getChannelData(0);
|
|
drawWave(input);
|
|
updateLevel(input);
|
|
|
|
if (!isRunning || muteMic || !socket || socket.readyState !== WebSocket.OPEN) return;
|
|
|
|
const downsampled = downsample(input, audioContext.sampleRate, TARGET_SAMPLE_RATE);
|
|
const pcm = floatTo16BitPcm(downsampled);
|
|
socket.send(pcm);
|
|
}
|
|
|
|
function handleServerMessage(event) {
|
|
const data = JSON.parse(event.data);
|
|
|
|
if (data.type === "ready") {
|
|
setStatus(data.model || "ready", "live");
|
|
} else if (data.type === "stt_ready") {
|
|
partialLine.textContent = "Слушаю...";
|
|
} else if (data.type === "stt_partial") {
|
|
partialLine.textContent = data.text || "Слушаю...";
|
|
} else if (data.type === "user_final") {
|
|
currentAssistantLine = null;
|
|
addMessage("user", data.text);
|
|
partialLine.textContent = "Думаю...";
|
|
} else if (data.type === "assistant_started") {
|
|
muteMic = true;
|
|
updateMuteLabel();
|
|
currentAssistantLine = addMessage("assistant", "");
|
|
currentAssistantText = "";
|
|
currentAssistantAudioChunks = 0;
|
|
currentAssistantTtsFailed = false;
|
|
currentAssistantTtsMessage = "";
|
|
setStatus("answering", "busy");
|
|
} else if (data.type === "assistant_delta") {
|
|
if (!currentAssistantLine) currentAssistantLine = addMessage("assistant", "");
|
|
currentAssistantLine.textContent += data.text;
|
|
currentAssistantText += data.text || "";
|
|
scrollTranscript();
|
|
} else if (data.type === "assistant_text_done") {
|
|
currentAssistantText = data.text || currentAssistantText;
|
|
partialLine.textContent = "Озвучиваю...";
|
|
} else if (data.type === "tts_audio") {
|
|
currentAssistantAudioChunks += 1;
|
|
if ((data.format || "").startsWith("mp3") || (data.mime_type || "").includes("mpeg")) {
|
|
playEncodedAudio(data.audio, data.mime_type || "audio/mpeg");
|
|
} else {
|
|
playPcmAudio(data.audio, data.sample_rate || TARGET_SAMPLE_RATE);
|
|
}
|
|
} else if (data.type === "tts_failed") {
|
|
currentAssistantTtsFailed = true;
|
|
currentAssistantTtsMessage = data.message || "ElevenLabs TTS не прислал аудио.";
|
|
partialLine.textContent = "ElevenLabs TTS недоступен, включаю резервную озвучку.";
|
|
addMessage("system", "ElevenLabs TTS недоступен, включаю резервную озвучку.");
|
|
} else if (data.type === "assistant_done") {
|
|
const fallbackStarted = maybeSpeakWithBrowserFallback();
|
|
if (!fallbackStarted) {
|
|
scheduleUnmuteAfterPlayback();
|
|
setStatus(currentAssistantTtsFailed ? "tts fallback" : "live", currentAssistantTtsFailed ? "busy" : "live");
|
|
partialLine.textContent = currentAssistantTtsFailed ? currentAssistantTtsMessage : "Слушаю...";
|
|
}
|
|
} else if (data.type === "reset_done") {
|
|
partialLine.textContent = "Контекст очищен.";
|
|
if (isRunning && socket && socket.readyState === WebSocket.OPEN) {
|
|
socket.send(JSON.stringify({ type: "start_greeting" }));
|
|
}
|
|
} else if (data.type === "error") {
|
|
lastErrorMessage = data.message || "Ошибка.";
|
|
partialLine.textContent = lastErrorMessage;
|
|
addMessage("system", lastErrorMessage);
|
|
setStatus("error", "error");
|
|
}
|
|
}
|
|
|
|
function addMessage(role, text) {
|
|
const line = document.createElement("div");
|
|
line.className = `line ${role}`;
|
|
line.textContent = text;
|
|
messagesEl.appendChild(line);
|
|
scrollTranscript();
|
|
return line;
|
|
}
|
|
|
|
function scrollTranscript() {
|
|
const transcript = document.querySelector(".transcript");
|
|
transcript.scrollTop = transcript.scrollHeight;
|
|
}
|
|
|
|
function setStatus(text, className) {
|
|
statusEl.textContent = text;
|
|
statusEl.className = `status-pill ${className || ""}`.trim();
|
|
}
|
|
|
|
function maybeSpeakWithBrowserFallback() {
|
|
if (currentAssistantAudioChunks > 0) return false;
|
|
if (!currentAssistantText.trim()) return false;
|
|
if (!("speechSynthesis" in window) || typeof SpeechSynthesisUtterance === "undefined") {
|
|
partialLine.textContent = currentAssistantTtsFailed
|
|
? currentAssistantTtsMessage
|
|
: "TTS не прислал аудио.";
|
|
return false;
|
|
}
|
|
|
|
window.speechSynthesis.cancel();
|
|
const utterance = new SpeechSynthesisUtterance(currentAssistantText);
|
|
utterance.lang = "ru-RU";
|
|
utterance.rate = 1.04;
|
|
utterance.pitch = 1;
|
|
utterance.onend = finishBrowserFallbackSpeech;
|
|
utterance.onerror = finishBrowserFallbackSpeech;
|
|
|
|
muteMic = true;
|
|
updateMuteLabel();
|
|
setStatus("browser voice", "busy");
|
|
partialLine.textContent = currentAssistantTtsFailed
|
|
? "Резервная озвучка включена: ElevenLabs TTS не дал аудио."
|
|
: "Резервная озвучка включена.";
|
|
window.speechSynthesis.speak(utterance);
|
|
return true;
|
|
}
|
|
|
|
function finishBrowserFallbackSpeech() {
|
|
muteMic = false;
|
|
updateMuteLabel();
|
|
setStatus("live", "live");
|
|
partialLine.textContent = "Слушаю...";
|
|
}
|
|
|
|
function updateMuteLabel() {
|
|
muteLabel.textContent = muteMic ? "muted" : "open";
|
|
}
|
|
|
|
function waitForSocketOpen(ws) {
|
|
return new Promise((resolve, reject) => {
|
|
ws.addEventListener("open", resolve, { once: true });
|
|
ws.addEventListener("error", reject, { once: true });
|
|
});
|
|
}
|
|
|
|
function waitForServerReady(ws) {
|
|
return new Promise((resolve, reject) => {
|
|
const timeout = window.setTimeout(() => {
|
|
cleanup();
|
|
reject(new Error("Сервер не ответил."));
|
|
}, 6000);
|
|
|
|
function cleanup() {
|
|
window.clearTimeout(timeout);
|
|
ws.removeEventListener("message", onMessage);
|
|
ws.removeEventListener("close", onClose);
|
|
ws.removeEventListener("error", onError);
|
|
}
|
|
|
|
function onMessage(event) {
|
|
const data = JSON.parse(event.data);
|
|
if (data.type === "ready") {
|
|
cleanup();
|
|
resolve(data);
|
|
} else if (data.type === "error") {
|
|
cleanup();
|
|
reject(new Error(data.message || "Сервер вернул ошибку."));
|
|
}
|
|
}
|
|
|
|
function onClose() {
|
|
cleanup();
|
|
reject(new Error("WebSocket закрыт."));
|
|
}
|
|
|
|
function onError() {
|
|
cleanup();
|
|
reject(new Error("Ошибка WebSocket."));
|
|
}
|
|
|
|
ws.addEventListener("message", onMessage);
|
|
ws.addEventListener("close", onClose);
|
|
ws.addEventListener("error", onError);
|
|
});
|
|
}
|
|
|
|
async function cleanupAfterFailedStart() {
|
|
if (processorNode) {
|
|
processorNode.disconnect();
|
|
processorNode.onaudioprocess = null;
|
|
}
|
|
if (sourceNode) sourceNode.disconnect();
|
|
if (mediaStream) mediaStream.getTracks().forEach((track) => track.stop());
|
|
if (socket && socket.readyState === WebSocket.OPEN) socket.close();
|
|
if (audioContext) await audioContext.close();
|
|
|
|
processorNode = null;
|
|
sourceNode = null;
|
|
mediaStream = null;
|
|
audioContext = null;
|
|
socket = null;
|
|
}
|
|
|
|
function downsample(buffer, inputRate, outputRate) {
|
|
if (outputRate === inputRate) return buffer;
|
|
const ratio = inputRate / outputRate;
|
|
const newLength = Math.round(buffer.length / ratio);
|
|
const result = new Float32Array(newLength);
|
|
let offsetResult = 0;
|
|
let offsetBuffer = 0;
|
|
|
|
while (offsetResult < result.length) {
|
|
const nextOffsetBuffer = Math.round((offsetResult + 1) * ratio);
|
|
let accumulator = 0;
|
|
let count = 0;
|
|
for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i += 1) {
|
|
accumulator += buffer[i];
|
|
count += 1;
|
|
}
|
|
result[offsetResult] = accumulator / Math.max(count, 1);
|
|
offsetResult += 1;
|
|
offsetBuffer = nextOffsetBuffer;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
function floatTo16BitPcm(floatBuffer) {
|
|
const output = new Int16Array(floatBuffer.length);
|
|
for (let i = 0; i < floatBuffer.length; i += 1) {
|
|
const sample = Math.max(-1, Math.min(1, floatBuffer[i]));
|
|
output[i] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
|
|
}
|
|
return output.buffer;
|
|
}
|
|
|
|
function playPcmAudio(base64Audio, sampleRate) {
|
|
if (!audioContext) return;
|
|
if (audioContext.state === "suspended") audioContext.resume();
|
|
const bytes = Uint8Array.from(atob(base64Audio), (char) => char.charCodeAt(0));
|
|
const samples = new Int16Array(bytes.buffer);
|
|
const audioBuffer = audioContext.createBuffer(1, samples.length, sampleRate);
|
|
const channel = audioBuffer.getChannelData(0);
|
|
for (let i = 0; i < samples.length; i += 1) {
|
|
channel[i] = samples[i] / 32768;
|
|
}
|
|
|
|
const source = audioContext.createBufferSource();
|
|
source.buffer = audioBuffer;
|
|
source.connect(audioContext.destination);
|
|
const startAt = Math.max(audioContext.currentTime + 0.03, playbackCursor);
|
|
source.start(startAt);
|
|
playbackCursor = startAt + audioBuffer.duration;
|
|
}
|
|
|
|
function playEncodedAudio(base64Audio, mimeType) {
|
|
pendingEncodedAudioCount += 1;
|
|
encodedPlaybackChain = encodedPlaybackChain.then(
|
|
() =>
|
|
new Promise((resolve) => {
|
|
const bytes = Uint8Array.from(atob(base64Audio), (char) => char.charCodeAt(0));
|
|
const blob = new Blob([bytes], { type: mimeType });
|
|
const url = URL.createObjectURL(blob);
|
|
const audio = new Audio(url);
|
|
activeEncodedAudios.add(audio);
|
|
|
|
function cleanup() {
|
|
URL.revokeObjectURL(url);
|
|
activeEncodedAudios.delete(audio);
|
|
pendingEncodedAudioCount = Math.max(0, pendingEncodedAudioCount - 1);
|
|
if (pendingEncodedAudioCount === 0 && muteMic) {
|
|
scheduleUnmuteAfterPlayback();
|
|
}
|
|
resolve();
|
|
}
|
|
|
|
audio.onended = cleanup;
|
|
audio.onerror = cleanup;
|
|
audio.play().catch(cleanup);
|
|
}),
|
|
);
|
|
}
|
|
|
|
function scheduleUnmuteAfterPlayback() {
|
|
if (pendingEncodedAudioCount > 0) return;
|
|
if (pendingUnmuteTimer) window.clearTimeout(pendingUnmuteTimer);
|
|
const remainingMs = audioContext ? Math.max(0, (playbackCursor - audioContext.currentTime) * 1000) : 0;
|
|
pendingUnmuteTimer = window.setTimeout(() => {
|
|
muteMic = false;
|
|
updateMuteLabel();
|
|
}, remainingMs + 120);
|
|
}
|
|
|
|
function updateLevel(buffer) {
|
|
let sum = 0;
|
|
for (let i = 0; i < buffer.length; i += 1) sum += buffer[i] * buffer[i];
|
|
const rms = Math.sqrt(sum / buffer.length);
|
|
const percent = Math.min(100, Math.round(rms * 260));
|
|
levelEl.style.width = `${percent}%`;
|
|
}
|
|
|
|
function drawIdleWave() {
|
|
canvasCtx.fillStyle = "#101820";
|
|
canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
|
|
canvasCtx.strokeStyle = "#2f9e80";
|
|
canvasCtx.lineWidth = 3;
|
|
canvasCtx.beginPath();
|
|
const mid = canvas.height / 2;
|
|
for (let x = 0; x < canvas.width; x += 1) {
|
|
const y = mid + Math.sin(x / 28) * 10 + Math.sin(x / 83) * 18;
|
|
if (x === 0) canvasCtx.moveTo(x, y);
|
|
else canvasCtx.lineTo(x, y);
|
|
}
|
|
canvasCtx.stroke();
|
|
}
|
|
|
|
function drawWave(buffer) {
|
|
canvasCtx.fillStyle = "#101820";
|
|
canvasCtx.fillRect(0, 0, canvas.width, canvas.height);
|
|
canvasCtx.strokeStyle = muteMic ? "#d9563f" : "#36c28f";
|
|
canvasCtx.lineWidth = 3;
|
|
canvasCtx.beginPath();
|
|
const slice = Math.max(1, Math.floor(buffer.length / canvas.width));
|
|
const mid = canvas.height / 2;
|
|
|
|
for (let x = 0; x < canvas.width; x += 1) {
|
|
const sample = buffer[x * slice] || 0;
|
|
const y = mid + sample * mid * 0.84;
|
|
if (x === 0) canvasCtx.moveTo(x, y);
|
|
else canvasCtx.lineTo(x, y);
|
|
}
|
|
canvasCtx.stroke();
|
|
}
|