I tried putting together a scenario using Twilio ConversationRelay with a restaurant as the subject, where "first-line phone responses are handled by AI"

// Generates TwiML including ConversationRelay.

import twilio from 'twilio';

const { VoiceResponse } = twilio.twiml;

export const DEFAULT_VOICE = 'ja-JP-Chirp3-HD-Kore';

export function buildConnectTwiml({
  wsUrl,
  actionUrl,
  welcomeGreeting = 'お電話ありがとうございます。AI が一次対応いたします。ご用件をお話しください。',
  voice = DEFAULT_VOICE,
}) {
  const response = new VoiceResponse();
  const connect = actionUrl ? response.connect({ action: actionUrl }) : response.connect();
  connect.conversationRelay({
    url: wsUrl,
    welcomeGreeting,
    language: 'ja-JP',
    ttsProvider: 'Google',
    voice,
    transcriptionProvider: 'Google',
    // Enable interruption while AI is speaking (barge-in).
    interruptible: 'speech',
    reportInputDuringAgentSpeech: 'speech',
  });
  return response.toString();
}

// Builds the FAQ knowledge for a fictional demo restaurant and the AI's system prompt.

export const RESTAURANT = {
  name: 'ビストロ クラスメソッド',
  hours: '11:00〜15:00 と 17:00〜22:00 (ラストオーダー 21:00)。水曜定休。',
  location: '東京都千代田区丸の内 1-1-1。JR 東京駅 丸の内南口から徒歩 3 分。',
  budget: 'ランチはお一人 1,200 円前後、ディナーはお一人 4,000 円前後です。',
  reservation: 'お電話またはこの AI で承ります。ご希望の日時と人数をお知らせください。',
  parking: '専用駐車場はありません。近隣のコインパーキングをご利用ください。',
};

export function buildSystemPrompt(faq = RESTAURANT) {
  return [
    `あなたは飲食店「${faq.name}」の電話一次対応 AI です。電話越しにお客様と話しています。`,
    '',
    '店舗情報:',
    `- 営業時間: ${faq.hours}`,
    `- 場所: ${faq.location}`,
    `- 予算: ${faq.budget}`,
    `- 予約: ${faq.reservation}`,
    `- 駐車場: ${faq.parking}`,
    '',
    '話し方の決まり:',
    '- 音声で読み上げられます。記号や markdown は使わないでください。アスタリスクや番号付き・箇条書きの記号を出力してはいけません。',
    '- 英語やアルファベットをそのまま書かず、カタカナで言ってください。',
    '- 自然で聞き取りやすい丁寧語で話してください。不自然な言い回し (たとえば「ご件」) は使わないでください。',
    '- 一度の返答は 1 文から 2 文までにし、短く答えてください。',
    '',
    '対応の方針:',
    '- 店舗情報で答えられることは、その範囲で正確に答えてください。',
    '- 次のいずれかに当てはまる場合は、返答の一番最後に [[ESCALATE]] という印だけを付けてください。担当者に引き継ぐ合図です。',
    '  - 店舗情報にない内容で、確かな回答ができないとき',
    '  - クレームや強い不満を示されたとき',
    '  - 契約・解約・返金など、意思決定を伴う判断が必要なとき',
    '- [[ESCALATE]] を付けるときは、その前に「担当者から折り返しご連絡します」と一言だけ伝えてください。',
  ].join('\n');
}

// Calls Amazon Bedrock's Claude Haiku 4.5 with ConverseStream.

import { BedrockRuntimeClient, ConverseStreamCommand } from '@aws-sdk/client-bedrock-runtime';

import { ESCALATION_MARKER } from './escalation.js';
import { sanitizeForSpeech, collapseWhitespace } from './sanitize.js';

export function createBedrockClient({ region }) {
  return new BedrockRuntimeClient({ region });
}

// Streaming version. Passes generated tokens to onToken incrementally to start playback sooner.
// Holds back the end by the marker length and removes it to prevent escalation markers from leaking into playback.
export async function streamReply({ client, modelId, systemPrompt, history, onToken, maxTokens = 300, temperature = 0.3 }) {
  const command = new ConverseStreamCommand({
    modelId,
    system: [{ text: systemPrompt }],
    messages: history.map((m) => ({ role: m.role, content: [{ text: m.content }] })),
    inferenceConfig: { maxTokens, temperature },
  });
  const res = await client.send(command);

  const marker = ESCALATION_MARKER;
  let full = '';
  let hold = '';

  const emit = (raw) => {
    const speak = sanitizeForSpeech(raw.split(marker).join(''));
    if (speak && onToken) onToken(speak);
  };

  for await (const ev of res?.stream ?? []) {
    const delta = ev?.contentBlockDelta?.delta?.text;
    if (!delta) continue;
    full += delta;
    hold += delta;
    // Leave the marker length at the end to prevent markers at the end from leaking into playback.
    if (hold.length > marker.length) {
      const cut = hold.length - marker.length;
      emit(hold.slice(0, cut));
      hold = hold.slice(cut);
    }
  }
  emit(hold);

  const escalate = full.includes(marker);
  const fullText = collapseWhitespace(sanitizeForSpeech(full.split(marker).join('')));
  return { fullText, escalate };
}

// Handles the ConversationRelay WebSocket protocol conversation logic.
// Injects sending (emit), LLM streaming, notifications, and SMS to make unit testing possible.

export function createConversation(deps) {
  const session = { history: [], from: null, to: null, callSid: null, lastAssistant: null };

  async function onMessage(msg) {
    if (!msg || typeof msg !== 'object') return {};

    if (msg.type === 'setup') {
      session.from = msg.from ?? null;
      session.to = msg.to ?? null;
      session.callSid = msg.callSid ?? null;
      return {};
    }

    if (msg.type === 'prompt') {
      session.history.push({ role: 'user', content: msg.voicePrompt });

      const { fullText, escalate } = await deps.streamReply({
        client: deps.client,
        modelId: deps.modelId,
        systemPrompt: deps.systemPrompt,
        history: session.history,
        onToken: (token) => deps.emit({ type: 'text', token, last: false }),
      });

      // Notify end of playback turn.
      deps.emit({ type: 'text', token: '', last: true });

      session.history.push({ role: 'assistant', content: fullText });
      session.lastAssistant = fullText;

      if (escalate) {
        const lastUser = [...session.history].reverse().find((m) => m.role === 'user');
        await deps.notifyEscalation({
          from: session.from,
          to: session.to,
          request: lastUser?.content ?? '',
          reply: fullText,
          transcript: session.history,
          time: deps.now ? deps.now() : null,
        });
        deps.emit({
          type: 'end',
          handoffData: JSON.stringify({ reasonCode: 'live-agent-handoff', reason: fullText }),
        });
      }

      return { escalate, reply: fullText };
    }

    return {};
  }

  async function onClose() {
    if (deps.sendSms && session.from && session.lastAssistant) {
      await deps.sendSms({ to: session.from, summary: session.lastAssistant });
    }
    if (deps.sendEmail && session.lastAssistant) {
      await deps.sendEmail({
        caller: session.from,
        called: session.to,
        summary: session.lastAssistant,
        transcript: session.history,
        time: deps.now ? deps.now() : null,
      });
    }
  }

  return { session, onMessage, onClose };
}

// Fastify server. Handles TwiML-returning HTTP and ConversationRelay WebSocket in a single process.

import Fastify from 'fastify';
import formbody from '@fastify/formbody';
import websocket from '@fastify/websocket';
import twilioSdk from 'twilio';

import { loadConfig } from './config.js';
import { buildConnectTwiml } from './twiml.js';
import { buildSystemPrompt } from './faq.js';
import { createBedrockClient, streamReply } from './bedrock.js';
import { notifyEscalation } from './slack.js';
import { sendSms } from './sms.js';
import { buildSummaryEmail, sendSummaryEmail } from './email.js';
import { createConversation } from './ws-handler.js';

export function buildServer(overrides = {}) {
  const config = overrides.config ?? loadConfig();
  const fastify = Fastify({ logger: overrides.logger ?? true });

  fastify.register(formbody);
  fastify.register(websocket);

  const systemPrompt = buildSystemPrompt();
  const bedrockClient = overrides.bedrockClient ?? createBedrockClient({ region: config.awsRegion });
  const twilioClient =
    overrides.twilioClient ??
    (config.twilioAccountSid && config.twilioAuthToken
      ? twilioSdk(config.twilioAccountSid, config.twilioAuthToken)
      : null);

  const wsUrl = () => (config.publicBaseUrl ? config.publicBaseUrl.replace(/^http/, 'ws') : '') + '/ws';
  const actionUrl = () => (config.publicBaseUrl ? config.publicBaseUrl : '') + '/voice/action';

  fastify.get('/', async () => 'ok');

  const voiceHandler = async (request, reply) => {
    const twiml = buildConnectTwiml({ wsUrl: wsUrl(), actionUrl: actionUrl() });
    reply.header('content-type', 'text/xml').send(twiml);
  };
  fastify.get('/voice', voiceHandler);
  fastify.post('/voice', voiceHandler);

  // action callback when escalation ends. Since AI has already delivered the message in voice, only disconnect here.
  fastify.post('/voice/action', async (request, reply) => {
    const response = new twilioSdk.twiml.VoiceResponse();
    response.hangup();
    reply.header('content-type', 'text/xml').send(response.toString());
  });

  fastify.register(async (instance) => {
    instance.get('/ws', { websocket: true }, (socket) => {
      const conversation = createConversation({
        client: bedrockClient,
        modelId: config.bedrockModelId,
        systemPrompt,
        streamReply,
        emit: (obj) => socket.send(JSON.stringify(obj)),
        now: () => new Date().toISOString(),
        notifyEscalation: (args) =>
          config.slackWebhookUrl ? notifyEscalation({ webhookUrl: config.slackWebhookUrl, ...args }) : Promise.resolve(false),
        sendSms: (twilioClient && config.twilioFromNumber)
          ? ({ to, summary }) =>
              sendSms({ client: twilioClient, from: config.twilioFromNumber, to, body: `本日はお電話ありがとうございました。${summary}` })
          : null,
        sendEmail: (config.sendgridApiKey && config.emailFrom && config.emailTo)
          ? async ({ caller, called, summary, transcript, time }) => {
              const { subject, body } = buildSummaryEmail({ from: caller, to: called, summary, transcript, time });
              return sendSummaryEmail({ apiKey: config.sendgridApiKey, fromEmail: config.emailFrom, toEmail: config.emailTo, subject, body });
            }
          : null,
      });

      socket.on('message', async (data) => {
        let msg;
        try {
          msg = JSON.parse(data.toString());
        } catch {
          return;
        }
        await conversation.onMessage(msg); // Responses are sent via emit inside onMessage
      });

      socket.on('close', () => {
        conversation.onClose().catch((err) => fastify.log.error(err));
      });
    });
  });

  return fastify;
}