Muse Gadget SDKにAmazon Pollyで日本語読み上げを追加する ― M5Stack Core2 for AWS

diff --git a/esp32/components/muse/Kconfig b/esp32/components/muse/Kconfig
index cab98f5..b7dcb77 100644
--- a/esp32/components/muse/Kconfig
+++ b/esp32/components/muse/Kconfig
@@ -202,6 +202,27 @@ menu "Muse"
             memory and preview decoding shares CPU time with voice and Link.
             Leave disabled to build without camera support.

+    config MUSE_HTTP_TTS
+        bool "Speak replies using an HTTPS TTS API"
+        depends on MUSE_HATCH
+        default n
+        help
+            POST reply text to a TTS API and play its MP3 response.
+            Requires PSRAM. Failed requests fall back to captions.
+
+    config MUSE_HTTP_TTS_URL
+        string "TTS API HTTPS endpoint"
+        depends on MUSE_HTTP_TTS
+        default ""
+
+    config MUSE_HTTP_TTS_API_KEY
+        string "TTS API key (x-api-key)"
+        depends on MUSE_HTTP_TTS
+        default ""
+        help
+            Stored in the local sdkconfig and compiled firmware. Never commit
+            or share these files with the key included.
+
     config MUSE_CJK_FONT
         bool "Show Chinese and Japanese in captions"
         depends on MUSE_ENABLED

diff --git a/esp32/components/muse/muse_tts_http.h b/esp32/components/muse/muse_tts_http.h
new file mode 100644
--- /dev/null
+++ b/esp32/components/muse/muse_tts_http.h
@@ -0,0 +1,14 @@
+#pragma once
+#include <stddef.h>
+#include <stdint.h>
+
+struct muse_tts_result_t {
+    uint32_t generation;
+    int message;
+    uint8_t *audio;  // caller frees; null on failure
+    size_t length;
+};
+// Called only from the chat task. The worker never touches chat state.
+bool muse_tts_http_start(const char *text, uint32_t generation, int message, size_t capacity);
+bool muse_tts_http_poll(muse_tts_result_t *result);
+void muse_tts_http_cancel(void);
diff --git a/esp32/components/muse/muse_tts_http.cpp b/esp32/components/muse/muse_tts_http.cpp
new file mode 100644
--- /dev/null
+++ b/esp32/components/muse/muse_tts_http.cpp
@@ -0,0 +1,156 @@
+#include "sdkconfig.h"
+#if CONFIG_MUSE_HTTP_TTS
+#include "muse_tts_http.h"
+#include <atomic>
+#include <cstdlib>
+#include <cstring>
+#include <strings.h>
+#include "cJSON.h"
+#include "esp_crt_bundle.h"
+#include "esp_heap_caps.h"
+#include "esp_http_client.h"
+#include "esp_log.h"
+#include "esp_timer.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/queue.h"
+#include "freertos/task.h"
+
+static const char *TAG = "muse_tts_http";
+struct request_t { char *body; uint32_t generation; int message; size_t capacity; uint32_t ticket; };
+static QueueHandle_t s_requests, s_results;
+static bool s_busy;
+static std::atomic<uint32_t> s_ticket{0};
+struct download_t {
+    uint8_t *audio;
+    size_t length, capacity;
+    uint32_t ticket;
+    int64_t deadline;
+    bool failed, audio_type;
+};
+
+static esp_err_t on_http(esp_http_client_event_t *event)
+{
+    auto *d = static_cast<download_t *>(event->user_data);
+    if (d->ticket != s_ticket.load() || esp_timer_get_time() > d->deadline) {
+        d->failed = true;
+        return ESP_FAIL;
+    }
+    if (event->event_id == HTTP_EVENT_ON_HEADER &&
+        !strcasecmp(event->header_key, "Content-Type")) {
+        const char *v = event->header_value;
+        d->audio_type = !strncasecmp(v, "audio/mpeg", 10) && (v[10] == 0 || v[10] == ';');
+    }
+    if (event->event_id == HTTP_EVENT_ON_DATA) {
+        if (event->data_len < 0 || static_cast<size_t>(event->data_len) > d->capacity - d->length) {
+            d->failed = true;
+            return ESP_FAIL;
+        }
+        if (esp_http_client_get_status_code(event->client) != 200 || !d->audio_type) {
+            d->failed = true;
+            return ESP_FAIL;
+        }
+        memcpy(d->audio + d->length, event->data, event->data_len);
+        d->length += event->data_len;
+    }
+    return ESP_OK;
+}
+
+static void worker(void *)
+{
+    request_t request;
+    for (;;) {
+        if (xQueueReceive(s_requests, &request, portMAX_DELAY) != pdTRUE) continue;
+        muse_tts_result_t result{request.generation, request.message, nullptr, 0};
+        download_t d{};
+        d.capacity = request.capacity;
+        d.ticket = request.ticket;
+        d.deadline = esp_timer_get_time() + 30000000LL;
+        d.audio = static_cast<uint8_t *>(heap_caps_malloc(d.capacity, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
+        esp_http_client_config_t config{};
+        config.url = CONFIG_MUSE_HTTP_TTS_URL;
+        config.method = HTTP_METHOD_POST;
+        config.timeout_ms = 25000;
+        config.crt_bundle_attach = esp_crt_bundle_attach;
+        config.disable_auto_redirect = true; // Never forward the key to another endpoint.
+        config.event_handler = on_http;
+        config.user_data = &d;
+        auto client = d.audio ? esp_http_client_init(&config) : nullptr;
+        if (client && request.ticket == s_ticket.load()) {
+            bool configured = esp_http_client_set_header(client, "Content-Type", "application/json") == ESP_OK &&
+                esp_http_client_set_header(client, "Accept", "audio/mpeg") == ESP_OK &&
+                esp_http_client_set_header(client, "x-api-key", CONFIG_MUSE_HTTP_TTS_API_KEY) == ESP_OK &&
+                esp_http_client_set_post_field(client, request.body, strlen(request.body)) == ESP_OK;
+            esp_err_t err = configured ? esp_http_client_perform(client) : ESP_FAIL;
+            int status = esp_http_client_get_status_code(client);
+            if (err == ESP_OK && status == 200 && !d.failed && d.audio_type && d.length &&
+                request.ticket == s_ticket.load() && esp_http_client_is_complete_data_received(client)) {
+                result.audio = d.audio;
+                result.length = d.length;
+                d.audio = nullptr;
+            } else {
+                ESP_LOGW(TAG, "TTS request failed (HTTP %d, %s)", status, esp_err_to_name(err));
+            }
+        }
+        if (client) esp_http_client_cleanup(client);
+        free(d.audio);
+        free(request.body);
+        // One in-flight request; the owner drains this before starting another.
+        xQueueSend(s_results, &result, portMAX_DELAY);
+    }
+}
+
+bool muse_tts_http_start(const char *text, uint32_t generation, int message, size_t capacity)
+{
+    if (s_busy || !text || !text[0] || !capacity ||
+        strncmp(CONFIG_MUSE_HTTP_TTS_URL, "https://", 8) || !CONFIG_MUSE_HTTP_TTS_API_KEY[0]) return false;
+    if (!s_requests) {
+        s_requests = xQueueCreate(1, sizeof(request_t));
+        s_results = xQueueCreate(1, sizeof(muse_tts_result_t));
+        if (!s_requests || !s_results || xTaskCreate(worker, "muse_tts", 8192, nullptr, 4, nullptr) != pdPASS) {
+            if (s_requests) vQueueDelete(s_requests);
+            if (s_results) vQueueDelete(s_results);
+            s_requests = s_results = nullptr;
+            return false;
+        }
+    }
+    // Caption storage can end mid-UTF-8 sequence; trim only that partial suffix.
+    char copy[1024];
+    size_t n = strnlen(text, sizeof(copy) - 1);
+    memcpy(copy, text, n);
+    copy[n] = 0;
+    if (n) {
+        size_t start = n - 1;
+        while (start && (static_cast<unsigned char>(copy[start]) & 0xc0) == 0x80) --start;
+        unsigned char lead = copy[start];
+        size_t width = lead < 0x80 ? 1 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4;
+        if (n - start < width) copy[start] = 0;
+    }
+    // The API accepts at most 1,000 Unicode characters (not UTF-8 bytes).
+    size_t chars = 0;
+    for (size_t j = 0; copy[j]; ++j) {
+        if ((static_cast<unsigned char>(copy[j]) & 0xc0) != 0x80 && ++chars > 1000) {
+            copy[j] = 0;
+            break;
+        }
+    }
+    cJSON *json = cJSON_CreateObject();
+    if (!json) return false;
+    bool added = cJSON_AddStringToObject(json, "text", copy) != nullptr;
+    char *body = added ? cJSON_PrintUnformatted(json) : nullptr;
+    cJSON_Delete(json);
+    if (!body) return false;
+    request_t request{body, generation, message, capacity, ++s_ticket};
+    if (xQueueSend(s_requests, &request, 0) != pdTRUE) { free(body); return false; }
+    s_busy = true;
+    return true;
+}
+
+bool muse_tts_http_poll(muse_tts_result_t *result)
+{
+    if (!s_results || xQueueReceive(s_results, result, 0) != pdTRUE) return false;
+    s_busy = false;
+    return true;
+}
+
+void muse_tts_http_cancel(void) { ++s_ticket; }
+#endif

diff --git a/esp32/components/muse/muse_chat_session.cpp b/esp32/components/muse/muse_chat_session.cpp
index 2866f5d..50a9560 100644
--- a/esp32/components/muse/muse_chat_session.cpp
+++ b/esp32/components/muse/muse_chat_session.cpp
@@ -31,7 +31,8 @@
  * A turn has no explicit end event; like Sidekick, it settles once every
  * message is done and nothing has arrived for a few seconds.
  *
- * Everything network-side runs on one task. The voice task talks to it through
+ * The Muse connection runs on one task; optional HTTPS TTS runs on a worker
+ * and hands completed audio back through a queue. The voice task talks through
  * a command queue, a stream buffer of mic audio, an event queue (captions) and
  * a stream buffer of reply audio. A generation number tags each turn so that
  * events and audio of a cancelled turn are dropped.
@@ -72,6 +73,9 @@ extern "C" {
 #include "muse_wifi.h"
 }
 #include "muse_chat_priv.h"
+#if CONFIG_MUSE_HTTP_TTS
+#include "muse_tts_http.h"
+#endif

 #include <xplat/noise/core/ClientSession.h>
 #include <xplat/noise/core/PsaCryptoBackend.h>
@@ -960,6 +964,9 @@ static void turn_reset_streams(void)

 static void turn_finish(void)
 {
+#if CONFIG_MUSE_HTTP_TTS
+    muse_tts_http_cancel();
+#endif
     turn_reset_streams();
     s_turn.phase = P_IDLE;
     s_turn.dict_id = s_turn.chat_id = 0;
@@ -1508,38 +1515,66 @@ static void on_chat_ack(stream_t *s)

 /* ---- Turn: speech ---- */

-static void start_tts(void)
+static void show_tts_as_text(int i)
 {
-    if (s_turn.tts_msg >= 0) {
-        return;
+    msg_t &m = s_turn.msgs[i];
+    m.pcm_start = s_turn.pcm_out;
+    m.pcm_frames = (uint32_t)(m.len * MIC_RATE / TEXT_CHARS_PER_S);
+    m.tts = TTS_ACTIVE;
+    s_turn.tts_msg = i;
+    s_turn.silent = true;
+    s_turn.mp3_len = 0;
+    s_turn.mp3_ended = false;
+    show_reply_start(m);
+}
+
+#if CONFIG_MUSE_HTTP_TTS
+static void poll_http_tts(void)
+{
+    muse_tts_result_t result{};
+    if (!muse_tts_http_poll(&result)) return;
+    if (s_turn.phase == P_WAIT_REPLY && result.generation == s_turn.gen &&
+        result.generation == s_gen.load() && result.message == s_turn.tts_msg) {
+        if (result.audio && result.length <= MP3_BUF && muse_settings_speaker_on()) {
+            memcpy(s_turn.mp3, result.audio, result.length);
+            s_turn.mp3_len = result.length;
+            s_turn.mp3_ended = true;
+            mark(M_MP3);
+            ESP_LOGI(TAG, "TTS API audio received (%u bytes)", (unsigned)result.length);
+        } else {
+            ESP_LOGW(TAG, "TTS unavailable; showing captions");
+            show_tts_as_text(result.message);
+        }
     }
+    free(result.audio);
+}
+#endif
+
+static void start_tts(void)
+{
+    if (s_turn.tts_msg >= 0) return;
     for (int i = 0; i < s_turn.nmsgs; i++) {
         msg_t &m = s_turn.msgs[i];
-        if (m.tts != TTS_QUEUED) {
-            continue;
+        if (m.tts != TTS_QUEUED) continue;
+#if CONFIG_MUSE_HTTP_TTS
+        if (s_turn.texts && muse_settings_speaker_on() && muse_settings_volume() > 0 &&
+            muse_tts_http_start(s_turn.texts + i * TEXT_MAX, s_turn.gen, i, MP3_BUF)) {
+            m.pcm_start = s_turn.pcm_out;
+            m.pcm_frames = 0;
+            m.tts = TTS_ACTIVE;
+            s_turn.tts_msg = i;
+            s_turn.silent = false;
+            s_turn.mp3_len = 0;
+            s_turn.mp3_ended = false;
+            s_turn.kbps = 0;
+            s_turn.down_rate = 0;
+            mp3dec_init(&s_turn.dec);
+            mark(M_TTS);
+            show_reply_start(m);
+            return;
         }
-        /*
-         * Replies are text, shown at reading pace: silence in place of speech
-         * paces the captions and ends the turn. To speak them instead, send
-         * the message's text (s_turn.texts + i * TEXT_MAX, if texts was
-         * allocated; up to TEXT_MAX - 1 bytes) to a TTS API of your choice and
-         * play the MP3 it returns. In place of the silence below: keep
-         * m.tts = TTS_ACTIVE and s_turn.tts_msg = i, set s_turn.silent = false,
-         * m.pcm_start = s_turn.pcm_out, m.pcm_frames = 0, s_turn.mp3_len = 0,
-         * s_turn.mp3_ended = false, s_turn.kbps = 0, s_turn.down_rate = 0 and
-         * mp3dec_init(&s_turn.dec). Then, on this task, pass the MP3 to
-         * tts_data() as it arrives (it buffers up to MP3_BUF and drops the
-         * rest, so hold off while it's full) and set s_turn.mp3_ended at the
-         * end. decode() plays it at the speaker's volume, captions following,
-         * and finishes the message once it's drained.
-         */
-        m.pcm_start = s_turn.pcm_out;
-        m.pcm_frames = (uint32_t)(m.len * MIC_RATE / TEXT_CHARS_PER_S);
-        m.tts = TTS_ACTIVE;
-        s_turn.tts_msg = i;
-        s_turn.silent = true;
-        ESP_LOGI(TAG, "showing message %s (%u chars)", m.id, (unsigned)m.len);
-        show_reply_start(m);
+#endif
+        show_tts_as_text(i);
         return;
     }
 }
@@ -1648,6 +1683,10 @@ static void decode(void)
         m.pcm_frames = s_turn.pcm_out - m.pcm_start + rest;
     }
     if (s_turn.mp3_ended && !s_turn.mp3_len) {
+        if (s_turn.kbps == 0) {
+            show_tts_as_text(s_turn.tts_msg);
+            return;
+        }
         m.pcm_frames = s_turn.pcm_out - m.pcm_start;
         m.tts = TTS_FINISHED;
         s_turn.tts_msg = -1;
@@ -1961,6 +2000,9 @@ static void hatch_task(void *arg)
                 handle(cmd);
             }
         }
+#if CONFIG_MUSE_HTTP_TTS
+        poll_http_tts();
+#endif
         if (!s_connected) {
             if (!muse_wifi_connected()) {
                 s_auto_next_us = 0;

diff --git a/esp32/components/muse/CMakeLists.txt b/esp32/components/muse/CMakeLists.txt
index d20c459..09c487b 100644
--- a/esp32/components/muse/CMakeLists.txt
+++ b/esp32/components/muse/CMakeLists.txt
@@ -43,7 +43,7 @@ if(CONFIG_MUSE_ENABLED)
         "muse_link.c"
         "muse_ble.c")
     if(CONFIG_MUSE_HATCH)
-        list(APPEND srcs "muse_chat.c" "muse_account_api.c" "muse_chat_session.cpp")
+        list(APPEND srcs "muse_chat.c" "muse_account_api.c" "muse_chat_session.cpp" "muse_tts_http.cpp")
         list(APPEND embed "test_reply.mp3")   # muse_hatch_mp3_selftest
     else()
         list(APPEND srcs "muse_chat_link.c")   # text replies over Link's session
@@ -133,7 +133,7 @@ if(CONFIG_MUSE_ENABLED)
 elseif(CONFIG_HOMEHUB_VOICE)
     # The Voice PE board uses just the Hatch voice session; the project's main/
     # voice_muse_chat.c stands in for the Muse settings, Wi-Fi and Link hooks.
-    list(APPEND srcs "muse_chat.c" "muse_account_api.c" "muse_chat_session.cpp" "muse_chat_text.c" "muse_text.c")
+    list(APPEND srcs "muse_chat.c" "muse_account_api.c" "muse_chat_session.cpp" "muse_tts_http.cpp" "muse_chat_text.c" "muse_text.c")
     list(APPEND embed "test_reply.mp3")
 endif()

diff --git a/esp32/tests/test_muse_tts_http.py b/esp32/tests/test_muse_tts_http.py
new file mode 100644
--- /dev/null
+++ b/esp32/tests/test_muse_tts_http.py
@@ -0,0 +1,99 @@
+"""Host tests of the production HTTPS response guards and playback handoff."""
+from pathlib import Path
+import os
+import shlex
+import subprocess
+import tempfile
+import unittest
+ROOT = Path(__file__).resolve().parents[1]
+
+class HttpTts(unittest.TestCase):
+    def test_guards_and_playback(self):
+        http = (ROOT / 'components/muse/muse_tts_http.cpp').read_text()
+        chat = (ROOT / 'components/muse/muse_chat_session.cpp').read_text()
+        guard = http[http.index('struct download_t'):http.index('static void worker')]
+        playback = chat[chat.index('static void show_tts_as_text'):chat.index('static void tts_data')]
+        code = r'''
+#include <atomic>
+#include <cassert>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <strings.h>
+#include "muse_tts_http.h"
+#define CONFIG_MUSE_HTTP_TTS 1
+#define ESP_LOGI(...) ((void)0)
+#define ESP_LOGW(...) ((void)0)
+#define MIC_RATE 16000
+#define TEXT_CHARS_PER_S 16
+#define TEXT_MAX 1024
+#define MP3_BUF 128
+using esp_err_t = int;
+const int ESP_OK = 0, ESP_FAIL = -1, HTTP_EVENT_ON_HEADER = 1, HTTP_EVENT_ON_DATA = 2;
+struct esp_http_client_event_t { void *user_data; int event_id; const char *header_key; const char *header_value; int data_len; void *client; const char *data; };
+static std::atomic<uint32_t> s_ticket{1};
+static int64_t clock_us;
+static int status = 200;
+static int64_t esp_timer_get_time() { return clock_us; }
+static int esp_http_client_get_status_code(void *) { return status; }
+''' + guard + r'''
+enum { P_IDLE, P_WAIT_REPLY, TTS_QUEUED, TTS_ACTIVE, M_MP3, M_TTS };
+struct msg_t { unsigned pcm_start, pcm_frames, len; int tts; };
+struct {
+ int phase, tts_msg, nmsgs, kbps, down_rate, dec; uint32_t gen, pcm_out;
+ bool silent, mp3_ended; uint8_t mp3[MP3_BUF]; size_t mp3_len; char *texts; msg_t msgs[2];
+} s_turn;
+static std::atomic<uint32_t> s_gen{7};
+static bool speaker = true, accept = true, pending;
+static int requests;
+static muse_tts_result_t result;
+static bool muse_settings_speaker_on() { return speaker; }
+static int muse_settings_volume() { return 80; }
+static void mark(int) {}
+static void show_reply_start(const msg_t &) {}
+static void mp3dec_init(int *) {}
+bool muse_tts_http_start(const char *, uint32_t, int, size_t) { ++requests; return accept; }
+bool muse_tts_http_poll(muse_tts_result_t *r) { if (!pending) return false; *r = result; pending = false; return true; }
+''' + playback + r'''
+int main() {
+ uint8_t buf[4] = {};
+ download_t d{buf, 0, sizeof(buf), 1, 100, false, false};
+ esp_http_client_event_t e{&d, HTTP_EVENT_ON_HEADER, "Content-Type", "audio/mpeg", 0, nullptr, "abcd"};
+ assert(on_http(&e) == ESP_OK && d.audio_type);
+ e.event_id = HTTP_EVENT_ON_DATA; e.data_len = 4;
+ assert(on_http(&e) == ESP_OK && d.length == 4 && !memcmp(buf, "abcd", 4));
+ e.data_len = 1;
+ assert(on_http(&e) == ESP_FAIL && d.length == 4);
+ d.length = 0; d.failed = false; d.audio_type = false;
+ assert(on_http(&e) == ESP_FAIL && !d.length);
+ d.audio_type = true; status = 403;
+ assert(on_http(&e) == ESP_FAIL && !d.length);
+ status = 302; assert(on_http(&e) == ESP_FAIL && !d.length);
+ status = 200; s_ticket = 2;
+ assert(on_http(&e) == ESP_FAIL && !d.length);
+ s_ticket = 1; clock_us = 101;
+ assert(on_http(&e) == ESP_FAIL && !d.length);
+ char text[] = "reply";
+ s_turn.phase = P_WAIT_REPLY; s_turn.gen = 7; s_turn.tts_msg = -1;
+ s_turn.nmsgs = 1; s_turn.texts = text; s_turn.msgs[0].tts = TTS_QUEUED;
+ start_tts(); assert(requests == 1 && !s_turn.silent && s_turn.tts_msg == 0);
+ start_tts(); assert(requests == 1);
+ auto complete = [](uint32_t gen, bool ok) {
+   result = {gen, 0, ok ? static_cast<uint8_t *>(malloc(4)) : nullptr, ok ? 4u : 0u};
+   if (result.audio) memcpy(result.audio, "mp3!", 4);
+   pending = true; poll_http_tts();
+ };
+ complete(6, true); assert(!s_turn.mp3_len);
+ complete(7, true); assert(s_turn.mp3_len == 4 && s_turn.mp3_ended);
+ complete(7, false); assert(s_turn.silent && !s_turn.mp3_len);
+ s_turn.tts_msg = -1; s_turn.msgs[0].tts = TTS_QUEUED; speaker = false;
+ start_tts(); assert(requests == 1 && s_turn.silent);
+ speaker = true; s_turn.tts_msg = -1; s_turn.msgs[0].tts = TTS_QUEUED; accept = false;
+ start_tts(); assert(s_turn.silent);
+}
+'''
+        with tempfile.TemporaryDirectory() as tmp:
+            source, binary = Path(tmp) / 'test.cpp', Path(tmp) / 'test'
+            source.write_text(code)
+            subprocess.run([*shlex.split(os.environ.get('CXX', 'c++')), '-std=c++17', '-Wall', '-Wextra', '-Werror', '-I', str(ROOT / 'components/muse'), str(source), '-o', str(binary)], check=True)
+            subprocess.run([str(binary)], check=True)