Next.js の LLM ストリーミングを外部 API なしで Playwright E2E テストしてみた

import { defineConfig } from "@playwright/test";
import path from "node:path";

const preloadPath = path.resolve("tests/e2e/mock-openrouter.cjs");

export default defineConfig({
    testDir: "./tests/e2e",
    fullyParallel: false,
    workers: 1,
    reporter: "line",
    outputDir: "test-results",
    use: {
        baseURL: "http://127.0.0.1:3100",
        browserName: "chromium",
        channel: "chrome",
        headless: true,
        trace: "retain-on-failure",
    },
    webServer: {
        command: "npm run dev -- --hostname 127.0.0.1 --port 3100",
        url: "http://127.0.0.1:3100",
        reuseExistingServer: false,
        timeout: 120_000,
        env: {
            ...process.env,
            NODE_ENV: "development",
            NODE_OPTIONS: `--require=${preloadPath}`,
            LLM_E2E_FETCH_MOCK: "1",
            OPENROUTER_API_KEY: "e2e-placeholder-not-a-secret",
        },
    },
});

"use strict";

if (process.env.NODE_ENV === "production") {
    throw new Error("E2E fetch mock must never run with NODE_ENV=production");
}
if (process.env.LLM_E2E_FETCH_MOCK !== "1") {
    throw new Error("E2E fetch mock requires an explicit local-only opt-in");
}

const OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions";
const originalFetch = globalThis.fetch;
const encoder = new TextEncoder();

function delayedStream(steps) {
    return new ReadableStream({
        start(controller) {
            let elapsed = 0;
            for (const step of steps) {
                elapsed += step.afterMs;
                setTimeout(() => {
                    if (step.error) {
                        controller.error(new Error(step.error));
                    } else if (step.close) {
                        controller.close();
                    } else {
                        controller.enqueue(encoder.encode(step.data));
                    }
                }, elapsed);
            }
        },
    });
}

function streamResponse(steps) {
    return new Response(delayedStream(steps), {
        status: 200,
        headers: { "Content-Type": "text/event-stream; charset=utf-8" },
    });
}

globalThis.fetch = async function controlledFetch(input, init) {
    if (String(input) !== OPENROUTER_URL) {
        return originalFetch(input, init);
    }

    const request = JSON.parse(String(init?.body ?? "{}"));
    const messages = Array.isArray(request.messages) ? request.messages : [];
    const lastUser = [...messages].reverse().find((message) => message?.role === "user");
    const scenario = String(lastUser?.content ?? "");

    if (scenario.includes("[[e2e:success]]")) {
        return streamResponse([
            { afterMs: 100, data: "data: {\"choices\":[{\"delta\":{\"content\":\"第一段階\"}}]}\r\n\r\n" },
            { afterMs: 250, data: "data: this-is-not-json\n\n" },
            { afterMs: 650, data: "data: {\"choices\":[{\"delta\":{\"content\":\"第二段階\"}}]}\n\n" },
            { afterMs: 100, data: "data: [DONE]\n\n" },
            { afterMs: 10, close: true },
        ]);
    }

    if (scenario.includes("[[e2e:pre-error]]")) {
        return new Response(JSON.stringify({ error: { message: "synthetic rate limit" } }), {
            status: 429,
            headers: { "Content-Type": "application/json", "Retry-After": "3" },
        });
    }

    if (scenario.includes("[[e2e:disconnect]]")) {
        return streamResponse([
            { afterMs: 100, data: "data: {\"choices\":[{\"delta\":{\"content\":\"部分応答\"}}]}\n\n" },
            { afterMs: 250, error: "synthetic mid-stream disconnect" },
        ]);
    }

    if (scenario.includes("[[e2e:missing-done]]")) {
        return streamResponse([
            { afterMs: 100, data: "data: {\"choices\":[{\"delta\":{\"content\":\"完了前\"}}]}\n\n" },
            { afterMs: 100, close: true },
        ]);
    }

    return new Response(JSON.stringify({ error: { message: "unknown E2E scenario" } }), {
        status: 400,
        headers: { "Content-Type": "application/json" },
    });
};

import { expect, test } from "@playwright/test";

async function send(page: import("@playwright/test").Page, prompt: string) {
    await page.goto("/");
    await page.getByTestId("prompt-input").fill(prompt);
    await page.getByTestId("send-button").click();
}

test("renders token-equivalent deltas incrementally and completes despite a malformed event", async ({ page }) => {
    await send(page, "[[e2e:success]] 公開可能な合成入力");

    const assistant = page.getByTestId("assistant-message").last();
    await expect(page.getByTestId("stream-status")).toBeVisible();
    await expect(assistant).toHaveText("第一段階");
    await expect(assistant).not.toContainText("第二段階");
    await expect(assistant).toHaveText("第一段階第二段階");
    await expect(page.getByTestId("stream-status")).toBeHidden();
    await expect(page.getByTestId("error-panel")).toHaveCount(0);
});

test("shows a pre-stream provider error separately from a stream interruption", async ({ page }) => {
    await send(page, "[[e2e:pre-error]] 公開可能な合成入力");

    await expect(page.getByTestId("error-panel")).toContainText("レート制限中");
    await expect(page.getByText("サーバからの推奨待機秒: 3s")).toBeVisible();
    await expect(page.getByTestId("assistant-message").last()).toContainText("レート制限中");
    await expect(page.getByTestId("stream-status")).toBeHidden();
});

test("retains partial output and identifies a mid-stream disconnect", async ({ page }) => {
    await send(page, "[[e2e:disconnect]] 公開可能な合成入力");

    const assistant = page.getByTestId("assistant-message").last();
    await expect(assistant).toContainText("部分応答");
    await expect(page.getByTestId("error-panel")).toContainText("途中で切断");
    await expect(assistant).toContainText("途中で切断");
    await expect(page.getByTestId("retry-button")).toBeVisible();
    await expect(page.getByTestId("stream-status")).toBeHidden();
});

test("treats EOF without the completion event as an incomplete stream", async ({ page }) => {
    await send(page, "[[e2e:missing-done]] 公開可能な合成入力");

    const assistant = page.getByTestId("assistant-message").last();
    await expect(assistant).toContainText("完了前");
    await expect(page.getByTestId("error-panel")).toContainText("完了通知前");
    await expect(page.getByTestId("stream-status")).toBeHidden();
});

import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import path from "node:path";
import test from "node:test";

const preload = path.resolve("tests/e2e/mock-openrouter.cjs");

function requireMock(environment) {
    return spawnSync(process.execPath, ["--require", preload, "--eval", "void 0"], {
        cwd: process.cwd(),
        encoding: "utf8",
        env: { ...process.env, ...environment },
    });
}

test("the E2E fetch mock refuses production", () => {
    const result = requireMock({ NODE_ENV: "production", LLM_E2E_FETCH_MOCK: "1" });
    assert.notEqual(result.status, 0);
    assert.match(result.stderr, /must never run with NODE_ENV=production/);
});
test("the E2E fetch mock requires explicit local opt-in", () => {
    const result = requireMock({ NODE_ENV: "development", LLM_E2E_FETCH_MOCK: "0" });
    assert.notEqual(result.status, 0);
    assert.match(result.stderr, /requires an explicit local-only opt-in/);
});

test("the E2E fetch mock can preload only with both development guards", () => {
    const result = requireMock({ NODE_ENV: "development", LLM_E2E_FETCH_MOCK: "1" });
    assert.equal(result.status, 0, result.stderr);
});