developersIO
I tried E2E testing Next.js LLM streaming without external APIs using Playwright

I tried E2E testing Next.js LLM streaming without external APIs using Playwright

# Next.js LLM Streaming Browser E2E Testing Without External Provider Calls ## Architecture Overview ``` Browser (Playwright) → Next.js API Route → Mock Provider (test-only) ↓ ReadableStream → SSE → UI rendering ``` ## Project Structure ``` ├── app/ │ └── api/ │ └── chat/ │ └── route.ts # Production API route ├── lib/ │ └── llm/ │ ├── provider.ts # Provider interface │ ├── openai-provider.ts # Production provider │ └── provider-factory.ts # Factory (no test code) ├── tests/ │ ├── e2e/ │ │ └── streaming.spec.ts │ └── fixtures/ │ └── mock-provider.ts # Test-only provider ├── playwright.config.ts └── next.config.ts ``` ## Core Implementation ### Provider Interface ```typescript // lib/llm/provider.ts export interface LLMProvider { stream(prompt: string): ReadableStream<string>; } export interface StreamChunk { type: 'token' | 'done' | 'error'; content?: string; error?: string; } ``` ### Production Provider Factory ```typescript // lib/llm/provider-factory.ts import { LLMProvider } from './provider'; import { OpenAIProvider } from './openai-provider'; // Production code never references mock provider export function createProvider(): LLMProvider { return new OpenAIProvider(process.env.OPENAI_API_KEY!); } ``` ### API Route ```typescript // app/api/chat/route.ts import { NextRequest } from 'next/server'; import { createProvider } from '@/lib/llm/provider-factory'; export async function POST(req: NextRequest) { const { prompt } = await req.json(); // Allow provider injection via header (only in non-production) const providerOverride = req.headers.get('x-test-provider-url'); if (providerOverride && process.env.NODE_ENV !== 'production') { return proxyToTestProvider(prompt, providerOverride); } const provider = createProvider(); const encoder = new TextEncoder(); const stream = new ReadableStream({ async start(controller) { try { const tokenStream = provider.stream(prompt); const reader = tokenStream.getReader(); while (true) { const { done, value } = await reader.read(); if (done) break; const sseData = `data: ${JSON.stringify({ type: 'token', content: value })}\n\n`; controller.enqueue(encoder.encode(sseData)); } controller.enqueue( encoder.encode(`data: ${JSON.stringify({ type: 'done' })}\n\n`) ); } catch (error: any) { if (error.status === 429) { controller.enqueue( encoder.encode( `data: ${JSON.stringify({ type: 'error', error: 'rate_limited' })}\n\n` ) ); } else { controller.enqueue( encoder.encode( `data: ${JSON.stringify({ type: 'error', error: 'stream_failed' })}\n\n` ) ); } } finally { controller.close(); } }, }); return new Response(stream, { headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive', }, }); } async function proxyToTestProvider(prompt: string, providerUrl: string) { const response = await fetch(providerUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ prompt }), }); return new Response(response.body, { headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive', }, }); } ``` ## Test-Only Mock Provider Server ```typescript // tests/fixtures/mock-provider.ts import { createServer, IncomingMessage, ServerResponse } from 'http'; import { AddressInfo } from 'net'; export type MockScenario = | { type: 'success'; tokens: string[]; delayMs?: number } | { type: 'rate_limit' } | { type: 'mid_stream_disconnect'; tokens: string[]; disconnectAfter: number } | { type: 'missing_done'; tokens: string[] }; export class MockLLMProviderServer { private server: ReturnType<typeof createServer>; private scenario: MockScenario = { type: 'success', tokens: [] }; private requestLog: Array<{ prompt: string; timestamp: number }> = []; constructor() { this.server = createServer(this.handleRequest.bind(this)); } private async handleRequest(req: IncomingMessage, res: ServerResponse) { if (req.method !== 'POST') { res.writeHead(405); res.end(); return; } // Parse request body const body = await this.readBody(req); const { prompt } = JSON.parse(body); this.requestLog.push({ prompt, timestamp: Date.now() }); const scenario = this.scenario; switch (scenario.type) { case 'rate_limit': res.writeHead(429, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: 'rate_limited' })); break; case 'success': await this.streamTokens(res, scenario.tokens, scenario.delayMs ?? 50, true); break; case 'mid_stream_disconnect': await this.streamTokens( res, scenario.tokens.slice(0, scenario.disconnectAfter), 50, false // no done event ); res.destroy(); // Forcefully close connection break; case 'missing_done': await this.streamTokens(res, scenario.tokens, 50, false); // omit done res.end(); break; } } private async streamTokens( res: ServerResponse, tokens: string[], delayMs: number, sendDone: boolean ) { res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive', }); for (const token of tokens) { await this.delay(delayMs); res.write( `data: ${JSON.stringify({ type: 'token', content: token })}\n\n` ); } if (sendDone) { res.write(`data: ${JSON.stringify({ type: 'done' })}\n\n`); } res.end(); } private readBody(req: IncomingMessage): Promise<string> { return new Promise((resolve, reject) => { let body = ''; req.on('data', chunk => (body += chunk)); req.on('end', () => resolve(body)); req.on('error', reject); }); } private delay(ms: number): Promise<void> { return new Promise(resolve => setTimeout(resolve, ms)); } async start(): Promise<string> { return new Promise(resolve => { this.server.listen(0, '127.0.0.1', () => { const { port } = this.server.address() as AddressInfo; resolve(`http://127.0.0.1:${port}`); }); }); } async stop(): Promise<void> { return new Promise((resolve, reject) => { this.server.close(err => (err ? reject(err) : resolve())); }); } setScenario(scenario: MockScenario) { this.scenario = scenario; return this; } getRequestLog() { return [...this.requestLog]; } clearRequestLog() { this.requestLog = []; } } ``` ## UI Component ```typescript // app/components/ChatStream.tsx 'use client'; import { useState, useCallback } from 'react'; type StreamState = 'idle' | 'streaming' | 'done' | 'error' | 'rate_limited' | 'disconnected'; export function ChatStream() { const [text, setText] = useState(''); const [state, setState] = useState<StreamState>('idle'); const sendMessage = useCallback(async (prompt: string) => { setText(''); setState('streaming'); try { const response = await fetch('/api/chat', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ prompt }), }); if (!response.ok) { setState(response.status === 429 ? 'rate_limited' : 'error'); return; } const reader = response.body!.getReader(); const decoder = new TextDecoder(); let buffer = ''; while (true) { const { done, value } = await reader.read(); if (done) { // Stream ended without explicit done event if (state !== 'done') { setState('disconnected'); } break; } buffer += decoder.decode(value, { stream: true }); const lines = buffer.split('\n\n'); buffer = lines.pop() ?? ''; for (const line of lines) { if (!line.startsWith('data: ')) continue; const data = JSON.parse(line.slice(6)); if (data.type === 'token') { setText(prev => prev + data.content); } else if (data.type === 'done') { setState('done'); } else if (data.type === 'error') { setState(data.error === 'rate_limited' ? 'rate_limited' : 'error'); } } } } catch { setState('disconnected'); } }, [state]); return ( <div> <button data-testid="send-button" onClick={() => sendMessage('Hello')} disabled={state === 'streaming'} > Send </button> <div data-testid="stream-output">{text}</div> <div data-testid="stream-state">{state}</div> {state === 'rate_limited' && ( <div data-testid="rate-limit-message"> Too many requests. Please wait. </div> )} {state === 'disconnected' && ( <div data-testid="disconnect-message"> Connection was interrupted. </div> )} </div> ); } ``` ## E2E Tests ```typescript // tests/e2e/streaming.spec.ts import { test, expect, Page } from '@playwright/test'; import { MockLLMProviderServer } from '../fixtures/mock-provider'; // Shared mock server across all tests in file let mockServer: MockLLMProviderServer; let mockServerUrl: string; test.beforeAll(async () => { mockServer = new MockLLMProviderServer(); mockServerUrl = await mockServer.start(); }); test.afterAll(async () => { await mockServer.stop(); }); test.beforeEach(() => { mockServer.clearRequestLog(); }); // Helper: inject mock provider URL via Next.js middleware workaround async function setupMockProvider(page: Page, url: string) { // Intercept the fetch to /api/chat and add the test header await page.route('/api/chat', async route => { const request = route.request(); await route.continue({ headers: { ...request.headers(), 'x-test-provider-url': url, }, }); }); } // ───────────────────────────────────────────── // Scenario 1: Sequential token display // ───────────────────────────────────────────── test('displays tokens incrementally as they stream', async ({ page }) => { mockServer.setScenario({ type: 'success', tokens: ['Hello', ', ', 'world', '!'], delayMs: 100, }); await setupMockProvider(page, mockServerUrl); await page.goto('/'); const output = page.getByTestId('stream-output'); const stateEl = page.getByTestId('stream-state'); await page.getByTestId('send-button').click(); // Verify intermediate states (tokens arrive one by one) await expect(output).toHaveText('Hello', { timeout: 500 }); await expect(output).toHaveText('Hello, ', { timeout: 500 }); await expect(output).toHaveText('Hello, world', { timeout: 500 }); await expect(output).toHaveText('Hello, world!', { timeout: 500 }); // Verify final state await expect(stateEl).toHaveText('done', { timeout: 1000 }); // Verify no external calls were made const logs = mockServer.getRequestLog(); expect(logs).toHaveLength(1); expect(logs[0].prompt).toBe('Hello'); }); // ───────────────────────────────────────────── // Scenario 2: 429 Rate Limit // ───────────────────────────────────────────── test('shows rate limit message on 429 response', async ({ page }) => { mockServer.setScenario({ type: 'rate_limit' }); await setupMockProvider(page, mockServerUrl); await page.goto('/'); await page.getByTestId('send-button').click(); await expect(page.getByTestId('stream-state')).toHaveText('rate_limited', { timeout: 2000, }); await expect(page.getByTestId('rate-limit-message')).toBeVisible(); // Output should remain empty await expect(page.getByTestId('stream-output')).toBeEmpty(); }); // ───────────────────────────────────────────── // Scenario 3: Mid-stream disconnection // ───────────────────────────────────────────── test('handles mid-stream disconnection gracefully', async ({ page }) => { mockServer.setScenario({ type: 'mid_stream_disconnect', tokens: ['Partial', ' response', ' text'], disconnectAfter: 2, // Only send first 2 tokens then disconnect }); await setupMockProvider(page, mockServerUrl); await page.goto('/'); await page.getByTestId('send-button').click(); // Partial content should be displayed await expect(page.getByTestId('stream-output')).toHaveText( 'Partial response', { timeout: 2000 } ); // State should indicate disconnection await expect(page.getByTestId('stream-state')).toHaveText('disconnected', { timeout: 3000, }); await expect(page.getByTestId('disconnect-message')).toBeVisible(); }); // ───────────────────────────────────────────── // Scenario 4: Missing done notification // ───────────────────────────────────────────── test('detects missing done event and marks as disconnected', async ({ page }) => { mockServer.setScenario({ type: 'missing_done', tokens: ['Complete', ' text', ' without', ' done'], }); await setupMockProvider(page, mockServerUrl); await page.goto('/'); await page.getByTestId('send-button').click(); // All tokens should appear await expect(page.getByTestId('stream-output')).toHaveText( 'Complete text without done', { timeout: 2000 } ); // But state should NOT be 'done' since we never received the done event await expect(page.getByTestId('stream-state')).toHaveText('disconnected', { timeout: 2000, }); }); // ───────────────────────────────────────────── // Scenario 5: Button disabled during streaming // ───────────────────────────────────────────── test('disables send button during active stream', async ({ page }) => { mockServer.setScenario({ type: 'success', tokens: ['Slow', ' response'], delayMs: 300, }); await setupMockProvider(page, mockServerUrl); await page.goto('/'); const button = page.getByTestId('send-button'); await button.click(); // Button should be disabled while streaming await expect(button).toBeDisabled(); // Button should re-enable after completion await expect(button).toBeEnabled({ timeout: 5000 }); }); ``` ## Playwright Configuration ```typescript // playwright.config.ts import { defineConfig, devices } from '@playwright/test'; export default defineConfig({ testDir: './tests/e2e', fullyParallel: false, // Sequential to avoid mock server port conflicts timeout: 30_000, use: { baseURL: 'http://localhost:3000', trace: 'on-first-retry', }, projects: [ { name: 'chromium', use: { ...devices['Desktop Chrome'] }, }, ], webServer: { command: 'NODE_ENV=test npm run dev', url: 'http://localhost:3000', reuseExistingServer: !process.env.CI, timeout: 60_000, }, }); ``` ## Key Design Decisions | Decision | Rationale | |---|---| | Mock server runs in Node.js test process | Avoids `page.route` SSE limitations in Playwright | | `x-test-provider-url` header injection | Keeps production code clean; guard with `NODE_ENV !== 'production'` | | `page.route` adds header transparently | Browser never knows about mock; tests real fetch/SSE path | | `server.destroy()` for disconnection | Simulates real TCP disconnection, not just empty response | | Sequential test execution | Single mock server instance; scenario state is not concurrent-safe | ## Running Tests ```bash # Start dev server and run tests npm run dev & npx playwright test # With detailed trace npx playwright test --trace on # Single scenario npx playwright test --grep "mid-stream disconnection" ```
PlaywrightNext.jsLLME2Eテストテスト自動化
2026.07.20

This page has been translated by machine translation. View original

Introduction

In chat interfaces using LLMs, it is important for users to see the generated content appearing gradually, rather than displaying all at once after the response is complete.

On the other hand, connecting streaming E2E tests to actual LLM providers incurs API usage costs. Response content and speed are also inconsistent. It is also difficult to reproduce HTTP 429 errors or mid-stream disconnections at intended moments.

This time, using a Next.js application that utilizes OpenRouter as a subject, I created browser E2E tests that do not communicate with external LLM APIs. In addition to verifying incremental display in the browser, I also verify errors before stream start, mid-stream disconnections, and missing completion notifications.

The test provider is not added to the production code. The configuration that passes through the Next.js Route Handler and the existing OpenRouter adapter is maintained.

Target Audience

  • Those who are displaying LLM responses via streaming in Next.js
  • Those considering switching LLM providers
  • Those who want to run browser E2E tests without using paid APIs or actual API keys
  • Those who want to reproduce not only happy paths, but also mid-stream disconnections and missing completion notifications

Verification Environment

Item Version
OS Windows
Node.js 24.14.0
npm 11.9.0
Next.js 15.5.9
React 19.1.0
Playwright 1.61.1
Google Chrome 150.0.7871.125

References

Architecture

Communication Path Being Tested

Next.js Route Handlers can use the web-standard Request and Response APIs. The POST /api/generate in this case also relays a Response with an upstream ReadableStream to the browser.

/api/generate, which the browser connects to, is not mocked. The application executes the same Route Handler as usual, and the OpenRouter adapter assembles the request. Only the OpenRouter URL that the adapter calls last is replaced within the Next.js server process.

Fixing the Stream Contract Visible from the Browser

Before replacing the backend, I fixed the stream contract visible from the browser.

request:
  POST /api/generate
  { "messages": [{ "role": "user|assistant", "content": "..." }] }

success:
  Content-Type: text/event-stream
  data: {"choices":[{"delta":{"content":"..."}}]}

completion:
  data: [DONE]

pre-stream failure:
  HTTP status + JSON { "error": "...", "retryAfter"?: "..." }

On the server side, provider-specific processing is separated into the following interface.

src/lib/llm/contract.ts
export type StreamChatRequest = {
  messages: ChatMessage[];
  signal?: AbortSignal;
};

export interface StreamingLlmBackend {
  readonly id: string;
  streamChat(request: StreamChatRequest): Promise<Response>;
}

The OpenRouter adapter sends requests with stream: true specified. If another provider returns a different event format, the adapter converts it to the application's common SSE format.

The browser side does not know the provider name. It only processes the following 3 types of events.

src/lib/llm/sse.ts
export type AppStreamEvent =
  | { type: "delta"; text: string }
  | { type: "error"; message: string }
  | { type: "done" };

With this boundary in place, even if the backend's authentication, URL, request format, or proprietary events change, the React side's incremental rendering does not need to be changed.

Implementation

Replacing fetch in the Next.js Server Process

Using Playwright's webServer configuration, the Next.js development server is started when tests begin.

Playwright can mock requests issued by browser pages using page.route() or browserContext.route().

In this case, requests issued by the browser only go as far as /api/generate. Requests to OpenRouter are issued from the Next.js server process. Mocking /api/generate on the browser side would bypass the Route Handler, prompt generation, OpenRouter adapter, and SSE relay.

Therefore, I configured a CommonJS preload for the Next.js process started by Playwright. Node.js's --require has the ability to load a specified module at startup. See the Node.js CLI documentation for details.

playwright.config.ts
import { defineConfig } from "@playwright/test";
import path from "node:path";

const preloadPath = path.resolve("tests/e2e/mock-openrouter.cjs");

export default defineConfig({
  testDir: "./tests/e2e",
  workers: 1,
  use: {
    baseURL: "http://127.0.0.1:3100",
    browserName: "chromium",
    channel: "chrome",
    headless: true,
  },
  webServer: {
    command: "npm run dev -- --hostname 127.0.0.1 --port 3100",
    url: "http://127.0.0.1:3100",
    reuseExistingServer: false,
    env: {
      ...process.env,
      NODE_ENV: "development",
      NODE_OPTIONS: `--require=${preloadPath}`,
      LLM_E2E_FETCH_MOCK: "1",
      OPENROUTER_API_KEY: "e2e-placeholder-not-a-secret",
    },
  },
});

Preventing the Test Preload from Running in Production

Since the preload replaces fetch in the server process, it must not be loaded in production environments.

At the beginning of the preload, the following 2 conditions are checked.

tests/e2e/mock-openrouter.cjs
"use strict";

if (process.env.NODE_ENV === "production") {
  throw new Error("E2E fetch mock must never run with NODE_ENV=production");
}

if (process.env.LLM_E2E_FETCH_MOCK !== "1") {
  throw new Error("E2E fetch mock requires an explicit local-only opt-in");
}

The URL to replace is matched exactly against the OpenRouter Chat Completions endpoint. All other fetch calls are passed to the original implementation.

tests/e2e/mock-openrouter.cjs
const OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions";
const originalFetch = globalThis.fetch;

globalThis.fetch = async function controlledFetch(input, init) {
  if (String(input) !== OPENROUTER_URL) {
    return originalFetch(input, init);
  }

  // Return a response according to the synthesized scenario
};

Unknown scenarios that reach the OpenRouter URL are not allowed to fall back to the actual network.

tests/e2e/mock-openrouter.cjs
return new Response(
  JSON.stringify({ error: { message: "unknown E2E scenario" } }),
  {
    status: 400,
    headers: { "Content-Type": "application/json" },
  },
);

This closes the path to the paid API in case of incorrect test input. The preload startup conditions are also verified with separate unit tests.

Creating a Time-Delayed Stream

Simply returning an SSE string all at once is not enough to verify that React rendered intermediate states. I created a ReadableStream that enqueue()s each chunk after a specified delay.

tests/e2e/mock-openrouter.cjs
const encoder = new TextEncoder();

function delayedStream(steps) {
  return new ReadableStream({
    start(controller) {
      let elapsed = 0;
      for (const step of steps) {
        elapsed += step.afterMs;
        setTimeout(() => {
          if (step.error) {
            controller.error(new Error(step.error));
          } else if (step.close) {
            controller.close();
          } else {
            controller.enqueue(encoder.encode(step.data));
          }
        }, elapsed);
      }
    },
  });
}

In the happy path, the first delta, a malformed JSON event, the next delta, and [DONE] are returned with time delays. CRLF and LF are also mixed in.

tests/e2e/mock-openrouter.cjs
return streamResponse([
  {
    afterMs: 100,
    data: 'data: {"choices":[{"delta":{"content":"Phase 1"}}]}\r\n\r\n',
  },
  { afterMs: 250, data: "data: this-is-not-json\n\n" },
  {
    afterMs: 650,
    data: 'data: {"choices":[{"delta":{"content":"Phase 2"}}]}\n\n',
  },
  { afterMs: 100, data: "data: [DONE]\n\n" },
  { afterMs: 10, close: true },
]);

In separate unit tests, UTF-8 Japanese strings and SSE delimiters were split in the middle of network chunks. This confirms that reconstruction does not depend on provider-side chunk boundaries.

Verifying Intermediate States in the Browser

In the E2E tests, not only the final result but also the state between the first delta and the next delta is verified.

tests/e2e/streaming.spec.ts
test("renders token-equivalent deltas incrementally", async ({ page }) => {
  await page.goto("/");
  await page.getByTestId("prompt-input")
    .fill("[[e2e:success]] publicly available synthetic input");
  await page.getByTestId("send-button").click();

  const assistant = page.getByTestId("assistant-message").last();

  await expect(page.getByTestId("stream-status")).toBeVisible();
  await expect(assistant).toHaveText("Phase 1");
  await expect(assistant).not.toContainText("Phase 2");

  await expect(assistant).toHaveText("Phase 1Phase 2");
  await expect(page.getByTestId("stream-status")).toBeHidden();
});

At the point when the first toHaveText("Phase 1") succeeds, the in-progress indicator is still visible. "Phase 2" is not yet in the DOM. After that, the second delta and [DONE] are received, and the state becomes complete.

A test that only verifies the final string would also pass an implementation that buffers everything. By verifying intermediate states, the user experience that we want to maintain can be tested directly.

Preparing Scenarios Beyond the Happy Path

When replacing the backend, the point at which failures occur is also important. The following 4 scenarios were prepared.

Scenario Test Response Expected Screen Display
Normal completion 2 deltas, malformed event, [DONE] Hide in-progress indicator after incremental display
Pre-stream error HTTP 429, JSON, Retry-After: 3 Display rate limit message and wait time in seconds
Mid-stream disconnection controller.error() after 1 delta Retain partial response, display mid-stream disconnection and retry button
Missing completion notification Normal close after 1 delta, no [DONE] Do not treat as normal completion, display pre-completion-notification termination and retry button

HTTP errors before stream start are handled before SSE reading begins. On mid-stream disconnection, the partial response already received is not discarded.

Even without a transport error, a stream may close without [DONE] arriving. Under the contract in this case, a normal EOF is not treated the same as [DONE]. The partial response is retained and the fact that the stream ended before the completion notification is displayed.

Results

Running the Tests

The following commands were executed.

npm test
npm run lint
npx tsc --noEmit
npm run build
npm run test:e2e
Verification Result
Stream contract and production startup prevention 10 / 10 passed
ESLint Passed
TypeScript --noEmit Passed
Next.js production build Passed
Playwright browser E2E 4 / 4 passed, 16.1 seconds
External LLM provider calls 0 
Running 4 tests using 1 worker
4 passed (16.1s)

During the mid-stream disconnection test, failed to pipe response was output to the Next.js development server. This is the result of the synthesized disconnection reaching the server's SSE relay. This is the expected diagnostic for this scenario.

When I first created the E2E tests, only 3 out of 4 passed. This was because the application displays an existing Japanese message for HTTP 429, but the test was expecting the string HTTP 429.

In this case, I did not modify the application to match the test — I confirmed the current screen specification and only corrected the expected values in the test. When an E2E test fails, it does not always mean the production code is wrong.

What I Learned

In these E2E tests, I was able to verify the entire flow from browser input through the Next.js Route Handler, OpenRouter adapter, and SSE relay to where responses appear incrementally on screen. Not only normal completion, but also pre-stream errors, mid-stream disconnections, and stream closure without [DONE] can be distinguished.

On the other hand, it is not clear whether Vercel or a CDN would buffer the response, or how deployment environment timeouts would behave. Response speed and error formats for the actual backend are also unverified. These aspects need to be verified in a preview environment after deciding on a backend candidate.

When adding another streaming backend, it seems manageable to confine the differences in authentication and request format within the adapter and return common delta, error, and done events to the browser. However, this is a design policy considered from the current results, not a fact verified with another backend. If an asynchronous job approach is chosen, rather than forcing it into the same shape, it should be considered as a separate API and screen design.

Summary

I created browser E2E tests that do not communicate with external LLM APIs by replacing only the fetch calls destined for OpenRouter within the Next.js server process. With this mechanism in place, even when changing the LLM backend in the future, the current streaming experience can be verified without depending on external API costs or response variability.

I hope this serves as a reference for those considering E2E testing of LLM streaming.

Appendix

The complete code used on the E2E test side is provided below. On the application side, data-testid attributes for prompt-input, send-button, assistant-message, stream-status, error-panel, and retry-button are required.

Complete Playwright Configuration
playwright.config.ts
import { defineConfig } from "@playwright/test";
import path from "node:path";

const preloadPath = path.resolve("tests/e2e/mock-openrouter.cjs");

export default defineConfig({
    testDir: "./tests/e2e",
    fullyParallel: false,
    workers: 1,
    reporter: "line",
    outputDir: "test-results",
    use: {
        baseURL: "http://127.0.0.1:3100",
        browserName: "chromium",
        channel: "chrome",
        headless: true,
        trace: "retain-on-failure",
    },
    webServer: {
        command: "npm run dev -- --hostname 127.0.0.1 --port 3100",
        url: "http://127.0.0.1:3100",
        reuseExistingServer: false,
        timeout: 120_000,
        env: {
            ...process.env,
            NODE_ENV: "development",
            NODE_OPTIONS: `--require=${preloadPath}`,
            LLM_E2E_FETCH_MOCK: "1",
            OPENROUTER_API_KEY: "e2e-placeholder-not-a-secret",
        },
    },
});
Complete OpenRouter fetch Mock
tests/e2e/mock-openrouter.cjs
"use strict";

if (process.env.NODE_ENV === "production") {
    throw new Error("E2E fetch mock must never run with NODE_ENV=production");
}
if (process.env.LLM_E2E_FETCH_MOCK !== "1") {
    throw new Error("E2E fetch mock requires an explicit local-only opt-in");
}

const OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions";
const originalFetch = globalThis.fetch;
const encoder = new TextEncoder();

function delayedStream(steps) {
    return new ReadableStream({
        start(controller) {
            let elapsed = 0;
            for (const step of steps) {
                elapsed += step.afterMs;
                setTimeout(() => {
                    if (step.error) {
                        controller.error(new Error(step.error));
                    } else if (step.close) {
                        controller.close();
                    } else {
                        controller.enqueue(encoder.encode(step.data));
                    }
                }, elapsed);
            }
        },
    });
}

function streamResponse(steps) {
    return new Response(delayedStream(steps), {
        status: 200,
        headers: { "Content-Type": "text/event-stream; charset=utf-8" },
    });
}

globalThis.fetch = async function controlledFetch(input, init) {
    if (String(input) !== OPENROUTER_URL) {
        return originalFetch(input, init);
    }

    const request = JSON.parse(String(init?.body ?? "{}"));
    const messages = Array.isArray(request.messages) ? request.messages : [];
    const lastUser = [...messages].reverse().find((message) => message?.role === "user");
    const scenario = String(lastUser?.content ?? "");

    if (scenario.includes("[[e2e:success]]")) {
        return streamResponse([
            { afterMs: 100, data: "data: {\"choices\":[{\"delta\":{\"content\":\"Phase 1\"}}]}\r\n\r\n" },
            { afterMs: 250, data: "data: this-is-not-json\n\n" },
            { afterMs: 650, data: "data: {\"choices\":[{\"delta\":{\"content\":\"Phase 2\"}}]}\n\n" },
            { afterMs: 100, data: "data: [DONE]\n\n" },
            { afterMs: 10, close: true },
        ]);
    }

    if (scenario.includes("[[e2e:pre-error]]")) {
        return new Response(JSON.stringify({ error: { message: "synthetic rate limit" } }), {
            status: 429,
            headers: { "Content-Type": "application/json", "Retry-After": "3" },
        });
    }

    if (scenario.includes("[[e2e:disconnect]]")) {
        return streamResponse([
            { afterMs: 100, data: "data: {\"choices\":[{\"delta\":{\"content\":\"partial response\"}}]}\n\n" },
            { afterMs: 250, error: "synthetic mid-stream disconnect" },
        ]);
    }

    if (scenario.includes("[[e2e:missing-done]]")) {
        return streamResponse([
            { afterMs: 100, data: "data: {\"choices\":[{\"delta\":{\"content\":\"before completion\"}}]}\n\n" },
            { afterMs: 100, close: true },
        ]);
    }

    return new Response(JSON.stringify({ error: { message: "unknown E2E scenario" } }), {
        status: 400,
        headers: { "Content-Type": "application/json" },
    });
};
Complete Browser E2E Tests
tests/e2e/streaming.spec.ts
import { expect, test } from "@playwright/test";

async function send(page: import("@playwright/test").Page, prompt: string) {
    await page.goto("/");
    await page.getByTestId("prompt-input").fill(prompt);
    await page.getByTestId("send-button").click();
}

test("renders token-equivalent deltas incrementally and completes despite a malformed event", async ({ page }) => {
    await send(page, "[[e2e:success]] publicly available synthetic input");

    const assistant = page.getByTestId("assistant-message").last();
    await expect(page.getByTestId("stream-status")).toBeVisible();
    await expect(assistant).toHaveText("Phase 1");
    await expect(assistant).not.toContainText("Phase 2");
    await expect(assistant).toHaveText("Phase 1Phase 2");
    await expect(page.getByTestId("stream-status")).toBeHidden();
    await expect(page.getByTestId("error-panel")).toHaveCount(0);
});

test("shows a pre-stream provider error separately from a stream interruption", async ({ page }) => {
    await send(page, "[[e2e:pre-error]] publicly available synthetic input");

    await expect(page.getByTestId("error-panel")).toContainText("Rate limited");
    await expect(page.getByText("Server recommended wait time: 3s")).toBeVisible();
    await expect(page.getByTestId("assistant-message").last()).toContainText("Rate limited");
    await expect(page.getByTestId("stream-status")).toBeHidden();
});

test("retains partial output and identifies a mid-stream disconnect", async ({ page }) => {
    await send(page, "[[e2e:disconnect]] publicly available synthetic input");

    const assistant = page.getByTestId("assistant-message").last();
    await expect(assistant).toContainText("partial response");
    await expect(page.getByTestId("error-panel")).toContainText("disconnected mid-stream");
    await expect(assistant).toContainText("disconnected mid-stream");
    await expect(page.getByTestId("retry-button")).toBeVisible();
    await expect(page.getByTestId("stream-status")).toBeHidden();
});

test("treats EOF without the completion event as an incomplete stream", async ({ page }) => {
    await send(page, "[[e2e:missing-done]] publicly available synthetic input");

    const assistant = page.getByTestId("assistant-message").last();
    await expect(assistant).toContainText("before completion");
    await expect(page.getByTestId("error-panel")).toContainText("before completion notification");
    await expect(page.getByTestId("stream-status")).toBeHidden();
});
Complete Preload Startup Condition Tests
tests/unit/e2e-fetch-guard.test.mjs
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import path from "node:path";
import test from "node:test";

const preload = path.resolve("tests/e2e/mock-openrouter.cjs");

function requireMock(environment) {
    return spawnSync(process.execPath, ["--require", preload, "--eval", "void 0"], {
        cwd: process.cwd(),
        encoding: "utf8",
        env: { ...process.env, ...environment },
    });
}

test("the E2E fetch mock refuses production", () => {
    const result = requireMock({ NODE_ENV: "production", LLM_E2E_FETCH_MOCK: "1" });
    assert.notEqual(result.status, 0);
    assert.match(result.stderr, /must never run with NODE_ENV=production/);
});
test("the E2E fetch mock requires explicit local opt-in", () => {
    const result = requireMock({ NODE_ENV: "development", LLM_E2E_FETCH_MOCK: "0" });
    assert.notEqual(result.status, 0);
    assert.match(result.stderr, /requires an explicit local-only opt-in/);
});

test("the E2E fetch mock can preload only with both development guards", () => {
    const result = requireMock({ NODE_ENV: "development", LLM_E2E_FETCH_MOCK: "1" });
    assert.equal(result.status, 0, result.stderr);
});

Share this article