feat(preview): SSE-streamed generation, no CF 100s edge cap
All checks were successful
Deploy to Production / deploy (push) Successful in 1m27s

Architectural fix for "spec_too_large" / preview_timeout — the sync
endpoint had to fit the whole model run into Cloudflare's ~100s edge
window, which made the system fragile against any prompt that produced
a verbose spec. The new streaming path pipes Anthropic's token deltas
as Server-Sent Events; every chunk resets CF's idle timer and a 15s
keepalive comment guarantees activity even during slow first-token
windows.

@bmm/llm: new streamSpecFromAnthropic() exposes the SDK's .stream()
flow with the same typed-error contract as generateSpec — same
SpecTruncatedError / SpecValidationError / SpecTimeoutError raised from
the relevant moment.

API: POST /v1/servers/preview/stream returns text/event-stream with
events 'text' (deltas), 'spec' (final success payload, same shape as
the sync endpoint), 'error' (typed). Anthropic-only — GLM/hobby falls
back to the sync route via 409 streaming_unavailable.

Frontend: apiSseStream() handles the POST + ReadableStream + SSE
parser. The wizard's analyze() prefers the stream and only uses the
sync endpoint on the explicit 409 fallback.

nginx (api.buildmymcpserver.com): the /v1/builds/ location block (which
already had proxy_buffering off + 600s read timeout for the WS build
stream) now also matches /v1/servers/preview/stream so the SSE
response isn't buffered.
This commit is contained in:
Marco Sadjadi
2026-05-28 21:11:05 +02:00
parent b930a454e8
commit 0c6d738a6b
4 changed files with 385 additions and 12 deletions

View File

@@ -5,7 +5,7 @@ import { Input, Label, Textarea } from '@/components/input';
import { InstallSnippets } from '@/components/install-snippets';
import { StreamingLogs } from '@/components/streaming-logs';
import { Button } from '@/components/ui/button';
import { apiFetch } from '@/lib/api';
import { apiFetch, apiSseStream } from '@/lib/api';
import { Loader2, RotateCcw, X } from 'lucide-react';
import Link from 'next/link';
import { useRouter, useSearchParams } from 'next/navigation';
@@ -241,19 +241,69 @@ function NewServerPageInner() {
return;
}
setStep('analyzing');
try {
const res = await apiFetch<PreviewResponse>('/v1/servers/preview', {
method: 'POST',
body: JSON.stringify({ prompt }),
});
setPreview(res);
setEditable(null); // will re-init via useEffect
// Streaming preview: pipes Anthropic's token deltas back as SSE. Cloudflare's
// ~100s edge cap doesn't bite because every chunk we receive resets the
// idle timer; the only practical limit is the model's own runtime. If the
// backend returns 409 streaming_unavailable (e.g. hobby/GLM tier), we fall
// back to the sync endpoint so the wizard still works there.
let finalResolved = false;
let sseError: { error?: string; detail?: string } | null = null;
let sseSpec: PreviewResponse | null = null;
await apiSseStream(
'/v1/servers/preview/stream',
{ prompt },
{
onEvent: (event, data) => {
if (event === 'spec') {
finalResolved = true;
sseSpec = data as PreviewResponse;
} else if (event === 'error') {
finalResolved = true;
sseError = data as { error?: string; detail?: string };
}
// 'text' deltas are ignored for now — the wizard already shows a
// spinner. We could surface partial JSON later if useful.
},
onError: (err) => {
sseError = { detail: err.message };
},
},
);
if (finalResolved && sseSpec) {
setPreview(sseSpec);
setEditable(null);
setStep('confirm');
} catch (e) {
const detail = (e as { detail?: { error?: string; detail?: string } }).detail;
setError(detail?.detail ?? detail?.error ?? (e as Error).message);
setStep('prompt');
return;
}
if (sseError && (sseError as { error?: string }).error === 'streaming_unavailable') {
// GLM / mock tier — fall back to sync.
try {
const res = await apiFetch<PreviewResponse>('/v1/servers/preview', {
method: 'POST',
body: JSON.stringify({ prompt }),
});
setPreview(res);
setEditable(null);
setStep('confirm');
return;
} catch (e) {
const detail = (e as { detail?: { error?: string; detail?: string } }).detail;
setError(detail?.detail ?? detail?.error ?? (e as Error).message);
setStep('prompt');
return;
}
}
setError(
(sseError as { detail?: string } | null)?.detail ??
(sseError as { error?: string } | null)?.error ??
'Spec generation failed.',
);
setStep('prompt');
}
function updateTool(i: number, patch: Partial<EditableTool>) {

View File

@@ -33,3 +33,96 @@ export function apiWebSocketURL(path: string): string {
const wsBase = httpBase.replace(/^http/, 'ws');
return `${wsBase}${path}`;
}
export interface SseHandlers {
onEvent: (event: string, data: unknown) => void;
onError?: (err: Error) => void;
}
/**
* POST with SSE response. Used by the streaming preview where Cloudflare's
* sync edge timeout (~100s) is fatal — every server-side chunk resets it,
* so as long as the model emits text we never hit the cap.
*
* Native EventSource doesn't support POST or auth-cookie credentials, hence
* the manual fetch + ReadableStream consumer. Caller controls aborting via
* the supplied AbortSignal (we attach it to the underlying fetch).
*/
export async function apiSseStream(
path: string,
body: unknown,
handlers: SseHandlers,
signal?: AbortSignal,
): Promise<void> {
let res: Response;
try {
res = await fetch(`${API_BASE}${path}`, {
method: 'POST',
credentials: 'include',
cache: 'no-store',
headers: {
'Content-Type': 'application/json',
Accept: 'text/event-stream',
},
body: JSON.stringify(body),
signal,
});
} catch (err) {
handlers.onError?.(err as Error);
return;
}
if (!res.ok) {
let detail: unknown = undefined;
try {
detail = await res.json();
} catch {}
const err = new Error(`api_error_${res.status}`);
(err as unknown as { detail?: unknown }).detail = detail;
(err as unknown as { status?: number }).status = res.status;
handlers.onError?.(err);
return;
}
const reader = res.body?.getReader();
if (!reader) {
handlers.onError?.(new Error('no_response_body'));
return;
}
const decoder = new TextDecoder();
let buffer = '';
try {
while (true) {
const { value, done } = await reader.read();
if (done) break;
buffer += decoder.decode(value, { stream: true });
// SSE events are separated by a blank line. Parse one event at a time.
let idx = buffer.indexOf('\n\n');
while (idx >= 0) {
const raw = buffer.slice(0, idx);
buffer = buffer.slice(idx + 2);
let event = 'message';
const dataLines: string[] = [];
for (const line of raw.split('\n')) {
if (line.startsWith(':')) continue; // SSE comment
if (line.startsWith('event:')) event = line.slice(6).trim();
else if (line.startsWith('data:')) dataLines.push(line.slice(5).trim());
}
if (dataLines.length > 0) {
const data = dataLines.join('\n');
let parsed: unknown = data;
try {
parsed = JSON.parse(data);
} catch {
// Treat as a raw string event.
}
handlers.onEvent(event, parsed);
}
idx = buffer.indexOf('\n\n');
}
}
} catch (err) {
handlers.onError?.(err as Error);
}
}