feat(preview): SSE-streamed generation, no CF 100s edge cap
All checks were successful
Deploy to Production / deploy (push) Successful in 1m27s

Architectural fix for "spec_too_large" / preview_timeout — the sync
endpoint had to fit the whole model run into Cloudflare's ~100s edge
window, which made the system fragile against any prompt that produced
a verbose spec. The new streaming path pipes Anthropic's token deltas
as Server-Sent Events; every chunk resets CF's idle timer and a 15s
keepalive comment guarantees activity even during slow first-token
windows.

@bmm/llm: new streamSpecFromAnthropic() exposes the SDK's .stream()
flow with the same typed-error contract as generateSpec — same
SpecTruncatedError / SpecValidationError / SpecTimeoutError raised from
the relevant moment.

API: POST /v1/servers/preview/stream returns text/event-stream with
events 'text' (deltas), 'spec' (final success payload, same shape as
the sync endpoint), 'error' (typed). Anthropic-only — GLM/hobby falls
back to the sync route via 409 streaming_unavailable.

Frontend: apiSseStream() handles the POST + ReadableStream + SSE
parser. The wizard's analyze() prefers the stream and only uses the
sync endpoint on the explicit 409 fallback.

nginx (api.buildmymcpserver.com): the /v1/builds/ location block (which
already had proxy_buffering off + 600s read timeout for the WS build
stream) now also matches /v1/servers/preview/stream so the SSE
response isn't buffered.
This commit is contained in:
Marco Sadjadi
2026-05-28 21:11:05 +02:00
parent b930a454e8
commit 0c6d738a6b
4 changed files with 385 additions and 12 deletions

View File

@@ -19,6 +19,7 @@ import {
generateSpec,
pickPreviewModel,
scanForInjection,
streamSpecFromAnthropic,
} from '@bmm/llm';
import {
BuildEvent,
@@ -174,6 +175,162 @@ export async function serverRoutes(app: FastifyInstance): Promise<void> {
}
});
// Streaming preview — pipes the model's text deltas back as Server-Sent
// Events so Cloudflare's ~100s edge cap is irrelevant: every chunk we
// write resets the idle timer. The final event is either `spec` (success)
// or `error` (any of the typed errors raised by the LLM layer).
//
// Anthropic-only for now: GLM doesn't ship a clean streaming JSON
// contract that justifies the duplication, and hobby's 4096-token budget
// already fits the sync path comfortably. The route falls back to the
// sync endpoint if the request comes from a non-Anthropic tier.
app.post('/v1/servers/preview/stream', { preHandler: requireAuth }, async (req, reply) => {
const user = req.user!;
const parsed = PreviewInput.safeParse(req.body);
if (!parsed.success) {
return reply.code(400).send({ error: 'invalid_input', issues: parsed.error.flatten() });
}
const billing = await getOrgBilling(user.orgId);
if (billing.suspended) {
return reply.code(402).send({
error: 'subscription_suspended',
detail:
billing.suspendedReason === 'payment_failed'
? 'Your subscription is paused due to a payment issue. Update your payment method in /settings/billing.'
: 'Your subscription is paused. Visit /settings/billing for details.',
suspendedReason: billing.suspendedReason,
});
}
const plan = billing.plan;
const rl = await checkDailyLimit('preview', user.userId, PREVIEW_DAILY_LIMIT[plan]);
if (!rl.ok) {
return reply.code(429).send({
error: 'rate_limited',
detail: `Daily preview limit reached for plan "${plan}" (${PREVIEW_DAILY_LIMIT[plan]}/day). Resets in ${Math.ceil(rl.resetIn / 3600)}h.`,
plan,
limit: PREVIEW_DAILY_LIMIT[plan],
resetIn: rl.resetIn,
});
}
const choice = pickPreviewModel(plan);
if (choice.provider !== 'anthropic' || !config.ANTHROPIC_API_KEY) {
return reply.code(409).send({
error: 'streaming_unavailable',
detail: 'Streaming preview is only available for Anthropic-backed tiers. Use POST /v1/servers/preview instead.',
});
}
// SSE response. X-Accel-Buffering disables nginx's response buffering
// so each chunk lands at the client immediately rather than after the
// full response is built — critical for the keepalive-vs-CF-100s logic
// to actually work.
reply.raw.setHeader('Content-Type', 'text/event-stream');
reply.raw.setHeader('Cache-Control', 'no-cache, no-transform');
reply.raw.setHeader('Connection', 'keep-alive');
reply.raw.setHeader('X-Accel-Buffering', 'no');
reply.raw.flushHeaders();
const send = (event: string, data: unknown) => {
reply.raw.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
};
// Heartbeat comment every 15s. Cloudflare's edge keeps the connection
// open as long as bytes flow; comments are SSE-noop but count as bytes.
const keepalive = setInterval(() => reply.raw.write(`: ping\n\n`), 15_000);
const abort = new AbortController();
req.raw.on('close', () => abort.abort());
let resolved = false;
await streamSpecFromAnthropic(
parsed.data.prompt,
{
apiKey: config.ANTHROPIC_API_KEY,
model: choice.model,
maxTokens: choice.maxTokens,
signal: abort.signal,
},
{
onText: (delta) => send('text', delta),
onSpec: async ({ spec, source }) => {
resolved = true;
const previewId = await cacheSpec(spec);
send('spec', {
previewId,
source,
plan,
modelDisplayName: choice.displayName,
modelBadge: choice.displayBadge,
upgradeHint: plan === 'hobby',
spec: {
name: spec.name,
description: spec.description,
tools: spec.tools.map((t) => ({
name: t.name,
description: t.description,
inputSchema: t.inputSchema,
})),
requiredSecrets: spec.requiredSecrets,
scopes: spec.scopes,
},
});
},
onError: (err) => {
resolved = true;
if (err instanceof SpecTruncatedError) {
app.log.warn(
{
reason: err.message,
prompt: parsed.data.prompt.slice(0, 200),
model: choice.displayName,
},
'preview_spec_truncated',
);
send('error', {
error: 'spec_too_large',
detail:
'The spec for this prompt exceeded the maximum response size. Split it into fewer tools or describe one capability per prompt.',
});
return;
}
if (err instanceof SpecValidationError) {
app.log.warn(
{
zod_message: err.message,
prompt: parsed.data.prompt.slice(0, 200),
model: choice.displayName,
},
'preview_spec_invalid',
);
send('error', { error: 'spec_invalid', detail: err.message });
return;
}
if (err instanceof BannedPatternError) {
send('error', { error: 'banned_pattern', detail: err.message });
return;
}
if (err instanceof SpecTimeoutError) {
send('error', {
error: 'preview_timeout',
detail: 'Spec generation took too long. Try a shorter, more specific prompt.',
});
return;
}
app.log.error(err);
send('error', { error: 'preview_failed', detail: err.message });
},
},
);
if (!resolved) {
send('error', { error: 'preview_failed', detail: 'stream ended without a final event' });
}
clearInterval(keepalive);
reply.raw.end();
});
app.post('/v1/servers', { preHandler: requireAuth }, async (req, reply) => {
const user = req.user!;
const parsed = CreateServerInput.safeParse(req.body);