feat(preview): SSE-streamed generation, no CF 100s edge cap
All checks were successful
Deploy to Production / deploy (push) Successful in 1m27s
All checks were successful
Deploy to Production / deploy (push) Successful in 1m27s
Architectural fix for "spec_too_large" / preview_timeout — the sync endpoint had to fit the whole model run into Cloudflare's ~100s edge window, which made the system fragile against any prompt that produced a verbose spec. The new streaming path pipes Anthropic's token deltas as Server-Sent Events; every chunk resets CF's idle timer and a 15s keepalive comment guarantees activity even during slow first-token windows. @bmm/llm: new streamSpecFromAnthropic() exposes the SDK's .stream() flow with the same typed-error contract as generateSpec — same SpecTruncatedError / SpecValidationError / SpecTimeoutError raised from the relevant moment. API: POST /v1/servers/preview/stream returns text/event-stream with events 'text' (deltas), 'spec' (final success payload, same shape as the sync endpoint), 'error' (typed). Anthropic-only — GLM/hobby falls back to the sync route via 409 streaming_unavailable. Frontend: apiSseStream() handles the POST + ReadableStream + SSE parser. The wizard's analyze() prefers the stream and only uses the sync endpoint on the explicit 409 fallback. nginx (api.buildmymcpserver.com): the /v1/builds/ location block (which already had proxy_buffering off + 600s read timeout for the WS build stream) now also matches /v1/servers/preview/stream so the SSE response isn't buffered.
This commit is contained in:
@@ -19,6 +19,7 @@ import {
|
||||
generateSpec,
|
||||
pickPreviewModel,
|
||||
scanForInjection,
|
||||
streamSpecFromAnthropic,
|
||||
} from '@bmm/llm';
|
||||
import {
|
||||
BuildEvent,
|
||||
@@ -174,6 +175,162 @@ export async function serverRoutes(app: FastifyInstance): Promise<void> {
|
||||
}
|
||||
});
|
||||
|
||||
// Streaming preview — pipes the model's text deltas back as Server-Sent
|
||||
// Events so Cloudflare's ~100s edge cap is irrelevant: every chunk we
|
||||
// write resets the idle timer. The final event is either `spec` (success)
|
||||
// or `error` (any of the typed errors raised by the LLM layer).
|
||||
//
|
||||
// Anthropic-only for now: GLM doesn't ship a clean streaming JSON
|
||||
// contract that justifies the duplication, and hobby's 4096-token budget
|
||||
// already fits the sync path comfortably. The route falls back to the
|
||||
// sync endpoint if the request comes from a non-Anthropic tier.
|
||||
app.post('/v1/servers/preview/stream', { preHandler: requireAuth }, async (req, reply) => {
|
||||
const user = req.user!;
|
||||
const parsed = PreviewInput.safeParse(req.body);
|
||||
if (!parsed.success) {
|
||||
return reply.code(400).send({ error: 'invalid_input', issues: parsed.error.flatten() });
|
||||
}
|
||||
|
||||
const billing = await getOrgBilling(user.orgId);
|
||||
if (billing.suspended) {
|
||||
return reply.code(402).send({
|
||||
error: 'subscription_suspended',
|
||||
detail:
|
||||
billing.suspendedReason === 'payment_failed'
|
||||
? 'Your subscription is paused due to a payment issue. Update your payment method in /settings/billing.'
|
||||
: 'Your subscription is paused. Visit /settings/billing for details.',
|
||||
suspendedReason: billing.suspendedReason,
|
||||
});
|
||||
}
|
||||
const plan = billing.plan;
|
||||
|
||||
const rl = await checkDailyLimit('preview', user.userId, PREVIEW_DAILY_LIMIT[plan]);
|
||||
if (!rl.ok) {
|
||||
return reply.code(429).send({
|
||||
error: 'rate_limited',
|
||||
detail: `Daily preview limit reached for plan "${plan}" (${PREVIEW_DAILY_LIMIT[plan]}/day). Resets in ${Math.ceil(rl.resetIn / 3600)}h.`,
|
||||
plan,
|
||||
limit: PREVIEW_DAILY_LIMIT[plan],
|
||||
resetIn: rl.resetIn,
|
||||
});
|
||||
}
|
||||
|
||||
const choice = pickPreviewModel(plan);
|
||||
if (choice.provider !== 'anthropic' || !config.ANTHROPIC_API_KEY) {
|
||||
return reply.code(409).send({
|
||||
error: 'streaming_unavailable',
|
||||
detail: 'Streaming preview is only available for Anthropic-backed tiers. Use POST /v1/servers/preview instead.',
|
||||
});
|
||||
}
|
||||
|
||||
// SSE response. X-Accel-Buffering disables nginx's response buffering
|
||||
// so each chunk lands at the client immediately rather than after the
|
||||
// full response is built — critical for the keepalive-vs-CF-100s logic
|
||||
// to actually work.
|
||||
reply.raw.setHeader('Content-Type', 'text/event-stream');
|
||||
reply.raw.setHeader('Cache-Control', 'no-cache, no-transform');
|
||||
reply.raw.setHeader('Connection', 'keep-alive');
|
||||
reply.raw.setHeader('X-Accel-Buffering', 'no');
|
||||
reply.raw.flushHeaders();
|
||||
|
||||
const send = (event: string, data: unknown) => {
|
||||
reply.raw.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
|
||||
};
|
||||
|
||||
// Heartbeat comment every 15s. Cloudflare's edge keeps the connection
|
||||
// open as long as bytes flow; comments are SSE-noop but count as bytes.
|
||||
const keepalive = setInterval(() => reply.raw.write(`: ping\n\n`), 15_000);
|
||||
const abort = new AbortController();
|
||||
req.raw.on('close', () => abort.abort());
|
||||
|
||||
let resolved = false;
|
||||
await streamSpecFromAnthropic(
|
||||
parsed.data.prompt,
|
||||
{
|
||||
apiKey: config.ANTHROPIC_API_KEY,
|
||||
model: choice.model,
|
||||
maxTokens: choice.maxTokens,
|
||||
signal: abort.signal,
|
||||
},
|
||||
{
|
||||
onText: (delta) => send('text', delta),
|
||||
onSpec: async ({ spec, source }) => {
|
||||
resolved = true;
|
||||
const previewId = await cacheSpec(spec);
|
||||
send('spec', {
|
||||
previewId,
|
||||
source,
|
||||
plan,
|
||||
modelDisplayName: choice.displayName,
|
||||
modelBadge: choice.displayBadge,
|
||||
upgradeHint: plan === 'hobby',
|
||||
spec: {
|
||||
name: spec.name,
|
||||
description: spec.description,
|
||||
tools: spec.tools.map((t) => ({
|
||||
name: t.name,
|
||||
description: t.description,
|
||||
inputSchema: t.inputSchema,
|
||||
})),
|
||||
requiredSecrets: spec.requiredSecrets,
|
||||
scopes: spec.scopes,
|
||||
},
|
||||
});
|
||||
},
|
||||
onError: (err) => {
|
||||
resolved = true;
|
||||
if (err instanceof SpecTruncatedError) {
|
||||
app.log.warn(
|
||||
{
|
||||
reason: err.message,
|
||||
prompt: parsed.data.prompt.slice(0, 200),
|
||||
model: choice.displayName,
|
||||
},
|
||||
'preview_spec_truncated',
|
||||
);
|
||||
send('error', {
|
||||
error: 'spec_too_large',
|
||||
detail:
|
||||
'The spec for this prompt exceeded the maximum response size. Split it into fewer tools or describe one capability per prompt.',
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (err instanceof SpecValidationError) {
|
||||
app.log.warn(
|
||||
{
|
||||
zod_message: err.message,
|
||||
prompt: parsed.data.prompt.slice(0, 200),
|
||||
model: choice.displayName,
|
||||
},
|
||||
'preview_spec_invalid',
|
||||
);
|
||||
send('error', { error: 'spec_invalid', detail: err.message });
|
||||
return;
|
||||
}
|
||||
if (err instanceof BannedPatternError) {
|
||||
send('error', { error: 'banned_pattern', detail: err.message });
|
||||
return;
|
||||
}
|
||||
if (err instanceof SpecTimeoutError) {
|
||||
send('error', {
|
||||
error: 'preview_timeout',
|
||||
detail: 'Spec generation took too long. Try a shorter, more specific prompt.',
|
||||
});
|
||||
return;
|
||||
}
|
||||
app.log.error(err);
|
||||
send('error', { error: 'preview_failed', detail: err.message });
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (!resolved) {
|
||||
send('error', { error: 'preview_failed', detail: 'stream ended without a final event' });
|
||||
}
|
||||
clearInterval(keepalive);
|
||||
reply.raw.end();
|
||||
});
|
||||
|
||||
app.post('/v1/servers', { preHandler: requireAuth }, async (req, reply) => {
|
||||
const user = req.user!;
|
||||
const parsed = CreateServerInput.safeParse(req.body);
|
||||
|
||||
Reference in New Issue
Block a user