gitflarePublicPartly agent-ready50
cf-workflows-pipeline
Durable content pipeline on Cloudflare Workflows: ingest, extract, summarise with Workers AI via AI Gateway, human approval, publish.
- LicenseApache-2.0
- LanguageTypeScript
- Last landed
629b2faOct 20, 2026
Fix pipeline runs failing at ingest (steps.attempt NOT NULL)
629b2faimport { WorkflowEntrypoint, WorkflowStep, WorkflowEvent } from "cloudflare:workers";import type { Env, RunParams } from "./env";import { extract } from "./lib/extract";import { parsePlan, shouldFail } from "./lib/inject";import { extractiveSummary } from "./lib/fallback";import { validateUrl } from "./lib/validate"; const MODEL = "@cf/meta/llama-3.3-70b-instruct-fp8-fast";const CFG = { retries: { limit: 3, delay: "2 seconds", backoff: "exponential" }, timeout: "30 seconds" } as const;const MAX_BYTES = 2 * 1024 * 1024;const LEN: Record<string, string> = { short: "2 sentences", medium: "one paragraph", long: "three paragraphs" }; export async function setStep(env: Env, runId: string, name: string, state: string, f: { attempt?: number; out?: string; err?: string; start?: boolean; end?: boolean } = {}) { const now = Date.now(); await env.DB.prepare( `INSERT INTO steps (id, run_id, name, state, attempt, started_at, ended_at, output_excerpt, error) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ON CONFLICT(id) DO UPDATE SET state=?4, attempt=COALESCE(?10, attempt), started_at=COALESCE(started_at, ?6), ended_at=COALESCE(?7, ended_at), output_excerpt=COALESCE(?8, output_excerpt), error=?9`, ).bind(`${runId}:${name}`, runId, name, state, f.attempt ?? 1, f.start ? now : null, f.end ? now : null, f.out?.slice(0, 600) ?? null, f.err ?? null, f.attempt ?? null).run();}async function setRun(env: Env, runId: string, status: string) { await env.DB.prepare("UPDATE runs SET status=?1, updated_at=?2 WHERE id=?3").bind(status, Date.now(), runId).run();} export class ContentPipeline extends WorkflowEntrypoint<Env, RunParams> { async run(event: WorkflowEvent<RunParams>, step: WorkflowStep) { const p = event.payload; const env = this.env; const plan = parsePlan(p.failures); const id = p.runId; const guarded = async <T>(name: string, ctx: { attempt: number }, fn: () => Promise<T>): Promise<T> => { await setStep(env, id, name, (ctx?.attempt ?? 1) > 1 ? "retrying" : "running", { attempt: (ctx?.attempt ?? 1), start: true }); try { if (shouldFail(plan, name, (ctx?.attempt ?? 1))) throw new Error(`injected failure (attempt ${(ctx?.attempt ?? 1)})`); const r = await fn(); return r; } catch (e) { await setStep(env, id, name, (ctx?.attempt ?? 1) >= 4 ? "failed" : "retrying", { attempt: (ctx?.attempt ?? 1), err: String(e instanceof Error ? e.message : e) }); throw e; } }; try { await setRun(env, id, "running"); const raw = await step.do("ingest", CFG, async (ctx) => guarded("ingest", ctx, async () => { let body = p.text ?? ""; if (!p.text) { const v = validateUrl(p.url ?? ""); if (!v.ok) throw new Error(v.error); const res = await fetch(v.url, { signal: AbortSignal.timeout(10_000), redirect: "follow" }); const ct = res.headers.get("content-type") ?? ""; if (!/text\/(html|plain)/i.test(ct)) throw new Error(`unsupported content-type: ${ct}`); const buf = await res.arrayBuffer(); if (buf.byteLength > MAX_BYTES) throw new Error("response over 2 MB"); body = new TextDecoder().decode(buf); if (/text\/plain/i.test(ct)) body = `<main>${body.replace(/</g, "<")}</main>`; } else body = `<main>${body.replace(/</g, "<")}</main>`; await setStep(env, id, "ingest", "done", { out: `${body.length} chars`, end: true }); return body; })); const ex = await step.do("extract", CFG, async (ctx) => guarded("extract", ctx, async () => { const r = extract(raw); await setStep(env, id, "extract", "done", { out: `${r.title}\n${r.text}`, end: true }); return r; })); const summary = await step.do("summarise", CFG, async (ctx) => guarded("summarise", ctx, async () => { let s = ""; let fallback = false; try { const out = (await env.AI.run(MODEL as keyof AiModels, { messages: [ { role: "system", content: `Summarise the text in ${LEN[p.length] ?? LEN.medium}.` }, { role: "user", content: ex.text.slice(0, 12000) }, ], } as never, { gateway: { id: env.AI_GATEWAY_ID } })) as { response?: string }; s = (out.response ?? "").trim(); if (!s) throw new Error("empty summary"); } catch (e) { console.log(JSON.stringify({ level: "warn", msg: "ai_fallback", runId: id, error: String(e) })); s = extractiveSummary(ex.text, p.length); fallback = true; } await setStep(env, id, "summarise", "done", { out: fallback ? `[fallback: extractive] ${s}` : s, end: true }); return s; })); await setStep(env, id, "approval", "waiting", { start: true }); await setRun(env, id, "awaiting_approval"); let approval; try { approval = await step.waitForEvent<{ approved: boolean; note?: string }>("approval", { type: "approval", timeout: "24 hours" }); } catch (e) { await setStep(env, id, "approval", "failed", { err: "approval timed out", end: true }); await setRun(env, id, "failed"); throw e; } if (!approval.payload?.approved) { await setStep(env, id, "approval", "failed", { err: `rejected: ${approval.payload?.note ?? ""}`, end: true }); await setRun(env, id, "rejected"); return { status: "rejected" }; } await setStep(env, id, "approval", "done", { end: true }); await setRun(env, id, "running"); await step.do("publish", CFG, async (ctx) => guarded("publish", ctx, async () => { await env.DB.prepare("INSERT OR REPLACE INTO published (id,title,summary,source_url,published_at) VALUES (?1,?2,?3,?4,?5)") .bind(id, ex.title || "Untitled", summary, p.url ?? null, Date.now()).run(); await setStep(env, id, "publish", "done", { out: `/p/${id}`, end: true }); })); await setRun(env, id, "published"); return { status: "published" }; } catch (e) { const row = await env.DB.prepare("SELECT status FROM runs WHERE id=?1").bind(id).first<{ status: string }>(); if (row?.status !== "rejected") await setRun(env, id, "failed"); throw e; } }}