Skip to content

gitflarePublicPartly agent-ready50

cf-workflows-pipeline

Durable content pipeline on Cloudflare Workflows: ingest, extract, summarise with Workers AI via AI Gateway, human approval, publish.

Fix pipeline runs failing at ingest (steps.attempt NOT NULL)629b2fa
TypeScript118 lines6.2 KB
import { WorkflowEntrypoint, WorkflowStep, WorkflowEvent } from "cloudflare:workers";import type { Env, RunParams } from "./env";import { extract } from "./lib/extract";import { parsePlan, shouldFail } from "./lib/inject";import { extractiveSummary } from "./lib/fallback";import { validateUrl } from "./lib/validate"; const MODEL = "@cf/meta/llama-3.3-70b-instruct-fp8-fast";const CFG = { retries: { limit: 3, delay: "2 seconds", backoff: "exponential" }, timeout: "30 seconds" } as const;const MAX_BYTES = 2 * 1024 * 1024;const LEN: Record<string, string> = { short: "2 sentences", medium: "one paragraph", long: "three paragraphs" }; export async function setStep(env: Env, runId: string, name: string, state: string, f: { attempt?: number; out?: string; err?: string; start?: boolean; end?: boolean } = {}) {  const now = Date.now();  await env.DB.prepare(    `INSERT INTO steps (id, run_id, name, state, attempt, started_at, ended_at, output_excerpt, error)     VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)     ON CONFLICT(id) DO UPDATE SET state=?4, attempt=COALESCE(?10, attempt),       started_at=COALESCE(started_at, ?6), ended_at=COALESCE(?7, ended_at),       output_excerpt=COALESCE(?8, output_excerpt), error=?9`,  ).bind(`${runId}:${name}`, runId, name, state, f.attempt ?? 1, f.start ? now : null, f.end ? now : null, f.out?.slice(0, 600) ?? null, f.err ?? null, f.attempt ?? null).run();}async function setRun(env: Env, runId: string, status: string) {  await env.DB.prepare("UPDATE runs SET status=?1, updated_at=?2 WHERE id=?3").bind(status, Date.now(), runId).run();} export class ContentPipeline extends WorkflowEntrypoint<Env, RunParams> {  async run(event: WorkflowEvent<RunParams>, step: WorkflowStep) {    const p = event.payload;    const env = this.env;    const plan = parsePlan(p.failures);    const id = p.runId;    const guarded = async <T>(name: string, ctx: { attempt: number }, fn: () => Promise<T>): Promise<T> => {      await setStep(env, id, name, (ctx?.attempt ?? 1) > 1 ? "retrying" : "running", { attempt: (ctx?.attempt ?? 1), start: true });      try {        if (shouldFail(plan, name, (ctx?.attempt ?? 1))) throw new Error(`injected failure (attempt ${(ctx?.attempt ?? 1)})`);        const r = await fn();        return r;      } catch (e) {        await setStep(env, id, name, (ctx?.attempt ?? 1) >= 4 ? "failed" : "retrying", { attempt: (ctx?.attempt ?? 1), err: String(e instanceof Error ? e.message : e) });        throw e;      }    };    try {      await setRun(env, id, "running");      const raw = await step.do("ingest", CFG, async (ctx) => guarded("ingest", ctx, async () => {        let body = p.text ?? "";        if (!p.text) {          const v = validateUrl(p.url ?? "");          if (!v.ok) throw new Error(v.error);          const res = await fetch(v.url, { signal: AbortSignal.timeout(10_000), redirect: "follow" });          const ct = res.headers.get("content-type") ?? "";          if (!/text\/(html|plain)/i.test(ct)) throw new Error(`unsupported content-type: ${ct}`);          const buf = await res.arrayBuffer();          if (buf.byteLength > MAX_BYTES) throw new Error("response over 2 MB");          body = new TextDecoder().decode(buf);          if (/text\/plain/i.test(ct)) body = `<main>${body.replace(/</g, "&lt;")}</main>`;        } else body = `<main>${body.replace(/</g, "&lt;")}</main>`;        await setStep(env, id, "ingest", "done", { out: `${body.length} chars`, end: true });        return body;      }));      const ex = await step.do("extract", CFG, async (ctx) => guarded("extract", ctx, async () => {        const r = extract(raw);        await setStep(env, id, "extract", "done", { out: `${r.title}\n${r.text}`, end: true });        return r;      }));      const summary = await step.do("summarise", CFG, async (ctx) => guarded("summarise", ctx, async () => {        let s = "";        let fallback = false;        try {          const out = (await env.AI.run(MODEL as keyof AiModels, {            messages: [              { role: "system", content: `Summarise the text in ${LEN[p.length] ?? LEN.medium}.` },              { role: "user", content: ex.text.slice(0, 12000) },            ],          } as never, { gateway: { id: env.AI_GATEWAY_ID } })) as { response?: string };          s = (out.response ?? "").trim();          if (!s) throw new Error("empty summary");        } catch (e) {          console.log(JSON.stringify({ level: "warn", msg: "ai_fallback", runId: id, error: String(e) }));          s = extractiveSummary(ex.text, p.length);          fallback = true;        }        await setStep(env, id, "summarise", "done", { out: fallback ? `[fallback: extractive] ${s}` : s, end: true });        return s;      }));      await setStep(env, id, "approval", "waiting", { start: true });      await setRun(env, id, "awaiting_approval");      let approval;      try {        approval = await step.waitForEvent<{ approved: boolean; note?: string }>("approval", { type: "approval", timeout: "24 hours" });      } catch (e) {        await setStep(env, id, "approval", "failed", { err: "approval timed out", end: true });        await setRun(env, id, "failed");        throw e;      }      if (!approval.payload?.approved) {        await setStep(env, id, "approval", "failed", { err: `rejected: ${approval.payload?.note ?? ""}`, end: true });        await setRun(env, id, "rejected");        return { status: "rejected" };      }      await setStep(env, id, "approval", "done", { end: true });      await setRun(env, id, "running");      await step.do("publish", CFG, async (ctx) => guarded("publish", ctx, async () => {        await env.DB.prepare("INSERT OR REPLACE INTO published (id,title,summary,source_url,published_at) VALUES (?1,?2,?3,?4,?5)")          .bind(id, ex.title || "Untitled", summary, p.url ?? null, Date.now()).run();        await setStep(env, id, "publish", "done", { out: `/p/${id}`, end: true });      }));      await setRun(env, id, "published");      return { status: "published" };    } catch (e) {      const row = await env.DB.prepare("SELECT status FROM runs WHERE id=?1").bind(id).first<{ status: string }>();      if (row?.status !== "rejected") await setRun(env, id, "failed");      throw e;    }  }}