Skip to content

gitflarePublicPartly agent-ready50

cf-d1-queues-pulse

Event analytics on Cloudflare D1 and Queues: ingestion API, batched consumer, rollups, live dashboard, load generator, dead-letter view.

Build the full D1 + Queues pulse app: dashboard, load generator, dead letters, percentiles, docs8a65841
TypeScript61 lines2.4 KB
import { validateEvent, type PulseEvent } from "./lib/validate";import { aggregate } from "./lib/aggregate";import * as Q from "./lib/sql";import { reservoir } from "./lib/percentile";import { route, isPoison } from "./routes";import type { Env } from "./env"; export type { Env };const SAMPLE_CAP = 100;const log = (event: string, data: Record<string, unknown> = {}) => console.log(JSON.stringify({ event, ...data })); export default {  async fetch(req: Request, env: Env): Promise<Response> {    try { return await route(req, env); }    catch (e) {      log("error", { name: e instanceof Error ? e.name : "unknown" });      return Response.json({ error: { code: "internal", message: "internal error" } }, { status: 500 });    }  },   async queue(batch: MessageBatch<unknown>, env: Env): Promise<void> {    const t0 = Date.now();    if (batch.queue.includes("-dlq")) {      await env.DB.batch(batch.messages.map((m) =>        env.DB.prepare(Q.INSERT_DEAD).bind(JSON.stringify(m.body), "exhausted retries", m.attempts, t0)));      batch.ackAll();      log("dlq_stored", { size: batch.messages.length });      return;    }    const good: PulseEvent[] = [];    const goodMsgs: Message<unknown>[] = [];    for (const m of batch.messages) {      const e = isPoison(m.body) ? null : validateEvent(m.body, t0);      if (e) { good.push(e); goodMsgs.push(m); } else m.retry();    }    const stmts = aggregate(good).map((r) => env.DB.prepare(Q.UPSERT_ROLLUP).bind(...Q.upsertParams(r)));    const byMin = new Map<number, number[]>();    for (const e of good) {      const k = Math.floor(e.ts / 60000);      const a = byMin.get(k) ?? [];      a.push(e.value);      byMin.set(k, a);    }    for (const [min, vs] of byMin)      for (const v of reservoir(vs, SAMPLE_CAP)) stmts.push(env.DB.prepare(Q.INSERT_SAMPLE).bind(min, v));    stmts.push(env.DB.prepare(Q.INSERT_STAT).bind(t0, batch.messages.length, Date.now() - t0));    await env.DB.batch(stmts);    goodMsgs.forEach((m) => m.ack());    log("batch", { size: batch.messages.length, good: good.length, ms: Date.now() - t0 });  },   async scheduled(_c: ScheduledController, env: Env): Promise<void> {    const min = Math.floor(Date.now() / 60000) - 7 * 1440;    await env.DB.batch([      env.DB.prepare(Q.PURGE_OLD).bind(min),      env.DB.prepare(Q.PURGE_SAMPLES).bind(min),      env.DB.prepare(Q.PURGE_STATS).bind(min * 60000),    ]);  },};