diff --git a/.env.example b/.env.example index 1bab4eb6e..813614875 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,9 @@ BULLMQ_REDIS_URL="redis://localhost:6379" INSIGHTS_PORT="4002" INSIGHTS_BULLMQ_REDIS_URL="" INSIGHTS_DISPATCH_INTERVAL_MS="300000" +INSIGHTS_SCHEDULED_DISPATCH_ENABLED="false" +# Comma-separated organization IDs for canaries; use * only after rollout. +INSIGHTS_SCHEDULED_ORGANIZATION_IDS="" INSIGHTS_MAINTENANCE_INTERVAL_MS="300000" INSIGHTS_STALE_ITEM_MS="900000" INSIGHTS_WORKER_CONCURRENCY="2" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 873e7d5d7..daf2d49a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,6 @@ jobs: name: Check Types runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 - env: - CLICKHOUSE_READONLY_URL: ${{ secrets.CLICKHOUSE_READONLY_URL }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 @@ -61,8 +59,12 @@ jobs: - name: ClickHouse types match DDL run: bun run --cwd packages/db ch:check - name: ClickHouse schema matches cluster - if: env.CLICKHOUSE_READONLY_URL != '' - run: bun run --cwd packages/db ch:verify + env: + CLICKHOUSE_READONLY_URL: ${{ secrets.CLICKHOUSE_READONLY_URL }} + run: | + if [[ -n "$CLICKHOUSE_READONLY_URL" ]]; then + bun run --cwd packages/db ch:verify + fi test: name: Test @@ -127,10 +129,12 @@ jobs: ports: - 8123:8123 options: >- + --ulimit nofile=262144:262144 --health-cmd "clickhouse-client --query 'SELECT 1'" --health-interval 10s --health-timeout 5s - --health-retries 5 + --health-start-period 30s + --health-retries 12 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -158,6 +162,8 @@ jobs: run: bun run --cwd packages/db clickhouse:init - name: Test env: + CLICKHOUSE_INTEGRATION_TESTS: "true" + CLICKHOUSE_URL: http://default:@localhost:8123 NODE_ENV: test run: bun run test - name: Insights integration diff --git a/apps/api/src/billing/autumn-webhook-replay.test.ts b/apps/api/src/billing/autumn-webhook-replay.test.ts new file mode 100644 index 000000000..5305bb6c6 --- /dev/null +++ b/apps/api/src/billing/autumn-webhook-replay.test.ts @@ -0,0 +1,136 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ error: vi.fn(), warn: vi.fn() })); + +vi.mock("evlog", () => ({ log: { error: state.error, warn: state.warn } })); +vi.mock("@/routes/webhooks/autumn", () => ({ + replayDeferredAutumnWebhooks: vi.fn(async () => ({ + completed: 0, + deadLettered: 0, + deferred: 0, + failed: [], + })), +})); +vi.mock("@/routes/webhooks/autumn-inbox", () => ({ + deleteCompletedAutumnWebhooks: vi.fn(async () => 0), + deleteDeadLetterAutumnWebhooks: vi.fn(async () => 0), + listUnalertedAutumnWebhookDeadLetters: vi.fn(async () => []), + markAutumnWebhookDeadLettersAlerted: vi.fn(async () => 0), +})); + +import { startAutumnWebhookReplayLoop } from "./autumn-webhook-replay"; +import { runAutumnWebhookMaintenance } from "./autumn-webhook-replay"; +import { replayDeferredAutumnWebhooks } from "@/routes/webhooks/autumn"; +import { + deleteCompletedAutumnWebhooks, + deleteDeadLetterAutumnWebhooks, + listUnalertedAutumnWebhookDeadLetters, + markAutumnWebhookDeadLettersAlerted, +} from "@/routes/webhooks/autumn-inbox"; + +beforeEach(() => { + vi.useFakeTimers(); + state.error.mockClear(); + state.warn.mockClear(); + vi.mocked(deleteCompletedAutumnWebhooks).mockClear(); + vi.mocked(deleteDeadLetterAutumnWebhooks).mockClear(); + vi.mocked(listUnalertedAutumnWebhookDeadLetters).mockClear(); + vi.mocked(markAutumnWebhookDeadLettersAlerted).mockClear(); + vi.mocked(replayDeferredAutumnWebhooks).mockClear(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("Autumn webhook replay loop", () => { + it("runs bounded replay and retention maintenance in one shot", async () => { + await expect(runAutumnWebhookMaintenance()).resolves.toEqual({ + completed: 0, + deadLettered: 0, + deadLetters: 0, + deferred: 0, + deleted: 0, + failed: [], + }); + expect(replayDeferredAutumnWebhooks).toHaveBeenCalledWith(100); + expect(deleteCompletedAutumnWebhooks).toHaveBeenCalledWith({ limit: 100 }); + expect(deleteDeadLetterAutumnWebhooks).toHaveBeenCalledWith({ limit: 100 }); + }); + + it("reports item-level replay failures that remain queued", async () => { + vi.mocked(replayDeferredAutumnWebhooks).mockResolvedValueOnce({ + completed: 0, + deadLettered: 0, + deferred: 0, + failed: ["msg-failed"], + }); + + await runAutumnWebhookMaintenance(); + + expect(state.warn).toHaveBeenCalledWith( + expect.objectContaining({ + component: "autumn_webhook_replay", + failed_count: 1, + }) + ); + }); + + it("alerts once for newly dead-lettered webhooks before retention", async () => { + vi.mocked(listUnalertedAutumnWebhookDeadLetters).mockResolvedValueOnce([ + { + attempts: 12, + deadLetteredAt: new Date("2026-07-01T00:00:00.000Z"), + errorMessage: "provider unavailable", + id: "msg-dead", + type: "balances.limit_reached", + }, + ]); + + await runAutumnWebhookMaintenance(); + + expect(state.error).toHaveBeenCalledWith( + expect.objectContaining({ + component: "autumn_webhook_replay", + dead_letter_count: 1, + dead_letter_ids: ["msg-dead"], + }) + ); + expect(markAutumnWebhookDeadLettersAlerted).toHaveBeenCalledWith([ + "msg-dead", + ]); + }); + + it("runs immediately, repeats every minute, and stops deterministically", async () => { + const maintenance = vi.fn(async () => undefined); + const loop = startAutumnWebhookReplayLoop(maintenance); + + await loop.run(); + expect(maintenance).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(60_000); + expect(maintenance).toHaveBeenCalledTimes(2); + + loop.stop(); + await vi.advanceTimersByTimeAsync(120_000); + expect(maintenance).toHaveBeenCalledTimes(2); + }); + + it("logs maintenance failures without rejecting or stopping the loop", async () => { + const maintenance = vi.fn(async () => { + throw new Error("database unavailable"); + }); + const loop = startAutumnWebhookReplayLoop(maintenance); + + await expect(loop.run()).resolves.toBeUndefined(); + expect(state.error).toHaveBeenCalledWith( + expect.objectContaining({ + component: "autumn_webhook_replay", + error_message: "database unavailable", + }) + ); + + await vi.advanceTimersByTimeAsync(60_000); + expect(maintenance).toHaveBeenCalledTimes(2); + loop.stop(); + }); +}); diff --git a/apps/api/src/billing/autumn-webhook-replay.ts b/apps/api/src/billing/autumn-webhook-replay.ts new file mode 100644 index 000000000..4444fb65a --- /dev/null +++ b/apps/api/src/billing/autumn-webhook-replay.ts @@ -0,0 +1,103 @@ +import { log } from "evlog"; +import { + deleteCompletedAutumnWebhooks, + deleteDeadLetterAutumnWebhooks, + listUnalertedAutumnWebhookDeadLetters, + markAutumnWebhookDeadLettersAlerted, +} from "@/routes/webhooks/autumn-inbox"; +import { replayDeferredAutumnWebhooks } from "@/routes/webhooks/autumn"; + +const REPLAY_INTERVAL_MS = 60_000; +const REPLAY_BATCH_SIZE = 100; + +export interface AutumnWebhookReplayLoop { + run(): Promise; + stop(): void; +} + +export async function runAutumnWebhookMaintenance(): Promise<{ + completed: number; + deadLettered: number; + deadLetters: number; + deferred: number; + deleted: number; + failed: string[]; +}> { + const replay = await replayDeferredAutumnWebhooks(REPLAY_BATCH_SIZE); + if (replay.failed.length > 0) { + log.warn({ + service: "api", + component: "autumn_webhook_replay", + message: "Autumn webhook replay batch had failures", + failed_count: replay.failed.length, + }); + } + + const deadLetters = await listUnalertedAutumnWebhookDeadLetters({ + limit: REPLAY_BATCH_SIZE, + }); + if (deadLetters.length > 0) { + log.error({ + service: "api", + component: "autumn_webhook_replay", + dead_letter_count: deadLetters.length, + dead_letter_ids: deadLetters.map((row) => row.id), + error_message: "Autumn webhooks exhausted replay attempts", + }); + await markAutumnWebhookDeadLettersAlerted(deadLetters.map((row) => row.id)); + } + + const deletedRows = await Promise.all([ + deleteCompletedAutumnWebhooks({ limit: REPLAY_BATCH_SIZE }), + deleteDeadLetterAutumnWebhooks({ limit: REPLAY_BATCH_SIZE }), + ]); + return { + ...replay, + deadLetters: deadLetters.length, + deleted: deletedRows.reduce((total, count) => total + count, 0), + }; +} + +export function startAutumnWebhookReplayLoop( + maintenance: () => Promise = runAutumnWebhookMaintenance +): AutumnWebhookReplayLoop { + let active: Promise | null = null; + let stopped = false; + + const run = (): Promise => { + if (stopped) { + return Promise.resolve(); + } + if (active) { + return active; + } + active = Promise.resolve() + .then(maintenance) + .then(() => undefined) + .catch((error) => { + log.error({ + service: "api", + component: "autumn_webhook_replay", + error_message: error instanceof Error ? error.message : String(error), + }); + }) + .finally(() => { + active = null; + }); + return active; + }; + + run().catch(() => undefined); + const timer = setInterval(() => { + run().catch(() => undefined); + }, REPLAY_INTERVAL_MS); + timer.unref?.(); + + return { + run, + stop: () => { + stopped = true; + clearInterval(timer); + }, + }; +} diff --git a/apps/api/src/bootstrap/shutdown.ts b/apps/api/src/bootstrap/shutdown.ts index 71b4c208d..09721a7d6 100644 --- a/apps/api/src/bootstrap/shutdown.ts +++ b/apps/api/src/bootstrap/shutdown.ts @@ -15,12 +15,12 @@ export function warmPostgresPool() { ); } -export function registerShutdownHooks() { - process.on("SIGINT", () => shutdownApi("SIGINT")); - process.on("SIGTERM", () => shutdownApi("SIGTERM")); +export function registerShutdownHooks(beforeShutdown?: () => void) { + process.on("SIGINT", () => shutdownApi("SIGINT", beforeShutdown)); + process.on("SIGTERM", () => shutdownApi("SIGTERM", beforeShutdown)); } -async function shutdownApi(signal: string) { +async function shutdownApi(signal: string, beforeShutdown?: () => void) { if (shuttingDown) { log.info({ lifecycle: "shutdown", @@ -31,6 +31,7 @@ async function shutdownApi(signal: string) { } shuttingDown = true; + beforeShutdown?.(); const timeout = setTimeout(() => { log.error({ lifecycle: "shutdown", diff --git a/apps/api/src/http/errors.test.ts b/apps/api/src/http/errors.test.ts index eaf20a710..8239012e9 100644 --- a/apps/api/src/http/errors.test.ts +++ b/apps/api/src/http/errors.test.ts @@ -83,7 +83,29 @@ describe("handleAppError", () => { requestId: "req_test_validation", }); expect(payload.details).toEqual([ - expect.objectContaining({ field: "body.name" }), + { field: "body.name", message: "Invalid value" }, ]); }); + + it("does not reflect Elysia validation values or messages in production", async () => { + process.env.NODE_ENV = "production"; + const response = handleAppError({ + code: "VALIDATION", + requestId: "req_test_reflection", + error: new ValidationError( + "body", + t.Object({ profile: t.Object({ email: t.String() }) }), + { profile: { email: 867_530_900 } } + ), + }); + const payload = await readPayload(response); + const serialized = JSON.stringify(payload); + + expect(payload.details).toEqual([ + { field: "body.profile.email", message: "Invalid value" }, + ]); + expect(serialized).not.toContain("867530900"); + expect(serialized).not.toContain("Expected"); + expect(serialized).not.toContain("found"); + }); }); diff --git a/apps/api/src/http/errors.ts b/apps/api/src/http/errors.ts index 94759eef2..8f817dff6 100644 --- a/apps/api/src/http/errors.ts +++ b/apps/api/src/http/errors.ts @@ -59,7 +59,7 @@ export function handleAppError({ isClientError, statusCode, }); - const validationDetails = getValidationDetails(error); + const validationDetails = getValidationDetails(error, isDevelopment); const headers: Record = { "Content-Type": "application/json", "X-Request-ID": responseRequestId, @@ -91,7 +91,10 @@ interface ValidationDetail { message: string; } -function getValidationDetails(error: unknown): ValidationDetail[] { +function getValidationDetails( + error: unknown, + isDevelopment: boolean +): ValidationDetail[] { if (!(error instanceof ValidationError) || error.type === "response") { return []; } @@ -111,10 +114,9 @@ function getValidationDetails(error: unknown): ValidationDetail[] { seenFields.add(field); details.push({ field, - message: - typeof issue.summary === "string" && issue.summary - ? issue.summary - : issue.message, + message: isDevelopment + ? getDevelopmentValidationMessage(issue) + : "Invalid value", }); if (details.length === 20) { break; @@ -123,6 +125,15 @@ function getValidationDetails(error: unknown): ValidationDetail[] { return details; } +function getDevelopmentValidationMessage(issue: { + message: string; + summary?: string; +}): string { + return typeof issue.summary === "string" && issue.summary + ? issue.summary + : issue.message; +} + function getErrorCode({ explicitCode, parsedCode, diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 444f58eed..885439b60 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -4,6 +4,7 @@ import cors from "@elysiajs/cors"; import { Elysia } from "elysia"; import { evlog } from "evlog/elysia"; import { handleAutumnRequest } from "@/billing/autumn"; +import { startAutumnWebhookReplayLoop } from "@/billing/autumn-webhook-replay"; import { configureApiInstrumentation } from "@/bootstrap/instrumentation"; import { configureApiLogger } from "@/bootstrap/logger"; import { registerProcessErrorHandlers } from "@/bootstrap/process-errors"; @@ -115,8 +116,9 @@ const app = new Elysia({ precompile: true }) .all("/*", handleOpenApiEndpoint, { parse: "none" }) .onError(handleAppError); +const autumnWebhookReplay = startAutumnWebhookReplayLoop(); warmPostgresPool(); -registerShutdownHooks(); +registerShutdownHooks(autumnWebhookReplay.stop); export default { fetch: app.fetch, diff --git a/apps/api/src/integration/autumn-webhook-inbox.test.ts b/apps/api/src/integration/autumn-webhook-inbox.test.ts new file mode 100644 index 000000000..743cbfee8 --- /dev/null +++ b/apps/api/src/integration/autumn-webhook-inbox.test.ts @@ -0,0 +1,248 @@ +import "@databuddy/test/env"; + +import { afterAll, afterEach, describe, expect, it } from "vitest"; +import { db, eq, inArray, shutdownPostgres } from "@databuddy/db"; +import { autumnWebhookEvents } from "@databuddy/db/schema"; +import { hasTestDb } from "@databuddy/test"; +import { + AUTUMN_WEBHOOK_MAX_ATTEMPTS, + autumnWebhookRetryDelayMs, + claimAutumnWebhook, + deleteCompletedAutumnWebhooks, + deleteDeadLetterAutumnWebhooks, + getAutumnWebhook, + listReplayableAutumnWebhookIds, + listUnalertedAutumnWebhookDeadLetters, + markAutumnWebhookDeadLettersAlerted, + recordAutumnWebhookAttempt, + storeAutumnWebhook, +} from "../routes/webhooks/autumn-inbox"; + +const iit = hasTestDb ? it : it.skip; +const ids = new Set(); + +function webhookId(label: string): string { + const id = `test-autumn-${label}-${crypto.randomUUID()}`; + ids.add(id); + return id; +} + +async function store(id: string): Promise { + await storeAutumnWebhook({ + id, + payload: { + customer_id: "user-example", + feature_id: "events", + limit_type: "included", + }, + type: "balances.limit_reached", + }); +} + +afterEach(async () => { + if (ids.size > 0) { + await db + .delete(autumnWebhookEvents) + .where(inArray(autumnWebhookEvents.id, [...ids])); + ids.clear(); + } +}); + +afterAll(() => shutdownPostgres()); + +describe("Autumn webhook inbox", () => { + iit("keeps the first payload and moves one claim through replay states", async () => { + const id = webhookId("lifecycle"); + const first = await storeAutumnWebhook({ + id, + payload: { + customer_id: "user-example", + feature_id: "events", + limit_type: "included", + }, + type: "balances.limit_reached", + }); + const duplicate = await storeAutumnWebhook({ + id, + payload: { customer_id: "different-customer" }, + type: "balances.usage_alert_triggered", + }); + + expect(first).toEqual(duplicate); + expect(duplicate.type).toBe("balances.limit_reached"); + expect(duplicate.payload).toEqual({ + customer_id: "user-example", + feature_id: "events", + limit_type: "included", + }); + + const firstClaim = await claimAutumnWebhook({ id }); + expect(firstClaim).not.toBeNull(); + await recordAutumnWebhookAttempt({ + attempts: firstClaim!.attempts, + claimToken: firstClaim!.claimToken, + errorMessage: "organization could not be resolved", + id, + status: "deferred", + }); + + const replayAt = new Date(Date.now() + 7 * 60 * 60 * 1000); + expect( + await listReplayableAutumnWebhookIds({ now: replayAt }) + ).toContain(id); + const replayClaim = await claimAutumnWebhook({ id, now: replayAt }); + expect(replayClaim).not.toBeNull(); + await recordAutumnWebhookAttempt({ + attempts: replayClaim!.attempts, + claimToken: replayClaim!.claimToken, + id, + now: replayAt, + status: "completed", + }); + + expect(await listReplayableAutumnWebhookIds({ now: replayAt })).not.toContain( + id + ); + expect((await getAutumnWebhook(id))?.status).toBe("completed"); + + const [row] = await db + .select({ + attempts: autumnWebhookEvents.attempts, + completedAt: autumnWebhookEvents.completedAt, + errorMessage: autumnWebhookEvents.errorMessage, + }) + .from(autumnWebhookEvents) + .where(eq(autumnWebhookEvents.id, id)); + expect(row).toEqual({ + attempts: 2, + completedAt: replayAt, + errorMessage: null, + }); + }); + + iit("allows one claim and keeps both completion orders terminal", async () => { + const completedId = webhookId("concurrent-completed"); + const failedId = webhookId("concurrent-failed"); + await store(completedId); + await store(failedId); + const claims = await Promise.all( + Array.from({ length: 8 }, () => claimAutumnWebhook({ id: completedId })) + ); + const claim = claims.find((row) => row !== null); + expect(claims.filter((row) => row !== null)).toHaveLength(1); + expect(claim).toBeDefined(); + + await recordAutumnWebhookAttempt({ + attempts: claim!.attempts, + claimToken: claim!.claimToken, + id: completedId, + status: "completed", + }); + await recordAutumnWebhookAttempt({ + attempts: claim!.attempts, + claimToken: claim!.claimToken, + errorMessage: "late worker failure", + id: completedId, + status: "pending", + }); + expect((await getAutumnWebhook(completedId))?.status).toBe("completed"); + + const failedClaim = await claimAutumnWebhook({ id: failedId }); + expect(failedClaim).not.toBeNull(); + await recordAutumnWebhookAttempt({ + attempts: failedClaim!.attempts, + claimToken: failedClaim!.claimToken, + errorMessage: "provider failed", + id: failedId, + status: "pending", + }); + await recordAutumnWebhookAttempt({ + attempts: failedClaim!.attempts, + claimToken: failedClaim!.claimToken, + id: failedId, + status: "completed", + }); + expect((await getAutumnWebhook(failedId))?.status).toBe("pending"); + }); + + iit("bounds retry delay and retains alerted dead letters for audit", async () => { + expect(autumnWebhookRetryDelayMs(1)).toBe(5 * 60 * 1000); + expect(autumnWebhookRetryDelayMs(100)).toBe(6 * 60 * 60 * 1000); + + const oldId = webhookId("dead-old"); + const recentId = webhookId("dead-recent"); + const unalertedId = webhookId("dead-unalerted"); + for (const id of [oldId, recentId, unalertedId]) { + await store(id); + await db + .update(autumnWebhookEvents) + .set({ attempts: AUTUMN_WEBHOOK_MAX_ATTEMPTS - 1 }) + .where(eq(autumnWebhookEvents.id, id)); + const claim = await claimAutumnWebhook({ id }); + expect(claim).not.toBeNull(); + expect( + await recordAutumnWebhookAttempt({ + attempts: claim!.attempts, + claimToken: claim!.claimToken, + errorMessage: "provider unavailable", + id, + status: "pending", + }) + ).toBe("dead_letter"); + } + + const selected = await listUnalertedAutumnWebhookDeadLetters(); + expect(selected.map((row) => row.id)).toEqual( + expect.arrayContaining([oldId, recentId, unalertedId]) + ); + await markAutumnWebhookDeadLettersAlerted([oldId, recentId]); + expect( + (await listUnalertedAutumnWebhookDeadLetters()).map((row) => row.id) + ).toContain(unalertedId); + + const now = new Date(); + const old = new Date(now.getTime() - 91 * 24 * 60 * 60 * 1000); + const cutoff = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000); + await db + .update(autumnWebhookEvents) + .set({ deadLetteredAt: old }) + .where(inArray(autumnWebhookEvents.id, [oldId, unalertedId])); + + expect(await deleteDeadLetterAutumnWebhooks({ olderThan: cutoff })).toBe(1); + expect(await getAutumnWebhook(oldId)).toBeNull(); + expect(await getAutumnWebhook(recentId)).not.toBeNull(); + expect(await getAutumnWebhook(unalertedId)).not.toBeNull(); + }); + + iit("deletes only completed rows older than the retention window", async () => { + const oldId = webhookId("completed-old"); + const recentId = webhookId("completed-recent"); + const pendingId = webhookId("completed-pending"); + for (const id of [oldId, recentId, pendingId]) { + await store(id); + } + for (const id of [oldId, recentId]) { + const claim = await claimAutumnWebhook({ id }); + expect(claim).not.toBeNull(); + await recordAutumnWebhookAttempt({ + attempts: claim!.attempts, + claimToken: claim!.claimToken, + id, + status: "completed", + }); + } + + const now = new Date(); + const old = new Date(now.getTime() - 31 * 24 * 60 * 60 * 1000); + const cutoff = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000); + await db + .update(autumnWebhookEvents) + .set({ completedAt: old }) + .where(inArray(autumnWebhookEvents.id, [oldId, pendingId])); + + expect(await deleteCompletedAutumnWebhooks({ olderThan: cutoff })).toBe(1); + expect(await getAutumnWebhook(oldId)).toBeNull(); + expect(await getAutumnWebhook(recentId)).not.toBeNull(); + expect(await getAutumnWebhook(pendingId)).not.toBeNull(); + }); +}); diff --git a/apps/api/src/integration/profile-handlers.test.ts b/apps/api/src/integration/profile-handlers.test.ts index 4cd882487..56a634788 100644 --- a/apps/api/src/integration/profile-handlers.test.ts +++ b/apps/api/src/integration/profile-handlers.test.ts @@ -331,13 +331,69 @@ describe("getTraitDistribution", () => { }); const distribution = await getTraitDistribution(website.id); - expect(distribution.identifiedProfiles).toBe(3); + expect(distribution).toMatchObject({ + hasMoreKeys: false, + hasMoreValues: false, + identifiedProfiles: 3, + returnedTraitKeys: 2, + totalTraitKeys: 2, + valuesPerKey: 20, + }); expect(distribution.traits).toEqual([ { key: "beta", value: "true", profiles: 1 }, { key: "plan", value: "pro", profiles: 2 }, { key: "plan", value: "free", profiles: 1 }, ]); }); + + iit("gives every returned key a value before adding more values per key", async () => { + const org = await insertOrganization(); + const website = await insertWebsite({ organizationId: org.id }); + const keys = Array.from({ length: 11 }, (_, index) => `trait_${index}`); + + await Promise.all( + Array.from({ length: 20 }, (_, valueIndex) => + seedProfile(website.id, `user_${valueIndex}`, { + traits: Object.fromEntries( + keys.map((key) => [key, `value_${valueIndex}`]) + ), + }) + ) + ); + + const distribution = await getTraitDistribution(website.id); + expect(distribution).toMatchObject({ + hasMoreKeys: false, + hasMoreValues: true, + returnedTraitKeys: 11, + totalTraitKeys: 11, + valuesPerKey: 18, + }); + expect(distribution.traits).toHaveLength(198); + expect(new Set(distribution.traits.map((trait) => trait.key))).toEqual( + new Set(keys) + ); + }); + + iit("reports when lower-coverage keys are omitted by the row bound", async () => { + const org = await insertOrganization(); + const website = await insertWebsite({ organizationId: org.id }); + await seedProfile(website.id, "user_1", { + traits: Object.fromEntries( + Array.from({ length: 201 }, (_, index) => [`trait_${index}`, true]) + ), + }); + + const distribution = await getTraitDistribution(website.id); + expect(distribution).toMatchObject({ + hasMoreKeys: true, + hasMoreValues: false, + returnedTraitKeys: 200, + totalTraitKeys: 201, + valuesPerKey: 1, + }); + expect(distribution.traits).toHaveLength(200); + }); }); describe("resolveTraitSegment", () => { diff --git a/apps/api/src/integration/public-flags-http.test.ts b/apps/api/src/integration/public-flags-http.test.ts index f3377ac11..340fdcceb 100644 --- a/apps/api/src/integration/public-flags-http.test.ts +++ b/apps/api/src/integration/public-flags-http.test.ts @@ -259,7 +259,7 @@ describe("public flags HTTP integration", () => { `/v1/flags/bulk?clientId=${website.id}&userId=${user.id}&keys=global-a,personal-c,missing` ); expect(response.status).toBe(200); - expect(response.headers.get("cache-control")).toContain("public"); + expect(response.headers.get("cache-control")).toBe("private, no-store"); const body = await json(response); expect(body.count).toBe(2); @@ -269,6 +269,40 @@ describe("public flags HTTP integration", () => { ]); expect(body.flags["global-a"]).toMatchObject({ enabled: true }); expect(body.flags["personal-c"]).toMatchObject({ enabled: true }); + + const omittedGet = await json( + await get(`/v1/flags/bulk?clientId=${website.id}&userId=${user.id}`) + ); + const omittedPost = await json( + await post("/v1/flags/bulk", { + clientId: website.id, + userId: user.id, + }) + ); + for (const result of [omittedGet, omittedPost]) { + expect(result.count).toBe(3); + expect(Object.keys(result.flags).sort()).toEqual([ + "global-a", + "global-b", + "personal-c", + ]); + } + + const emptyGet = await json( + await get( + `/v1/flags/bulk?clientId=${website.id}&userId=${user.id}&keys=` + ) + ); + const emptyPost = await json( + await post("/v1/flags/bulk", { + clientId: website.id, + keys: [], + userId: user.id, + }) + ); + for (const result of [emptyGet, emptyPost]) { + expect(result).toEqual({ count: 0, flags: {} }); + } }); iit("returns safe defaults for missing params and malformed properties", async () => { @@ -461,7 +495,7 @@ describe("public flags HTTP integration", () => { { "x-api-key": keyA.secret } ) ).status - ).toBe(403); + ).toBe(404); }); iit("requires user-associated API keys for writes", async () => { @@ -485,7 +519,7 @@ describe("public flags HTTP integration", () => { expect(response.status).toBe(403); expect(await json(response)).toEqual({ - error: "API key must be associated with a user", + error: "Forbidden", }); }); }); diff --git a/apps/api/src/routes/public/flags-boundary.test.ts b/apps/api/src/routes/public/flags-boundary.test.ts new file mode 100644 index 000000000..e8c33b877 --- /dev/null +++ b/apps/api/src/routes/public/flags-boundary.test.ts @@ -0,0 +1,114 @@ +import "@databuddy/test/env"; +import { Elysia } from "elysia"; +import { describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ + flags: [ + { + defaultValue: true, + dependencies: null, + flagsToTargetGroups: [], + key: "enabled-for-everyone", + payload: null, + rolloutBy: null, + rolloutPercentage: null, + rules: null, + status: "active", + type: "boolean", + variants: null, + }, + ], +})); + +vi.mock("@databuddy/db", async (importOriginal) => ({ + ...(await importOriginal()), + db: { + query: { + flags: { + findMany: vi.fn(async () => state.flags), + }, + }, + }, +})); + +vi.mock("@databuddy/redis", async (importOriginal) => ({ + ...(await importOriginal()), + cacheable: (fn: (...args: never[]) => unknown) => fn, +})); + +vi.mock("@databuddy/redis/rate-limit", () => ({ + getRateLimitHeaders: () => ({}), + ratelimit: async () => ({ success: true }), +})); + +const { flagsRoute } = await import("./flags"); +const app = new Elysia().use(flagsRoute); + +function request(path: string, body?: unknown) { + return app.handle( + new Request(`http://localhost${path}`, { + body: body === undefined ? undefined : JSON.stringify(body), + headers: + body === undefined ? undefined : { "content-type": "application/json" }, + method: body === undefined ? "GET" : "POST", + }) + ); +} + +describe("public bulk flags boundary", () => { + it("returns all flags only when the key filter is omitted", async () => { + for (const response of [ + await request("/v1/flags/bulk?clientId=site_1"), + await request("/v1/flags/bulk", { clientId: "site_1" }), + ]) { + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + count: 1, + flags: { "enabled-for-everyone": { enabled: true } }, + }); + } + }); + + it("returns no flags for explicitly empty or blank key lists", async () => { + for (const response of [ + await request("/v1/flags/bulk?clientId=site_1&keys="), + await request("/v1/flags/bulk?clientId=site_1&keys=%20,%20"), + await request("/v1/flags/bulk", { clientId: "site_1", keys: [] }), + await request("/v1/flags/bulk", { + clientId: "site_1", + keys: ["", " "], + }), + ]) { + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ count: 0, flags: {} }); + } + }); + + it("rejects more than 100 requested keys", async () => { + const keys = Array.from({ length: 101 }, (_, index) => `flag-${index}`); + const getResponse = await request( + `/v1/flags/bulk?clientId=site_1&keys=${keys.join(",")}` + ); + expect(getResponse.status).toBe(400); + + const postResponse = await request("/v1/flags/bulk", { + clientId: "site_1", + keys, + }); + expect(postResponse.status).toBe(422); + }); + + it("rejects keys longer than 128 characters", async () => { + const key = "x".repeat(129); + const getResponse = await request( + `/v1/flags/bulk?clientId=site_1&keys=${key}` + ); + expect(getResponse.status).toBe(400); + + const postResponse = await request("/v1/flags/bulk", { + clientId: "site_1", + keys: [key], + }); + expect(postResponse.status).toBe(422); + }); +}); diff --git a/apps/api/src/routes/public/flags.ts b/apps/api/src/routes/public/flags.ts index c4f3d701c..13244e499 100644 --- a/apps/api/src/routes/public/flags.ts +++ b/apps/api/src/routes/public/flags.ts @@ -86,6 +86,11 @@ interface EvaluableFlag { variants?: FlagVariant[] | null; } +const MAX_BULK_FLAG_KEYS = 100; +const MAX_FLAG_KEY_LENGTH = 128; +const MAX_BULK_FLAG_QUERY_LENGTH = + MAX_BULK_FLAG_KEYS * (MAX_FLAG_KEY_LENGTH + 1); + const flagQuerySchema = t.Object({ key: t.String(), clientId: t.String(), @@ -99,7 +104,7 @@ const flagQuerySchema = t.Object({ const bulkFlagQuerySchema = t.Object({ clientId: t.String(), - keys: t.Optional(t.String()), + keys: t.Optional(t.String({ maxLength: MAX_BULK_FLAG_QUERY_LENGTH })), userId: t.Optional(t.String()), email: t.Optional(t.String()), organizationId: t.Optional(t.String()), @@ -110,7 +115,11 @@ const bulkFlagQuerySchema = t.Object({ const bulkFlagBodySchema = t.Object({ clientId: t.String(), - keys: t.Optional(t.Array(t.String())), + keys: t.Optional( + t.Array(t.String({ maxLength: MAX_FLAG_KEY_LENGTH }), { + maxItems: MAX_BULK_FLAG_KEYS, + }) + ), userId: t.Optional(t.String()), email: t.Optional(t.String()), organizationId: t.Optional(t.String()), @@ -776,6 +785,25 @@ async function evaluateBulkFlags( }; } + if (input.keys && input.keys.length > MAX_BULK_FLAG_KEYS) { + set.status = 400; + return { + flags: {}, + count: 0, + error: `A maximum of ${MAX_BULK_FLAG_KEYS} flag keys is allowed`, + }; + } + + const normalizedKeys = input.keys?.map((key) => key.trim()); + if (normalizedKeys?.some((key) => key.length > MAX_FLAG_KEY_LENGTH)) { + set.status = 400; + return { + flags: {}, + count: 0, + error: `Flag keys must be at most ${MAX_FLAG_KEY_LENGTH} characters`, + }; + } + const context: UserContext = { userId: input.userId, email: input.email, @@ -783,9 +811,17 @@ async function evaluateBulkFlags( teamId: input.teamId, properties: input.properties, }; - const requestedKeys = input.keys - ? new Set(input.keys.map((key) => key.trim()).filter(Boolean)) - : null; + const filteredKeys = normalizedKeys?.filter(Boolean); + const requestedKeys = + filteredKeys === undefined ? null : new Set(filteredKeys); + if (requestedKeys?.size === 0) { + mergeWideEvent({ + flag_total_flags: 0, + flag_evaluated: 0, + flag_count: 0, + }); + return { flags: {}, count: 0 }; + } const clientFlags = await fromMemory( `fc:${input.clientId}:${input.environment || ""}`, diff --git a/apps/api/src/routes/webhooks/autumn-inbox.ts b/apps/api/src/routes/webhooks/autumn-inbox.ts new file mode 100644 index 000000000..b58778efd --- /dev/null +++ b/apps/api/src/routes/webhooks/autumn-inbox.ts @@ -0,0 +1,411 @@ +import { randomUUID } from "node:crypto"; +import { + and, + asc, + db, + eq, + gte, + inArray, + isNotNull, + isNull, + lt, + lte, + or, + sql, +} from "@databuddy/db"; +import { + autumnWebhookEvents, + type AutumnWebhookStatus, +} from "@databuddy/db/schema"; + +const MAX_ERROR_LENGTH = 500; +const MAX_MAINTENANCE_BATCH = 100; +const COMPLETED_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; +const DEAD_LETTER_RETENTION_MS = 90 * 24 * 60 * 60 * 1000; +const RETRY_BASE_MS = 5 * 60 * 1000; +const RETRY_MAX_MS = 6 * 60 * 60 * 1000; + +export const AUTUMN_WEBHOOK_LEASE_MS = 5 * 60 * 1000; +export const AUTUMN_WEBHOOK_MAX_ATTEMPTS = 12; + +export interface StoredAutumnWebhook { + id: string; + payload: Record; + status: AutumnWebhookStatus; + type: string; +} + +export interface ClaimedAutumnWebhook extends StoredAutumnWebhook { + attempts: number; + claimToken: string; +} + +export interface AutumnWebhookDeadLetter { + attempts: number; + deadLetteredAt: Date; + errorMessage: string | null; + id: string; + type: string; +} + +type AttemptStatus = "completed" | "deferred" | "pending"; + +function batchLimit(value: number | undefined): number { + return Math.min( + Math.max(value ?? MAX_MAINTENANCE_BATCH, 1), + MAX_MAINTENANCE_BATCH + ); +} + +export function autumnWebhookRetryDelayMs(attempts: number): number { + return Math.min(RETRY_BASE_MS * 2 ** Math.max(0, attempts - 1), RETRY_MAX_MS); +} + +export async function storeAutumnWebhook(input: { + id: string; + type: string; + payload: Record; +}): Promise { + await db + .insert(autumnWebhookEvents) + .values({ + id: input.id, + type: input.type, + payload: input.payload, + }) + .onConflictDoNothing({ target: autumnWebhookEvents.id }); + + const stored = await getAutumnWebhook(input.id); + if (!stored) { + throw new Error("Autumn webhook was not persisted"); + } + return stored; +} + +export async function getAutumnWebhook( + id: string +): Promise { + const [row] = await db + .select({ + id: autumnWebhookEvents.id, + payload: autumnWebhookEvents.payload, + status: autumnWebhookEvents.status, + type: autumnWebhookEvents.type, + }) + .from(autumnWebhookEvents) + .where(eq(autumnWebhookEvents.id, id)) + .limit(1); + return row ?? null; +} + +export async function claimAutumnWebhook(input: { + id: string; + leaseMs?: number; + now?: Date; +}): Promise { + const now = input.now ?? new Date(); + const claimToken = randomUUID(); + const leaseExpiresAt = new Date( + now.getTime() + (input.leaseMs ?? AUTUMN_WEBHOOK_LEASE_MS) + ); + const [row] = await db + .update(autumnWebhookEvents) + .set({ + attempts: sql`${autumnWebhookEvents.attempts} + 1`, + leaseExpiresAt, + leaseToken: claimToken, + nextAttemptAt: null, + status: "processing", + updatedAt: now, + }) + .where( + and( + eq(autumnWebhookEvents.id, input.id), + lt(autumnWebhookEvents.attempts, AUTUMN_WEBHOOK_MAX_ATTEMPTS), + or( + and( + inArray(autumnWebhookEvents.status, ["pending", "deferred"]), + or( + isNull(autumnWebhookEvents.nextAttemptAt), + lte(autumnWebhookEvents.nextAttemptAt, now) + ) + ), + and( + eq(autumnWebhookEvents.status, "processing"), + or( + isNull(autumnWebhookEvents.leaseExpiresAt), + lte(autumnWebhookEvents.leaseExpiresAt, now) + ) + ) + ) + ) + ) + .returning({ + attempts: autumnWebhookEvents.attempts, + id: autumnWebhookEvents.id, + payload: autumnWebhookEvents.payload, + status: autumnWebhookEvents.status, + type: autumnWebhookEvents.type, + }); + return row ? { ...row, claimToken } : null; +} + +export async function recordAutumnWebhookAttempt(input: { + attempts: number; + claimToken: string; + errorMessage?: string; + id: string; + now?: Date; + status: AttemptStatus; +}): Promise { + const now = input.now ?? new Date(); + if (input.status === "completed") { + const [completed] = await db + .update(autumnWebhookEvents) + .set({ + alertedAt: null, + completedAt: now, + deadLetteredAt: null, + errorMessage: null, + leaseExpiresAt: null, + leaseToken: null, + nextAttemptAt: null, + status: "completed", + updatedAt: now, + }) + .where( + and( + eq(autumnWebhookEvents.id, input.id), + eq(autumnWebhookEvents.leaseToken, input.claimToken), + eq(autumnWebhookEvents.status, "processing") + ) + ) + .returning({ status: autumnWebhookEvents.status }); + if (completed) { + return completed.status; + } + } else { + const deadLetter = input.attempts >= AUTUMN_WEBHOOK_MAX_ATTEMPTS; + const status = deadLetter ? "dead_letter" : input.status; + const [retry] = await db + .update(autumnWebhookEvents) + .set({ + completedAt: null, + deadLetteredAt: deadLetter ? now : null, + errorMessage: input.errorMessage?.slice(0, MAX_ERROR_LENGTH) ?? null, + leaseExpiresAt: null, + leaseToken: null, + nextAttemptAt: deadLetter + ? null + : new Date(now.getTime() + autumnWebhookRetryDelayMs(input.attempts)), + status, + updatedAt: now, + }) + .where( + and( + eq(autumnWebhookEvents.id, input.id), + eq(autumnWebhookEvents.status, "processing"), + eq(autumnWebhookEvents.leaseToken, input.claimToken) + ) + ) + .returning({ status: autumnWebhookEvents.status }); + if (retry) { + return retry.status; + } + } + + const stored = await getAutumnWebhook(input.id); + if (!stored) { + throw new Error("Autumn webhook disappeared before status update"); + } + return stored.status; +} + +export async function deadLetterExhaustedAutumnWebhooks( + now = new Date() +): Promise { + const rows = await db + .update(autumnWebhookEvents) + .set({ + alertedAt: null, + deadLetteredAt: now, + errorMessage: "Maximum replay attempts exhausted", + leaseExpiresAt: null, + leaseToken: null, + nextAttemptAt: null, + status: "dead_letter", + updatedAt: now, + }) + .where( + and( + gte(autumnWebhookEvents.attempts, AUTUMN_WEBHOOK_MAX_ATTEMPTS), + or( + inArray(autumnWebhookEvents.status, ["pending", "deferred"]), + and( + eq(autumnWebhookEvents.status, "processing"), + or( + isNull(autumnWebhookEvents.leaseExpiresAt), + lte(autumnWebhookEvents.leaseExpiresAt, now) + ) + ) + ) + ) + ) + .returning({ id: autumnWebhookEvents.id }); + return rows.length; +} + +export async function listReplayableAutumnWebhookIds(input?: { + limit?: number; + now?: Date; +}): Promise { + const now = input?.now ?? new Date(); + const rows = await db + .select({ id: autumnWebhookEvents.id }) + .from(autumnWebhookEvents) + .where( + and( + lt(autumnWebhookEvents.attempts, AUTUMN_WEBHOOK_MAX_ATTEMPTS), + or( + and( + inArray(autumnWebhookEvents.status, ["pending", "deferred"]), + or( + isNull(autumnWebhookEvents.nextAttemptAt), + lte(autumnWebhookEvents.nextAttemptAt, now) + ) + ), + and( + eq(autumnWebhookEvents.status, "processing"), + or( + isNull(autumnWebhookEvents.leaseExpiresAt), + lte(autumnWebhookEvents.leaseExpiresAt, now) + ) + ) + ) + ) + ) + .orderBy(asc(autumnWebhookEvents.updatedAt)) + .limit(batchLimit(input?.limit)); + return rows.map((row) => row.id); +} + +export async function listUnalertedAutumnWebhookDeadLetters(input?: { + limit?: number; +}): Promise { + const rows = await db + .select({ + attempts: autumnWebhookEvents.attempts, + deadLetteredAt: autumnWebhookEvents.deadLetteredAt, + errorMessage: autumnWebhookEvents.errorMessage, + id: autumnWebhookEvents.id, + type: autumnWebhookEvents.type, + }) + .from(autumnWebhookEvents) + .where( + and( + eq(autumnWebhookEvents.status, "dead_letter"), + isNull(autumnWebhookEvents.alertedAt), + isNotNull(autumnWebhookEvents.deadLetteredAt) + ) + ) + .orderBy(asc(autumnWebhookEvents.deadLetteredAt)) + .limit(batchLimit(input?.limit)); + return rows.filter( + (row): row is AutumnWebhookDeadLetter => row.deadLetteredAt !== null + ); +} + +export async function markAutumnWebhookDeadLettersAlerted( + ids: string[], + now = new Date() +): Promise { + if (ids.length === 0) { + return 0; + } + const rows = await db + .update(autumnWebhookEvents) + .set({ alertedAt: now, updatedAt: now }) + .where( + and( + inArray(autumnWebhookEvents.id, ids), + eq(autumnWebhookEvents.status, "dead_letter"), + isNull(autumnWebhookEvents.alertedAt) + ) + ) + .returning({ id: autumnWebhookEvents.id }); + return rows.length; +} + +export async function deleteCompletedAutumnWebhooks(input?: { + limit?: number; + olderThan?: Date; +}): Promise { + const olderThan = + input?.olderThan ?? new Date(Date.now() - COMPLETED_RETENTION_MS); + const rows = await db + .select({ id: autumnWebhookEvents.id }) + .from(autumnWebhookEvents) + .where( + and( + eq(autumnWebhookEvents.status, "completed"), + lt(autumnWebhookEvents.completedAt, olderThan) + ) + ) + .orderBy(asc(autumnWebhookEvents.completedAt)) + .limit(batchLimit(input?.limit)); + if (rows.length === 0) { + return 0; + } + const deleted = await db + .delete(autumnWebhookEvents) + .where( + and( + inArray( + autumnWebhookEvents.id, + rows.map((row) => row.id) + ), + eq(autumnWebhookEvents.status, "completed"), + lt(autumnWebhookEvents.completedAt, olderThan) + ) + ) + .returning({ id: autumnWebhookEvents.id }); + return deleted.length; +} + +export async function deleteDeadLetterAutumnWebhooks(input?: { + limit?: number; + olderThan?: Date; +}): Promise { + const olderThan = + input?.olderThan ?? new Date(Date.now() - DEAD_LETTER_RETENTION_MS); + const rows = await db + .select({ id: autumnWebhookEvents.id }) + .from(autumnWebhookEvents) + .where( + and( + eq(autumnWebhookEvents.status, "dead_letter"), + isNotNull(autumnWebhookEvents.alertedAt), + lt(autumnWebhookEvents.deadLetteredAt, olderThan) + ) + ) + .orderBy(asc(autumnWebhookEvents.deadLetteredAt)) + .limit(batchLimit(input?.limit)); + if (rows.length === 0) { + return 0; + } + const deleted = await db + .delete(autumnWebhookEvents) + .where( + and( + inArray( + autumnWebhookEvents.id, + rows.map((row) => row.id) + ), + eq(autumnWebhookEvents.status, "dead_letter"), + isNotNull(autumnWebhookEvents.alertedAt), + lt(autumnWebhookEvents.deadLetteredAt, olderThan) + ) + ) + .returning({ id: autumnWebhookEvents.id }); + return deleted.length; +} diff --git a/apps/api/src/routes/webhooks/autumn.test.ts b/apps/api/src/routes/webhooks/autumn.test.ts index 21e645805..33ccbd1aa 100644 --- a/apps/api/src/routes/webhooks/autumn.test.ts +++ b/apps/api/src/routes/webhooks/autumn.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { beforeEach, describe, expect, it, vi } from "vitest"; const state = vi.hoisted(() => ({ @@ -11,13 +12,16 @@ const state = vi.hoisted(() => ({ usage: 8000, }, })), + cooldownConditions: [] as unknown[], inserted: [] as Record[], + insertFailure: null as Error | null, log: { error: vi.fn(), info: vi.fn(), warn: vi.fn(), }, operations: [] as string[], + locks: [] as string[], ownedOrganizations: [] as Array<{ organizationId: string; organization: { @@ -27,6 +31,23 @@ const state = vi.hoisted(() => ({ }; }>, recentRows: [] as Array<{ id: string }>, + storeFailure: null as Error | null, + storedWebhooks: new Map< + string, + { + attempts: number; + claimToken: string | null; + id: string; + payload: Record; + status: + | "pending" + | "processing" + | "deferred" + | "completed" + | "dead_letter"; + type: string; + } + >(), send: vi.fn(async () => ({ data: { id: "email-1" }, error: null })), userRow: { email: "customer@example.com", name: "Customer" } as { email: string | null; @@ -34,12 +55,83 @@ const state = vi.hoisted(() => ({ } | null, })); +vi.mock("./autumn-inbox", () => ({ + claimAutumnWebhook: vi.fn(async ({ id }: { id: string }) => { + const stored = state.storedWebhooks.get(id); + if (!(stored && ["pending", "deferred"].includes(stored.status))) { + return null; + } + stored.attempts += 1; + stored.claimToken = `claim-${stored.attempts}`; + stored.status = "processing"; + return { + ...stored, + claimToken: stored.claimToken, + }; + }), + deadLetterExhaustedAutumnWebhooks: vi.fn(async () => 0), + getAutumnWebhook: vi.fn(async (id: string) => state.storedWebhooks.get(id) ?? null), + listReplayableAutumnWebhookIds: vi.fn(async () => + [...state.storedWebhooks.values()] + .filter((row) => ["pending", "deferred"].includes(row.status)) + .map((row) => row.id) + ), + recordAutumnWebhookAttempt: vi.fn( + async (input: { + attempts: number; + claimToken: string; + errorMessage?: string; + id: string; + status: "pending" | "deferred" | "completed"; + }) => { + const stored = state.storedWebhooks.get(input.id); + if (!stored) { + throw new Error("missing stored webhook"); + } + if (stored.status === "completed") { + return stored.status; + } + if (input.status === "completed") { + stored.status = "completed"; + stored.claimToken = null; + return stored.status; + } + stored.status = input.attempts >= 12 ? "dead_letter" : input.status; + stored.claimToken = null; + return stored.status; + } + ), + storeAutumnWebhook: vi.fn( + async (input: { + id: string; + payload: Record; + type: string; + }) => { + if (state.storeFailure) { + throw state.storeFailure; + } + const existing = state.storedWebhooks.get(input.id); + if (existing) { + return existing; + } + const stored = { + ...input, + attempts: 0, + claimToken: null, + status: "pending" as const, + }; + state.storedWebhooks.set(input.id, stored); + return stored; + } + ), +})); + vi.mock("@databuddy/db", () => ({ and: (...conditions: unknown[]) => ({ conditions }), db: { query: { member: { - findMany: vi.fn(async () => state.ownedOrganizations), + findMany: vi.fn(async () => state.ownedOrganizations), }, organization: { findFirst: vi.fn(async () => null) }, user: { findFirst: vi.fn(async () => state.userRow) }, @@ -47,34 +139,44 @@ vi.mock("@databuddy/db", () => ({ }, eq: (field: unknown, value: unknown) => ({ field, op: "eq", value }), gt: (field: unknown, value: unknown) => ({ field, op: "gt", value }), + isNull: (field: unknown) => ({ field, op: "isNull" }), normalizeEmailNotificationSettings: (raw?: { billing?: { usageWarnings?: boolean }; }) => ({ billing: { usageWarnings: raw?.billing?.usageWarnings ?? true }, }), + or: (...conditions: unknown[]) => ({ conditions, op: "or" }), sql: (strings: TemplateStringsArray, ...values: unknown[]) => ({ strings: Array.from(strings), values, }), withTransaction: vi.fn(async (fn: (tx: unknown) => Promise) => fn({ - execute: vi.fn(async () => { + execute: vi.fn(async (query: unknown) => { state.operations.push("lock"); + const values = (query as { values?: unknown[] }).values; + state.locks.push(String(values?.[0] ?? "")); }), insert: vi.fn(() => ({ values: vi.fn(async (value: Record) => { + if (state.insertFailure) { + throw state.insertFailure; + } state.operations.push("insert"); state.inserted.push(value); }), })), select: vi.fn(() => ({ from: vi.fn(() => ({ - where: vi.fn(() => ({ + where: vi.fn((condition: unknown) => { + state.cooldownConditions.push(condition); + return { limit: vi.fn(async () => { state.operations.push("select"); return state.recentRows; }), - })), + }; + }), })), })), }) @@ -88,6 +190,7 @@ vi.mock("@databuddy/db/schema", () => ({ emailSentTo: "emailSentTo", featureId: "featureId", id: "id", + organizationId: "organizationId", userId: "userId", }, })); @@ -154,20 +257,27 @@ vi.mock("../../lib/tracing", () => ({ import { UsageAlertEmail, UsageLimitEmail } from "@databuddy/email"; import { + handleVerifiedAutumnEvent, handleLimitReached, handleUsageAlert, + replayDeferredAutumnWebhook, sendAlertEmail, } from "./autumn"; beforeEach(() => { process.env.RESEND_API_KEY = "test-resend-key"; state.inserted = []; + state.insertFailure = null; + state.locks = []; state.operations = []; state.ownedOrganizations = []; state.recentRows = []; + state.storeFailure = null; + state.storedWebhooks.clear(); state.userRow = { email: "customer@example.com", name: "Customer" }; state.send.mockClear(); state.check.mockClear(); + state.cooldownConditions = []; state.check.mockResolvedValue({ allowed: true, balance: { @@ -197,13 +307,34 @@ describe("sendAlertEmail", () => { alertType: "included", cooldownKey: "events", customerId: "user-1", + organizationId: "org-1", react: { type: "email" } as never, recipient: { email: "member@example.com" }, subject: "Limit reached", }); expect(result).toEqual({ success: true, message: "Already sent recently" }); - expect(state.operations).toEqual(["lock", "select"]); + expect(state.operations).toEqual(["lock", "lock", "select"]); + expect(state.locks).toEqual([ + "usage-alert:user-1:events", + "usage-alert:org-1:user-1:events", + ]); + expect(state.cooldownConditions).toEqual([ + { + conditions: [ + { field: "userId", op: "eq", value: "user-1" }, + { field: "featureId", op: "eq", value: "events" }, + expect.objectContaining({ field: "createdAt", op: "gt" }), + { + conditions: [ + { field: "organizationId", op: "eq", value: "org-1" }, + { field: "organizationId", op: "isNull" }, + ], + op: "or", + }, + ], + }, + ]); expect(state.send).not.toHaveBeenCalled(); expect(state.inserted).toEqual([]); }); @@ -213,13 +344,20 @@ describe("sendAlertEmail", () => { alertType: "included", cooldownKey: "events", customerId: "user-1", + organizationId: "org-1", react: { type: "email" } as never, recipient: { email: "member@example.com" }, subject: "Limit reached", }); expect(result).toEqual({ success: true, message: "Email sent" }); - expect(state.operations).toEqual(["lock", "select", "send", "insert"]); + expect(state.operations).toEqual([ + "lock", + "lock", + "select", + "send", + "insert", + ]); expect(state.send).toHaveBeenCalledWith({ from: "alerts@databuddy.cc", to: "member@example.com", @@ -230,8 +368,9 @@ describe("sendAlertEmail", () => { expect(state.inserted).toEqual([ expect.objectContaining({ alertType: "included", - emailSentTo: "member@example.com", + emailSentTo: "member@example.com", featureId: "events", + organizationId: "org-1", userId: "user-1", }), ]); @@ -247,6 +386,7 @@ describe("sendAlertEmail", () => { alertType: "included", cooldownKey: "events", customerId: "user-1", + organizationId: "org-1", react: { type: "email" } as never, recipient: { email: "member@example.com" }, subject: "Limit reached", @@ -256,7 +396,7 @@ describe("sendAlertEmail", () => { success: false, message: "Alert email delivery failed", }); - expect(state.operations).toEqual(["lock", "select"]); + expect(state.operations).toEqual(["lock", "lock", "select"]); expect(state.inserted).toEqual([]); }); @@ -267,6 +407,7 @@ describe("sendAlertEmail", () => { alertType: "included", cooldownKey: "events", customerId: "user-1", + organizationId: "org-1", react: { type: "email" } as never, recipient: { email: "member@example.com" }, subject: "Limit reached", @@ -325,7 +466,7 @@ describe("Autumn usage emails", () => { organizationName: "Acme", remainingAmount: 62, usageAmount: 288, - usageUnit: "allowance units", + usageUnit: "usage units", }) ); expect(UsageAlertEmail).not.toHaveBeenCalledWith( @@ -337,6 +478,57 @@ describe("Autumn usage emails", () => { to: "recipient@example.com", }) ); + expect(state.check).toHaveBeenCalledWith({ + customerId: "user-1", + entityId: "org-1", + featureId: "agent_credits", + }); + }); + + it("keeps one owner's organization balances and cooldowns independent", async () => { + state.ownedOrganizations.push({ + organizationId: "org-2", + organization: { + emailNotifications: { billing: { usageWarnings: true } }, + id: "org-2", + name: "Second organization", + }, + }); + + await handleLimitReached({ + customer_id: "user-1", + entity_id: "org-1", + feature_id: "events", + limit_type: "included", + }); + await handleLimitReached({ + customer_id: "user-1", + entity_id: "org-2", + feature_id: "events", + limit_type: "included", + }); + + expect(state.check).toHaveBeenNthCalledWith(1, { + customerId: "user-1", + entityId: "org-1", + featureId: "events", + }); + expect(state.check).toHaveBeenNthCalledWith(2, { + customerId: "user-1", + entityId: "org-2", + featureId: "events", + }); + expect(state.locks).toEqual([ + "usage-alert:user-1:events:limit:included", + "usage-alert:org-1:user-1:events:limit:included", + "usage-alert:user-1:events:limit:included", + "usage-alert:org-2:user-1:events:limit:included", + ]); + expect(state.inserted.map((row) => row.organizationId)).toEqual([ + "org-1", + "org-2", + ]); + expect(state.send).toHaveBeenCalledTimes(2); }); it("handles an actual hard limit and tells the template whether use is paused", async () => { @@ -393,7 +585,7 @@ describe("Autumn usage emails", () => { expect(state.send).not.toHaveBeenCalled(); }); - it("defers usage alerts when multiple organizations are ambiguous", async () => { + it("defers usage alerts when organizations are ambiguous", async () => { state.ownedOrganizations.push({ organizationId: "org-2", organization: { @@ -413,6 +605,7 @@ describe("Autumn usage emails", () => { }); expect(result).toEqual({ + disposition: "deferred", success: false, message: "Billing usage email deferred: organization could not be resolved", }); @@ -430,6 +623,7 @@ describe("Autumn usage emails", () => { }); expect(result).toEqual({ + disposition: "deferred", success: false, message: "Billing usage email deferred: organization could not be resolved", }); @@ -438,3 +632,171 @@ describe("Autumn usage emails", () => { expect(state.send).not.toHaveBeenCalled(); }); }); + +describe("Autumn webhook inbox", () => { + const organization = { + organizationId: "org-1", + organization: { + emailNotifications: { billing: { usageWarnings: true } }, + id: "org-1", + name: "Acme", + }, + }; + + beforeEach(() => { + state.ownedOrganizations = [organization]; + }); + + it("stores ambiguous multi-organization events and acknowledges the durable deferral", async () => { + state.ownedOrganizations.push({ + organizationId: "org-2", + organization: { + emailNotifications: { billing: { usageWarnings: true } }, + id: "org-2", + name: "Second organization", + }, + }); + + const result = await handleVerifiedAutumnEvent("msg-ambiguous", { + data: { + customer_id: "user-1", + feature_id: "events", + usage_alert: { + name: "Do not retain this provider label", + threshold: 80, + threshold_type: "usage_percentage", + }, + }, + type: "balances.usage_alert_triggered", + }); + + expect(result).toEqual({ + disposition: "deferred", + message: "Webhook stored for replay", + success: true, + }); + expect(state.storedWebhooks.get("msg-ambiguous")).toEqual({ + attempts: 1, + claimToken: null, + id: "msg-ambiguous", + payload: { + customer_id: "user-1", + feature_id: "events", + usage_alert: { + threshold: 80, + threshold_type: "usage_percentage", + }, + }, + status: "deferred", + type: "balances.usage_alert_triggered", + }); + expect(state.send).not.toHaveBeenCalled(); + }); + + it("does not repeat a completed delivery for a duplicate Svix ID", async () => { + const event = { + data: { + customer_id: "user-1", + entity_id: "org-1", + feature_id: "events", + limit_type: "included", + }, + type: "balances.limit_reached", + }; + + await handleVerifiedAutumnEvent("msg-duplicate", event); + const duplicate = await handleVerifiedAutumnEvent("msg-duplicate", event); + + expect(duplicate).toEqual({ + disposition: "duplicate", + message: "Webhook already processed", + success: true, + }); + expect(state.send).toHaveBeenCalledTimes(1); + expect(state.storedWebhooks.get("msg-duplicate")?.status).toBe( + "completed" + ); + }); + + it("reuses a hashed provider idempotency key after post-send persistence failure", async () => { + const svixId = "msg-post-send-failure"; + const key = createHash("sha256").update(svixId).digest("hex"); + state.insertFailure = new Error("usage alert log unavailable"); + const event = { + data: { + customer_id: "user-1", + entity_id: "org-1", + feature_id: "events", + limit_type: "included", + }, + type: "balances.limit_reached", + }; + + await expect(handleVerifiedAutumnEvent(svixId, event)).rejects.toThrow( + "usage alert log unavailable" + ); + expect(state.storedWebhooks.get(svixId)?.status).toBe("pending"); + + state.insertFailure = null; + await expect(replayDeferredAutumnWebhook(svixId)).resolves.toEqual({ + message: "Email sent", + success: true, + }); + + expect(key).toMatch(/^[0-9a-f]{64}$/); + expect(state.send).toHaveBeenNthCalledWith( + 1, + expect.any(Object), + { idempotencyKey: key } + ); + expect(state.send).toHaveBeenNthCalledWith( + 2, + expect.any(Object), + { idempotencyKey: key } + ); + expect(state.storedWebhooks.get(svixId)?.status).toBe("completed"); + }); + + it("keeps provider retry semantics when persistence fails", async () => { + state.storeFailure = new Error("database unavailable"); + + await expect( + handleVerifiedAutumnEvent("msg-storage-failure", { + data: { + customer_id: "user-1", + entity_id: "org-1", + feature_id: "events", + limit_type: "included", + }, + type: "balances.limit_reached", + }) + ).rejects.toThrow("database unavailable"); + expect(state.send).not.toHaveBeenCalled(); + }); + + it("replays a deferred event after its organization becomes resolvable", async () => { + state.ownedOrganizations.push({ + organizationId: "org-2", + organization: { + emailNotifications: { billing: { usageWarnings: true } }, + id: "org-2", + name: "Second organization", + }, + }); + await handleVerifiedAutumnEvent("msg-replay", { + data: { + customer_id: "user-1", + feature_id: "events", + limit_type: "included", + }, + type: "balances.limit_reached", + }); + + state.ownedOrganizations = [organization]; + const result = await replayDeferredAutumnWebhook("msg-replay"); + + expect(result).toEqual({ success: true, message: "Email sent" }); + expect(state.send).toHaveBeenCalledTimes(1); + expect(state.storedWebhooks.get("msg-replay")?.status).toBe("completed"); + }); +}); diff --git a/apps/api/src/routes/webhooks/autumn.ts b/apps/api/src/routes/webhooks/autumn.ts index 7c243e96f..f49b98087 100644 --- a/apps/api/src/routes/webhooks/autumn.ts +++ b/apps/api/src/routes/webhooks/autumn.ts @@ -1,10 +1,12 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { and, db, eq, gt, + isNull, normalizeEmailNotificationSettings, + or, sql, withTransaction, } from "@databuddy/db"; @@ -19,13 +21,24 @@ import { } from "@databuddy/redis"; import { getAutumn } from "@databuddy/rpc"; import { recordPlanChange } from "@databuddy/services/billing-lifecycle"; +import { DATABUNNY_USAGE } from "@databuddy/shared/billing"; import { Elysia } from "elysia"; +import { log } from "evlog"; import { useLogger } from "evlog/elysia"; import { Resend } from "resend"; import { Webhook } from "svix"; // biome-ignore lint/performance/noNamespaceImport: vitest+bun fails to bind zod's named `z` export; namespace import is the reliable form import * as z from "zod"; import { mergeWideEvent } from "@databuddy/ai/lib/tracing"; +import { + claimAutumnWebhook, + deadLetterExhaustedAutumnWebhooks, + getAutumnWebhook, + listReplayableAutumnWebhookIds, + recordAutumnWebhookAttempt, + storeAutumnWebhook, + type ClaimedAutumnWebhook, +} from "./autumn-inbox"; const COOLDOWN_MS = 7 * 24 * 60 * 60 * 1000; @@ -35,21 +48,59 @@ const SLACK_URL = process.env.SLACK_WEBHOOK_URL ?? ""; const svix = SVIX_SECRET ? new Webhook(SVIX_SECRET) : null; const slack = SLACK_URL ? new SlackProvider({ webhookUrl: SLACK_URL }) : null; +const billingIdentifierSchema = z.string().min(1).max(200); + +interface AutumnLogger { + error(error: Error, fields?: Record): void; + info(message: string, fields?: Record): void; + warn(message: string, fields?: Record): void; +} + +function getAutumnLogger(): AutumnLogger { + try { + return useLogger(); + } catch { + return { + error: (error, fields) => + log.error({ + service: "api", + component: "autumn_webhook", + error_message: error.message, + ...fields, + }), + info: (message, fields) => + log.info({ + service: "api", + component: "autumn_webhook", + message, + ...fields, + }), + warn: (message, fields) => + log.warn({ + service: "api", + component: "autumn_webhook", + message, + ...fields, + }), + }; + } +} + const limitReachedSchema = z.object({ - customer_id: z.string(), - entity_id: z.string().optional(), - feature_id: z.string(), + customer_id: billingIdentifierSchema, + entity_id: billingIdentifierSchema.optional(), + feature_id: billingIdentifierSchema, limit_type: z.enum(["included", "max_purchase", "spend_limit"]), }); const usageAlertSchema = z.object({ - customer_id: z.string(), - entity_id: z.string().optional(), - feature_id: z.string(), + customer_id: billingIdentifierSchema, + entity_id: billingIdentifierSchema.optional(), + feature_id: billingIdentifierSchema, usage_alert: z.object({ - name: z.string().optional(), + name: z.string().max(200).optional(), threshold: z.number(), - threshold_type: z.string(), + threshold_type: z.string().min(1).max(100), }), }); @@ -90,10 +141,18 @@ interface RawAutumnEvent { } interface WebhookResult { + disposition?: "dead_letter" | "deferred" | "duplicate"; message: string; success: boolean; } +type ReplayableAutumnEvent = + | { data: LimitReachedData; type: "balances.limit_reached" } + | { + data: UsageAlertData; + type: "balances.usage_alert_triggered"; + }; + interface BillingRecipient { email: string | null; } @@ -170,11 +229,10 @@ export async function resolveBillingOrganization( function getFeatureCopy(featureId: string): BillingFeatureCopy { if (featureId === "agent_credits") { return { - description: - "This allowance powers Databunny questions and AI analysis. More complex work uses more of it.", - name: "Databunny usage", - pausedActivity: "Databunny questions and AI analysis", - unit: "allowance units", + description: DATABUNNY_USAGE.description, + name: DATABUNNY_USAGE.name, + pausedActivity: DATABUNNY_USAGE.pausedActivity, + unit: DATABUNNY_USAGE.unit, }; } @@ -199,10 +257,15 @@ function getFeatureCopy(featureId: string): BillingFeatureCopy { async function getUsageSnapshot( customerId: string, - featureId: string + featureId: string, + entityId?: string ): Promise { try { - const response = await getAutumn().check({ customerId, featureId }); + const response = await getAutumn().check({ + customerId, + featureId, + ...(entityId ? { entityId } : {}), + }); const balance = response.balance; if (!balance) { return null; @@ -216,22 +279,28 @@ async function getUsageSnapshot( usage: balance.usage, }; } catch (error) { - useLogger().error( + getAutumnLogger().error( error instanceof Error ? error : new Error(String(error)), - { autumn: { step: "load_usage", customerId, featureId } } + { autumn: { step: "load_usage", customerId, entityId, featureId } } ); return null; } } function formatUsageNumber(value: number): string { + if (!Number.isFinite(value)) { + return "—"; + } return new Intl.NumberFormat("en-US", { maximumFractionDigits: 1 }).format( value ); } function usagePercentage(snapshot: UsageSnapshot): number | null { - if (snapshot.granted <= 0) { + if ( + !(Number.isFinite(snapshot.granted) && Number.isFinite(snapshot.usage)) || + snapshot.granted <= 0 + ) { return null; } return Math.round((snapshot.usage / snapshot.granted) * 100); @@ -241,34 +310,54 @@ export async function sendAlertEmail(opts: { customerId: string; cooldownKey: string; alertType: string; + idempotencyKey?: string; + organizationId: string; subject: string; react: React.ReactElement; recipient: BillingRecipient; }): Promise { - const log = useLogger(); - const { customerId, cooldownKey, alertType, subject, react, recipient } = - opts; + const log = getAutumnLogger(); + const { + customerId, + cooldownKey, + alertType, + idempotencyKey, + organizationId, + subject, + react, + recipient, + } = opts; const { email } = recipient; if (!email) { log.warn("No email for customer", { - autumn: { customerId, cooldownKey }, + autumn: { customerId, cooldownKey, organizationId }, }); return { success: true, message: "No notification recipient found" }; } const resendApiKey = process.env.RESEND_API_KEY; if (!resendApiKey) { log.error(new Error("RESEND_API_KEY is not configured"), { - autumn: { customerId, cooldownKey, step: "email_configuration" }, + autumn: { + customerId, + cooldownKey, + organizationId, + step: "email_configuration", + }, }); return { success: false, message: "Alert email delivery unavailable" }; } const resend = new Resend(resendApiKey); return await withTransaction(async (tx) => { + // Old API replicas use this organization-agnostic lock and write NULL + // organization IDs. Take it first until the mixed-version window closes. await tx.execute( sql`SELECT pg_advisory_xact_lock(hashtextextended(${`usage-alert:${customerId}:${cooldownKey}`}, 0))` ); + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtextextended(${`usage-alert:${organizationId}:${customerId}:${cooldownKey}`}, 0))` + ); const since = new Date(Date.now() - COOLDOWN_MS); const [recent] = await tx @@ -278,14 +367,18 @@ export async function sendAlertEmail(opts: { and( eq(usageAlertLog.userId, customerId), eq(usageAlertLog.featureId, cooldownKey), - gt(usageAlertLog.createdAt, since) + gt(usageAlertLog.createdAt, since), + or( + eq(usageAlertLog.organizationId, organizationId), + isNull(usageAlertLog.organizationId) + ) ) ) .limit(1); if (recent) { log.info("Skipping alert - sent recently", { - autumn: { customerId, cooldownKey }, + autumn: { customerId, cooldownKey, organizationId }, }); return { success: true, message: "Already sent recently" }; } @@ -294,17 +387,20 @@ export async function sendAlertEmail(opts: { render(react), render(react, { plainText: true }), ]); - const result = await resend.emails.send({ + const message = { from: config.email.alertsFrom, to: email, subject, html, text, - }); + }; + const result = idempotencyKey + ? await resend.emails.send(message, { idempotencyKey }) + : await resend.emails.send(message); if (result.error) { log.error(new Error(result.error.message), { - autumn: { customerId, resend: result.error }, + autumn: { customerId, organizationId, resend: result.error }, }); return { success: false, message: "Alert email delivery failed" }; } @@ -312,13 +408,19 @@ export async function sendAlertEmail(opts: { await tx.insert(usageAlertLog).values({ id: randomUUID(), userId: customerId, + organizationId, featureId: cooldownKey, alertType, emailSentTo: email, }); log.info("Alert email sent", { - autumn: { customerId, cooldownKey, emailId: result.data?.id }, + autumn: { + customerId, + cooldownKey, + emailId: result.data?.id, + organizationId, + }, }); return { success: true, message: "Email sent" }; }); @@ -344,22 +446,27 @@ async function invalidatePlanCaches(customerId: string | null): Promise { ), ]); } catch (error) { - useLogger().info("Plan cache invalidation failed (best-effort)", { + getAutumnLogger().info("Plan cache invalidation failed (best-effort)", { autumn: { customerId, error }, }); } } export async function handleLimitReached( - data: LimitReachedData + data: LimitReachedData, + idempotencyKey?: string ): Promise { const { customer_id, entity_id, feature_id, limit_type } = data; const organization = await resolveBillingOrganization(customer_id, entity_id); if (!organization) { - useLogger().warn("Could not resolve billing organization", { - autumn: { customerId: customer_id, entityId: entity_id }, - }); + getAutumnLogger().warn( + "Deferring billing usage email without one organization", + { + autumn: { customerId: customer_id, entityId: entity_id }, + } + ); return { + disposition: "deferred", success: false, message: "Billing usage email deferred: organization could not be resolved", @@ -368,7 +475,7 @@ export async function handleLimitReached( const [recipient, snapshot] = await Promise.all([ getBillingRecipient(customer_id), - getUsageSnapshot(customer_id, feature_id), + getUsageSnapshot(customer_id, feature_id, entity_id), ]); if ( @@ -392,6 +499,8 @@ export async function handleLimitReached( customerId: customer_id, cooldownKey: `${feature_id}:limit:${limit_type}`, alertType: limit_type, + idempotencyKey, + organizationId: organization.id, subject, react: UsageLimitEmail({ featureDescription: feature.description, @@ -412,15 +521,20 @@ export async function handleLimitReached( } export async function handleUsageAlert( - data: UsageAlertData + data: UsageAlertData, + idempotencyKey?: string ): Promise { const { customer_id, entity_id, feature_id, usage_alert } = data; const organization = await resolveBillingOrganization(customer_id, entity_id); if (!organization) { - useLogger().warn("Could not resolve billing organization", { - autumn: { customerId: customer_id, entityId: entity_id }, - }); + getAutumnLogger().warn( + "Deferring billing usage email without one organization", + { + autumn: { customerId: customer_id, entityId: entity_id }, + } + ); return { + disposition: "deferred", success: false, message: "Billing usage email deferred: organization could not be resolved", @@ -429,7 +543,7 @@ export async function handleUsageAlert( const [recipient, snapshot] = await Promise.all([ getBillingRecipient(customer_id), - getUsageSnapshot(customer_id, feature_id), + getUsageSnapshot(customer_id, feature_id, entity_id), ]); if ( !normalizeEmailNotificationSettings(organization.emailNotifications).billing @@ -459,6 +573,8 @@ export async function handleUsageAlert( customerId: customer_id, cooldownKey: `${feature_id}:alert:${usage_alert.threshold_type}:${usage_alert.threshold}`, alertType: `usage_alert_${usage_alert.threshold_type}`, + idempotencyKey, + organizationId: organization.id, subject, react: UsageAlertEmail({ featureDescription: feature.description, @@ -513,7 +629,7 @@ const SCENARIO_LABELS: Record< async function handleProductsUpdated( data: ProductsUpdatedData ): Promise { - const log = useLogger(); + const log = getAutumnLogger(); const { scenario, customer, updated_product } = data; const productLabel = updated_product.name ?? updated_product.id; @@ -605,18 +721,256 @@ function verifySvix( } } -function dispatch( +function replayableAutumnEvent( + event: RawAutumnEvent +): ReplayableAutumnEvent | null { + switch (event.type) { + case "balances.limit_reached": + return { + data: limitReachedSchema.parse(event.data), + type: event.type, + }; + case "balances.usage_alert_triggered": { + const data = usageAlertSchema.parse(event.data); + return { + data: { + customer_id: data.customer_id, + ...(data.entity_id ? { entity_id: data.entity_id } : {}), + feature_id: data.feature_id, + usage_alert: { + threshold: data.usage_alert.threshold, + threshold_type: data.usage_alert.threshold_type, + }, + }, + type: event.type, + }; + } + default: + return null; + } +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function webhookIdempotencyKey(svixId: string): string { + return createHash("sha256").update(svixId).digest("hex"); +} + +async function processClaimedAutumnWebhook( + stored: ClaimedAutumnWebhook +): Promise { + let result: WebhookResult; + try { + const event = replayableAutumnEvent({ + data: stored.payload, + type: stored.type, + }); + if (!event) { + throw new Error(`Unsupported stored Autumn webhook type: ${stored.type}`); + } + result = await dispatch(event, webhookIdempotencyKey(stored.id)); + } catch (error) { + const status = await recordAutumnWebhookAttempt({ + attempts: stored.attempts, + claimToken: stored.claimToken, + errorMessage: errorMessage(error), + id: stored.id, + status: "pending", + }); + if (status === "completed") { + return { + disposition: "duplicate", + message: "Webhook already processed", + success: true, + }; + } + if (status === "dead_letter") { + return { + disposition: "dead_letter", + message: "Webhook moved to dead letter", + success: true, + }; + } + throw error; + } + + if (result.disposition === "deferred") { + const status = await recordAutumnWebhookAttempt({ + attempts: stored.attempts, + claimToken: stored.claimToken, + errorMessage: result.message, + id: stored.id, + status: "deferred", + }); + if (status === "completed") { + return { + disposition: "duplicate", + message: "Webhook already processed", + success: true, + }; + } + if (status === "dead_letter") { + return { + disposition: "dead_letter", + message: "Webhook moved to dead letter", + success: true, + }; + } + return { + disposition: "deferred", + message: "Webhook stored for replay", + success: true, + }; + } + + if (!result.success) { + const status = await recordAutumnWebhookAttempt({ + attempts: stored.attempts, + claimToken: stored.claimToken, + errorMessage: result.message, + id: stored.id, + status: "pending", + }); + if (status === "completed") { + return { + disposition: "duplicate", + message: "Webhook already processed", + success: true, + }; + } + if (status === "dead_letter") { + return { + disposition: "dead_letter", + message: "Webhook moved to dead letter", + success: true, + }; + } + return result; + } + + await recordAutumnWebhookAttempt({ + attempts: stored.attempts, + claimToken: stored.claimToken, + id: stored.id, + status: "completed", + }); + return result; +} + +async function processStoredAutumnWebhook( + svixId: string +): Promise { + const claimed = await claimAutumnWebhook({ id: svixId }); + if (claimed) { + return processClaimedAutumnWebhook(claimed); + } + + const stored = await getAutumnWebhook(svixId); + if (!stored) { + return { success: false, message: "Stored webhook not found" }; + } + if (stored.status === "completed") { + return { + disposition: "duplicate", + message: "Webhook already processed", + success: true, + }; + } + if (stored.status === "dead_letter") { + return { + disposition: "duplicate", + message: "Webhook retained for investigation", + success: true, + }; + } + return { + disposition: "deferred", + message: "Webhook already queued for replay", + success: true, + }; +} + +export async function handleVerifiedAutumnEvent( + svixId: string, event: RawAutumnEvent +): Promise { + const replayable = replayableAutumnEvent(event); + if (!replayable) { + return dispatch(event); + } + + await storeAutumnWebhook({ + id: svixId, + payload: replayable.data, + type: replayable.type, + }); + return processStoredAutumnWebhook(svixId); +} + +export function replayDeferredAutumnWebhook( + svixId: string +): Promise { + return processStoredAutumnWebhook(svixId); +} + +export async function replayDeferredAutumnWebhooks(limit = 25): Promise<{ + completed: number; + deadLettered: number; + deferred: number; + failed: string[]; +}> { + const exhausted = await deadLetterExhaustedAutumnWebhooks(); + const ids = await listReplayableAutumnWebhookIds({ limit }); + const report = { + completed: 0, + deadLettered: exhausted, + deferred: 0, + failed: [] as string[], + }; + for (const id of ids) { + try { + const result = await replayDeferredAutumnWebhook(id); + if (result.disposition === "dead_letter") { + report.deadLettered += 1; + continue; + } + if (result.disposition === "deferred") { + report.deferred += 1; + continue; + } + if (result.success) { + report.completed += 1; + continue; + } + report.failed.push(id); + } catch { + report.failed.push(id); + } + } + return report; +} + +function dispatch( + event: RawAutumnEvent, + idempotencyKey?: string ): Promise | WebhookResult { switch (event.type) { case "balances.limit_reached": - return handleLimitReached(limitReachedSchema.parse(event.data)); + return handleLimitReached( + limitReachedSchema.parse(event.data), + idempotencyKey + ); case "balances.usage_alert_triggered": - return handleUsageAlert(usageAlertSchema.parse(event.data)); + return handleUsageAlert( + usageAlertSchema.parse(event.data), + idempotencyKey + ); case "customer.products.updated": return handleProductsUpdated(productsUpdatedSchema.parse(event.data)); default: - useLogger().warn("Unknown webhook type", { + getAutumnLogger().warn("Unknown webhook type", { autumn: { type: event.type }, }); return { success: true, message: "Unknown event type" }; @@ -626,7 +980,7 @@ function dispatch( export const autumnWebhook = new Elysia().post( "/autumn", async ({ headers, request, set }) => { - const log = useLogger(); + const log = getAutumnLogger(); const rawBody = await request.text(); const verify = verifySvix(rawBody, { @@ -664,14 +1018,22 @@ export const autumnWebhook = new Elysia().post( } const svixId = headers["svix-id"]; + if (!svixId) { + set.status = 401; + return { success: false, message: "Invalid signature" }; + } mergeWideEvent({ webhook_type: event.type, - ...(svixId ? { svix_id: svixId } : {}), + svix_id: svixId, }); log.info("Autumn webhook", { autumn: { type: event.type } }); try { - const result = await dispatch(event); + const result = await handleVerifiedAutumnEvent(svixId, event); + if (result.disposition === "deferred") { + set.status = 202; + return result; + } if (!result.success) { set.status = 502; } diff --git a/apps/basket/src/hooks/auth.test.ts b/apps/basket/src/hooks/auth.test.ts new file mode 100644 index 000000000..f88b1720d --- /dev/null +++ b/apps/basket/src/hooks/auth.test.ts @@ -0,0 +1,87 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const state = vi.hoisted(() => ({ + captureError: vi.fn(), + findOwner: vi.fn(), +})); + +vi.mock("@databuddy/db", () => ({ + db: { + query: { + member: { findFirst: state.findOwner }, + websites: { findFirst: vi.fn() }, + }, + }, +})); + +vi.mock("@databuddy/redis/cache-invalidation", () => ({ + cacheNamespaces: { + apiKeyOwnerId: "api-key-owner-id", + websiteWithOwner: "website-with-owner", + }, +})); + +vi.mock("@databuddy/redis/cacheable", () => ({ + cacheable: (fn: unknown) => fn, +})); + +vi.mock("@lib/structured-errors", () => ({ + basketErrors: { + billingCheckUnavailable: () => new Error("billing check unavailable"), + websiteLookupUnavailable: () => new Error("website lookup unavailable"), + }, +})); + +vi.mock("@lib/tracing", () => ({ + captureError: state.captureError, + record: (_name: string, callback: () => unknown) => callback(), +})); + +vi.mock("@utils/origin-ip-validation", () => ({ + isValidOriginFromSettings: vi.fn(() => false), +})); + +const { resolveApiKeyOwnerId } = await import("./auth"); + +describe("website owner lookup", () => { + beforeEach(() => { + state.captureError.mockClear(); + state.findOwner.mockReset(); + }); + + test("returns the organization owner when lookup succeeds", async () => { + state.findOwner.mockResolvedValueOnce({ userId: "owner-1" }); + + await expect(resolveApiKeyOwnerId("org-1")).resolves.toBe("owner-1"); + }); + + test("does not look up an owner for a personal website", async () => { + await expect(resolveApiKeyOwnerId(null)).resolves.toBeNull(); + expect(state.findOwner).not.toHaveBeenCalled(); + }); + + test("fails closed when the owner lookup is unavailable", async () => { + const error = new Error("database unavailable"); + state.findOwner.mockRejectedValueOnce(error); + + await expect(resolveApiKeyOwnerId("org-1")).rejects.toThrow( + "billing check unavailable" + ); + expect(state.captureError).toHaveBeenCalledWith(error, { + message: "Workspace owner lookup failed", + organizationId: "org-1", + }); + }); + + test("fails closed when an organization has no owner", async () => { + state.findOwner.mockResolvedValueOnce(null); + + await expect(resolveApiKeyOwnerId("org-1")).rejects.toThrow( + "billing check unavailable" + ); + expect(state.captureError).toHaveBeenCalledWith(expect.any(Error), { + message: "Workspace owner lookup returned no owner", + organizationId: "org-1", + }); + }); +}); diff --git a/apps/basket/src/hooks/auth.ts b/apps/basket/src/hooks/auth.ts index 885ff0d8f..318072765 100644 --- a/apps/basket/src/hooks/auth.ts +++ b/apps/basket/src/hooks/auth.ts @@ -46,17 +46,19 @@ function _resolveOwnerId( return orgMember.userId; } } catch (error) { - if (error instanceof EvlogError) { - throw error; - } captureError(error, { - message: "Failed to fetch workspace owner", + message: "Workspace owner lookup failed", organizationId, }); - throw basketErrors.websiteLookupUnavailable(); + throw basketErrors.billingCheckUnavailable(); } - return null; + const error = new Error("Organization has no owner member"); + captureError(error, { + message: "Workspace owner lookup returned no owner", + organizationId, + }); + throw basketErrors.billingCheckUnavailable(); }); } diff --git a/apps/basket/src/index.ts b/apps/basket/src/index.ts index 185132900..26dfbd8a8 100644 --- a/apps/basket/src/index.ts +++ b/apps/basket/src/index.ts @@ -17,6 +17,7 @@ import { handleUncaughtException, handleUnhandledRejection, } from "@lib/process-errors"; +import { sanitizeRequestId } from "@lib/request-id"; import { buildBasketErrorPayload } from "@lib/structured-errors"; import { captureError } from "@lib/tracing"; import basketRouter from "@routes/basket"; @@ -104,7 +105,8 @@ const app = new Elysia() } const requestId = - request.headers.get("x-request-id") ?? crypto.randomUUID(); + sanitizeRequestId(request.headers.get("x-request-id")) ?? + crypto.randomUUID(); captureError(error, { requestId }); const { status, payload } = buildBasketErrorPayload(error, { diff --git a/apps/basket/src/lib/event-service.ts b/apps/basket/src/lib/event-service.ts index b54bcb334..d4da14945 100644 --- a/apps/basket/src/lib/event-service.ts +++ b/apps/basket/src/lib/event-service.ts @@ -240,9 +240,7 @@ export function insertOutgoingLink( }); } -export function insertTrackEventsBatch( - events: EventsInsert[] -): Promise { +export function insertTrackEventsBatch(events: EventsInsert[]): Promise { return record("insertTrackEventsBatch", async () => { if (events.length === 0) { return; diff --git a/apps/basket/src/lib/request-id.test.ts b/apps/basket/src/lib/request-id.test.ts new file mode 100644 index 000000000..3d5c84115 --- /dev/null +++ b/apps/basket/src/lib/request-id.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { sanitizeRequestId } from "./request-id"; + +describe("sanitizeRequestId", () => { + it("keeps bounded request identifiers used by common tracing systems", () => { + expect(sanitizeRequestId(" req_123:trace-456.7 ")).toBe( + "req_123:trace-456.7" + ); + }); + + it.each([ + null, + "", + "request id", + "request\r\nid", + "x".repeat(129), + ])("rejects unsafe inbound request IDs: %j", (value) => { + expect(sanitizeRequestId(value)).toBeNull(); + }); +}); diff --git a/apps/basket/src/lib/request-id.ts b/apps/basket/src/lib/request-id.ts new file mode 100644 index 000000000..4e85dcfb2 --- /dev/null +++ b/apps/basket/src/lib/request-id.ts @@ -0,0 +1,6 @@ +const REQUEST_ID_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/; + +export function sanitizeRequestId(value: string | null): string | null { + const requestId = value?.trim(); + return requestId && REQUEST_ID_PATTERN.test(requestId) ? requestId : null; +} diff --git a/apps/basket/src/routes/webhooks/paddle.test.ts b/apps/basket/src/routes/webhooks/paddle.test.ts index cf1cf9fd7..ca7b35893 100644 --- a/apps/basket/src/routes/webhooks/paddle.test.ts +++ b/apps/basket/src/routes/webhooks/paddle.test.ts @@ -22,7 +22,6 @@ const VALID_PAYLOAD = JSON.stringify({ details: { totals: { total: "1000" } }, }, }); - describe("verifyPaddleSignature", () => { test("valid Paddle Billing signature -> accepted", () => { const result = verifyPaddleSignature(VALID_PAYLOAD, sign(VALID_PAYLOAD), SECRET); diff --git a/apps/basket/src/routes/webhooks/stripe-normalization.ts b/apps/basket/src/routes/webhooks/stripe-normalization.ts new file mode 100644 index 000000000..ec0944853 --- /dev/null +++ b/apps/basket/src/routes/webhooks/stripe-normalization.ts @@ -0,0 +1,803 @@ +export interface ExpandableObject { + id: string; +} + +interface WebhookContextObject extends ExpandableObject { + customer?: string | ExpandableObject | null; + description?: string | null; + metadata?: Record; +} + +interface WebhookPaymentError { + code?: unknown; + decline_code?: unknown; + message?: unknown; + type?: unknown; +} + +interface WebhookPaymentContext extends WebhookContextObject { + cancellation_reason?: unknown; + last_payment_error?: WebhookPaymentError | null; +} + +interface WebhookInvoiceContext extends WebhookContextObject { + parent?: { + subscription_details?: { + metadata?: Record | null; + } | null; + } | null; + subscription_details?: { + metadata?: Record | null; + } | null; +} + +export interface WebhookPaymentIntent extends WebhookContextObject { + amount: number; + amount_received?: number; + cancellation_reason?: unknown; + created: number; + currency: string; + invoice?: string | WebhookInvoiceContext | null; + last_payment_error?: WebhookPaymentError | null; +} + +export interface WebhookInvoicePayment { + amount_paid?: number | null; + amount_requested?: number | null; + created: number; + currency: string; + id: string; + invoice: string | WebhookInvoiceContext; + is_default?: boolean; + payment?: { + charge?: string | WebhookContextObject | null; + payment_intent?: string | WebhookPaymentContext | null; + payment_record?: string | WebhookContextObject | null; + type: "charge" | "payment_intent" | "payment_record"; + }; + status: "canceled" | "open" | "paid"; +} + +export interface WebhookInvoice extends WebhookInvoiceContext { + amount_due?: number; + amount_paid: number; + amount_remaining?: number; + billing_reason?: string | null; + created: number; + currency: string; + payment_intent?: string | WebhookPaymentContext | null; + payments?: { + data: WebhookInvoicePayment[]; + has_more?: boolean; + } | null; + status?: string; + subscription?: string | null; + total?: number; +} + +export interface WebhookCharge extends WebhookContextObject { + amount_refunded: number; + currency: string; + payment_intent?: string | WebhookContextObject | null; + refunds?: { + data: Array<{ + amount: number; + created: number; + id: string; + }>; + }; +} + +export interface StripeWebhookEvent { + api_version?: string | null; + created: number; + data: { + object: + | WebhookCharge + | WebhookInvoice + | WebhookInvoicePayment + | WebhookPaymentIntent; + }; + id: string; + type: string; +} + +export type StripeRecordKind = "attempt" | "link" | "money"; + +const STRIPE_API_DATE_PREFIX = /^(\d{4}-\d{2}-\d{2})/; +const STRIPE_ZERO_DECIMAL_CURRENCIES = new Set([ + "BIF", + "CLP", + "DJF", + "GNF", + "JPY", + "KMF", + "KRW", + "MGA", + "PYG", + "RWF", + "UGX", + "VND", + "VUV", + "XAF", + "XOF", + "XPF", +]); +// Stripe keeps charge amounts for these nominally zero-decimal currencies in +// two-decimal form for API compatibility (for example, 5 UGX is sent as 500). +const STRIPE_TWO_DECIMAL_COMPATIBILITY_CURRENCIES = new Set(["ISK", "UGX"]); + +export interface NormalizedStripeRecord { + amount: number; + context: { + apiVersion?: string; + cancellationReason?: string; + eventCreated: number; + eventId: string; + eventType?: string; + failureCode?: string; + failureDeclineCode?: string; + failureType?: string; + invoiceId?: string; + invoicePaymentId?: string; + moneyKind?: + | "invoice" + | "invoice_fallback" + | "invoice_payment" + | "refund" + | "standalone_candidate"; + paymentIntentId?: string; + recordKind: StripeRecordKind; + }; + createdUnix: number; + currency: string; + customerId?: string; + productName?: string; + rawMetadata: Record; + status: "canceled" | "completed" | "failed" | "linked" | "refunded"; + transactionId: string; + type: "refund" | "sale" | "subscription" | "subscription_event"; +} + +function validUnixSeconds(value: unknown): value is number { + return Number.isSafeInteger(value) && Number(value) > 0; +} + +function requireUnixSeconds(value: unknown, label: string): number { + if (!validUnixSeconds(value)) { + throw new Error(`${label} must be a positive Unix timestamp`); + } + return value; +} + +function amountFromMinorUnits( + value: unknown, + currency: string, + label: string +): number { + if (!(Number.isSafeInteger(value) && Number(value) > 0)) { + throw new Error(`${label} must be a positive integer`); + } + const normalizedCurrency = currency.toUpperCase(); + const exponent = + STRIPE_ZERO_DECIMAL_CURRENCIES.has(normalizedCurrency) && + !STRIPE_TWO_DECIMAL_COMPATIBILITY_CURRENCIES.has(normalizedCurrency) + ? 0 + : 2; + return Number(value) / 10 ** exponent; +} + +function nonNegativeInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && Number(value) >= 0; +} + +export function expandableId( + value: string | ExpandableObject | null | undefined +): string | undefined { + if (!value) { + return; + } + return typeof value === "string" ? value : value.id; +} + +function customerId( + value: string | ExpandableObject | null | undefined +): string | undefined { + return expandableId(value); +} + +function expandedObject( + value: string | T | null | undefined +): T | undefined { + return typeof value === "object" && value !== null ? value : undefined; +} + +const STRIPE_REASON_TOKEN = /^[a-z0-9][a-z0-9_-]{0,63}$/; + +function reasonToken(value: unknown): string | undefined { + if (typeof value !== "string") { + return; + } + const token = value.trim().toLowerCase(); + return STRIPE_REASON_TOKEN.test(token) ? token : undefined; +} + +function paymentFailureContext( + ...sources: Array +): Pick< + NormalizedStripeRecord["context"], + "cancellationReason" | "failureCode" | "failureDeclineCode" | "failureType" +> { + const first = (read: (source: WebhookPaymentContext) => unknown) => + sources + .map((source) => (source ? reasonToken(read(source)) : undefined)) + .find((value): value is string => value !== undefined); + const cancellationReason = first((source) => source.cancellation_reason); + const failureCode = first((source) => source.last_payment_error?.code); + const failureDeclineCode = first( + (source) => source.last_payment_error?.decline_code + ); + const failureType = first((source) => source.last_payment_error?.type); + return { + ...(cancellationReason ? { cancellationReason } : {}), + ...(failureCode ? { failureCode } : {}), + ...(failureDeclineCode ? { failureDeclineCode } : {}), + ...(failureType ? { failureType } : {}), + }; +} + +export function invoiceMetadataSources( + invoice: WebhookInvoiceContext +): Record { + return { + ...invoice.parent?.subscription_details?.metadata, + ...invoice.subscription_details?.metadata, + ...invoice.metadata, + }; +} + +function eventContext( + event: StripeWebhookEvent, + recordKind: StripeRecordKind, + extra: Omit< + NormalizedStripeRecord["context"], + "apiVersion" | "eventCreated" | "eventId" | "recordKind" + > = {} +): NormalizedStripeRecord["context"] { + return { + ...(event.api_version ? { apiVersion: event.api_version } : {}), + eventCreated: requireUnixSeconds(event.created, "Stripe event.created"), + eventId: event.id, + eventType: event.type, + recordKind, + ...extra, + }; +} + +function linkRecord( + event: StripeWebhookEvent, + input: { + createdUnix?: number; + currency: string; + customerId?: string; + invoiceId?: string; + invoicePaymentId?: string; + paymentIntentId?: string; + productName?: string; + rawMetadata?: Record; + transactionId?: string; + } +): NormalizedStripeRecord { + return { + amount: 0, + context: eventContext(event, "link", { + ...(input.invoiceId ? { invoiceId: input.invoiceId } : {}), + ...(input.invoicePaymentId + ? { invoicePaymentId: input.invoicePaymentId } + : {}), + ...(input.paymentIntentId + ? { paymentIntentId: input.paymentIntentId } + : {}), + }), + createdUnix: requireUnixSeconds( + input.createdUnix ?? event.created, + "Stripe link timestamp" + ), + currency: input.currency.toUpperCase(), + ...(input.customerId ? { customerId: input.customerId } : {}), + ...(input.productName ? { productName: input.productName } : {}), + rawMetadata: input.rawMetadata ?? {}, + status: "linked", + transactionId: input.transactionId ?? event.id, + type: "subscription_event", + }; +} + +function attemptRecord( + event: StripeWebhookEvent, + input: { + amountMinorUnits: number; + createdUnix?: number; + currency: string; + customerId?: string; + invoiceId?: string; + paymentIntentId?: string; + productName?: string; + rawMetadata?: Record; + reason?: Pick< + NormalizedStripeRecord["context"], + | "cancellationReason" + | "failureCode" + | "failureDeclineCode" + | "failureType" + >; + status: "canceled" | "failed"; + } +): NormalizedStripeRecord { + return { + amount: + Number.isSafeInteger(input.amountMinorUnits) && input.amountMinorUnits > 0 + ? amountFromMinorUnits( + input.amountMinorUnits, + input.currency, + "Stripe attempt amount" + ) + : 0, + context: eventContext(event, "attempt", { + ...(input.invoiceId ? { invoiceId: input.invoiceId } : {}), + ...(input.paymentIntentId + ? { paymentIntentId: input.paymentIntentId } + : {}), + ...input.reason, + }), + createdUnix: requireUnixSeconds( + input.createdUnix ?? event.created, + "Stripe attempt timestamp" + ), + currency: input.currency.toUpperCase(), + ...(input.customerId ? { customerId: input.customerId } : {}), + ...(input.productName ? { productName: input.productName } : {}), + rawMetadata: input.rawMetadata ?? {}, + status: input.status, + transactionId: event.id, + type: "subscription_event", + }; +} + +function invoicePaymentContext(payment: WebhookInvoicePayment): { + customerId?: string; + productName?: string; + rawMetadata: Record; +} { + const invoice = expandedObject(payment.invoice); + const paymentObject = + expandedObject(payment.payment?.payment_intent) ?? + expandedObject(payment.payment?.charge) ?? + expandedObject(payment.payment?.payment_record); + const productName = + invoice?.description ?? paymentObject?.description ?? undefined; + const resolvedCustomerId = + customerId(invoice?.customer) ?? customerId(paymentObject?.customer); + return { + ...(resolvedCustomerId ? { customerId: resolvedCustomerId } : {}), + ...(productName ? { productName } : {}), + rawMetadata: { + ...paymentObject?.metadata, + ...(invoice ? invoiceMetadataSources(invoice) : {}), + }, + }; +} + +function invoicePaymentRecord( + event: StripeWebhookEvent, + payment: WebhookInvoicePayment, + rawMetadata: Record = {}, + fallbackCustomerId?: string, + fallbackProductName?: string +): NormalizedStripeRecord | null { + if (payment.status !== "paid" || !payment.amount_paid) { + return null; + } + const invoiceId = expandableId(payment.invoice); + if (!invoiceId) { + throw new Error("Stripe InvoicePayment is missing invoice identity"); + } + const paymentIntentId = expandableId(payment.payment?.payment_intent); + const expandedContext = invoicePaymentContext(payment); + const resolvedCustomerId = fallbackCustomerId ?? expandedContext.customerId; + const resolvedProductName = + fallbackProductName ?? expandedContext.productName; + return { + amount: amountFromMinorUnits( + payment.amount_paid, + payment.currency, + "Stripe InvoicePayment.amount_paid" + ), + context: eventContext(event, "money", { + invoiceId, + invoicePaymentId: payment.id, + moneyKind: "invoice_payment", + ...(paymentIntentId ? { paymentIntentId } : {}), + }), + createdUnix: requireUnixSeconds(event.created, "Stripe payment time"), + currency: payment.currency.toUpperCase(), + ...(resolvedCustomerId ? { customerId: resolvedCustomerId } : {}), + ...(resolvedProductName ? { productName: resolvedProductName } : {}), + rawMetadata: { ...expandedContext.rawMetadata, ...rawMetadata }, + status: "completed", + transactionId: payment.id, + type: "subscription", + }; +} + +function invoicePaymentLinkRecord( + event: StripeWebhookEvent, + payment: WebhookInvoicePayment, + rawMetadata: Record, + fallbackCustomerId?: string, + fallbackProductName?: string +): NormalizedStripeRecord | null { + if (payment.status !== "paid") { + return null; + } + const invoiceId = expandableId(payment.invoice); + const paymentIntentId = expandableId(payment.payment?.payment_intent); + if (!(invoiceId && paymentIntentId)) { + return null; + } + const expandedContext = invoicePaymentContext(payment); + return linkRecord(event, { + createdUnix: event.created, + currency: payment.currency, + customerId: fallbackCustomerId ?? expandedContext.customerId, + invoiceId, + invoicePaymentId: payment.id, + paymentIntentId, + productName: fallbackProductName ?? expandedContext.productName, + rawMetadata: { ...expandedContext.rawMetadata, ...rawMetadata }, + // A relation and a later direct InvoicePayment event must survive FINAL + // independently. Stripe event + allocation IDs are immutable on retries. + transactionId: `${event.id}:${payment.id}`, + }); +} + +function apiDate(apiVersion: string | null | undefined): string | null { + const match = apiVersion?.match(STRIPE_API_DATE_PREFIX); + return match?.[1] ?? null; +} + +export function usesInvoicePayments( + apiVersion: string | null | undefined +): boolean { + const date = apiDate(apiVersion); + return date !== null && date >= "2025-03-31"; +} + +export function emitsInvoicePaymentPaidEvents( + apiVersion: string | null | undefined +): boolean { + const date = apiDate(apiVersion); + return date !== null && date >= "2025-05-28"; +} + +function normalizePaymentIntent( + event: StripeWebhookEvent, + status: "canceled" | "failed" | "succeeded" +): NormalizedStripeRecord[] { + const intent = event.data.object as WebhookPaymentIntent; + const invoiceId = expandableId(intent.invoice); + const common = { + createdUnix: event.created, + currency: intent.currency, + customerId: customerId(intent.customer), + invoiceId, + paymentIntentId: intent.id, + productName: intent.description ?? undefined, + rawMetadata: intent.metadata, + }; + if (status !== "succeeded") { + return [ + attemptRecord(event, { + ...common, + amountMinorUnits: intent.amount, + reason: paymentFailureContext(intent), + status, + }), + ]; + } + if (invoiceId) { + return [linkRecord(event, common)]; + } + const amountMinorUnits = + intent.amount_received && intent.amount_received > 0 + ? intent.amount_received + : intent.amount; + return [ + { + amount: amountFromMinorUnits( + amountMinorUnits, + intent.currency, + "Stripe PaymentIntent amount" + ), + context: eventContext(event, "money", { + moneyKind: "standalone_candidate", + paymentIntentId: intent.id, + }), + createdUnix: requireUnixSeconds(event.created, "Stripe payment time"), + currency: intent.currency.toUpperCase(), + ...(common.customerId ? { customerId: common.customerId } : {}), + ...(common.productName ? { productName: common.productName } : {}), + rawMetadata: intent.metadata ?? {}, + status: "completed", + transactionId: intent.id, + type: "sale", + }, + ]; +} + +function invoiceMoneyRecord( + event: StripeWebhookEvent, + invoice: WebhookInvoice, + amountMinorUnits: number, + input: { + customerId?: string; + moneyKind?: "invoice" | "invoice_fallback"; + paymentIntentId?: string; + productName?: string; + rawMetadata: Record; + } +): NormalizedStripeRecord { + return { + amount: amountFromMinorUnits( + amountMinorUnits, + invoice.currency, + "Stripe Invoice.amount_paid" + ), + context: eventContext(event, "money", { + invoiceId: invoice.id, + moneyKind: input.moneyKind ?? "invoice", + ...(input.paymentIntentId + ? { paymentIntentId: input.paymentIntentId } + : {}), + }), + createdUnix: requireUnixSeconds(event.created, "Stripe payment time"), + currency: invoice.currency.toUpperCase(), + ...(input.customerId ? { customerId: input.customerId } : {}), + ...(input.productName ? { productName: input.productName } : {}), + rawMetadata: input.rawMetadata, + status: "completed", + transactionId: invoice.id, + // Old readers exclude subscription events. The reconciler promotes marked + // fallbacks to subscription money, making Basket/API rollout order safe. + type: + input.moneyKind === "invoice_fallback" + ? "subscription_event" + : "subscription", + }; +} + +function paidInvoiceAllocationMinorUnits(invoice: WebhookInvoice): number { + return (invoice.payments?.data ?? []).reduce((total, payment) => { + if ( + payment.status !== "paid" || + !Number.isSafeInteger(payment.amount_paid) || + Number(payment.amount_paid) <= 0 + ) { + return total; + } + return total + Number(payment.amount_paid); + }, 0); +} + +function normalizePaidInvoice( + event: StripeWebhookEvent +): NormalizedStripeRecord[] { + const invoice = event.data.object as WebhookInvoice; + const rawMetadata = invoiceMetadataSources(invoice); + const invoiceCustomerId = customerId(invoice.customer); + const productName = invoice.description ?? undefined; + const context = linkRecord(event, { + currency: invoice.currency, + customerId: invoiceCustomerId, + invoiceId: invoice.id, + productName, + rawMetadata, + }); + if (invoice.status !== "paid" || invoice.amount_paid <= 0) { + return [context]; + } + + const supportsInvoicePayments = usesInvoicePayments(event.api_version); + const legacyPaymentIntentId = expandableId(invoice.payment_intent); + if (!supportsInvoicePayments && legacyPaymentIntentId) { + return [ + context, + invoiceMoneyRecord(event, invoice, invoice.amount_paid, { + customerId: invoiceCustomerId, + paymentIntentId: legacyPaymentIntentId, + productName, + rawMetadata, + }), + ]; + } + + const needsFallback = + emitsInvoicePaymentPaidEvents(event.api_version) || + (supportsInvoicePayments && + (invoice.payments == null || invoice.payments.has_more === true)); + if (needsFallback) { + const paymentLinks = (invoice.payments?.data ?? []) + .map((payment) => + invoicePaymentLinkRecord( + event, + payment, + rawMetadata, + invoiceCustomerId, + productName + ) + ) + .filter((record): record is NormalizedStripeRecord => record !== null); + // Modern endpoints may not have subscribed to invoice_payment.paid yet, + // while transition snapshots can be paginated. Reads subtract any exact + // allocations from this total, so neither case loses or duplicates money. + return [ + context, + ...paymentLinks, + invoiceMoneyRecord(event, invoice, invoice.amount_paid, { + customerId: invoiceCustomerId, + moneyKind: "invoice_fallback", + paymentIntentId: legacyPaymentIntentId, + productName, + rawMetadata, + }), + ]; + } + + // InvoicePayment objects shipped before their paid webhook. During that + // transition window, embedded allocations are the only exact money source. + // Older snapshots can expose the same shape, so retain those allocations too. + const allocations = (invoice.payments?.data ?? []) + .map((payment) => + invoicePaymentRecord( + event, + payment, + rawMetadata, + invoiceCustomerId, + productName + ) + ) + .filter((record): record is NormalizedStripeRecord => record !== null); + if (supportsInvoicePayments) { + const outOfBandMinorUnits = Math.max( + 0, + invoice.amount_paid - paidInvoiceAllocationMinorUnits(invoice) + ); + return [ + context, + ...allocations, + ...(outOfBandMinorUnits > 0 + ? [ + invoiceMoneyRecord(event, invoice, outOfBandMinorUnits, { + customerId: invoiceCustomerId, + productName, + rawMetadata, + }), + ] + : []), + ]; + } + if (allocations.length > 0) { + return [context, ...allocations]; + } + + // Old snapshot versions can represent out-of-band invoices with no PaymentIntent. + return [ + context, + invoiceMoneyRecord(event, invoice, invoice.amount_paid, { + customerId: invoiceCustomerId, + productName, + rawMetadata, + }), + ]; +} + +function normalizeFailedInvoice( + event: StripeWebhookEvent +): NormalizedStripeRecord[] { + const invoice = event.data.object as WebhookInvoice; + const openPayments = (invoice.payments?.data ?? []).filter( + (payment) => + payment.status === "open" && nonNegativeInteger(payment.amount_requested) + ); + const requestedPayment = + openPayments.find((payment) => payment.is_default) ?? + (openPayments.length === 1 ? openPayments[0] : undefined); + const requestedPaymentIntentId = expandableId( + requestedPayment?.payment?.payment_intent + ); + const requestedPaymentIntent = expandedObject( + requestedPayment?.payment?.payment_intent + ); + const invoicePaymentIntent = expandedObject(invoice.payment_intent); + const amountMinorUnits = + requestedPayment?.amount_requested ?? + (nonNegativeInteger(invoice.amount_remaining) + ? invoice.amount_remaining + : undefined) ?? + (nonNegativeInteger(invoice.amount_due) ? invoice.amount_due : undefined) ?? + (nonNegativeInteger(invoice.total) ? invoice.total : undefined) ?? + invoice.amount_paid; + return [ + attemptRecord(event, { + amountMinorUnits, + currency: invoice.currency, + customerId: customerId(invoice.customer), + invoiceId: invoice.id, + paymentIntentId: + requestedPaymentIntentId ?? expandableId(invoice.payment_intent), + productName: invoice.description ?? undefined, + rawMetadata: invoiceMetadataSources(invoice), + reason: paymentFailureContext( + requestedPaymentIntent, + invoicePaymentIntent + ), + status: "failed", + }), + ]; +} + +function normalizeRefund(event: StripeWebhookEvent): NormalizedStripeRecord[] { + const charge = event.data.object as WebhookCharge; + const paymentIntentId = expandableId(charge.payment_intent); + return (charge.refunds?.data ?? []).map((refund) => ({ + amount: -amountFromMinorUnits( + refund.amount, + charge.currency, + "Stripe Refund.amount" + ), + context: eventContext(event, "money", { + moneyKind: "refund", + ...(paymentIntentId ? { paymentIntentId } : {}), + }), + createdUnix: requireUnixSeconds(refund.created, "Stripe refund time"), + currency: charge.currency.toUpperCase(), + ...(customerId(charge.customer) + ? { customerId: customerId(charge.customer) } + : {}), + productName: "Refund", + rawMetadata: charge.metadata ?? {}, + status: "refunded", + transactionId: refund.id, + type: "refund", + })); +} + +export function normalizeStripeEvent( + event: StripeWebhookEvent +): NormalizedStripeRecord[] { + requireUnixSeconds(event.created, "Stripe event.created"); + switch (event.type) { + case "payment_intent.succeeded": + return normalizePaymentIntent(event, "succeeded"); + case "payment_intent.payment_failed": + return normalizePaymentIntent(event, "failed"); + case "payment_intent.canceled": + return normalizePaymentIntent(event, "canceled"); + case "invoice.paid": + case "invoice.payment_succeeded": + return normalizePaidInvoice(event); + case "invoice.payment_failed": + return normalizeFailedInvoice(event); + case "invoice_payment.paid": { + const payment = invoicePaymentRecord( + event, + event.data.object as WebhookInvoicePayment + ); + return payment ? [payment] : []; + } + case "charge.refunded": + return normalizeRefund(event); + default: + return []; + } +} diff --git a/apps/basket/src/routes/webhooks/stripe.test.ts b/apps/basket/src/routes/webhooks/stripe.test.ts index d0ef0a1eb..99fe48692 100644 --- a/apps/basket/src/routes/webhooks/stripe.test.ts +++ b/apps/basket/src/routes/webhooks/stripe.test.ts @@ -1,6 +1,13 @@ import { describe, expect, test } from "vitest"; import { createHmac } from "node:crypto"; -import { invoiceMetadataSources, verifyStripeSignature } from "./stripe"; +import { stripeRecordMetadata, verifyStripeSignature } from "./stripe"; +import { + type StripeWebhookEvent, + emitsInvoicePaymentPaidEvents, + invoiceMetadataSources, + normalizeStripeEvent, + usesInvoicePayments, +} from "./stripe-normalization"; const SECRET = "whsec_test_secret_key"; @@ -13,6 +20,7 @@ function sign(payload: string, secret = SECRET, timestamp?: number): string { } const VALID_PAYLOAD = JSON.stringify({ + created: 1_700_000_123, id: "evt_1", type: "payment_intent.succeeded", data: { @@ -223,3 +231,900 @@ describe("invoiceMetadataSources", () => { ).toEqual({}); }); }); + +describe("normalizeStripeEvent", () => { + const acaciaIntent = { + api_version: "2024-10-28.acacia", + created: 1_700_000_200, + id: "evt_pi_paid", + type: "payment_intent.succeeded", + data: { + object: { + amount: 300, + created: 1_700_000_000, + currency: "usd", + id: "pi_1", + invoice: "in_1", + metadata: { + databuddy_profile_id: "profile-1", + databuddy_session_id: "session-1", + }, + }, + }, + } satisfies StripeWebhookEvent; + const acaciaInvoice = { + api_version: "2024-10-28.acacia", + created: 1_700_000_201, + id: "evt_invoice_paid", + type: "invoice.paid", + data: { + object: { + amount_paid: 300, + created: 1_699_900_000, + currency: "usd", + id: "in_1", + payment_intent: "pi_1", + status: "paid", + }, + }, + } satisfies StripeWebhookEvent; + + test("keeps Acacia PaymentIntent attribution as a link and counts the invoice once", () => { + const records = [ + ...normalizeStripeEvent(acaciaInvoice), + ...normalizeStripeEvent(acaciaIntent), + ]; + const money = records.filter( + (record) => record.context.recordKind === "money" + ); + const link = records.find( + (record) => record.context.eventId === "evt_pi_paid" + ); + + expect(money).toHaveLength(1); + expect(money[0]).toMatchObject({ + amount: 3, + createdUnix: 1_700_000_201, + transactionId: "in_1", + type: "subscription", + }); + expect(link).toMatchObject({ + context: { + invoiceId: "in_1", + paymentIntentId: "pi_1", + recordKind: "link", + }, + rawMetadata: { + databuddy_profile_id: "profile-1", + databuddy_session_id: "session-1", + }, + }); + }); + + test("is independent of Acacia webhook delivery order and retries", () => { + const forward = [ + ...normalizeStripeEvent(acaciaIntent), + ...normalizeStripeEvent(acaciaInvoice), + ]; + const reverse = [ + ...normalizeStripeEvent(acaciaInvoice), + ...normalizeStripeEvent(acaciaIntent), + ]; + expect( + forward.map((record) => record.transactionId).sort() + ).toEqual(reverse.map((record) => record.transactionId).sort()); + expect(normalizeStripeEvent(acaciaInvoice)).toEqual( + normalizeStripeEvent(acaciaInvoice) + ); + }); + + test("keeps direct InvoicePayment facts separate from the invoice fallback", () => { + const intent = { + ...acaciaIntent, + api_version: "2025-08-27.basil", + data: { + object: { ...acaciaIntent.data.object, invoice: undefined }, + }, + } satisfies StripeWebhookEvent; + const payment = { + api_version: "2025-08-27.basil", + created: 1_700_000_202, + id: "evt_inpay_paid", + type: "invoice_payment.paid", + data: { + object: { + amount_paid: 300, + created: 1_700_000_190, + currency: "usd", + id: "inpay_1", + invoice: "in_1", + payment: { type: "payment_intent", payment_intent: "pi_1" }, + status: "paid", + }, + }, + } satisfies StripeWebhookEvent; + const invoice = { + ...acaciaInvoice, + api_version: "2025-08-27.basil", + data: { + object: { ...acaciaInvoice.data.object, payment_intent: undefined }, + }, + } satisfies StripeWebhookEvent; + const records = [ + ...normalizeStripeEvent(intent), + ...normalizeStripeEvent(payment), + ...normalizeStripeEvent(invoice), + ]; + + expect( + records.filter((record) => record.context.moneyKind === "invoice") + ).toHaveLength(0); + expect( + records.find((record) => record.transactionId === "in_1") + ).toMatchObject({ + amount: 3, + context: { + invoiceId: "in_1", + moneyKind: "invoice_fallback", + }, + type: "subscription_event", + }); + expect( + records.find((record) => record.transactionId === "inpay_1") + ).toMatchObject({ + amount: 3, + context: { + invoiceId: "in_1", + moneyKind: "invoice_payment", + paymentIntentId: "pi_1", + }, + createdUnix: 1_700_000_202, + }); + expect( + records.find((record) => record.transactionId === "pi_1") + ).toMatchObject({ context: { moneyKind: "standalone_candidate" } }); + expect( + normalizeStripeEvent(invoice).filter( + (record) => record.context.moneyKind === "invoice_payment" + ) + ).toEqual([]); + }); + + test.each([ + ["usd", 500, 5], + ["jpy", 500, 500], + ["isk", 500, 5], + ["ugx", 500, 5], + ] as const)( + "converts %s Stripe minor units using the charge exponent", + (currency, amount, expected) => { + const [record] = normalizeStripeEvent({ + ...acaciaIntent, + id: `evt_${currency}`, + data: { + object: { + ...acaciaIntent.data.object, + amount, + currency, + id: `pi_${currency}`, + invoice: undefined, + }, + }, + }); + + expect(record?.amount).toBe(expected); + expect(record?.currency).toBe(currency.toUpperCase()); + } + ); + + test("applies zero-decimal conversion to attempts and refunds", () => { + const [attempt] = normalizeStripeEvent({ + ...acaciaIntent, + id: "evt_jpy_failed", + type: "payment_intent.payment_failed", + data: { + object: { + ...acaciaIntent.data.object, + amount: 500, + currency: "jpy", + invoice: undefined, + }, + }, + }); + const [refund] = normalizeStripeEvent({ + api_version: "2025-08-27.basil", + created: 1_700_000_301, + id: "evt_jpy_refund", + type: "charge.refunded", + data: { + object: { + amount_refunded: 250, + currency: "jpy", + id: "ch_jpy", + refunds: { + data: [{ amount: 250, created: 1_700_000_300, id: "re_jpy" }], + }, + }, + }, + }); + + expect(attempt?.amount).toBe(500); + expect(refund?.amount).toBe(-250); + }); + + test("keeps modern invoice totals safe until exact allocations arrive", () => { + const fullyOutOfBand = normalizeStripeEvent({ + api_version: "2025-08-27.basil", + created: 1_700_000_300, + id: "evt_oob_invoice", + type: "invoice.paid", + data: { + object: { + amount_paid: 10_000, + created: 1_699_000_000, + currency: "usd", + id: "in_oob", + payments: { data: [], has_more: false }, + status: "paid", + }, + }, + }); + const partiallyOutOfBand = normalizeStripeEvent({ + api_version: "2025-08-27.basil", + created: 1_700_000_301, + id: "evt_partial_oob_invoice", + type: "invoice.paid", + data: { + object: { + amount_paid: 10_000, + created: 1_699_000_000, + currency: "usd", + id: "in_partial_oob", + payments: { + data: [ + { + amount_paid: 6_000, + created: 1_700_000_290, + currency: "usd", + id: "inpay_partial", + invoice: "in_partial_oob", + payment: { + payment_intent: "pi_partial", + type: "payment_intent", + }, + status: "paid", + }, + ], + has_more: false, + }, + status: "paid", + }, + }, + }); + const fullMoney = fullyOutOfBand.filter( + (record) => record.context.recordKind === "money" + ); + const partialMoney = partiallyOutOfBand.filter( + (record) => record.context.recordKind === "money" + ); + + expect(fullMoney).toMatchObject([ + { + amount: 100, + context: { invoiceId: "in_oob", moneyKind: "invoice_fallback" }, + transactionId: "in_oob", + }, + ]); + expect(partialMoney).toMatchObject([ + { + amount: 100, + context: { + invoiceId: "in_partial_oob", + moneyKind: "invoice_fallback", + }, + transactionId: "in_partial_oob", + }, + ]); + expect(partiallyOutOfBand).toContainEqual( + expect.objectContaining({ + context: expect.objectContaining({ + invoiceId: "in_partial_oob", + paymentIntentId: "pi_partial", + recordKind: "link", + }), + transactionId: "evt_partial_oob_invoice:inpay_partial", + }) + ); + }); + + test("keeps exact allocation and out-of-band amounts in the transition window", () => { + const money = normalizeStripeEvent({ + api_version: "2025-05-27.basil", + created: 1_700_000_301, + id: "evt_transition_oob", + type: "invoice.paid", + data: { + object: { + amount_paid: 10_000, + created: 1_699_000_000, + currency: "usd", + id: "in_transition_oob", + payments: { + data: [ + { + amount_paid: 6_000, + created: 1_700_000_290, + currency: "usd", + id: "inpay_transition", + invoice: "in_transition_oob", + payment: { + payment_intent: "pi_transition", + type: "payment_intent", + }, + status: "paid", + }, + ], + has_more: false, + }, + status: "paid", + }, + }, + }).filter((record) => record.context.recordKind === "money"); + + expect(money).toMatchObject([ + { + amount: 60, + context: { moneyKind: "invoice_payment" }, + transactionId: "inpay_transition", + }, + { + amount: 40, + context: { moneyKind: "invoice" }, + transactionId: "in_transition_oob", + }, + ]); + }); + + test("falls back to the total and retains visible modern allocation links", () => { + const records = normalizeStripeEvent({ + api_version: "2025-08-27.basil", + created: 1_700_000_300, + id: "evt_partial_invoice", + type: "invoice.paid", + data: { + object: { + amount_paid: 400, + created: 1_699_000_000, + currency: "usd", + id: "in_partial", + status: "paid", + payments: { + has_more: true, + data: [ + { + amount_paid: 100, + created: 1_700_000_100, + currency: "usd", + id: "inpay_1", + invoice: "in_partial", + payment: { + type: "payment_intent", + payment_intent: "pi_1", + }, + status: "paid", + }, + { + amount_paid: 200, + created: 1_700_000_200, + currency: "usd", + id: "inpay_2", + invoice: "in_partial", + payment: { + type: "payment_intent", + payment_intent: "pi_2", + }, + status: "paid", + }, + ], + }, + }, + }, + }); + const money = records.filter( + (record) => record.context.recordKind === "money" + ); + + expect(money).toMatchObject([ + { + amount: 4, + context: { + invoiceId: "in_partial", + moneyKind: "invoice_fallback", + }, + transactionId: "in_partial", + }, + ]); + expect( + records + .filter((record) => record.context.recordKind === "link") + .map((record) => record.transactionId) + ).toEqual([ + "evt_partial_invoice", + "evt_partial_invoice:inpay_1", + "evt_partial_invoice:inpay_2", + ]); + }); + + test("uses requested and remaining invoice amounts for failed partial payments", () => { + const failedInvoice = ( + id: string, + object: StripeWebhookEvent["data"]["object"] + ) => + normalizeStripeEvent({ + api_version: "2025-08-27.basil", + created: 1_700_000_400, + id, + type: "invoice.payment_failed", + data: { object }, + })[0]; + const remaining = failedInvoice("evt_remaining", { + amount_due: 10_000, + amount_paid: 3_000, + amount_remaining: 7_000, + created: 1_699_000_000, + currency: "usd", + id: "in_remaining", + status: "open", + }); + const requested = failedInvoice("evt_requested", { + amount_due: 10_000, + amount_paid: 3_000, + amount_remaining: 7_000, + created: 1_699_000_000, + currency: "usd", + id: "in_requested", + payments: { + data: [ + { + amount_requested: 2_500, + created: 1_700_000_390, + currency: "usd", + id: "inpay_requested", + invoice: "in_requested", + is_default: true, + payment: { + type: "payment_intent", + payment_intent: "pi_requested", + }, + status: "open", + }, + ], + has_more: false, + }, + status: "open", + }); + + expect(remaining?.amount).toBe(70); + expect(requested?.amount).toBe(25); + expect(requested?.context.paymentIntentId).toBe("pi_requested"); + }); + + test("carries expanded invoice context on direct InvoicePayment events", () => { + const [record] = normalizeStripeEvent({ + api_version: "2025-08-27.basil", + created: 1_700_000_500, + id: "evt_expanded_inpay", + type: "invoice_payment.paid", + data: { + object: { + amount_paid: 300, + created: 1_700_000_490, + currency: "usd", + id: "inpay_expanded", + invoice: { + customer: "cus_invoice", + description: "Pro plan", + id: "in_expanded", + metadata: { databuddy_session_id: "session-invoice" }, + parent: { + subscription_details: { + metadata: { databuddy_profile_id: "profile-subscription" }, + }, + }, + }, + payment: { + payment_intent: { + customer: "cus_payment", + description: "Fallback plan", + id: "pi_expanded", + metadata: { databuddy_anonymous_id: "anon-payment" }, + }, + type: "payment_intent", + }, + status: "paid", + }, + }, + }); + + expect(record).toMatchObject({ + customerId: "cus_invoice", + productName: "Pro plan", + rawMetadata: { + databuddy_anonymous_id: "anon-payment", + databuddy_profile_id: "profile-subscription", + databuddy_session_id: "session-invoice", + }, + context: { + invoiceId: "in_expanded", + paymentIntentId: "pi_expanded", + }, + }); + }); + + test("retains failed and canceled attempts with intended amount", () => { + for (const [type, status] of [ + ["payment_intent.payment_failed", "failed"], + ["payment_intent.canceled", "canceled"], + ] as const) { + const [record] = normalizeStripeEvent({ + ...acaciaIntent, + id: `evt_${status}`, + type, + data: { + object: { + ...acaciaIntent.data.object, + amount_received: 0, + }, + }, + }); + expect(record).toMatchObject({ + amount: 3, + status, + transactionId: `evt_${status}`, + type: "subscription_event", + context: { eventType: type, recordKind: "attempt" }, + }); + } + }); + + test("keeps actionable failure codes without retaining provider messages", () => { + const [record] = normalizeStripeEvent({ + ...acaciaIntent, + id: "evt_declined", + type: "payment_intent.payment_failed", + data: { + object: { + ...acaciaIntent.data.object, + last_payment_error: { + code: "card_declined", + decline_code: "insufficient_funds", + message: "Do not persist this provider message", + type: "card_error", + }, + }, + }, + }); + if (!record) { + throw new Error("Expected a normalized failed payment"); + } + + expect(record.context).toMatchObject({ + failureCode: "card_declined", + failureDeclineCode: "insufficient_funds", + failureType: "card_error", + }); + expect(record.context).not.toHaveProperty("message"); + expect(stripeRecordMetadata({}, record.context)).toMatchObject({ + stripe_failure_code: "card_declined", + stripe_failure_decline_code: "insufficient_funds", + stripe_failure_type: "card_error", + }); + expect(JSON.stringify(stripeRecordMetadata({}, record.context))).not.toContain( + "provider message" + ); + }); + + test("rejects unbounded failure text but keeps a safe cancellation reason", () => { + const [record] = normalizeStripeEvent({ + ...acaciaIntent, + id: "evt_canceled_reason", + type: "payment_intent.canceled", + data: { + object: { + ...acaciaIntent.data.object, + cancellation_reason: " Requested_By_Customer ", + last_payment_error: { + code: "free-form failure text is not a code", + decline_code: "x".repeat(65), + type: 42, + }, + }, + }, + }); + if (!record) { + throw new Error("Expected a normalized canceled payment"); + } + + expect(record.context).toMatchObject({ + cancellationReason: "requested_by_customer", + }); + expect(record.context.failureCode).toBeUndefined(); + expect(record.context.failureDeclineCode).toBeUndefined(); + expect(record.context.failureType).toBeUndefined(); + expect(stripeRecordMetadata({}, record.context)).toMatchObject({ + stripe_cancellation_reason: "requested_by_customer", + }); + }); + + test("reads invoice failure codes from the expanded attempted payment", () => { + const [record] = normalizeStripeEvent({ + api_version: "2025-08-27.basil", + created: 1_700_000_401, + id: "evt_invoice_declined", + type: "invoice.payment_failed", + data: { + object: { + amount_due: 2500, + amount_paid: 0, + created: 1_700_000_390, + currency: "usd", + id: "in_declined", + payments: { + data: [ + { + amount_requested: 2500, + created: 1_700_000_400, + currency: "usd", + id: "inpay_declined", + invoice: "in_declined", + is_default: true, + payment: { + payment_intent: { + id: "pi_declined", + last_payment_error: { + code: "card_declined", + decline_code: "do_not_honor", + type: "card_error", + }, + }, + type: "payment_intent", + }, + status: "open", + }, + ], + has_more: false, + }, + status: "open", + }, + }, + }); + + expect(record).toMatchObject({ + amount: 25, + context: { + failureCode: "card_declined", + failureDeclineCode: "do_not_honor", + failureType: "card_error", + paymentIntentId: "pi_declined", + }, + }); + }); + + test("uses economic event time instead of object creation or retry arrival", () => { + const [record] = normalizeStripeEvent({ + ...acaciaIntent, + api_version: "2025-08-27.basil", + created: 1_700_172_800, + data: { + object: { ...acaciaIntent.data.object, created: 1_700_000_000, invoice: undefined }, + }, + }); + expect(record?.createdUnix).toBe(1_700_172_800); + }); + + test("retains embedded allocations before InvoicePayment paid webhooks exist", () => { + const records = normalizeStripeEvent({ + ...acaciaInvoice, + api_version: "2025-03-31.basil", + data: { + object: { + ...acaciaInvoice.data.object, + customer: "cus_invoice_only", + metadata: { databuddy_session_id: "session-invoice-only" }, + payment_intent: undefined, + payments: { + data: [ + { + amount_paid: 300, + created: 1_700_000_190, + currency: "usd", + id: "inpay_embedded", + invoice: "in_1", + payment: { + payment_intent: "pi_embedded", + type: "payment_intent", + }, + status: "paid", + }, + ], + has_more: false, + }, + }, + }, + }); + expect(records).toHaveLength(2); + expect(records[0]).toMatchObject({ + customerId: "cus_invoice_only", + rawMetadata: { databuddy_session_id: "session-invoice-only" }, + context: { + invoiceId: "in_1", + recordKind: "link", + }, + }); + expect(records[0]?.context.paymentIntentId).toBeUndefined(); + expect(records[1]).toMatchObject({ + amount: 3, + context: { + invoiceId: "in_1", + invoicePaymentId: "inpay_embedded", + moneyKind: "invoice_payment", + paymentIntentId: "pi_embedded", + }, + transactionId: "inpay_embedded", + }); + expect(usesInvoicePayments("2025-03-31.basil")).toBe(true); + expect(emitsInvoicePaymentPaidEvents("2025-03-31.basil")).toBe(false); + }); + + test("uses a modern total fallback even before the endpoint subscribes", () => { + expect(usesInvoicePayments("2025-03-30.acacia")).toBe(false); + expect(emitsInvoicePaymentPaidEvents("2025-05-27.basil")).toBe(false); + expect(emitsInvoicePaymentPaidEvents("2025-05-28.basil")).toBe(true); + + const records = normalizeStripeEvent({ + ...acaciaInvoice, + api_version: "2025-05-28.basil", + data: { + object: { + ...acaciaInvoice.data.object, + payment_intent: undefined, + payments: { + data: [ + { + amount_paid: 300, + created: 1_700_000_190, + currency: "usd", + id: "inpay_direct", + invoice: "in_1", + payment: { + payment_intent: "pi_direct", + type: "payment_intent", + }, + status: "paid", + }, + ], + has_more: false, + }, + }, + }, + }); + + expect(records).toHaveLength(3); + expect(records[0]).toMatchObject({ + context: { invoiceId: "in_1", recordKind: "link" }, + transactionId: "evt_invoice_paid", + }); + expect(records[1]).toMatchObject({ + context: { + invoiceId: "in_1", + invoicePaymentId: "inpay_direct", + paymentIntentId: "pi_direct", + recordKind: "link", + }, + transactionId: "evt_invoice_paid:inpay_direct", + }); + expect(records[2]).toMatchObject({ + amount: 3, + context: { invoiceId: "in_1", moneyKind: "invoice_fallback" }, + transactionId: "in_1", + }); + }); + + test("keeps transition-window totals when embedded allocations are paginated", () => { + const records = normalizeStripeEvent({ + ...acaciaInvoice, + api_version: "2025-05-27.basil", + data: { + object: { + ...acaciaInvoice.data.object, + payment_intent: undefined, + payments: { + data: [ + { + amount_paid: 100, + created: 1_700_000_190, + currency: "usd", + id: "inpay_partial_page", + invoice: "in_1", + payment: { + payment_intent: "pi_partial_page", + type: "payment_intent", + }, + status: "paid", + }, + ], + has_more: true, + }, + }, + }, + }); + + expect(records).toHaveLength(3); + expect(records[1]).toMatchObject({ + context: { + invoiceId: "in_1", + invoicePaymentId: "inpay_partial_page", + paymentIntentId: "pi_partial_page", + recordKind: "link", + }, + transactionId: "evt_invoice_paid:inpay_partial_page", + }); + expect(records[2]).toMatchObject({ + amount: 3, + context: { invoiceId: "in_1", moneyKind: "invoice_fallback" }, + transactionId: "in_1", + }); + }); + + test("keeps legacy out-of-band invoices as money", () => { + const records = normalizeStripeEvent({ + ...acaciaInvoice, + api_version: "2024-10-28.acacia", + data: { + object: { ...acaciaInvoice.data.object, payment_intent: undefined }, + }, + }); + expect(records.some((record) => record.context.moneyKind === "invoice")).toBe( + true + ); + }); + + test("serializes source identity without changing analytics attribution", () => { + expect( + stripeRecordMetadata( + { profile_id: "profile-1" }, + { + apiVersion: "2025-08-27.basil", + eventCreated: 123, + eventId: "evt_1", + eventType: "invoice_payment.paid", + invoiceId: "in_1", + paymentIntentId: "pi_1", + recordKind: "money", + } + ) + ).toEqual({ + databuddy_revenue_model: "stripe_events_v1", + profile_id: "profile-1", + stripe_api_version: "2025-08-27.basil", + stripe_event_created: 123, + stripe_event_id: "evt_1", + stripe_event_type: "invoice_payment.paid", + stripe_invoice_id: "in_1", + stripe_payment_intent_id: "pi_1", + stripe_record_kind: "money", + }); + expect( + stripeRecordMetadata( + {}, + { + eventCreated: 123, + eventId: "evt_invoice", + invoiceId: "in_1", + moneyKind: "invoice_fallback", + recordKind: "money", + } + ) + ).toMatchObject({ stripe_money_kind: "invoice_fallback" }); + }); +}); diff --git a/apps/basket/src/routes/webhooks/stripe.ts b/apps/basket/src/routes/webhooks/stripe.ts index 4222143b0..9ee247de8 100644 --- a/apps/basket/src/routes/webhooks/stripe.ts +++ b/apps/basket/src/routes/webhooks/stripe.ts @@ -4,6 +4,11 @@ import { Elysia } from "elysia"; import { evlog, useLogger } from "evlog/elysia"; import { getDailySalt, saltAnonymousId } from "@lib/security"; import { sanitizeString, VALIDATION_LIMITS } from "@utils/validation"; +import { + type NormalizedStripeRecord, + type StripeWebhookEvent, + normalizeStripeEvent, +} from "./stripe-normalization"; import { formatDate, getWebhookConfig, resolveWebsiteId } from "./shared"; const SIGNATURE_TOLERANCE_SECONDS = 300; @@ -14,162 +19,131 @@ interface WebhookConfig { websiteId: string | null; } -interface WebhookPaymentIntent { - amount: number; - amount_received?: number; - created: number; - currency: string; - customer?: string | { id: string } | null; - description?: string | null; - id: string; - invoice?: string | { id: string } | null; - metadata?: Record; -} - -interface WebhookCharge { - amount_refunded: number; - currency: string; - customer?: string | { id: string } | null; - id: string; - metadata?: Record; - payment_intent?: string | { id: string } | null; - refunds?: { - data: Array<{ - id: string; - amount: number; - created: number; - }>; - }; -} - -interface WebhookInvoice { - amount_paid: number; - billing_reason?: string | null; - created: number; - currency: string; - customer?: string | { id: string } | null; - description?: string | null; - id: string; - metadata?: Record; - parent?: { - subscription_details?: { metadata?: Record | null } | null; - } | null; - payment_intent?: string | { id: string } | null; - status?: string; - subscription?: string | null; - subscription_details?: { - metadata?: Record | null; - } | null; -} - -interface WebhookEvent { - data: { - object: WebhookPaymentIntent | WebhookCharge | WebhookInvoice; - }; - id: string; - type: string; +interface AnalyticsMetadata { + anonymous_id?: string; + client_id?: string; + profile_id?: string; + session_id?: string; } export function verifyStripeSignature( payload: string, header: string, secret: string -): { valid: true; event: WebhookEvent } | { valid: false; error: string } { +): + | { valid: true; event: StripeWebhookEvent } + | { valid: false; error: string } { const parts: Record = {}; - for (const item of header.split(",")) { const [key, value] = item.split("="); - if (key && value) { - if (!parts[key]) { - parts[key] = []; - } - parts[key].push(value); + if (!(key && value)) { + continue; } + const values = parts[key] ?? []; + values.push(value); + parts[key] = values; } const timestamp = parts.t?.[0]; - const signatures = parts.v1 || []; - + const signatures = parts.v1 ?? []; if (!timestamp) { return { valid: false, error: "Missing timestamp in signature header" }; } - if (signatures.length === 0) { return { valid: false, error: "No v1 signatures found in header" }; } - const timestampNum = Number.parseInt(timestamp, 10); + const timestampNumber = Number.parseInt(timestamp, 10); const now = Math.floor(Date.now() / 1000); - - if (Math.abs(now - timestampNum) > SIGNATURE_TOLERANCE_SECONDS) { + if ( + !Number.isSafeInteger(timestampNumber) || + Math.abs(now - timestampNumber) > SIGNATURE_TOLERANCE_SECONDS + ) { return { valid: false, error: "Timestamp outside tolerance zone" }; } - const signedPayload = `${timestamp}.${payload}`; - const expectedSignature = createHmac("sha256", secret) - .update(signedPayload, "utf8") + const expected = createHmac("sha256", secret) + .update(`${timestamp}.${payload}`, "utf8") .digest("hex"); - - const signatureMatch = signatures.some((sig) => { + const signatureMatch = signatures.some((signature) => { try { - return timingSafeEqual(Buffer.from(expectedSignature), Buffer.from(sig)); + return timingSafeEqual(Buffer.from(expected), Buffer.from(signature)); } catch { return false; } }); - if (!signatureMatch) { return { valid: false, error: "Signature mismatch" }; } try { - const event = JSON.parse(payload) as WebhookEvent; - return { valid: true, event }; + return { valid: true, event: JSON.parse(payload) as StripeWebhookEvent }; } catch { return { valid: false, error: "Invalid JSON payload" }; } } -interface AnalyticsMetadata { - anonymous_id?: string; - client_id?: string; - profile_id?: string; - session_id?: string; -} - -async function extractAnalyticsMetadata( - metadata: Record | undefined -): Promise { - if (!metadata) { - return {}; - } - - const rawAnonId = metadata.databuddy_anonymous_id; - let anonymousId: string | undefined; - if (rawAnonId) { - const salt = await getDailySalt(); - anonymousId = saltAnonymousId(rawAnonId, salt); - } - +function analyticsMetadata( + metadata: Record, + dailySalt: string | undefined +): AnalyticsMetadata { + const anonymousId = + metadata.databuddy_anonymous_id && dailySalt + ? saltAnonymousId(metadata.databuddy_anonymous_id, dailySalt) + : undefined; + const clientId = sanitizeString( + metadata.databuddy_client_id, + VALIDATION_LIMITS.USER_ID_MAX_LENGTH + ); + const profileId = sanitizeString( + metadata.databuddy_profile_id, + VALIDATION_LIMITS.USER_ID_MAX_LENGTH + ); + const sessionId = sanitizeString( + metadata.databuddy_session_id, + VALIDATION_LIMITS.SESSION_ID_MAX_LENGTH + ); return { - anonymous_id: anonymousId, - profile_id: - sanitizeString( - metadata.databuddy_profile_id, - VALIDATION_LIMITS.USER_ID_MAX_LENGTH - ) || undefined, - session_id: metadata.databuddy_session_id, - client_id: metadata.databuddy_client_id, + ...(anonymousId ? { anonymous_id: anonymousId } : {}), + ...(clientId ? { client_id: clientId } : {}), + ...(profileId ? { profile_id: profileId } : {}), + ...(sessionId ? { session_id: sessionId } : {}), }; } -function extractCustomerId( - customer: string | { id: string } | null | undefined -): string | undefined { - if (!customer) { - return; - } - return typeof customer === "string" ? customer : customer.id; +export function stripeRecordMetadata( + metadata: AnalyticsMetadata, + context: NormalizedStripeRecord["context"] +): Record { + return { + ...metadata, + databuddy_revenue_model: "stripe_events_v1", + ...(context.cancellationReason + ? { stripe_cancellation_reason: context.cancellationReason } + : {}), + stripe_event_created: context.eventCreated, + stripe_event_id: context.eventId, + ...(context.eventType ? { stripe_event_type: context.eventType } : {}), + ...(context.failureCode + ? { stripe_failure_code: context.failureCode } + : {}), + ...(context.failureDeclineCode + ? { stripe_failure_decline_code: context.failureDeclineCode } + : {}), + ...(context.failureType + ? { stripe_failure_type: context.failureType } + : {}), + stripe_record_kind: context.recordKind, + ...(context.apiVersion ? { stripe_api_version: context.apiVersion } : {}), + ...(context.invoiceId ? { stripe_invoice_id: context.invoiceId } : {}), + ...(context.invoicePaymentId + ? { stripe_invoice_payment_id: context.invoicePaymentId } + : {}), + ...(context.moneyKind ? { stripe_money_kind: context.moneyKind } : {}), + ...(context.paymentIntentId + ? { stripe_payment_intent_id: context.paymentIntentId } + : {}), + }; } function getConfig(hash: string): Promise { @@ -178,357 +152,65 @@ function getConfig(hash: string): Promise { >; } -export function invoiceMetadataSources( - invoice: WebhookInvoice -): Record { - return { - ...invoice.parent?.subscription_details?.metadata, - ...invoice.subscription_details?.metadata, - ...invoice.metadata, - }; -} - -async function existingAttribution( - ownerId: string, - transactionId: string -): Promise< - Pick -> { - try { - const result = await clickHouse.query({ - query: `SELECT - argMax(profile_id, synced_at) AS profile_id, - argMax(ifNull(anonymous_id, ''), synced_at) AS anonymous_id, - argMax(ifNull(session_id, ''), synced_at) AS session_id - FROM analytics.revenue - WHERE owner_id = {ownerId:String} AND transaction_id = {transactionId:String}`, - query_params: { ownerId, transactionId }, - format: "JSONEachRow", - }); - const [row] = await result.json<{ - profile_id: string; - anonymous_id: string; - session_id: string; - }>(); - if (!row) { - return {}; - } - return { - profile_id: row.profile_id || undefined, - anonymous_id: row.anonymous_id || undefined, - session_id: row.session_id || undefined, - }; - } catch { - return {}; - } -} - -async function withCarriedAttribution( - metadata: AnalyticsMetadata, - ownerId: string, - transactionId: string -): Promise { - if (metadata.profile_id || metadata.anonymous_id || metadata.session_id) { - return metadata; - } - const carried = await existingAttribution(ownerId, transactionId); - return { ...metadata, ...carried }; -} - -async function insertRevenueRow( +async function persistStripeRecords( config: WebhookConfig, - metadata: AnalyticsMetadata, - row: { - transactionId: string; - type: "sale" | "subscription" | "refund"; - status: string; - amount: number; - currency: string; - customerId: string | undefined; - productName: string | null | undefined; - createdUnix: number; - } + records: NormalizedStripeRecord[] ): Promise { - await clickHouse.insert({ - table: "analytics.revenue", - values: [ - { - owner_id: config.ownerId, - website_id: await resolveWebsiteId( - metadata.client_id, - config.websiteId, - config.ownerId - ), - transaction_id: row.transactionId, - provider: "stripe", - type: row.type, - status: row.status, - amount: row.amount, - original_amount: row.amount, - original_currency: row.currency, - currency: row.currency, - anonymous_id: metadata.anonymous_id || undefined, - profile_id: metadata.profile_id || undefined, - session_id: metadata.session_id || undefined, - customer_id: row.customerId, - product_name: row.productName || undefined, - metadata: JSON.stringify(metadata), - created: formatDate(new Date(row.createdUnix * 1000)), - synced_at: formatDate(new Date()), - }, - ], - format: "JSONEachRow", - }); -} - -async function handlePaymentIntent( - pi: WebhookPaymentIntent, - config: WebhookConfig -): Promise { - const log = useLogger(); - const metadata = await withCarriedAttribution( - await extractAnalyticsMetadata(pi.metadata), - config.ownerId, - pi.id - ); - const customerId = extractCustomerId(pi.customer); - const descLower = pi.description?.toLowerCase() ?? ""; - const isSubscription = !!pi.invoice || descLower.startsWith("subscription"); - const type: "sale" | "subscription" = isSubscription - ? "subscription" - : "sale"; - const amount = (pi.amount_received ?? pi.amount) / 100; - const currency = pi.currency.toUpperCase(); - const productName = - pi.description && !descLower.startsWith("subscription") - ? pi.description - : undefined; - - log.set({ - revenue: { - type, - status: "completed", - amount, - currency, - customerId, - transactionId: pi.id, - }, - }); - - await insertRevenueRow(config, metadata, { - transactionId: pi.id, - type, - status: "completed", - amount, - currency, - customerId, - productName, - createdUnix: pi.created, - }); -} - -async function handleFailedPayment( - pi: WebhookPaymentIntent, - config: WebhookConfig, - status: "failed" | "canceled" -): Promise { - const log = useLogger(); - const metadata = await withCarriedAttribution( - await extractAnalyticsMetadata(pi.metadata), - config.ownerId, - pi.id - ); - const customerId = extractCustomerId(pi.customer); - const amount = (pi.amount_received ?? pi.amount) / 100; - const currency = pi.currency.toUpperCase(); - const descLower = pi.description?.toLowerCase() ?? ""; - const isSubscription = !!pi.invoice || descLower.startsWith("subscription"); - const type: "sale" | "subscription" = isSubscription - ? "subscription" - : "sale"; - const productName = - pi.description && !descLower.startsWith("subscription") - ? pi.description - : undefined; - - log.set({ - revenue: { - type, - status, - amount, - currency, - customerId, - transactionId: pi.id, - }, - }); - - await insertRevenueRow(config, metadata, { - transactionId: pi.id, - type, - status, - amount, - currency, - customerId, - productName, - createdUnix: pi.created, - }); -} - -async function matchingPaymentIntentRow( - ownerId: string, - amount: number, - createdUnix: number -): Promise { - try { - const result = await clickHouse.query({ - query: `SELECT transaction_id - FROM analytics.revenue - WHERE owner_id = {ownerId:String} - AND startsWith(transaction_id, 'pi_') - AND amount = {amount:Float64} - AND abs(toUnixTimestamp(created) - {createdUnix:Int64}) <= 1 - ORDER BY synced_at DESC - LIMIT 1`, - query_params: { ownerId, amount, createdUnix }, - format: "JSONEachRow", - }); - const [row] = await result.json<{ transaction_id: string }>(); - return row?.transaction_id; - } catch { - return; - } -} - -async function handleInvoicePaid( - invoice: WebhookInvoice, - config: WebhookConfig -): Promise { - const log = useLogger(); - if (invoice.amount_paid === 0) { - log.set({ revenue: { skipped: "zero_amount", invoiceId: invoice.id } }); + if (records.length === 0) { return; } - - const customerId = extractCustomerId(invoice.customer); - const amount = invoice.amount_paid / 100; - const currency = invoice.currency.toUpperCase(); - const paymentIntentId = - typeof invoice.payment_intent === "string" - ? invoice.payment_intent - : invoice.payment_intent?.id; - const transactionId = - paymentIntentId || - (await matchingPaymentIntentRow(config.ownerId, amount, invoice.created)) || - invoice.id; - const metadata = await withCarriedAttribution( - await extractAnalyticsMetadata(invoiceMetadataSources(invoice)), - config.ownerId, - transactionId + const needsAnonymousSalt = records.some( + (record) => record.rawMetadata.databuddy_anonymous_id ); - - log.set({ - revenue: { - type: "subscription", - status: "completed", - amount, - currency, - customerId, - transactionId, - billingReason: invoice.billing_reason, - subscriptionId: invoice.subscription, - }, - }); - - await insertRevenueRow(config, metadata, { - transactionId, - type: "subscription", - status: "completed", - amount, - currency, - customerId, - productName: invoice.description || undefined, - createdUnix: invoice.created, - }); -} - -async function handleInvoiceFailed( - invoice: WebhookInvoice, - config: WebhookConfig -): Promise { - const log = useLogger(); - const metadata = await extractAnalyticsMetadata( - invoiceMetadataSources(invoice) - ); - const customerId = extractCustomerId(invoice.customer); - const amount = invoice.amount_paid / 100; - const currency = invoice.currency.toUpperCase(); - - log.set({ - revenue: { - type: "subscription", - status: "failed", - amount, - currency, - customerId, - transactionId: invoice.id, - billingReason: invoice.billing_reason, - subscriptionId: invoice.subscription, - }, - }); - - await insertRevenueRow(config, metadata, { - transactionId: invoice.id, - type: "subscription", - status: "failed", - amount, - currency, - customerId, - productName: invoice.description || undefined, - createdUnix: invoice.created, - }); -} - -async function handleRefund( - charge: WebhookCharge, - config: WebhookConfig -): Promise { - const log = useLogger(); - const chargePaymentIntentId = - typeof charge.payment_intent === "string" - ? charge.payment_intent - : charge.payment_intent?.id; - const metadata = await withCarriedAttribution( - await extractAnalyticsMetadata(charge.metadata), - config.ownerId, - chargePaymentIntentId || charge.id + const dailySalt = needsAnonymousSalt ? await getDailySalt() : undefined; + const websiteIds = new Map>(); + const resolveRecordWebsite = (metadata: AnalyticsMetadata) => { + const key = metadata.client_id ?? ""; + let pending = websiteIds.get(key); + if (!pending) { + pending = resolveWebsiteId( + metadata.client_id, + config.websiteId, + config.ownerId + ); + websiteIds.set(key, pending); + } + return pending; + }; + const syncedAt = formatDate(new Date()); + const values = await Promise.all( + records.map(async (record) => { + const metadata = analyticsMetadata(record.rawMetadata, dailySalt); + return { + owner_id: config.ownerId, + website_id: await resolveRecordWebsite(metadata), + transaction_id: record.transactionId, + provider: "stripe", + type: record.type, + status: record.status, + amount: record.amount, + original_amount: record.amount, + original_currency: record.currency, + currency: record.currency, + anonymous_id: metadata.anonymous_id, + profile_id: metadata.profile_id, + session_id: metadata.session_id, + customer_id: record.customerId, + product_name: record.productName, + metadata: JSON.stringify( + stripeRecordMetadata(metadata, record.context) + ), + created: formatDate(new Date(record.createdUnix * 1000)), + synced_at: syncedAt, + }; + }) ); - const customerId = extractCustomerId(charge.customer); - const currency = charge.currency.toUpperCase(); - const refunds = charge.refunds?.data || []; - log.set({ - revenue: { - type: "refund", - currency, - customerId, - refundCount: refunds.length, - }, + await clickHouse.insert({ + table: "analytics.revenue", + values, + format: "JSONEachRow", }); - - for (const refund of refunds) { - const amount = refund.amount / 100; - - await insertRevenueRow(config, metadata, { - transactionId: refund.id, - type: "refund", - status: "refunded", - amount: -amount, - currency, - customerId, - productName: "Refund", - createdUnix: refund.created, - }); - } } export const stripeWebhook = new Elysia().use(evlog()).post( @@ -537,30 +219,23 @@ export const stripeWebhook = new Elysia().use(evlog()).post( const log = useLogger(); log.set({ provider: "stripe", webhookHash: params.hash }); - const result = await getConfig(params.hash); - - if ("error" in result) { - log.set({ configError: result.error }); + const config = await getConfig(params.hash); + if ("error" in config) { + log.set({ configError: config.error }); set.status = 404; return { error: "Webhook endpoint not found" }; } - log.set({ ownerId: result.ownerId, websiteId: result.websiteId }); - const signature = request.headers.get("stripe-signature"); if (!signature) { - log.set({ signatureError: "missing_header" }); set.status = 400; return { error: "Missing stripe-signature header" }; } - - const body = await request.text(); const verification = verifyStripeSignature( - body, + await request.text(), signature, - result.stripeWebhookSecret + config.stripeWebhookSecret ); - if (!verification.valid) { log.warn("Stripe signature verification failed"); log.set({ signatureError: verification.error }); @@ -569,54 +244,23 @@ export const stripeWebhook = new Elysia().use(evlog()).post( } const event = verification.event; - log.set({ eventType: event.type, eventId: event.id }); - + log.set({ + eventId: event.id, + eventType: event.type, + stripeApiVersion: event.api_version, + }); try { - switch (event.type) { - case "payment_intent.succeeded": { - await handlePaymentIntent( - event.data.object as WebhookPaymentIntent, - result - ); - break; - } - case "payment_intent.payment_failed": { - await handleFailedPayment( - event.data.object as WebhookPaymentIntent, - result, - "failed" - ); - break; - } - case "payment_intent.canceled": { - await handleFailedPayment( - event.data.object as WebhookPaymentIntent, - result, - "canceled" - ); - break; - } - case "invoice.paid": - case "invoice.payment_succeeded": { - await handleInvoicePaid(event.data.object as WebhookInvoice, result); - break; - } - case "invoice.payment_failed": { - await handleInvoiceFailed( - event.data.object as WebhookInvoice, - result - ); - break; - } - case "charge.refunded": { - await handleRefund(event.data.object as WebhookCharge, result); - break; - } - default: { - log.set({ unhandled: true }); - } - } - + const records = normalizeStripeEvent(event); + await persistStripeRecords(config, records); + log.set({ + recordCount: records.length, + moneyRecordCount: records.filter( + (record) => record.context.recordKind === "money" + ).length, + attemptRecordCount: records.filter( + (record) => record.context.recordKind === "attempt" + ).length, + }); return { received: true, type: event.type }; } catch (error) { log.error(error instanceof Error ? error : new Error(String(error))); diff --git a/apps/dashboard/app/(auth)/auth/error/page.tsx b/apps/dashboard/app/(auth)/auth/error/page.tsx index e63552b90..9615ccb9c 100644 --- a/apps/dashboard/app/(auth)/auth/error/page.tsx +++ b/apps/dashboard/app/(auth)/auth/error/page.tsx @@ -3,6 +3,7 @@ import Link from "next/link"; import { parseAsString, useQueryState } from "nuqs"; import { Suspense } from "react"; +import { safeCallbackPath } from "@/lib/safe-callback"; import { ArrowLeftIcon, ShieldWarningIcon } from "@databuddy/ui/icons"; import { Button, Spinner, Text } from "@databuddy/ui"; @@ -91,6 +92,7 @@ function AuthErrorPage() { "callback", parseAsString.withDefault("/websites") ); + const safeCallback = safeCallbackPath(callback); const errorInfo = ERROR_MESSAGES[errorCode] ?? @@ -100,8 +102,8 @@ function AuthErrorPage() { errorCode.toLowerCase() ); const recoveryHref = isMagicLinkError - ? `/login/magic?callback=${encodeURIComponent(callback)}` - : `/login?callback=${encodeURIComponent(callback)}`; + ? `/login/magic?callback=${encodeURIComponent(safeCallback)}` + : `/login?callback=${encodeURIComponent(safeCallback)}`; return ( <> diff --git a/apps/dashboard/app/(auth)/login/magic/page.tsx b/apps/dashboard/app/(auth)/login/magic/page.tsx index b55be568d..11241cc79 100644 --- a/apps/dashboard/app/(auth)/login/magic/page.tsx +++ b/apps/dashboard/app/(auth)/login/magic/page.tsx @@ -6,6 +6,7 @@ import { useRouter } from "next/navigation"; import { parseAsString, useQueryState } from "nuqs"; import { Suspense, useState } from "react"; import { toast } from "sonner"; +import { safeCallbackPath } from "@/lib/safe-callback"; import { ArrowLeftIcon, EnvelopeSimpleIcon } from "@databuddy/ui/icons"; import { Button, Field, Input, Spinner, Text } from "@databuddy/ui"; @@ -17,7 +18,8 @@ function MagicLinkPage() { ); const [email, setEmail] = useState(""); const [isLoading, setIsLoading] = useState(false); - const loginHref = `/login?callback=${encodeURIComponent(callback)}`; + const safeCallback = safeCallbackPath(callback); + const loginHref = `/login?callback=${encodeURIComponent(safeCallback)}`; const handleMagicLinkLogin = async (e: React.FormEvent) => { e.preventDefault(); @@ -30,8 +32,8 @@ function MagicLinkPage() { try { const { error } = await authClient.signIn.magicLink({ email, - callbackURL: callback, - errorCallbackURL: `/auth/error?callback=${encodeURIComponent(callback)}`, + callbackURL: safeCallback, + errorCallbackURL: `/auth/error?callback=${encodeURIComponent(safeCallback)}`, }); if (error) { toast.error("We couldn't send the magic link. Try again in a moment."); @@ -39,7 +41,7 @@ function MagicLinkPage() { toast.success("Magic link sent. Check your email."); sessionStorage.setItem("databuddy:magic-email", email); router.push( - `/login/magic-sent?callback=${encodeURIComponent(callback)}` + `/login/magic-sent?callback=${encodeURIComponent(safeCallback)}` ); } } catch { diff --git a/apps/dashboard/app/(auth)/login/page.tsx b/apps/dashboard/app/(auth)/login/page.tsx index 3b48a0939..487561102 100644 --- a/apps/dashboard/app/(auth)/login/page.tsx +++ b/apps/dashboard/app/(auth)/login/page.tsx @@ -7,6 +7,7 @@ import { parseAsString, useQueryState } from "nuqs"; import { Suspense, useState } from "react"; import { toast } from "sonner"; import { GithubMark, GoogleMark } from "@/components/ui/brand-icons"; +import { safeCallbackPath } from "@/lib/safe-callback"; import { EnvelopeSimpleIcon, EyeIcon, EyeSlashIcon } from "@databuddy/ui/icons"; import { Badge, @@ -31,9 +32,10 @@ function LoginPage() { const [password, setPassword] = useState(""); const [showPassword, setShowPassword] = useState(false); const isHydrated = useHydrated(); + const safeCallback = safeCallbackPath(callback); const lastUsed = isHydrated ? authClient.getLastUsedLoginMethod() : null; - const callbackQuery = `?callback=${encodeURIComponent(callback)}`; + const callbackQuery = `?callback=${encodeURIComponent(safeCallback)}`; const getProviderLabel = (provider: "github" | "google") => provider === "github" ? "GitHub" : "Google"; @@ -41,12 +43,12 @@ function LoginPage() { const handleSocialLogin = async (provider: "github" | "google") => { setIsLoading(true); const newUserCallbackURL = - callback === "/websites" ? "/onboarding" : callback; + safeCallback === "/websites" ? "/onboarding" : safeCallback; try { const result = await authClient.signIn.social({ provider, - callbackURL: callback, + callbackURL: safeCallback, newUserCallbackURL, disableRedirect: true, }); @@ -89,7 +91,7 @@ function LoginPage() { await authClient.signIn.email({ email, password, - callbackURL: callback, + callbackURL: safeCallback, fetchOptions: { onError: (error) => { setIsLoading(false); @@ -99,7 +101,7 @@ function LoginPage() { ) { storeVerificationEmail(email); router.push( - `/login/verification-needed?callback=${encodeURIComponent(callback)}` + `/login/verification-needed?callback=${encodeURIComponent(safeCallback)}` ); } else { toast.error( diff --git a/apps/dashboard/app/(auth)/login/verification-needed/page.tsx b/apps/dashboard/app/(auth)/login/verification-needed/page.tsx index 62f4276ad..17be76a70 100644 --- a/apps/dashboard/app/(auth)/login/verification-needed/page.tsx +++ b/apps/dashboard/app/(auth)/login/verification-needed/page.tsx @@ -5,6 +5,7 @@ import Link from "next/link"; import { parseAsString, useQueryState } from "nuqs"; import { Suspense, useEffect, useState } from "react"; import { toast } from "sonner"; +import { safeCallbackPath } from "@/lib/safe-callback"; import { ArrowLeftIcon, WarningIcon } from "@databuddy/ui/icons"; import { Button, Spinner, Text } from "@databuddy/ui"; import { @@ -20,7 +21,8 @@ function VerificationNeededPage() { const [email, setEmail] = useState(""); const [isLoading, setIsLoading] = useState(false); const [isReady, setIsReady] = useState(false); - const loginHref = `/login?callback=${encodeURIComponent(callback)}`; + const safeCallback = safeCallbackPath(callback); + const loginHref = `/login?callback=${encodeURIComponent(safeCallback)}`; useEffect(() => { const params = new URLSearchParams(window.location.search); @@ -51,7 +53,7 @@ function VerificationNeededPage() { try { const { error } = await authClient.sendVerificationEmail({ email, - callbackURL: callback, + callbackURL: safeCallback, }); if (error) { toast.error( diff --git a/apps/dashboard/app/(auth)/register/page.tsx b/apps/dashboard/app/(auth)/register/page.tsx index 4f3c2c979..cf1029c61 100644 --- a/apps/dashboard/app/(auth)/register/page.tsx +++ b/apps/dashboard/app/(auth)/register/page.tsx @@ -16,6 +16,7 @@ import { type SignupMethod, trackAppEvent, } from "@/lib/app-events"; +import { safeCallbackPath } from "@/lib/safe-callback"; import { CaretLeftIcon, EyeIcon, @@ -38,6 +39,7 @@ function RegisterPageContent() { "callback", parseAsString.withDefault("/websites") ); + const safeCallback = safeCallbackPath(callback); const [isLoading, setIsLoading] = useState(false); const [formData, setFormData] = useState({ name: "", @@ -79,7 +81,7 @@ function RegisterPageContent() { localStorage.setItem("pendingPlanSelection", selectedPlan); return `/billing/plans?plan=${selectedPlan}`; } - return callback; + return safeCallback; }; const getProviderLabel = (provider: "github" | "google") => @@ -110,10 +112,7 @@ function RegisterPageContent() { fetchOptions: { onSuccess: () => { trackSignup(APP_EVENTS.signupCompleted, signupProperties); - trackOpenAiRegistrationCompleted({ - email: formData.email, - properties: signupProperties, - }); + trackOpenAiRegistrationCompleted(); toast.success( "Account created! Please check your email to verify your account." ); @@ -457,11 +456,7 @@ function RegisterPageContent() { Already have an account?{" "} Sign in diff --git a/apps/dashboard/app/(main)/billing/components/topup-card.tsx b/apps/dashboard/app/(main)/billing/components/topup-card.tsx index 73f737081..35ba9a122 100644 --- a/apps/dashboard/app/(main)/billing/components/topup-card.tsx +++ b/apps/dashboard/app/(main)/billing/components/topup-card.tsx @@ -1,12 +1,9 @@ "use client"; import { CreditArcSlider } from "@/components/ui/credit-arc-slider"; -import { - DATABUNNY_USAGE_EXPLANATION, - DATABUNNY_USAGE_UNIT, -} from "@/lib/databunny-usage"; import { cn } from "@/lib/utils"; import { getUserFacingErrorMessage } from "@/lib/user-facing-error"; +import { DATABUNNY_USAGE } from "@databuddy/shared/billing"; import { blendedRatePerCredit, calculateTopupCost, @@ -86,7 +83,7 @@ export function TopupCard() { Top up Databunny usage - {DATABUNNY_USAGE_EXPLANATION} Purchased usage stacks with your plan + {DATABUNNY_USAGE.description} Purchased usage stacks with your plan and does not expire. @@ -99,7 +96,7 @@ export function TopupCard() { min={TOPUP_MIN_QUANTITY} onValueChange={setQuantity} value={quantity} - unit={DATABUNNY_USAGE_UNIT} + unit={DATABUNNY_USAGE.unit} /> @@ -127,7 +124,7 @@ export function TopupCard() { {quantity.toLocaleString()} usage units - + ${blendedRate.toFixed(4)} @@ -183,22 +180,19 @@ export function TopupCard() {
- Graduated tiers: each usage unit is billed by the tier it falls - into. Buy 5,000 and only the first 100 cost $ - {BASE_RATE.toFixed(2)} — every unit above that drops to a - cheaper rate. + With graduated pricing, each unit is priced by its tier. + Reaching a lower rate does not change the price of earlier + units.
{TOPUP_TIERS.map((tier, idx) => { const prev = idx === 0 ? null : TOPUP_TIERS[idx - 1]; - const prevTop = - prev === null - ? TOPUP_MIN_QUANTITY - : prev.to === "inf" - ? 0 - : prev.to; - const topLabel = - tier.to === "inf" ? "∞" : tier.to.toLocaleString(); + const tierStart = + prev === null || prev.to === "inf" ? 1 : prev.to + 1; + const rangeLabel = + tier.to === "inf" + ? `${tierStart.toLocaleString()}+` + : `${tierStart.toLocaleString()} – ${tier.to.toLocaleString()}`; const isActive = tier.amount === tierInfo.currentRate; return (
- {prevTop.toLocaleString()} – {topLabel} + {rangeLabel} {isActive && ( You're here @@ -259,7 +253,7 @@ function NudgeSlot({ blendedRate, nudge, quantity, savings }: NudgeSlotProps) { {nudge.unitsUntilNextTier.toLocaleString()} {" "} - more and every usage unit drops to{" "} + more to reach the next tier. Usage units in that tier cost{" "} ${nudge.nextRate.toFixed(3)} @@ -278,7 +272,7 @@ function NudgeSlot({ blendedRate, nudge, quantity, savings }: NudgeSlotProps) { ${savings.toFixed(2)} {" "} - vs. buying one at a time — that's{" "} + compared with the first-tier rate — that's{" "} {Math.round((1 - blendedRate / BASE_RATE) * 100)}% {" "} @@ -293,11 +287,11 @@ function NudgeSlot({ blendedRate, nudge, quantity, savings }: NudgeSlotProps) { weight="duotone" /> - Volume discount kicks in at{" "} + After the first{" "} - {FIRST_TIER_TOP.toLocaleString()}+ + {FIRST_TIER_TOP.toLocaleString()} {" "} - usage units — every unit drops to{" "} + usage units, additional units cost{" "} ${SECOND_TIER_RATE.toFixed(3)} diff --git a/apps/dashboard/app/(main)/feedback/components/credits-panel.tsx b/apps/dashboard/app/(main)/feedback/components/credits-panel.tsx index d493271df..0135ca562 100644 --- a/apps/dashboard/app/(main)/feedback/components/credits-panel.tsx +++ b/apps/dashboard/app/(main)/feedback/components/credits-panel.tsx @@ -8,7 +8,7 @@ import { } from "@databuddy/ui/icons"; import { Button, Card, Skeleton } from "@databuddy/ui"; import { cn } from "@/lib/utils"; -import { DATABUNNY_USAGE_EXPLANATION } from "@/lib/databunny-usage"; +import { DATABUNNY_USAGE } from "@databuddy/shared/billing"; interface RewardTier { creditsRequired: number; @@ -239,7 +239,7 @@ export function CreditsPanel({ { - if (!organizationId) { - return; - } setVoteMutation.mutate({ insightId, vote }); }, - [organizationId, setVoteMutation] + [setVoteMutation] ); const feedbackById = votesQuery.data?.votes ?? {}; diff --git a/apps/dashboard/app/(main)/onboarding/page.tsx b/apps/dashboard/app/(main)/onboarding/page.tsx index 7f08bb724..b07e2d17a 100644 --- a/apps/dashboard/app/(main)/onboarding/page.tsx +++ b/apps/dashboard/app/(main)/onboarding/page.tsx @@ -85,7 +85,7 @@ export default function OnboardingPage() { trackAppEvent(APP_EVENTS.signupCompleted, signupProperties, { flush: true, }); - trackOpenAiRegistrationCompleted({ properties: signupProperties }); + trackOpenAiRegistrationCompleted(); } trackAppEvent(APP_EVENTS.onboardingStarted); }, []); diff --git a/apps/dashboard/app/(main)/websites/[id]/errors/_components/error-summary-stats.tsx b/apps/dashboard/app/(main)/websites/[id]/errors/_components/error-summary-stats.tsx index d0e8e4445..cc88c47c3 100644 --- a/apps/dashboard/app/(main)/websites/[id]/errors/_components/error-summary-stats.tsx +++ b/apps/dashboard/app/(main)/websites/[id]/errors/_components/error-summary-stats.tsx @@ -100,7 +100,7 @@ export const ErrorSummaryStats = ({ /> diff --git a/apps/dashboard/app/(main)/websites/[id]/funnels/_components/funnel-analytics.tsx b/apps/dashboard/app/(main)/websites/[id]/funnels/_components/funnel-analytics.tsx index 830b0435f..6cadd6e7c 100644 --- a/apps/dashboard/app/(main)/websites/[id]/funnels/_components/funnel-analytics.tsx +++ b/apps/dashboard/app/(main)/websites/[id]/funnels/_components/funnel-analytics.tsx @@ -110,9 +110,10 @@ export function FunnelAnalytics({ total_users_completed: selectedReferrerData.completed_users, overall_conversion_rate: selectedReferrerData.conversion_rate, avg_completion_time: 0, - avg_completion_time_formatted: "0s", + avg_completion_time_formatted: "—", biggest_dropoff_step: 1, biggest_dropoff_rate: 100 - selectedReferrerData.conversion_rate, + duration_available: false, steps_analytics: [], } : data; @@ -127,6 +128,7 @@ export function FunnelAnalytics({ const errorInsights = data?.error_insights; const hasErrorCorrelation = errorInsights && + errorInsights.available && errorInsights.dropoffs_with_errors > 0 && errorInsights.error_correlation_rate > 0; @@ -216,7 +218,7 @@ export function FunnelAnalytics({ v < 60 ? `${Math.round(v)}s` : `${Math.round(v / 60)}m` } icon={ClockIcon} - showChart={hasChartData} + showChart={hasChartData && displayData.duration_available} title="Avg Time" value={displayData.avg_completion_time_formatted || "—"} /> diff --git a/apps/dashboard/app/(main)/websites/[id]/funnels/_components/funnel-flow.tsx b/apps/dashboard/app/(main)/websites/[id]/funnels/_components/funnel-flow.tsx index 05b15ba4c..6e7cea905 100644 --- a/apps/dashboard/app/(main)/websites/[id]/funnels/_components/funnel-flow.tsx +++ b/apps/dashboard/app/(main)/websites/[id]/funnels/_components/funnel-flow.tsx @@ -157,7 +157,7 @@ export function FunnelFlow({ steps }: FunnelFlowProps) { {step.step_name} - {step.error_count > 0 && ( + {step.error_context_available && step.error_count > 0 && ( diff --git a/apps/dashboard/app/(main)/websites/[id]/revenue/_components/revenue-attribution-tables.tsx b/apps/dashboard/app/(main)/websites/[id]/revenue/_components/revenue-attribution-tables.tsx index 4d8321524..7c8426619 100644 --- a/apps/dashboard/app/(main)/websites/[id]/revenue/_components/revenue-attribution-tables.tsx +++ b/apps/dashboard/app/(main)/websites/[id]/revenue/_components/revenue-attribution-tables.tsx @@ -1,8 +1,8 @@ "use client"; import type { DateRange } from "@/types/analytics"; +import type { DynamicQueryFilter } from "@/types/api"; import type { ColumnDef } from "@tanstack/react-table"; -import { useAtom } from "jotai"; import { useMemo } from "react"; import { DataTable } from "@/components/table/data-table"; import { @@ -10,28 +10,30 @@ import { type RevenueEntry, } from "@/components/table/rows"; import { useBatchDynamicQuery } from "@/hooks/use-dynamic-query"; -import { dynamicQueryFiltersAtom } from "@/stores/jotai/filterAtoms"; interface RevenueAttributionTablesProps { + currency: string; dateRange: DateRange; - isLoading?: boolean; + enabled?: boolean; onAddFilter?: (field: string, value: string) => void; + queryFilters: DynamicQueryFilter[]; websiteId: string; } export function RevenueAttributionTables({ + currency, websiteId, dateRange, + enabled = true, onAddFilter, + queryFilters, }: RevenueAttributionTablesProps) { - const [filters] = useAtom(dynamicQueryFiltersAtom); - const queries = useMemo( () => [ { id: "revenue-products", parameters: ["revenue_by_product"], - filters, + filters: queryFilters, }, { id: "revenue-traffic", @@ -42,7 +44,7 @@ export function RevenueAttributionTables({ "revenue_by_utm_campaign", "revenue_by_entry_page", ], - filters, + filters: queryFilters, }, { id: "revenue-geo", @@ -51,7 +53,7 @@ export function RevenueAttributionTables({ "revenue_by_region", "revenue_by_city", ], - filters, + filters: queryFilters, }, { id: "revenue-tech", @@ -60,16 +62,17 @@ export function RevenueAttributionTables({ "revenue_by_browser", "revenue_by_os", ], - filters, + filters: queryFilters, }, ], - [filters] + [queryFilters] ); const { isLoading: queryLoading, getDataForQuery } = useBatchDynamicQuery( websiteId, dateRange, - queries + queries, + { enabled } ); const isLoading = queryLoading; @@ -153,39 +156,51 @@ export function RevenueAttributionTables({ ); const productColumns = useMemo( - () => createRevenueColumns({ type: "default", nameLabel: "Product" }), - [] + () => + createRevenueColumns({ currency, type: "default", nameLabel: "Product" }), + [currency] ); const referrerColumns = useMemo( - () => createRevenueColumns({ type: "referrer" }), - [] + () => createRevenueColumns({ currency, type: "referrer" }), + [currency] ); const utmColumns = useMemo( - () => createRevenueColumns({ type: "utm", nameLabel: "Source" }), - [] + () => createRevenueColumns({ currency, type: "utm", nameLabel: "Source" }), + [currency] ); const pageColumns = useMemo( - () => createRevenueColumns({ type: "default", nameLabel: "Entry Page" }), - [] + () => + createRevenueColumns({ + currency, + type: "default", + nameLabel: "Entry Page", + }), + [currency] ); const countryColumns = useMemo( - () => createRevenueColumns({ type: "country" }), - [] + () => createRevenueColumns({ currency, type: "country" }), + [currency] ); const regionColumns = useMemo( - () => createRevenueColumns({ type: "region" }), - [] + () => createRevenueColumns({ currency, type: "region" }), + [currency] + ); + const cityColumns = useMemo( + () => createRevenueColumns({ currency, type: "city" }), + [currency] ); - const cityColumns = useMemo(() => createRevenueColumns({ type: "city" }), []); const deviceColumns = useMemo( - () => createRevenueColumns({ type: "device" }), - [] + () => createRevenueColumns({ currency, type: "device" }), + [currency] ); const browserColumns = useMemo( - () => createRevenueColumns({ type: "browser" }), - [] + () => createRevenueColumns({ currency, type: "browser" }), + [currency] + ); + const osColumns = useMemo( + () => createRevenueColumns({ currency, type: "os" }), + [currency] ); - const osColumns = useMemo(() => createRevenueColumns({ type: "os" }), []); const trafficTabs = useMemo( () => [ diff --git a/apps/dashboard/app/(main)/websites/[id]/revenue/_components/revenue-chart.tsx b/apps/dashboard/app/(main)/websites/[id]/revenue/_components/revenue-chart.tsx index 642bbb885..2c5dc986e 100644 --- a/apps/dashboard/app/(main)/websites/[id]/revenue/_components/revenue-chart.tsx +++ b/apps/dashboard/app/(main)/websites/[id]/revenue/_components/revenue-chart.tsx @@ -14,6 +14,7 @@ import { chartSeriesColorAtIndex, chartSurfaceClassName, } from "@/lib/chart-presentation"; +import { formatRevenueCurrency } from "@/lib/revenue-currency"; import { cn } from "@/lib/utils"; import type { RevenueMetricVisibilityState } from "@/stores/jotai/chartAtoms"; import { @@ -49,65 +50,51 @@ interface RevenueChartMetric { label: string; } -const REVENUE_METRICS: RevenueChartMetric[] = [ - { - key: "revenue", - label: "Revenue", - color: chartSeriesColorAtIndex(0), - formatValue: (v) => - new Intl.NumberFormat("en-US", { - style: "currency", - currency: "USD", - minimumFractionDigits: 0, - maximumFractionDigits: 0, - }).format(v), - }, - { - key: "transactions", - label: "Transactions", - color: chartSeriesColorAtIndex(1), - formatValue: (v) => v.toLocaleString(), - }, - { - key: "avg_transaction", - label: "Avg Transaction", - color: chartSeriesColorAtIndex(2), - formatValue: (v) => - new Intl.NumberFormat("en-US", { - style: "currency", - currency: "USD", - minimumFractionDigits: 0, - maximumFractionDigits: 0, - }).format(v), - }, - { - key: "customers", - label: "Customers", - color: chartSeriesColorAtIndex(3), - formatValue: (v) => v.toLocaleString(), - }, - { - key: "refunds", - label: "Refunds", - color: chartSeriesColorAtIndex(4), - formatValue: (v) => - new Intl.NumberFormat("en-US", { - style: "currency", - currency: "USD", - minimumFractionDigits: 0, - maximumFractionDigits: 0, - }).format(Math.abs(v)), - }, -]; +function createRevenueMetrics(currency: string): RevenueChartMetric[] { + return [ + { + key: "revenue", + label: "Revenue", + color: chartSeriesColorAtIndex(0), + formatValue: (v) => formatRevenueCurrency(v, currency), + }, + { + key: "transactions", + label: "Transactions", + color: chartSeriesColorAtIndex(1), + formatValue: (v) => v.toLocaleString(), + }, + { + key: "avg_transaction", + label: "Avg Transaction", + color: chartSeriesColorAtIndex(2), + formatValue: (v) => formatRevenueCurrency(v, currency), + }, + { + key: "customers", + label: "Customers", + color: chartSeriesColorAtIndex(3), + formatValue: (v) => v.toLocaleString(), + }, + { + key: "refunds", + label: "Refunds", + color: chartSeriesColorAtIndex(4), + formatValue: (v) => formatRevenueCurrency(Math.abs(v), currency), + }, + ]; +} interface RevenueChartProps { className?: string; + currency: string; data: RevenueChartDataPoint[]; height?: number; isLoading: boolean; } export function RevenueChart({ + currency, data, isLoading, height = 350, @@ -115,16 +102,17 @@ export function RevenueChart({ }: RevenueChartProps) { const [visibleMetrics] = useAtom(revenueMetricVisibilityAtom); const [, toggleMetric] = useAtom(toggleRevenueMetricAtom); + const metrics = useMemo(() => createRevenueMetrics(currency), [currency]); const hiddenMetrics = useMemo( () => Object.fromEntries( - REVENUE_METRICS.map((m) => [ + metrics.map((m) => [ m.key, !visibleMetrics[m.key as keyof RevenueMetricVisibilityState], ]) ), - [visibleMetrics] + [metrics, visibleMetrics] ); const hasData = data.length > 0; @@ -177,7 +165,7 @@ export function RevenueChart({ }} > - {REVENUE_METRICS.map((metric) => ( + {metrics.map((metric) => ( ( @@ -226,7 +214,7 @@ export function RevenueChart({ { - const metric = REVENUE_METRICS.find((m) => m.label === label); + const metric = metrics.find((m) => m.label === label); const isHidden = metric ? hiddenMetrics[metric.key] : false; return ( { - const metric = REVENUE_METRICS.find( - (m) => m.label === payload.value - ); + const metric = metrics.find((m) => m.label === payload.value); if (metric) { toggleMetric( metric.key as keyof RevenueMetricVisibilityState @@ -253,7 +239,7 @@ export function RevenueChart({ verticalAlign="bottom" wrapperStyle={chartRechartsLegendInteractiveWrapperStyle} /> - {REVENUE_METRICS.map((metric) => ( + {metrics.map((metric) => ( ( data: T[], startDate: string, @@ -191,6 +178,7 @@ function RevenueSettingsSheet({ const queryClient = useQueryClient(); const [stripeSecret, setStripeSecret] = useState(""); const [paddleSecret, setPaddleSecret] = useState(""); + const [currencyDraft, setCurrencyDraft] = useState(null); const [showStripeSecret, setShowStripeSecret] = useState(false); const [showPaddleSecret, setShowPaddleSecret] = useState(false); const [expandedSection, setExpandedSection] = @@ -208,9 +196,30 @@ function RevenueSettingsSheet({ queryKey: ["revenue-config", websiteId], queryFn: () => orpc.revenue.get.call({ websiteId }), }); + const savedCurrency = normalizeRevenueCurrency(config?.currency); + const configuredCurrency = + typeof config?.currency === "string" + ? config.currency.trim().toUpperCase() + : "USD"; + const currencyValue = currencyDraft ?? configuredCurrency; + const normalizedCurrency = normalizeRevenueCurrency(currencyValue); + const currencyInvalid = normalizedCurrency === null; + const hasChanges = Boolean( + stripeSecret || + paddleSecret || + (normalizedCurrency && normalizedCurrency !== savedCurrency) + ); const createWebhookMutation = useMutation({ - mutationFn: () => orpc.revenue.upsert.call({ websiteId }), + mutationFn: () => { + if (!normalizedCurrency) { + throw new Error("Invalid revenue currency"); + } + return orpc.revenue.upsert.call({ + websiteId, + currency: normalizedCurrency, + }); + }, onSuccess: () => { queryClient.invalidateQueries({ queryKey: ["revenue-config", websiteId], @@ -224,6 +233,7 @@ function RevenueSettingsSheet({ mutationFn: (data: { stripeWebhookSecret?: string; paddleWebhookSecret?: string; + currency: string; }) => orpc.revenue.upsert.call({ websiteId, ...data }), onSuccess: () => { queryClient.invalidateQueries({ @@ -231,6 +241,7 @@ function RevenueSettingsSheet({ }); setStripeSecret(""); setPaddleSecret(""); + setCurrencyDraft(null); toast.success("Configuration saved"); }, onError: () => toast.error("Failed to save"), @@ -248,19 +259,22 @@ function RevenueSettingsSheet({ }); const handleSave = () => { + if (!normalizedCurrency) { + return; + } + const updates: { stripeWebhookSecret?: string; paddleWebhookSecret?: string; - } = {}; + currency: string; + } = { currency: normalizedCurrency }; if (stripeSecret) { updates.stripeWebhookSecret = stripeSecret; } if (paddleSecret) { updates.paddleWebhookSecret = paddleSecret; } - if (Object.keys(updates).length > 0) { - upsertMutation.mutate(updates); - } + upsertMutation.mutate(updates); }; const toggleSection = (section: ExpandedSection) => { @@ -303,193 +317,207 @@ function RevenueSettingsSheet({
) : ( -
- - ) : undefined - } - icon={LinkIcon} - isExpanded={expandedSection === "webhooks"} - onToggleAction={() => toggleSection("webhooks")} - title="Webhook URLs" - > - {webhookHash ? ( -
-
-
-

- Stripe -

- - Open dashboard - - +
+ + Currency + + setCurrencyDraft(event.target.value.toUpperCase()) + } + placeholder="USD" + value={currencyValue} + /> + {currencyInvalid ? ( + + Enter a valid three-letter ISO currency code. + + ) : ( + + Only matching transactions are shown in revenue reports. + + )} + + +
+ + ) : undefined + } + icon={LinkIcon} + isExpanded={expandedSection === "webhooks"} + onToggleAction={() => toggleSection("webhooks")} + title="Webhook URLs" + > + {webhookHash ? ( +
+
+
+

+ Stripe +

+ + Open dashboard + + +
+
+ + {stripeUrl} + + +
-
- - {stripeUrl} - - + +
+
+

+ Paddle +

+ + Open dashboard + + +
+
+ + {paddleUrl} + + +
-
+ +
+ ) : ( +
+

+ Generate URLs to receive payment events. +

+ +
+ )} + + + + ) : undefined + } + icon={StripeLogoIcon} + isExpanded={expandedSection === "stripe"} + onToggleAction={() => toggleSection("stripe")} + title="Stripe" + > +
-
-

- Paddle -

- - Open dashboard - - -
+

+ Signing secret +

- - {paddleUrl} - + setStripeSecret(e.target.value)} + placeholder={ + config?.stripeConfigured ? "••••••••" : "whsec_..." + } + type={showStripeSecret ? "text" : "password"} + value={stripeSecret} + />
- -
- ) : ( -
-

- Generate URLs to receive payment events. -

- -
- )} - - - - ) : undefined - } - icon={StripeLogoIcon} - isExpanded={expandedSection === "stripe"} - onToggleAction={() => toggleSection("stripe")} - title="Stripe" - > -
-
-

- Signing secret -

-
- setStripeSecret(e.target.value)} - placeholder={ - config?.stripeConfigured ? "••••••••" : "whsec_..." - } - type={showStripeSecret ? "text" : "password"} - value={stripeSecret} - /> - -
-
- -
-

- Required events -

-
- {STRIPE_EVENTS.required.map((event) => ( - - {event} - - ))} -
-
- - {STRIPE_EVENTS.optional.length > 0 && (
-

Optional

+

+ Required events +

- {STRIPE_EVENTS.optional.map((event) => ( + {STRIPE_WEBHOOK_EVENTS.required.map(({ event }) => ( {event} @@ -497,78 +525,80 @@ function RevenueSettingsSheet({ ))}
- )} -
- - - - ) : undefined - } - icon={CurrencyDollarIcon} - isExpanded={expandedSection === "paddle"} - onToggleAction={() => toggleSection("paddle")} - title="Paddle" - > -
-
-

- Signing secret -

-
- setPaddleSecret(e.target.value)} - placeholder={ - config?.paddleConfigured - ? "••••••••" - : "pdl_ntfset_..." - } - type={showPaddleSecret ? "text" : "password"} - value={paddleSecret} - /> - -
-
-
-

- Required events -

-
- {PADDLE_EVENTS.required.map((event) => ( - 0 && ( +
+

+ Optional +

+
+ {STRIPE_WEBHOOK_EVENTS.optional.map(({ event }) => ( + + {event} + + ))} +
+
+ )} +
+ + + + ) : undefined + } + icon={CurrencyDollarIcon} + isExpanded={expandedSection === "paddle"} + onToggleAction={() => toggleSection("paddle")} + title="Paddle" + > +
+
+

+ Signing secret +

+
+ setPaddleSecret(e.target.value)} + placeholder={ + config?.paddleConfigured + ? "••••••••" + : "pdl_ntfset_..." + } + type={showPaddleSecret ? "text" : "password"} + value={paddleSecret} + /> + +
-
- {PADDLE_EVENTS.optional.length > 0 && (
-

Optional

+

+ Required events +

- {PADDLE_EVENTS.optional.map((event) => ( + {PADDLE_EVENTS.required.map((event) => ( {event} @@ -576,9 +606,27 @@ function RevenueSettingsSheet({ ))}
- )} -
- + + {PADDLE_EVENTS.optional.length > 0 && ( +
+

+ Optional +

+
+ {PADDLE_EVENTS.optional.map((event) => ( + + {event} + + ))} +
+
+ )} +
+ +
)} @@ -593,7 +641,7 @@ function RevenueSettingsSheet({
+ {hasPaymentData && ( +
+ + + + + +
+ )} +
@@ -766,6 +898,7 @@ export function RevenueContent({ websiteId }: RevenueContentProps) {
) : ( setSettingsOpen(true), }} description={ - isConfigured - ? "Revenue will appear here once your payment provider sends webhook events." - : "Connect Stripe or Paddle to start tracking revenue and attribution." + hasInvalidCurrency + ? "Choose the currency used to filter and format revenue reports." + : isConfigured + ? "Revenue will appear here once your payment provider sends webhook events." + : "Connect Stripe or Paddle to start tracking revenue and attribution." } icon={} title={ - isConfigured - ? "Waiting for transactions" - : "Set up revenue tracking" + hasInvalidCurrency + ? "Choose a valid currency" + : isConfigured + ? "Waiting for transactions" + : "Set up revenue tracking" } /> )} diff --git a/apps/dashboard/app/api/openai-ads/conversions/route.ts b/apps/dashboard/app/api/openai-ads/conversions/route.ts deleted file mode 100644 index 80ff9d4b1..000000000 --- a/apps/dashboard/app/api/openai-ads/conversions/route.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { config } from "@databuddy/env/app"; -import { type NextRequest, NextResponse } from "next/server"; -import { z } from "zod"; -import { buildOpenAiRegistrationCompletedEvent } from "@/lib/openai-ads-conversions"; - -const ConversionRequestSchema = z.object({ - email: z.string().email().max(320).optional(), - eventId: z.string().min(1).max(160), - oppref: z.string().min(1).max(512).optional(), - sourceUrl: z.string().url().max(2048), -}); - -function readClientIp(headers: Headers): string | undefined { - return headers.get("x-real-ip")?.trim() || undefined; -} - -async function readJson(request: NextRequest): Promise { - try { - return await request.json(); - } catch { - return null; - } -} - -export async function POST(request: NextRequest) { - const pixelId = config.integrations.openAiAdsPixelId; - const apiKey = config.integrations.openAiAdsConversionsApiKey; - - if (!(pixelId && apiKey)) { - return NextResponse.json({ skipped: true }); - } - - const parsed = ConversionRequestSchema.safeParse(await readJson(request)); - if (!parsed.success) { - return NextResponse.json( - { error: "Invalid conversion payload" }, - { status: 400 } - ); - } - - const event = buildOpenAiRegistrationCompletedEvent({ - ...parsed.data, - ipAddress: readClientIp(request.headers), - timestampMs: Date.now(), - userAgent: request.headers.get("user-agent") ?? undefined, - }); - - try { - const response = await fetch( - `https://bzr.openai.com/v1/events?pid=${encodeURIComponent(pixelId)}`, - { - body: JSON.stringify({ events: [event], validate_only: false }), - headers: { - Authorization: `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - method: "POST", - } - ); - if (!response.ok) { - return NextResponse.json({ ok: false }, { status: 502 }); - } - } catch { - return NextResponse.json({ ok: false }, { status: 502 }); - } - - return NextResponse.json({ ok: true }); -} diff --git a/apps/dashboard/autumn.config.ts b/apps/dashboard/autumn.config.ts index f71d9447b..dfa6e146c 100644 --- a/apps/dashboard/autumn.config.ts +++ b/apps/dashboard/autumn.config.ts @@ -1,6 +1,6 @@ import { AGENT_CREDIT_SCHEMA } from "./lib/credit-schema"; -import { DATABUNNY_USAGE_NAME } from "./lib/databunny-usage"; import { TOPUP_MAX_QUANTITY, TOPUP_TIERS } from "./lib/topup-math"; +import { DATABUNNY_USAGE, LEGACY_SCALE_PLAN } from "@databuddy/shared/billing"; import { feature, item, plan } from "atmn"; export const events = feature({ @@ -41,7 +41,7 @@ export const agent_cache_write_tokens = feature({ export const agent_credits = feature({ id: "agent_credits", - name: DATABUNNY_USAGE_NAME, + name: DATABUNNY_USAGE.name, type: "credit_system", creditSchema: [ { @@ -180,8 +180,8 @@ export const pro = plan({ * parity with Pro. No new bells, no overage tiers. */ export const scale = plan({ - id: "scale", - name: "Enterprise", + id: LEGACY_SCALE_PLAN.id, + name: LEGACY_SCALE_PLAN.name, addOn: false, autoEnable: false, price: { @@ -297,7 +297,7 @@ export const credits_booster = plan({ */ export const credits_topup = plan({ id: "credits_topup", - name: DATABUNNY_USAGE_NAME, + name: DATABUNNY_USAGE.name, addOn: true, autoEnable: false, items: [ diff --git a/apps/dashboard/components/layout/organization-selector.tsx b/apps/dashboard/components/layout/organization-selector.tsx index c0af860ef..1db41a3fd 100644 --- a/apps/dashboard/components/layout/organization-selector.tsx +++ b/apps/dashboard/components/layout/organization-selector.tsx @@ -17,6 +17,7 @@ import { import { CreateOrganizationDialog } from "@/components/organizations/create-organization-dialog"; import { useBillingContext } from "@/components/providers/billing-provider"; import { useOrganizationsContext } from "@/components/providers/organizations-provider"; +import { resetActiveOrganizationQueries } from "@/lib/active-organization-queries"; import { cn } from "@/lib/utils"; import { pendingActiveOrganizationIdAtom } from "@/stores/jotai/organizationsAtoms"; import { Avatar, DropdownMenu } from "@databuddy/ui/client"; @@ -161,8 +162,8 @@ export function OrganizationSelector({ return; } + await resetActiveOrganizationQueries(queryClient); router.push("/websites"); - queryClient.clear(); toast.success("Organization updated"); }; diff --git a/apps/dashboard/components/openai-ads-pixel.test.ts b/apps/dashboard/components/openai-ads-pixel.test.ts index 0fe677a07..658cf1193 100644 --- a/apps/dashboard/components/openai-ads-pixel.test.ts +++ b/apps/dashboard/components/openai-ads-pixel.test.ts @@ -1,6 +1,5 @@ import { describe, expect, it } from "bun:test"; import { - buildOpenAiRegistrationConversionPayload, buildOpenAiRegistrationMeasureArgs, isOpenAiAdsPixelHostAllowed, } from "./openai-ads-pixel"; @@ -19,26 +18,6 @@ describe("isOpenAiAdsPixelHostAllowed", () => { expect(isOpenAiAdsPixelHostAllowed("staging.databuddy.cc")).toBe(true); }); - it("builds a registration payload with the OpenAI reference", () => { - expect( - buildOpenAiRegistrationConversionPayload({ - email: "founder@example.com", - eventId: "conversion-1", - properties: { - method: "email", - oppref: "openai-reference", - utm_source: "openai_ads", - }, - sourceUrl: "https://app.databuddy.cc/register?utm_source=openai_ads", - }) - ).toEqual({ - email: "founder@example.com", - eventId: "conversion-1", - oppref: "openai-reference", - sourceUrl: "https://app.databuddy.cc/register?utm_source=openai_ads", - }); - }); - it("passes event_id as the pixel options argument for dedupe", () => { expect(buildOpenAiRegistrationMeasureArgs("conversion-1")).toEqual([ "measure", diff --git a/apps/dashboard/components/openai-ads-pixel.tsx b/apps/dashboard/components/openai-ads-pixel.tsx index de64d37dc..987adac43 100644 --- a/apps/dashboard/components/openai-ads-pixel.tsx +++ b/apps/dashboard/components/openai-ads-pixel.tsx @@ -2,25 +2,10 @@ import { useEffect } from "react"; import { publicConfig } from "@databuddy/env/public"; -import type { SignupEventProperties } from "@databuddy/shared/custom-events"; const PIXEL_ID = publicConfig.integrations.openAiAdsPixelId; const SCRIPT_SRC = "https://bzrcdn.openai.com/sdk/oaiq.min.js"; -interface RegistrationConversionInput { - email?: string; - eventId?: string; - properties?: SignupEventProperties; - sourceUrl?: string; -} - -interface RegistrationConversionPayload { - email?: string; - eventId: string; - oppref?: string; - sourceUrl: string; -} - type OpenAiAdsQueue = ((...args: unknown[]) => void) & { q?: unknown[][]; }; @@ -85,17 +70,6 @@ function createRegistrationEventId(): string { .slice(2)}`; } -export function buildOpenAiRegistrationConversionPayload( - input: RegistrationConversionInput -): RegistrationConversionPayload { - return { - ...(input.email ? { email: input.email } : {}), - eventId: input.eventId ?? createRegistrationEventId(), - ...(input.properties?.oppref ? { oppref: input.properties.oppref } : {}), - sourceUrl: input.sourceUrl ?? window.location.href, - }; -} - export function buildOpenAiRegistrationMeasureArgs( eventId?: string ): unknown[] { @@ -123,31 +97,6 @@ export function measureOpenAiRegistrationCompleted(eventId?: string) { window.oaiq?.(...buildOpenAiRegistrationMeasureArgs(eventId)); } -export function sendOpenAiRegistrationCompletedConversion( - payload: RegistrationConversionPayload -) { - if ( - typeof window === "undefined" || - !PIXEL_ID || - !isOpenAiAdsPixelHostAllowed(window.location.hostname) - ) { - return; - } - - fetch("/api/openai-ads/conversions", { - body: JSON.stringify(payload), - headers: { "Content-Type": "application/json" }, - keepalive: true, - method: "POST", - }).catch(() => { - // Signup should not depend on ad-platform reporting. - }); -} - -export function trackOpenAiRegistrationCompleted( - input: RegistrationConversionInput = {} -) { - const payload = buildOpenAiRegistrationConversionPayload(input); - measureOpenAiRegistrationCompleted(payload.eventId); - sendOpenAiRegistrationCompletedConversion(payload); +export function trackOpenAiRegistrationCompleted() { + measureOpenAiRegistrationCompleted(createRegistrationEventId()); } diff --git a/apps/dashboard/components/table/rows/revenue-row.tsx b/apps/dashboard/components/table/rows/revenue-row.tsx index 499ff5427..546a1d625 100644 --- a/apps/dashboard/components/table/rows/revenue-row.tsx +++ b/apps/dashboard/components/table/rows/revenue-row.tsx @@ -2,6 +2,7 @@ import type { CellContext, ColumnDef } from "@tanstack/react-table"; import { BrowserIcon, CountryFlag, OSIcon } from "@/components/icon"; import { ReferrerSourceCell } from "@/components/atomic/ReferrerSourceCell"; import { formatNumber } from "@/lib/formatters"; +import { formatRevenueCurrency } from "@/lib/revenue-currency"; import { CurrencyDollarIcon, MapPinIcon, @@ -19,15 +20,8 @@ export interface RevenueEntry { transactions: number; } -const formatCurrency = (amount: number, currency = "USD"): string => - new Intl.NumberFormat("en-US", { - style: "currency", - currency, - minimumFractionDigits: 0, - maximumFractionDigits: 0, - }).format(amount); - interface RevenueRowProps { + currency: string; nameLabel?: string; type?: | "default" @@ -42,9 +36,10 @@ interface RevenueRowProps { } export function createRevenueColumns({ + currency, type = "default", nameLabel, -}: RevenueRowProps = {}): ColumnDef[] { +}: RevenueRowProps): ColumnDef[] { const getNameColumn = (): ColumnDef => { const header = nameLabel || @@ -166,7 +161,7 @@ export function createRevenueColumns({ const value = info.getValue() as number; return ( - {formatCurrency(value)} + {formatRevenueCurrency(value, currency)} ); }, diff --git a/apps/dashboard/hooks/use-goals.ts b/apps/dashboard/hooks/use-goals.ts index be17580ac..57cf77b9f 100644 --- a/apps/dashboard/hooks/use-goals.ts +++ b/apps/dashboard/hooks/use-goals.ts @@ -17,7 +17,9 @@ export interface GoalAnalyticsData { avg_completion_time_formatted: string; biggest_dropoff_rate: number; biggest_dropoff_step: number; + duration_available: boolean; error_insights: { + available: boolean; total_errors: number; sessions_with_errors: number; dropoffs_with_errors: number; @@ -29,6 +31,7 @@ export interface GoalAnalyticsData { conversion_rate: number; dropoff_rate: number; dropoffs: number; + error_context_available: boolean; error_count: number; error_rate: number; step_name: string; diff --git a/apps/dashboard/lib/active-organization-queries.test.ts b/apps/dashboard/lib/active-organization-queries.test.ts new file mode 100644 index 000000000..22d42b260 --- /dev/null +++ b/apps/dashboard/lib/active-organization-queries.test.ts @@ -0,0 +1,29 @@ +import { QueryClient } from "@tanstack/react-query"; +import { describe, expect, test } from "vitest"; +import { AUTH_QUERY_KEYS } from "@/components/providers/organizations-provider"; +import { orpc } from "@/lib/orpc"; +import { resetActiveOrganizationQueries } from "./active-organization-queries"; + +describe("resetActiveOrganizationQueries", () => { + test("removes organization data without clearing unrelated query state", async () => { + const queryClient = new QueryClient(); + const unrelatedKey = ["global", "release-notes"] as const; + queryClient.setQueryData(unrelatedKey, "keep"); + queryClient.setQueryData(orpc.websites.key(), "old websites"); + queryClient.setQueryData(orpc.billing.key(), "old billing"); + queryClient.setQueryData(orpc.feedback.key(), "old feedback"); + queryClient.setQueryData(orpc.insights.key(), "old insights"); + queryClient.setQueryData(AUTH_QUERY_KEYS.activeOrganization, "old org"); + + await resetActiveOrganizationQueries(queryClient); + + expect(queryClient.getQueryData(orpc.websites.key())).toBeUndefined(); + expect(queryClient.getQueryData(orpc.billing.key())).toBeUndefined(); + expect(queryClient.getQueryData(orpc.feedback.key())).toBeUndefined(); + expect(queryClient.getQueryData(orpc.insights.key())).toBeUndefined(); + expect(queryClient.getQueryData(unrelatedKey)).toBe("keep"); + expect( + queryClient.getQueryState(AUTH_QUERY_KEYS.activeOrganization)?.isInvalidated + ).toBe(true); + }); +}); diff --git a/apps/dashboard/lib/active-organization-queries.ts b/apps/dashboard/lib/active-organization-queries.ts new file mode 100644 index 000000000..f75005ea8 --- /dev/null +++ b/apps/dashboard/lib/active-organization-queries.ts @@ -0,0 +1,46 @@ +import type { QueryClient, QueryKey } from "@tanstack/react-query"; +import { AUTH_QUERY_KEYS } from "@/components/providers/organizations-provider"; +import { orpc } from "@/lib/orpc"; + +const ACTIVE_ORGANIZATION_QUERY_ROOTS: QueryKey[] = [ + orpc.agentChats.key(), + orpc.alarms.key(), + orpc.annotations.key(), + orpc.anomalies.key(), + orpc.apikeys.key(), + orpc.autocomplete.key(), + orpc.billing.key(), + orpc.feedback.key(), + orpc.flags.key(), + orpc.funnels.key(), + orpc.goals.key(), + orpc.insightGeneration.key(), + orpc.insights.key(), + orpc.integrations.key(), + orpc.linkFolders.key(), + orpc.links.key(), + orpc.organizations.key(), + orpc.profiles.key(), + orpc.revenue.key(), + orpc.statusPage.key(), + orpc.targetGroups.key(), + orpc.tracker.key(), + orpc.uptime.key(), + orpc.websites.key(), +]; + +type OrganizationQueryClient = Pick< + QueryClient, + "invalidateQueries" | "removeQueries" +>; + +export async function resetActiveOrganizationQueries( + queryClient: OrganizationQueryClient +): Promise { + for (const queryKey of ACTIVE_ORGANIZATION_QUERY_ROOTS) { + queryClient.removeQueries({ queryKey }); + } + await queryClient.invalidateQueries({ + queryKey: AUTH_QUERY_KEYS.activeOrganization, + }); +} diff --git a/apps/dashboard/lib/autumn/customer-plan-name.test.ts b/apps/dashboard/lib/autumn/customer-plan-name.test.ts index d5205065b..e8cd63750 100644 --- a/apps/dashboard/lib/autumn/customer-plan-name.test.ts +++ b/apps/dashboard/lib/autumn/customer-plan-name.test.ts @@ -1,9 +1,12 @@ import { describe, expect, test } from "bun:test"; +import { LEGACY_SCALE_PLAN } from "@databuddy/shared/billing"; import { getCustomerPlanName } from "./customer-plan-name"; describe("getCustomerPlanName", () => { test("presents the internal scale plan as Enterprise", () => { - expect(getCustomerPlanName("scale", "Scale")).toBe("Enterprise"); + expect(getCustomerPlanName(LEGACY_SCALE_PLAN.id, "Scale")).toBe( + LEGACY_SCALE_PLAN.name + ); }); test("keeps other plan names unchanged", () => { diff --git a/apps/dashboard/lib/autumn/customer-plan-name.ts b/apps/dashboard/lib/autumn/customer-plan-name.ts index 39ef91c43..f37cac4ea 100644 --- a/apps/dashboard/lib/autumn/customer-plan-name.ts +++ b/apps/dashboard/lib/autumn/customer-plan-name.ts @@ -1,6 +1,4 @@ -const CUSTOMER_PLAN_NAMES: Record = { - scale: "Enterprise", -}; +import { LEGACY_SCALE_PLAN } from "@databuddy/shared/billing"; export function getCustomerPlanName( planId: string | null | undefined, @@ -10,5 +8,7 @@ export function getCustomerPlanName( return fallbackName; } - return CUSTOMER_PLAN_NAMES[planId] ?? fallbackName; + return planId === LEGACY_SCALE_PLAN.id + ? LEGACY_SCALE_PLAN.name + : fallbackName; } diff --git a/apps/dashboard/lib/databunny-usage.ts b/apps/dashboard/lib/databunny-usage.ts deleted file mode 100644 index f1d41fa39..000000000 --- a/apps/dashboard/lib/databunny-usage.ts +++ /dev/null @@ -1,6 +0,0 @@ -export const DATABUNNY_USAGE_NAME = "Databunny usage"; -export const DATABUNNY_ALLOWANCE_NAME = "AI analysis allowance"; -export const DATABUNNY_USAGE_UNIT = "usage units"; - -export const DATABUNNY_USAGE_EXPLANATION = - "Databunny uses this allowance to answer questions about your analytics. Complex analysis uses more than a simple question."; diff --git a/apps/dashboard/lib/openai-ads-conversions.test.ts b/apps/dashboard/lib/openai-ads-conversions.test.ts deleted file mode 100644 index f617cef20..000000000 --- a/apps/dashboard/lib/openai-ads-conversions.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { describe, expect, it } from "bun:test"; -import { - buildOpenAiRegistrationCompletedEvent, - hashOpenAiUserValue, -} from "./openai-ads-conversions"; - -describe("OpenAI Ads conversion helpers", () => { - it("normalizes and hashes user identifiers", () => { - expect(hashOpenAiUserValue(" Test@Example.com ")).toBe( - "973dfe463ec85785f5f95af5ba3906eedb2d931c24e69824a89ea65dba4e813b" - ); - }); - - it("builds a registration_completed customer action without raw email", () => { - const event = buildOpenAiRegistrationCompletedEvent({ - email: "founder@example.com", - eventId: "conversion-1", - ipAddress: "203.0.113.10", - oppref: "openai-reference", - sourceUrl: "https://app.databuddy.cc/register?utm_source=openai_ads", - timestampMs: 1_788_000_000_000, - userAgent: "Mozilla/5.0", - }); - - expect(event).toMatchObject({ - action_source: "web", - data: { type: "customer_action" }, - id: "conversion-1", - oppref: "openai-reference", - source_url: "https://app.databuddy.cc/register?utm_source=openai_ads", - timestamp_ms: 1_788_000_000_000, - type: "registration_completed", - user: { - ip_address: "203.0.113.10", - user_agent: "Mozilla/5.0", - }, - }); - expect(JSON.stringify(event)).not.toContain("founder@example.com"); - expect(event.user?.email_sha256).toHaveLength(64); - }); -}); diff --git a/apps/dashboard/lib/openai-ads-conversions.ts b/apps/dashboard/lib/openai-ads-conversions.ts deleted file mode 100644 index 120d95351..000000000 --- a/apps/dashboard/lib/openai-ads-conversions.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { createHash } from "node:crypto"; - -const REGISTRATION_EVENT_TYPE = "registration_completed"; -const CUSTOMER_ACTION_TYPE = "customer_action"; - -interface BuildRegistrationEventInput { - email?: string; - eventId: string; - ipAddress?: string; - oppref?: string; - sourceUrl: string; - timestampMs: number; - userAgent?: string; -} - -export interface OpenAiAdsRegistrationEvent { - action_source: "web"; - data: { - type: typeof CUSTOMER_ACTION_TYPE; - }; - id: string; - oppref?: string; - source_url: string; - timestamp_ms: number; - type: typeof REGISTRATION_EVENT_TYPE; - user?: { - email_sha256?: string; - ip_address?: string; - user_agent?: string; - }; -} - -export function hashOpenAiUserValue(value: string): string { - return createHash("sha256").update(value.trim().toLowerCase()).digest("hex"); -} - -export function buildOpenAiRegistrationCompletedEvent( - input: BuildRegistrationEventInput -): OpenAiAdsRegistrationEvent { - const user: NonNullable = {}; - - if (input.email) { - user.email_sha256 = hashOpenAiUserValue(input.email); - } - if (input.ipAddress) { - user.ip_address = input.ipAddress; - } - if (input.userAgent) { - user.user_agent = input.userAgent; - } - - return { - action_source: "web", - data: { type: CUSTOMER_ACTION_TYPE }, - id: input.eventId, - ...(input.oppref ? { oppref: input.oppref } : {}), - source_url: input.sourceUrl, - timestamp_ms: input.timestampMs, - type: REGISTRATION_EVENT_TYPE, - ...(Object.keys(user).length ? { user } : {}), - }; -} diff --git a/apps/dashboard/lib/revenue-currency.test.ts b/apps/dashboard/lib/revenue-currency.test.ts new file mode 100644 index 000000000..32efd9799 --- /dev/null +++ b/apps/dashboard/lib/revenue-currency.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from "vitest"; +import type { DynamicQueryFilter } from "@/types/api"; +import { + appendRevenueCurrencyFilter, + formatRevenueCurrency, + normalizeRevenueCurrency, +} from "./revenue-currency"; + +describe("revenue currency", () => { + test("normalizes valid configured currency without inventing a fallback", () => { + expect(normalizeRevenueCurrency(" eur ")).toBe("EUR"); + expect(normalizeRevenueCurrency(undefined)).toBeNull(); + expect(normalizeRevenueCurrency("US")).toBeNull(); + expect(normalizeRevenueCurrency("ZZZ")).toBeNull(); + }); + + test("formats money in the configured currency", () => { + expect(formatRevenueCurrency(1234.5, "EUR")).toBe("€1,235"); + expect(formatRevenueCurrency(1234.5, "JPY")).toBe("¥1,235"); + }); + + test("formats invalid configuration as a neutral number instead of USD", () => { + expect(formatRevenueCurrency(1234.5, "ZZZ")).toBe("1,235"); + expect(() => formatRevenueCurrency(1234.5, "not-a-code")).not.toThrow(); + }); + + test("appends an exact query-only filter without mutating visible filters", () => { + const visibleFilters: DynamicQueryFilter[] = [ + { field: "country", operator: "eq", value: "DE" }, + ]; + const queryFilters = appendRevenueCurrencyFilter(visibleFilters, "EUR"); + + expect(queryFilters).toEqual([ + { field: "country", operator: "eq", value: "DE" }, + { field: "currency", operator: "eq", value: "EUR" }, + ]); + expect(queryFilters).not.toBe(visibleFilters); + expect(visibleFilters).toEqual([ + { field: "country", operator: "eq", value: "DE" }, + ]); + }); + + test("does not add a currency filter for invalid configuration", () => { + const filters: DynamicQueryFilter[] = [ + { field: "country", operator: "eq", value: "DE" }, + ]; + + expect(appendRevenueCurrencyFilter(filters, "ZZZ")).toEqual(filters); + }); +}); diff --git a/apps/dashboard/lib/revenue-currency.ts b/apps/dashboard/lib/revenue-currency.ts new file mode 100644 index 000000000..c31f9f077 --- /dev/null +++ b/apps/dashboard/lib/revenue-currency.ts @@ -0,0 +1,43 @@ +import { normalizeCurrencyCode } from "@databuddy/shared/currency"; +import type { DynamicQueryFilter } from "@/types/api"; + +export const normalizeRevenueCurrency = normalizeCurrencyCode; + +export function formatRevenueCurrency( + amount: number, + currency: unknown +): string { + const normalizedCurrency = normalizeRevenueCurrency(currency); + if (!normalizedCurrency) { + return new Intl.NumberFormat("en-US", { + maximumFractionDigits: 0, + minimumFractionDigits: 0, + }).format(amount); + } + + return new Intl.NumberFormat("en-US", { + style: "currency", + currency: normalizedCurrency, + minimumFractionDigits: 0, + maximumFractionDigits: 0, + }).format(amount); +} + +export function appendRevenueCurrencyFilter( + filters: DynamicQueryFilter[], + currency: unknown +): DynamicQueryFilter[] { + const normalizedCurrency = normalizeRevenueCurrency(currency); + if (!normalizedCurrency) { + return [...filters]; + } + + return [ + ...filters, + { + field: "currency", + operator: "eq", + value: normalizedCurrency, + }, + ]; +} diff --git a/apps/dashboard/lib/revenue-overview.test.ts b/apps/dashboard/lib/revenue-overview.test.ts new file mode 100644 index 000000000..855b85bef --- /dev/null +++ b/apps/dashboard/lib/revenue-overview.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "bun:test"; +import type { RevenueOverview } from "./revenue-overview"; +import { + hasPaymentActivity, + hasRevenueActivity, + paymentFailureObservationDescription, + paymentFailureRateLabel, + paymentFailureReasonLabel, +} from "./revenue-overview"; + +const emptyOverview = { + attributed_revenue: 0, + attributed_transactions: 0, + canceled_payment_attempts: 0, + failed_payment_amount: 0, + failed_payment_attempts: 0, + observed_failure_event_types: 0, + payment_diagnostics_available: 1, + payment_failure_rate: 0, + recovered_payment_attempts: 0, + required_failure_event_types: 2, + refund_amount: 0, + refund_count: 0, + sale_count: 0, + sale_revenue: 0, + subscription_count: 0, + subscription_revenue: 0, + successful_payment_attempts: 0, + total_revenue: 0, + total_transactions: 0, + top_payment_cancellation_reason: "", + top_payment_failure_reason: "", + unique_customers: 0, +} satisfies RevenueOverview; + +describe("revenue overview activity", () => { + it("shows a failure-only overview instead of the empty state", () => { + const overview = { + ...emptyOverview, + failed_payment_amount: 25, + failed_payment_attempts: 1, + payment_failure_rate: 100, + }; + + expect(hasRevenueActivity(overview)).toBe(true); + expect(hasPaymentActivity(overview)).toBe(true); + }); + + it("keeps a truly empty overview in the setup state", () => { + expect(hasRevenueActivity(emptyOverview)).toBe(false); + expect(hasPaymentActivity(emptyOverview)).toBe(false); + }); + + it("does not treat unavailable payment diagnostics as zero activity", () => { + const overview = { + ...emptyOverview, + canceled_payment_attempts: null, + failed_payment_attempts: null, + payment_diagnostics_available: 0, + successful_payment_attempts: null, + }; + + expect(hasRevenueActivity(overview)).toBe(false); + expect(hasPaymentActivity(overview)).toBe(false); + expect(paymentFailureRateLabel(overview)).toBe("—"); + expect(paymentFailureObservationDescription(overview)).toBe( + "Stripe payment diagnostics are unavailable for the selected filters." + ); + }); +}); + +describe("payment failure observations", () => { + it("reports a tracked zero without inferring webhook coverage", () => { + const overview = { + ...emptyOverview, + successful_payment_attempts: 20, + }; + + expect(paymentFailureRateLabel(overview)).toBe("0%"); + expect(paymentFailureObservationDescription(overview)).toBe( + "No recognized Stripe failure event types were observed in this range." + ); + }); + + it("describes observed event types as occurrences", () => { + const overview = { + ...emptyOverview, + failed_payment_attempts: 2, + observed_failure_event_types: 1, + payment_failure_rate: 10, + successful_payment_attempts: 18, + }; + + expect(paymentFailureRateLabel(overview)).toBe("10%"); + expect(paymentFailureObservationDescription(overview)).toBe( + "1 distinct Stripe failure event type was observed in this range." + ); + }); + + it("formats safe provider codes for display", () => { + expect(paymentFailureReasonLabel("insufficient_funds")).toBe( + "insufficient funds" + ); + expect(paymentFailureReasonLabel(undefined)).toBe(""); + }); +}); diff --git a/apps/dashboard/lib/revenue-overview.ts b/apps/dashboard/lib/revenue-overview.ts new file mode 100644 index 000000000..2fa99c0ec --- /dev/null +++ b/apps/dashboard/lib/revenue-overview.ts @@ -0,0 +1,86 @@ +export interface RevenueOverview { + attributed_revenue: number; + attributed_transactions: number; + canceled_payment_attempts: number | null; + failed_payment_amount: number | null; + failed_payment_attempts: number | null; + observed_failure_event_types: number | null; + payment_diagnostics_available: number; + payment_failure_rate: number | null; + recovered_payment_attempts: number | null; + refund_amount: number; + refund_count: number; + required_failure_event_types: number | null; + sale_count: number; + sale_revenue: number; + subscription_count: number; + subscription_revenue: number; + successful_payment_attempts: number | null; + top_payment_cancellation_reason: string | null; + top_payment_failure_reason: string | null; + total_revenue: number; + total_transactions: number; + unique_customers: number; +} + +function finiteNumber(value: unknown): number { + const number = Number(value); + return Number.isFinite(number) ? number : 0; +} + +export function hasRevenueActivity( + overview: RevenueOverview | undefined +): boolean { + return Boolean( + overview && + (overview.total_transactions > 0 || + finiteNumber(overview.failed_payment_attempts) > 0 || + finiteNumber(overview.canceled_payment_attempts) > 0) + ); +} + +export function paymentFailureRateLabel( + overview: RevenueOverview | undefined +): string { + if (overview?.payment_diagnostics_available === 0) { + return "—"; + } + const measuredAttempts = + finiteNumber(overview?.failed_payment_attempts) + + finiteNumber(overview?.successful_payment_attempts); + if (measuredAttempts === 0) { + return "—"; + } + return `${finiteNumber(overview?.payment_failure_rate)}%`; +} + +export function paymentFailureObservationDescription( + overview: RevenueOverview | undefined +): string { + if (overview?.payment_diagnostics_available === 0) { + return "Stripe payment diagnostics are unavailable for the selected filters."; + } + const observed = finiteNumber(overview?.observed_failure_event_types); + if (observed === 0) { + return "No recognized Stripe failure event types were observed in this range."; + } + return `${observed} distinct Stripe failure event ${observed === 1 ? "type was" : "types were"} observed in this range.`; +} + +export function paymentFailureReasonLabel( + reason: null | string | undefined +): string { + return reason?.replaceAll("_", " ").replaceAll("-", " ") ?? ""; +} + +export function hasPaymentActivity( + overview: RevenueOverview | undefined +): boolean { + return Boolean( + overview && + overview.payment_diagnostics_available !== 0 && + (finiteNumber(overview.successful_payment_attempts) > 0 || + finiteNumber(overview.failed_payment_attempts) > 0 || + finiteNumber(overview.canceled_payment_attempts) > 0) + ); +} diff --git a/apps/dashboard/lib/safe-callback.test.ts b/apps/dashboard/lib/safe-callback.test.ts new file mode 100644 index 000000000..b0787b403 --- /dev/null +++ b/apps/dashboard/lib/safe-callback.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "bun:test"; +import { safeCallbackPath } from "./safe-callback"; + +describe("safeCallbackPath", () => { + it("keeps local paths, queries, and fragments", () => { + expect(safeCallbackPath("/websites/site-1?tab=goals#details")).toBe( + "/websites/site-1?tab=goals#details" + ); + }); + + it("rejects absolute and protocol-relative URLs", () => { + for (const callback of [ + "https://attacker.example/path", + "//attacker.example/path", + "///attacker.example/path", + ]) { + expect(safeCallbackPath(callback)).toBe("/websites"); + } + }); + + it("rejects browser-normalized authority bypasses", () => { + for (const callback of [ + "/\t/attacker.example", + "/\n/attacker.example", + "/\r/attacker.example", + "/\\attacker.example", + "/%5cattacker.example", + "/%255cattacker.example", + "/%2f%2fattacker.example", + ]) { + expect(safeCallbackPath(callback)).toBe("/websites"); + } + }); + + it("rejects raw and encoded control characters", () => { + for (const callback of [ + "/path\u0000suffix", + "/path\u001fsuffix", + "/path\u007fsuffix", + "/path%00suffix", + "/path%250asuffix", + ]) { + expect(safeCallbackPath(callback, "/login")).toBe("/login"); + } + }); + + it("rejects malformed encoding instead of guessing", () => { + expect(safeCallbackPath("/path/%not-encoding")).toBe("/websites"); + }); + + it("rejects deeply nested encoding instead of relying on a decode limit", () => { + let encoded = "%2f%2fattacker.example"; + for (let pass = 0; pass < 6; pass += 1) { + encoded = encodeURIComponent(encoded); + } + + expect(safeCallbackPath(`/${encoded}`)).toBe("/websites"); + }); +}); diff --git a/apps/dashboard/lib/safe-callback.ts b/apps/dashboard/lib/safe-callback.ts index f01833677..6d6a4906f 100644 --- a/apps/dashboard/lib/safe-callback.ts +++ b/apps/dashboard/lib/safe-callback.ts @@ -1,14 +1,61 @@ +const CALLBACK_ORIGIN = "https://callback.databuddy.invalid"; + export function safeCallbackPath( callback: string | null | undefined, fallback = "/websites" ): string { - if ( - typeof callback === "string" && - callback.startsWith("/") && - !callback.startsWith("//") && - !callback.includes("\\") - ) { - return callback; - } - return fallback; + if (typeof callback !== "string" || !isSafePath(callback)) { + return fallback; + } + + return callback; +} + +function isSafePath(value: string): boolean { + if (hasUnsafePathSyntax(value)) { + return false; + } + + let decoded = value; + for (let pass = 0; pass < 5; pass += 1) { + let next: string; + try { + next = decodeURIComponent(decoded); + } catch { + return false; + } + + if (hasUnsafePathSyntax(next)) { + return false; + } + if (next === decoded) { + break; + } + if (pass === 4) { + return false; + } + decoded = next; + } + + try { + return new URL(value, CALLBACK_ORIGIN).origin === CALLBACK_ORIGIN; + } catch { + return false; + } +} + +function hasUnsafePathSyntax(value: string): boolean { + return ( + !value.startsWith("/") || + value.startsWith("//") || + value.includes("\\") || + hasControlCharacter(value) + ); +} + +function hasControlCharacter(value: string): boolean { + return Array.from(value).some((character) => { + const code = character.charCodeAt(0); + return code <= 31 || (code >= 127 && code <= 159); + }); } diff --git a/apps/dashboard/types/funnels.ts b/apps/dashboard/types/funnels.ts index fa533ea3b..eea0aee80 100644 --- a/apps/dashboard/types/funnels.ts +++ b/apps/dashboard/types/funnels.ts @@ -43,6 +43,7 @@ export interface FunnelStepAnalytics { conversion_rate: number; dropoff_rate: number; dropoffs: number; + error_context_available: boolean; error_count: number; error_rate: number; step_name: string; @@ -53,6 +54,7 @@ export interface FunnelStepAnalytics { } export interface FunnelErrorInsights { + available: boolean; dropoffs_with_errors: number; error_correlation_rate: number; sessions_with_errors: number; @@ -74,6 +76,7 @@ export interface FunnelAnalyticsData { avg_completion_time_formatted: string; biggest_dropoff_rate: number; biggest_dropoff_step: number; + duration_available: boolean; error_insights?: FunnelErrorInsights; overall_conversion_rate: number; steps_analytics: FunnelStepAnalytics[]; diff --git a/apps/docs/app/(home)/pricing/data.ts b/apps/docs/app/(home)/pricing/data.ts index df6ea2d47..8fbec0b5f 100644 --- a/apps/docs/app/(home)/pricing/data.ts +++ b/apps/docs/app/(home)/pricing/data.ts @@ -1,3 +1,5 @@ +import { DATABUNNY_USAGE } from "@databuddy/shared/billing"; + export interface FeatureDisplay { plural: string; singular: string; @@ -47,9 +49,9 @@ export interface RawPlan { const AGENT_CREDITS_FEATURE: RawFeature = { id: "agent_credits", - name: "Databunny Usage", + name: DATABUNNY_USAGE.name, type: "single_use", - display: { singular: "usage unit", plural: "usage units" }, + display: { singular: "usage unit", plural: DATABUNNY_USAGE.unit }, }; const EVENTS_FEATURE: RawFeature = { diff --git a/apps/docs/content/docs/Integrations/gtm.mdx b/apps/docs/content/docs/Integrations/gtm.mdx index cf700d433..b5e4b600e 100644 --- a/apps/docs/content/docs/Integrations/gtm.mdx +++ b/apps/docs/content/docs/Integrations/gtm.mdx @@ -166,13 +166,7 @@ Send GTM data layer events to Databuddy: ## Triggers Configuration -### Page View Tracking - -Create a trigger for enhanced page views: - -1. **Trigger Type**: Page View - DOM Ready -2. **Trigger Name**: "Databuddy - Enhanced Page View" -3. **Conditions**: Page Path contains your domain +The analytics script records pageviews automatically, including SPA navigation. Use the **All Pages** trigger only to load the script; do not add a second pageview tag. ### Scroll Tracking diff --git a/apps/docs/content/docs/Integrations/payments.mdx b/apps/docs/content/docs/Integrations/payments.mdx index d72d16c02..58caff5ee 100644 --- a/apps/docs/content/docs/Integrations/payments.mdx +++ b/apps/docs/content/docs/Integrations/payments.mdx @@ -4,6 +4,7 @@ description: Attribute Stripe and Paddle revenue to sessions, campaigns, and ide --- import { Callout, CodeBlock, Tab, Tabs } from "@/components/docs"; +import { STRIPE_WEBHOOK_EVENTS } from "@databuddy/shared/stripe-webhooks"; Connect your payment provider and every transaction shows up in the Revenue dashboard, attributed to the session, referrer, and user that produced it. @@ -24,13 +25,19 @@ Connect your payment provider and every transaction shows up in the Revenue dash In the [Stripe dashboard](https://dashboard.stripe.com/webhooks), add an endpoint with your generated URL and subscribe to these events: -| Event | Purpose | -|---|---| -| `payment_intent.succeeded` | Records completed sales and subscriptions (required) | -| `invoice.paid` | Records recurring subscription payments with attribution (required) | -| `charge.refunded` | Records refunds (required) | -| `payment_intent.payment_failed` | Tracks failed payments (optional) | -| `payment_intent.canceled` | Tracks canceled payments (optional) | + + + +{STRIPE_WEBHOOK_EVENTS.required.map(({ event, purpose }) => ( + +))} +{STRIPE_WEBHOOK_EVENTS.optional.map(({ event, purpose }) => ( + +))} + +
EventPurpose
{event}{purpose} ({event === "invoice_payment.paid" ? "required on API 2025-05-28.basil or later" : "required"})
{event}{purpose} (optional)
+ +Stripe introduced `invoice_payment.paid` in `2025-05-28.basil`. Subscribe to it on that version or later for exact allocation details. Databuddy also keeps the total from `invoice.paid` as a safe fallback, so adding the newer event after an API upgrade does not drop or double-count revenue. Earlier versions, including the `2025-03-31.basil` transition, use embedded invoice allocations when the full list is available. Paste the endpoint's **signing secret** into the Revenue settings so Databuddy can verify each delivery. @@ -57,18 +64,29 @@ Metadata connects a payment to the visitor who made it. Include what you have: e const { anonId, sessionId } = getTrackingIds(); const profileId = getProfileId(); -// ...send them to your server, then create the payment: +// ...send them to your server and reuse the same metadata: +const databuddyMetadata = { + databuddy_client_id: websiteId, + databuddy_session_id: sessionId, + databuddy_anonymous_id: anonId, + databuddy_profile_id: profileId, +}; + await stripe.paymentIntents.create({ amount, currency, - metadata: { - databuddy_client_id: websiteId, - databuddy_session_id: sessionId, - databuddy_anonymous_id: anonId, - databuddy_profile_id: profileId, - }, + metadata: databuddyMetadata, +}); + +// For recurring billing, attach it to the Subscription too. +await stripe.subscriptions.create({ + customer, + items: [{ price: priceId }], + metadata: databuddyMetadata, });`} + +Stripe does not automatically copy PaymentIntent metadata to future subscription invoices. Put the IDs on the Subscription (or `subscription_data.metadata` when using Checkout) so recurring revenue stays attributable. diff --git a/apps/docs/content/docs/sdk/configuration.mdx b/apps/docs/content/docs/sdk/configuration.mdx index 4a75df94e..5bfc4ed00 100644 --- a/apps/docs/content/docs/sdk/configuration.mdx +++ b/apps/docs/content/docs/sdk/configuration.mdx @@ -21,7 +21,7 @@ This page documents all configuration options across the Databuddy SDK platforms scriptUrl="https://cdn.databuddy.cc/databuddy.js" // Core tracking - trackWebVitals // default: false + trackWebVitals={true} // opt in; default: false trackErrors // default: false trackOutgoingLinks // default: false trackInteractions // default: false @@ -65,13 +65,17 @@ This page documents all configuration options across the Databuddy SDK platforms | Prop | Type | Default | Description | |------|------|---------|-------------| -| `trackWebVitals` | `boolean` | `false` | Core Web Vitals (LCP, INP, CLS, TTFB) | +| `trackWebVitals` | `boolean` | `false` | FCP, LCP, INP, CLS, TTFB, and FPS | | `trackErrors` | `boolean` | `false` | JavaScript errors | | `trackOutgoingLinks` | `boolean` | `false` | External link clicks | | `trackInteractions` | `boolean` | `false` | Button clicks and form submissions | | `trackAttributes` | `boolean` | `false` | Elements with `data-track` attributes | | `trackHashChanges` | `boolean` | `false` | URL hash changes | + + Use `trackWebVitals` in React, Vue, and Nuxt, or `data-track-web-vitals` with the CDN script. The older `trackPerformance` option remains a deprecated compatibility alias; `trackWebVitals` takes precedence when both are set. + + #### Batching & Performance | Prop | Type | Default | Description | @@ -139,7 +143,7 @@ Vue uses the same props as React but with kebab-case naming: | Attribute | Type | Default | Description | |-----------|------|---------|-------------| | `data-client-id` | `string` | Required | Your client ID | -| `data-track-web-vitals` | flag | `false` | Track page performance and Core Web Vitals | +| `data-track-web-vitals` | flag | `false` | FCP, LCP, INP, CLS, TTFB, and FPS | | `data-track-errors` | flag | `false` | Track JavaScript errors | | `data-track-outgoing-links` | flag | `false` | Track external links | | `data-track-interactions` | flag | `false` | Track interactions | diff --git a/apps/docs/content/docs/sdk/nuxt.mdx b/apps/docs/content/docs/sdk/nuxt.mdx index f67ac673d..25fb90238 100644 --- a/apps/docs/content/docs/sdk/nuxt.mdx +++ b/apps/docs/content/docs/sdk/nuxt.mdx @@ -148,7 +148,7 @@ All options are passed under the `databuddy` key in `nuxt.config.ts`: | `clientId` | `string` | — | Your Databuddy client ID | | `disabled` | `boolean` | `false` | Disable all tracking | | `debug` | `boolean` | `false` | Enable verbose logging | -| `trackWebVitals` | `boolean` | `false` | Core Web Vitals (LCP, CLS, TTFB) | +| `trackWebVitals` | `boolean` | `false` | Core Web Vitals (LCP, INP, CLS, TTFB) | | `trackErrors` | `boolean` | `false` | JS errors + Vue component errors | | `trackOutgoingLinks` | `boolean` | `false` | Clicks on external links | | `trackInteractions` | `boolean` | `false` | Button clicks and form submissions | @@ -158,6 +158,10 @@ All options are passed under the `databuddy` key in `nuxt.config.ts`: | `samplingRate` | `number` | `1.0` | Event sampling rate (0.0–1.0) | | `flags` | `FlagsConfig` | — | Feature flag configuration | + + `trackPerformance` still works as a deprecated compatibility alias. Rename it to `trackWebVitals`; if both are set, `trackWebVitals` takes precedence. + + ## Privacy diff --git a/apps/docs/lib/pricing-copy.test.ts b/apps/docs/lib/pricing-copy.test.ts index 2cea64870..d266ec222 100644 --- a/apps/docs/lib/pricing-copy.test.ts +++ b/apps/docs/lib/pricing-copy.test.ts @@ -38,5 +38,6 @@ describe("public pricing copy", () => { expect(markdown).not.toContain("Assistant messages"); expect(markdown).not.toContain("Agent credits"); + expect(markdown).not.toContain("Scale"); }); }); diff --git a/apps/docs/lib/public-copy-contract.test.ts b/apps/docs/lib/public-copy-contract.test.ts index 580f992fd..07ccef694 100644 --- a/apps/docs/lib/public-copy-contract.test.ts +++ b/apps/docs/lib/public-copy-contract.test.ts @@ -23,6 +23,40 @@ describe("public copy contracts", () => { expect(publicDocs).not.toContain("trackSessions="); }); + it("lists every performance metric collected by trackWebVitals", async () => { + const configuration = await readFile( + join( + import.meta.dir, + "..", + "content", + "docs", + "sdk", + "configuration.mdx" + ), + "utf8" + ); + const vitalsPlugin = await readFile( + join( + import.meta.dir, + "..", + "..", + "..", + "packages", + "tracker", + "src", + "plugins", + "vitals.ts" + ), + "utf8" + ); + const description = "FCP, LCP, INP, CLS, TTFB, and FPS"; + + expect(configuration.split(description)).toHaveLength(3); + for (const metric of ["FCP", "LCP", "INP", "CLS", "TTFB", "FPS"]) { + expect(vitalsPlugin).toContain(`on${metric}(handleMetric)`); + } + }); + it("keeps the tracker-size claim aligned with the checked-in bundle", async () => { const bundle = await readFile( join( diff --git a/apps/docs/package.json b/apps/docs/package.json index 72095ebbf..e30c3f04b 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -91,6 +91,7 @@ }, "devDependencies": { "@biomejs/biome": "2.5.1", + "@databuddy/tracker": "workspace:*", "@tailwindcss/postcss": "^4.2.2", "@types/d3": "^7.4.3", "@types/d3-geo": "^3.1.0", diff --git a/apps/insights/src/delivery.test.ts b/apps/insights/src/delivery.test.ts index 0c4ad14f6..b9437f657 100644 --- a/apps/insights/src/delivery.test.ts +++ b/apps/insights/src/delivery.test.ts @@ -91,6 +91,41 @@ describe("Slack finding blocks", () => { ); }); + it("honors Retry-After values longer than five seconds", async () => { + const responses = [ + new Response("rate limited", { + headers: { "Retry-After": "120" }, + status: 429, + }), + Response.json({ ok: true, ts: "123.456" }), + ]; + const delays: number[] = []; + const fetcher = (async () => { + const response = responses.shift(); + if (!response) { + throw new Error("Unexpected Slack request"); + } + return response; + }) as typeof fetch; + + await postToSlack( + "token", + "channel-test", + [{ type: "divider" }], + "Finding", + "effect-test", + { + fetcher, + random: () => 0, + sleep: async (milliseconds) => { + delays.push(milliseconds); + }, + } + ); + + expect(delays).toEqual([120_000]); + }); + it("uses the durable effect ID as Slack's client message ID", () => { expect( buildSlackPostPayload( diff --git a/apps/insights/src/delivery.ts b/apps/insights/src/delivery.ts index 351907253..03b5689e7 100644 --- a/apps/insights/src/delivery.ts +++ b/apps/insights/src/delivery.ts @@ -15,7 +15,6 @@ const SLACK_POST_URL = "https://slack.com/api/chat.postMessage"; const SLACK_POST_TIMEOUT_MS = 10_000; const SLACK_RATE_LIMIT_ATTEMPTS = 3; const SLACK_RATE_LIMIT_FALLBACK_MS = 1000; -const SLACK_RATE_LIMIT_MAX_WAIT_MS = 5000; const MAX_DIGEST_INSIGHTS = 3; const MAX_ONGOING_LINES = 5; const SLACK_HEADER_MAX = 150; @@ -436,10 +435,7 @@ function rateLimitDelay(response: Response, random: () => number): number { Number.isFinite(seconds) && seconds > 0 ? seconds * 1000 : SLACK_RATE_LIMIT_FALLBACK_MS; - return ( - Math.min(requested, SLACK_RATE_LIMIT_MAX_WAIT_MS) + - Math.floor(random() * 250) - ); + return requested + Math.floor(random() * 250); } export async function postToSlack( diff --git a/apps/insights/src/detection.test.ts b/apps/insights/src/detection.test.ts index 92894a459..e2b7c001f 100644 --- a/apps/insights/src/detection.test.ts +++ b/apps/insights/src/detection.test.ts @@ -10,6 +10,7 @@ import { median, wowWindow, } from "./detection"; +import { computeResolutions } from "./resolution"; function makeDailyRows( values: { @@ -244,6 +245,55 @@ describe("detectSignals", () => { expect(diagnostics.failedFamilies).toBe(2); }); + it("detects an exact custom event disappearance", async () => { + const queryFn = createMockQueryFn( + [], + { sessions: 100 }, + { sessions: 100 }, + { + custom_events: [ + { name: "checkout_completed", unique_users: 0 }, + { name: "checkout_completed", unique_users: 30 }, + ], + } + ); + + const signals = await detectSignals( + BASE_PARAMS, + queryFn, + dayjs("2025-03-15") + ); + const signal = signals.find( + (candidate) => candidate.metric === "custom_event:checkout_completed" + ); + + expect(signal).toMatchObject({ + current: 0, + direction: "down", + kind: "missing_expected_data", + }); + }); + + it("ignores small custom event changes", async () => { + const queryFn = createMockQueryFn([], { sessions: 100 }, { sessions: 100 }, { + custom_events: [ + { name: "checkout_completed", unique_users: 18 }, + { name: "checkout_completed", unique_users: 20 }, + ], + }); + + const signals = await detectSignals( + BASE_PARAMS, + queryFn, + dayjs("2025-03-15") + ); + expect( + signals.find( + (candidate) => candidate.metric === "custom_event:checkout_completed" + ) + ).toBeUndefined(); + }); + it("returns no signals and reports an incomplete scan when one family fails", async () => { const diagnostics = { failedFamilies: 0 }; const queryFn: QueryFn = async (request) => { @@ -941,61 +991,6 @@ describe("detectSignals", () => { expect(volumeSignals.length).toBe(0); }); - it("filters rate metrics when the comparison has too few sessions", async () => { - const queryFn = createMockQueryFn( - [], - { - unique_visitors: 10, - sessions: 10, - pageviews: 10, - bounce_rate: 60, - median_session_duration: 120, - }, - { - unique_visitors: 5, - sessions: 5, - pageviews: 5, - bounce_rate: 30, - median_session_duration: 60, - } - ); - - const signals = await detectSignals(BASE_PARAMS, queryFn); - const rateSignals = signals.filter( - (s) => - s.metric === "bounce_rate" || s.metric === "session_duration" - ); - expect(rateSignals).toEqual([]); - }); - }); - - describe("rate metric absolute delta filter", () => { - it("filters rate metrics with less than 10pp absolute change", async () => { - const queryFn = createMockQueryFn( - [], - { - unique_visitors: 200, - sessions: 200, - pageviews: 200, - bounce_rate: 52, - median_session_duration: 67, - }, - { - unique_visitors: 200, - sessions: 200, - pageviews: 200, - bounce_rate: 45, - median_session_duration: 60, - } - ); - - const signals = await detectSignals(BASE_PARAMS, queryFn); - const rateSignals = signals.filter( - (s) => - s.metric === "bounce_rate" || s.metric === "session_duration" - ); - expect(rateSignals.length).toBe(0); - }); }); describe("impact floor filter", () => { @@ -1169,79 +1164,6 @@ describe("detectSignals", () => { }); }); - describe("low-traffic floor", () => { - const lowTrafficDays = () => { - const start = dayjs().subtract(27, "day"); - return makeDailyRows( - generateStableDays( - 28, - { - visitors: 3, - sessions: 4, - pageviews: 6, - bounce_rate: 40, - median_session_duration: 60, - }, - start - ) - ); - }; - - it("suppresses rate changes without enough sessions", async () => { - const queryFn = createMockQueryFn( - lowTrafficDays(), - { - bounce_rate: 60, - median_session_duration: 120, - }, - { - bounce_rate: 30, - median_session_duration: 60, - } - ); - - const signals = await detectSignals(BASE_PARAMS, queryFn); - expect(signals.find((s) => s.metric === "bounce_rate")).toBeUndefined(); - expect( - signals.find((s) => s.metric === "session_duration") - ).toBeUndefined(); - }); - - it("keeps rate changes when the site has enough sessions", async () => { - const start = dayjs().subtract(27, "day"); - const rows = makeDailyRows( - generateStableDays( - 28, - { - visitors: 100, - sessions: 120, - pageviews: 200, - bounce_rate: 40, - median_session_duration: 60, - }, - start - ) - ); - const queryFn = createMockQueryFn( - rows, - { - bounce_rate: 60, - median_session_duration: 120, - }, - { - bounce_rate: 30, - median_session_duration: 60, - } - ); - - const signals = await detectSignals(BASE_PARAMS, queryFn); - expect(signals.find((s) => s.metric === "bounce_rate")).toBeDefined(); - expect( - signals.find((s) => s.metric === "session_duration") - ).toBeDefined(); - }); - }); - describe("vitals detection", () => { it("ignores regressions that remain within the good threshold", async () => { const queryFn = createMockQueryFn([], {}, {}, { @@ -1283,6 +1205,23 @@ describe("detectSignals", () => { const signals = await detectSignals(BASE_PARAMS, queryFn); expect(signals.find((signal) => signal.metric === "lcp")).toBeUndefined(); }); + + it("ignores comparisons with a sparse baseline period", async () => { + const queryFn = createMockQueryFn( + [], + { sessions: 1000 }, + { sessions: 1000 }, + { + vitals_overview: [ + { metric_name: "INP", p75: 240, samples: 100 }, + { metric_name: "INP", p75: 150, samples: 1 }, + ], + } + ); + + const signals = await detectSignals(BASE_PARAMS, queryFn); + expect(signals.find((signal) => signal.metric === "inp")).toBeUndefined(); + }); }); describe("revenue detection", () => { @@ -1309,6 +1248,43 @@ describe("detectSignals", () => { expect(revSignal!.deltaPercent).toBe(-50); }); + it("evaluates each revenue currency independently", async () => { + let revenueCalls = 0; + const queryFn: QueryFn = async (request) => { + if (request.type !== "revenue_overview") { + return []; + } + revenueCalls += 1; + return revenueCalls === 1 + ? [ + { currency: "USD", total_revenue: 50 }, + { currency: "EUR", total_revenue: 200 }, + ] + : [ + { currency: "USD", total_revenue: 100 }, + { currency: "EUR", total_revenue: 100 }, + ]; + }; + + const signals = (await detectSignals(BASE_PARAMS, queryFn)).filter( + (signal) => signal.metric === "revenue" + ); + + expect(signals).toHaveLength(2); + expect(signals.find((signal) => signal.currency === "USD")).toMatchObject({ + current: 50, + baseline: 100, + direction: "down", + label: "Revenue (USD)", + }); + expect(signals.find((signal) => signal.currency === "EUR")).toMatchObject({ + current: 200, + baseline: 100, + direction: "up", + label: "Revenue (EUR)", + }); + }); + it("skips small revenue changes", async () => { const queryFn = createMockQueryFn([], {}, {}, { revenue_overview: [{ total_revenue: 110 }, { total_revenue: 100 }], @@ -1341,6 +1317,226 @@ describe("detectSignals", () => { const signals = await detectSignals(BASE_PARAMS, queryFn); expect(signals.find((s) => s.metric === "revenue")).toBeDefined(); }); + + it("flags a material payment failure-rate regression", async () => { + const queryFn = createMockQueryFn([], {}, {}, { + revenue_overview: [ + { + failed_payment_attempts: 10, + observed_failure_event_types: 2, + payment_failure_rate: 20, + required_failure_event_types: 2, + successful_payment_attempts: 40, + }, + { + failed_payment_attempts: 2, + observed_failure_event_types: 2, + payment_failure_rate: 5, + required_failure_event_types: 2, + successful_payment_attempts: 38, + }, + ], + }); + + const signals = await detectSignals(BASE_PARAMS, queryFn); + const failureSignal = signals.find( + (signal) => signal.metric === "payment_failure_rate" + ); + expect(failureSignal?.direction).toBe("up"); + expect(failureSignal?.current).toBe(20); + expect(failureSignal?.baseline).toBe(5); + }); + + it("detects a regression when only one failure event type occurs", async () => { + const queryFn = createMockQueryFn([], {}, {}, { + revenue_overview: [ + { + failed_payment_attempts: 10, + observed_failure_event_types: 1, + payment_failure_rate: 20, + required_failure_event_types: 2, + successful_payment_attempts: 40, + }, + { + failed_payment_attempts: 2, + observed_failure_event_types: 2, + payment_failure_rate: 5, + required_failure_event_types: 2, + successful_payment_attempts: 38, + }, + ], + }); + + const signals = await detectSignals(BASE_PARAMS, queryFn); + expect( + signals.find((signal) => signal.metric === "payment_failure_rate") + ).toMatchObject({ current: 20, baseline: 5, direction: "up" }); + }); + + it("detects a failure spike against a prior zero", async () => { + const queryFn = createMockQueryFn([], {}, {}, { + revenue_overview: [ + { + failed_payment_attempts: 10, + observed_failure_event_types: 1, + payment_failure_rate: 20, + successful_payment_attempts: 40, + }, + { + failed_payment_attempts: 0, + observed_failure_event_types: 0, + payment_failure_rate: 0, + successful_payment_attempts: 40, + }, + ], + }); + + const signals = await detectSignals(BASE_PARAMS, queryFn); + expect( + signals.find((signal) => signal.metric === "payment_failure_rate") + ).toMatchObject({ current: 20, baseline: 0, direction: "up" }); + }); + + it("does not claim recovery when no current failure events are observed", async () => { + const queryFn = createMockQueryFn([], {}, {}, { + revenue_overview: [ + { + failed_payment_attempts: 0, + observed_failure_event_types: 0, + payment_failure_rate: 0, + successful_payment_attempts: 40, + }, + { + failed_payment_attempts: 10, + observed_failure_event_types: 1, + payment_failure_rate: 20, + successful_payment_attempts: 40, + }, + ], + }); + + const signals = await detectSignals(BASE_PARAMS, queryFn); + expect( + signals.find((signal) => signal.metric === "payment_failure_rate") + ).toBeUndefined(); + expect( + computeResolutions({ + canRecover: true, + detectedSignals: signals, + now: new Date("2026-06-01T00:00:00.000Z"), + openInsights: [ + { + changePercent: 300, + createdAt: new Date("2026-05-01T00:00:00.000Z"), + id: "open-payment-failure", + sentiment: "negative", + subjectKey: "payment_failure_rate", + type: "quality_shift", + }, + ], + }) + ).toEqual([{ id: "open-payment-failure", reason: "stale" }]); + }); + + it("detects a recovery when failures occur in both periods", async () => { + const queryFn = createMockQueryFn([], {}, {}, { + revenue_overview: [ + { + failed_payment_attempts: 5, + observed_failure_event_types: 1, + payment_failure_rate: 10, + successful_payment_attempts: 45, + }, + { + failed_payment_attempts: 20, + observed_failure_event_types: 1, + payment_failure_rate: 40, + successful_payment_attempts: 30, + }, + ], + }); + + const signals = await detectSignals(BASE_PARAMS, queryFn); + expect( + signals.find((signal) => signal.metric === "payment_failure_rate") + ).toMatchObject({ current: 10, baseline: 40, direction: "down" }); + }); + + it("does not confuse higher payment volume with a failure regression", async () => { + const queryFn = createMockQueryFn([], {}, {}, { + revenue_overview: [ + { + failed_payment_attempts: 10, + payment_failure_rate: 10, + successful_payment_attempts: 90, + }, + { + failed_payment_attempts: 5, + payment_failure_rate: 10, + successful_payment_attempts: 45, + }, + ], + }); + + const signals = await detectSignals(BASE_PARAMS, queryFn); + expect( + signals.find((signal) => signal.metric === "payment_failure_rate") + ).toBeUndefined(); + }); + + it("ignores sparse payment failure samples", async () => { + const queryFn = createMockQueryFn([], {}, {}, { + revenue_overview: [ + { + failed_payment_attempts: 2, + payment_failure_rate: 40, + successful_payment_attempts: 3, + }, + { + failed_payment_attempts: 0, + payment_failure_rate: 0, + successful_payment_attempts: 4, + }, + ], + }); + + const signals = await detectSignals(BASE_PARAMS, queryFn); + expect( + signals.find((signal) => signal.metric === "payment_failure_rate") + ).toBeUndefined(); + }); + + it("does not combine failure and volume floors from different periods", async () => { + const queryFn = createMockQueryFn( + [], + { sessions: 100 }, + { sessions: 100 }, + { + revenue_overview: [ + { + failed_payment_attempts: 5, + payment_failure_rate: 100, + successful_payment_attempts: 0, + }, + { + failed_payment_attempts: 0, + payment_failure_rate: 0, + successful_payment_attempts: 10, + }, + ], + } + ); + + const signals = await detectSignals( + BASE_PARAMS, + queryFn, + dayjs("2025-03-15") + ); + + expect( + signals.find((signal) => signal.metric === "payment_failure_rate") + ).toBeUndefined(); + }); }); describe("correlated signal collapsing", () => { @@ -1388,12 +1584,17 @@ describe("detectSignals", () => { pageviews: 200, bounce_rate: 25, median_session_duration: 60, + }, + { + error_summary: [ + { totalErrors: 100, affectedUsers: 30, errorRate: 5 }, + { totalErrors: 20, affectedUsers: 10, errorRate: 2 }, + ], } ); const signals = await detectSignals(BASE_PARAMS, queryFn); - const downSignals = signals.filter((s) => s.direction === "down"); - expect(downSignals.some((s) => s.metric === "bounce_rate")).toBe(true); + expect(signals.some((s) => s.metric === "error_count")).toBe(true); }); it("does not collapse a single traffic metric", async () => { diff --git a/apps/insights/src/detection.ts b/apps/insights/src/detection.ts index 658811c63..0c36cbdcd 100644 --- a/apps/insights/src/detection.ts +++ b/apps/insights/src/detection.ts @@ -14,6 +14,7 @@ dayjs.extend(timezonePlugin); export interface DetectedSignal { baseline: number; baselineDates?: string[]; + currency?: string; current: number; definitionEvidence?: { metrics: InsightMetric[]; @@ -70,18 +71,6 @@ const ANOMALY_METRICS: AnomalyMetric[] = [ dailyField: "pageviews", summaryField: "pageviews", }, - { - key: "bounce_rate", - label: "Bounce rate", - dailyField: "bounce_rate", - summaryField: "bounce_rate", - }, - { - key: "session_duration", - label: "Avg. session duration", - dailyField: "median_session_duration", - summaryField: "median_session_duration", - }, ]; export function median(values: number[]): number { @@ -110,15 +99,20 @@ const ZSCORE_MIN_BASELINE = 6; const ZSCORE_HISTORY_DAYS = 22; const WOW_TRAFFIC_THRESHOLD = 40; const WOW_ERROR_THRESHOLD = 40; +const WOW_CUSTOM_EVENT_THRESHOLD = 60; const WOW_REVENUE_THRESHOLD = 30; +const WOW_PAYMENT_FAILURE_THRESHOLD = 50; const WOW_VITALS_THRESHOLD = 30; const REVENUE_MIN_ABSOLUTE_CHANGE = 25; const REVENUE_MIN_TRANSACTIONS = 5; -const FILTER_SESSION_DURATION_MIN_DELTA = 60; -const FILTER_SESSION_DURATION_MIN_PEAK = 20; -const FILTER_BOUNCE_MIN_DELTA = 10; +const PAYMENT_FAILURE_MIN_FAILED = 5; +const PAYMENT_FAILURE_MIN_TOTAL_ATTEMPTS = 10; +const PAYMENT_FAILURE_MIN_RATE = 5; +const PAYMENT_FAILURE_MIN_POINT_CHANGE = 5; const FILTER_ERROR_MIN_DELTA = 5; const FILTER_ERROR_MIN_PEAK = 10; +const FILTER_CUSTOM_EVENT_MIN_DELTA = 10; +const FILTER_CUSTOM_EVENT_MIN_PEAK = 20; const ERROR_MIN_AFFECTED_USERS = 5; const ERROR_SIGNIFICANT_AFFECTED_USERS = 20; const ERROR_MIN_SESSION_RATE = 1; @@ -192,15 +186,11 @@ export function adaptiveWowThreshold( type SignalFilter = (signal: DetectedSignal) => boolean; const METRIC_FILTERS: Record = { - session_duration: (s) => - Math.abs(s.current - s.baseline) >= FILTER_SESSION_DURATION_MIN_DELTA && - Math.max(s.current, s.baseline) >= FILTER_SESSION_DURATION_MIN_PEAK, - bounce_rate: (s) => - Math.abs(s.current - s.baseline) >= FILTER_BOUNCE_MIN_DELTA, error_count: (s) => Math.abs(s.current - s.baseline) >= FILTER_ERROR_MIN_DELTA && Math.max(s.current, s.baseline) >= FILTER_ERROR_MIN_PEAK, revenue: () => true, + payment_failure_rate: () => true, lcp: () => true, inp: () => true, }; @@ -232,11 +222,18 @@ export function makeWowSignal( } function passesImpactFilter(signal: DetectedSignal): boolean { + if (signal.metric.startsWith("custom_event:")) { + return ( + Math.abs(signal.current - signal.baseline) >= + FILTER_CUSTOM_EVENT_MIN_DELTA && + Math.max(signal.current, signal.baseline) >= FILTER_CUSTOM_EVENT_MIN_PEAK + ); + } const filter = METRIC_FILTERS[signal.metric]; return filter ? filter(signal) : DEFAULT_TRAFFIC_FILTER(signal); } -const RATE_METRICS = new Set(["bounce_rate", "session_duration", "lcp", "inp"]); +const RATE_METRICS = new Set(["lcp", "inp"]); export function passesLowTrafficFloor( signal: DetectedSignal, @@ -245,6 +242,9 @@ export function passesLowTrafficFloor( if (weeklySessions >= LOW_TRAFFIC_WEEKLY_SESSIONS) { return true; } + if (signal.metric === "payment_failure_rate") { + return true; + } if (RATE_METRICS.has(signal.metric)) { return false; } @@ -317,6 +317,16 @@ function mapRowsByStringField( return mapped; } +function mapRevenueRowsByCurrency( + rows: Record[] +): Map> { + const mapped = new Map>(); + for (const row of rows) { + mapped.set(stringField(row, "currency") ?? "", row); + } + return mapped; +} + function densifyDailyHistory( rows: Record[], from: string, @@ -338,8 +348,6 @@ function densifyDailyHistory( dense.push( byDate.get(key) ?? { date: key, - bounce_rate: 0, - median_session_duration: 0, pageviews: 0, sessions: 0, visitors: 0, @@ -398,7 +406,13 @@ function rethrowDetectionAbort( async function readDetectorFamily(params: { abortSignal?: AbortSignal; - family: "errors" | "history" | "revenue" | "summary" | "vitals"; + family: + | "custom_events" + | "errors" + | "history" + | "revenue" + | "summary" + | "vitals"; read: () => Promise; websiteId: string; }): Promise> { @@ -494,10 +508,10 @@ export async function detectSignals( const wowDirection = new Map(); for (const s of wowSignals) { - wowDirection.set(s.metric, s.direction); + wowDirection.set(`${s.metric}:${s.currency ?? ""}`, s.direction); } const reconciledZscore = zscoreSignals.filter((s) => { - const wow = wowDirection.get(s.metric); + const wow = wowDirection.get(`${s.metric}:${s.currency ?? ""}`); return wow === undefined || wow === s.direction; }); @@ -505,9 +519,10 @@ export async function detectSignals( const byMetric = new Map(); for (const signal of all) { - const prev = byMetric.get(signal.metric); + const key = `${signal.metric}:${signal.currency ?? ""}`; + const prev = byMetric.get(key); if (!prev || Math.abs(signal.deltaPercent) > Math.abs(prev.deltaPercent)) { - byMetric.set(signal.metric, signal); + byMetric.set(key, signal); } } @@ -648,7 +663,7 @@ async function detectWow( ); } - const [summary, errors, revenue, vitals] = await Promise.all([ + const [summary, customEvents, errors, revenue, vitals] = await Promise.all([ readDetectorFamily({ abortSignal, family: "summary", @@ -659,6 +674,16 @@ async function detectWow( query("summary_metrics", previousFrom, previousTo), ]), }), + readDetectorFamily({ + abortSignal, + family: "custom_events", + websiteId, + read: () => + Promise.all([ + query("custom_events", currentFrom, currentTo), + query("custom_events", previousFrom, previousTo), + ]), + }), readDetectorFamily({ abortSignal, family: "errors", @@ -691,11 +716,53 @@ async function detectWow( }), ]); const [currentSummary, previousSummary] = summary.value ?? [[], []]; + const [currentCustomEvents, previousCustomEvents] = customEvents.value ?? [ + [], + [], + ]; const [currentErrors, previousErrors] = errors.value ?? [[], []]; const [currentRevenue, previousRevenue] = revenue.value ?? [[], []]; const [currentVitals, previousVitals] = vitals.value ?? [[], []]; const signals: DetectedSignal[] = []; + const currentEventsByName = mapRowsByStringField(currentCustomEvents, "name"); + const previousEventsByName = mapRowsByStringField( + previousCustomEvents, + "name" + ); + const customEventNames = new Set([ + ...currentEventsByName.keys(), + ...previousEventsByName.keys(), + ]); + for (const eventName of customEventNames) { + const currentUsers = numberField( + currentEventsByName.get(eventName), + "unique_users" + ); + const previousUsers = numberField( + previousEventsByName.get(eventName), + "unique_users" + ); + const deltaPercent = safeDeltaPercent(currentUsers, previousUsers); + if ( + Math.abs(deltaPercent) < WOW_CUSTOM_EVENT_THRESHOLD || + Math.abs(currentUsers - previousUsers) < FILTER_CUSTOM_EVENT_MIN_DELTA || + Math.max(currentUsers, previousUsers) < FILTER_CUSTOM_EVENT_MIN_PEAK + ) { + continue; + } + const signal = makeWowSignal( + `custom_event:${eventName}`, + `“${eventName}” users`, + currentUsers, + previousUsers, + currentTo + ); + if (currentUsers === 0 && previousUsers > 0) { + signal.kind = "missing_expected_data"; + } + signals.push(signal); + } for (const metric of ANOMALY_METRICS) { const currentValue = numberField(currentSummary[0], metric.summaryField); @@ -760,24 +827,110 @@ async function detectWow( } } - const revNow = numberField(currentRevenue[0], "total_revenue"); - const revPrev = numberField(previousRevenue[0], "total_revenue"); - const revenueTransactions = Math.max( - numberField(currentRevenue[0], "total_transactions"), - numberField(previousRevenue[0], "total_transactions") - ); - const meaningfulRevenueChange = - Math.abs(revNow - revPrev) >= REVENUE_MIN_ABSOLUTE_CHANGE || - revenueTransactions >= REVENUE_MIN_TRANSACTIONS; - if ((revNow > 0 || revPrev > 0) && meaningfulRevenueChange) { - const pct = revPrev === 0 ? 100 : safeDeltaPercent(revNow, revPrev); + const currentRevenueByCurrency = mapRevenueRowsByCurrency(currentRevenue); + const previousRevenueByCurrency = mapRevenueRowsByCurrency(previousRevenue); + const revenueCurrencies = new Set([ + ...currentRevenueByCurrency.keys(), + ...previousRevenueByCurrency.keys(), + ]); + for (const currency of revenueCurrencies) { + const current = currentRevenueByCurrency.get(currency); + const previous = previousRevenueByCurrency.get(currency); + const currencySuffix = currency ? ` (${currency})` : ""; + const revNow = numberField(current, "total_revenue"); + const revPrev = numberField(previous, "total_revenue"); + const revenueTransactions = Math.max( + numberField(current, "total_transactions"), + numberField(previous, "total_transactions") + ); + const meaningfulRevenueChange = + Math.abs(revNow - revPrev) >= REVENUE_MIN_ABSOLUTE_CHANGE || + revenueTransactions >= REVENUE_MIN_TRANSACTIONS; + if ((revNow > 0 || revPrev > 0) && meaningfulRevenueChange) { + const pct = revPrev === 0 ? 100 : safeDeltaPercent(revNow, revPrev); + if ( + Math.abs(pct) >= WOW_REVENUE_THRESHOLD || + (revPrev === 0 && revNow > 0) + ) { + const signal = makeWowSignal( + "revenue", + `Revenue${currencySuffix}`, + revNow, + revPrev, + currentTo + ); + if (currency) { + signal.currency = currency; + } + signals.push(signal); + } + } + + const failedPaymentsNow = numberField(current, "failed_payment_attempts"); + const failedPaymentsPrev = numberField(previous, "failed_payment_attempts"); + const successfulPaymentsNow = numberField( + current, + "successful_payment_attempts" + ); + const successfulPaymentsPrev = numberField( + previous, + "successful_payment_attempts" + ); + const paymentFailureRateNow = numberField(current, "payment_failure_rate"); + const paymentFailureRatePrev = numberField( + previous, + "payment_failure_rate" + ); + const observedFailureEventTypesNow = numberField( + current, + "observed_failure_event_types" + ); + const observedFailureEventTypesPrev = numberField( + previous, + "observed_failure_event_types" + ); + const isFailureRateIncrease = + paymentFailureRateNow > paymentFailureRatePrev; + // Event types observed in a date range are occurrences, not proof of the + // endpoint's subscription coverage. A regression needs current evidence; + // a recovery needs failure observations in both periods. + const hasFailureObservations = + observedFailureEventTypesNow > 0 && + (isFailureRateIncrease || observedFailureEventTypesPrev > 0); + const paymentAttemptsNow = failedPaymentsNow + successfulPaymentsNow; + const paymentAttemptsPrev = failedPaymentsPrev + successfulPaymentsPrev; + const hasMaterialFailureSample = + (failedPaymentsNow >= PAYMENT_FAILURE_MIN_FAILED && + paymentAttemptsNow >= PAYMENT_FAILURE_MIN_TOTAL_ATTEMPTS) || + (failedPaymentsPrev >= PAYMENT_FAILURE_MIN_FAILED && + paymentAttemptsPrev >= PAYMENT_FAILURE_MIN_TOTAL_ATTEMPTS); + const failureRateDelta = safeDeltaPercent( + paymentFailureRateNow, + paymentFailureRatePrev + ); if ( - Math.abs(pct) >= WOW_REVENUE_THRESHOLD || - (revPrev === 0 && revNow > 0) + hasFailureObservations && + hasMaterialFailureSample && + paymentAttemptsNow >= PAYMENT_FAILURE_MIN_TOTAL_ATTEMPTS && + paymentAttemptsPrev >= PAYMENT_FAILURE_MIN_TOTAL_ATTEMPTS && + Math.max(paymentFailureRateNow, paymentFailureRatePrev) >= + PAYMENT_FAILURE_MIN_RATE && + Math.abs(paymentFailureRateNow - paymentFailureRatePrev) >= + PAYMENT_FAILURE_MIN_POINT_CHANGE && + Math.abs(failureRateDelta) >= WOW_PAYMENT_FAILURE_THRESHOLD ) { - signals.push( - makeWowSignal("revenue", "Revenue", revNow, revPrev, currentTo) + const signal = makeWowSignal( + "payment_failure_rate", + `Payment failure rate${currencySuffix}`, + paymentFailureRateNow, + paymentFailureRatePrev, + currentTo, + true ); + if (currency) { + signal.currency = currency; + } + signals.push(signal); } } @@ -790,9 +943,11 @@ async function detectWow( const curVal = numberField(cur, "p75"); const prevVal = numberField(prev, "p75"); const curSamples = numberField(cur, "samples"); + const prevSamples = numberField(prev, "samples"); if ( curSamples < 10 || + prevSamples < 10 || prevVal === 0 || curVal === 0 || curVal > VITALS_MAX_PLAUSIBLE[metricName] || @@ -815,7 +970,7 @@ async function detectWow( } return { - failedFamilies: [summary, errors, revenue, vitals].filter( + failedFamilies: [summary, customEvents, errors, revenue, vitals].filter( (result) => result.failed ).length, signals, diff --git a/apps/insights/src/funnel-detection.test.ts b/apps/insights/src/funnel-detection.test.ts index 20f2a4e94..ca8507daa 100644 --- a/apps/insights/src/funnel-detection.test.ts +++ b/apps/insights/src/funnel-detection.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "bun:test"; import { validateInvestigationDecision } from "@databuddy/ai/insights/validate"; +import type { InvestigationEvidence } from "@databuddy/shared/insights"; import dayjs from "dayjs"; import type { DetectSignalsParams } from "./detection"; import { @@ -10,6 +11,7 @@ import { type FunnelGoalDeps, type GoalConversion, type GoalDef, + UNLINKED_COMPLETIONS_QUERY, } from "./funnel-detection"; import { prepareInvestigation } from "./investigation"; import { terminalDecisionFromEvidence } from "./terminal-decision"; @@ -78,6 +80,25 @@ function makeDeps(overrides: Partial): FunnelGoalDeps { }; } +function withVerifiedRemediation( + investigation: ReturnType +): InvestigationEvidence[] { + const definition = investigation.evidence.find( + (item) => item.kind === "definition" + ); + if (!(definition && investigation.signal.expectation)) { + throw new Error("Expected a confirmed definition repair fixture"); + } + return [ + ...investigation.evidence, + { + ...definition, + evidenceId: `${definition.evidenceId}:verified`, + remediation: investigation.signal.expectation, + }, + ]; +} + describe("detectFunnelGoalSignals", () => { it("uses the goal filters for both completions and the visitor denominator", async () => { const filters = [ @@ -94,18 +115,13 @@ describe("detectFunnelGoalSignals", () => { observed.push(denominatorFilters); return 50; }, - processGoalAnalytics: async ( - _steps, + processGoalConversionCount: async ( + _step, completionFilters, - _params, - totalUsers + _params ) => { - observed.push(completionFilters, totalUsers); - return { - overall_conversion_rate: 20, - total_users_completed: 10, - total_users_entered: 50, - } as never; + observed.push(completionFilters); + return 10; }, }); @@ -114,19 +130,125 @@ describe("detectFunnelGoalSignals", () => { { from: "2026-05-22", to: "2026-05-28" } ); - expect(observed).toEqual([filters, filters, 50]); + expect(observed).toEqual([filters, filters]); expect(result).toEqual({ completions: 10, entrants: 50, rate: 20 }); }); it("does not infer a definition completion from site-wide revenue", () => { const deps = defaultFunnelGoalDeps("test-site", TODAY.toDate(), { getTotalWebsiteUsers: async () => 0, - processGoalAnalytics: async () => ({}) as never, + processGoalConversionCount: async () => 0, }); expect(deps.confirmCompletion).toBeUndefined(); }); + it("confirms only the exact unlinked event target", async () => { + const observed: unknown[] = []; + const deps = defaultFunnelGoalDeps("test-site", TODAY.toDate(), { + confirmUnlinkedCompletions: async (...args) => { + observed.push(args.slice(0, 3)); + return 10; + }, + getTotalWebsiteUsers: async () => 0, + processGoalConversionCount: async () => 0, + }); + + const result = await deps.confirmCompletion?.({ + definitionId: GOAL.id, + definitionType: "goal", + expectation: { + currentCompletions: 0, + currentEntrants: 100, + definitionUpdatedAt: GOAL.updatedAt.toISOString(), + eventName: GOAL.target, + instruction: "Restore tracking", + kind: "tracking", + previousCompletions: 20, + }, + filters: [], + range: { from: "2026-05-22", to: "2026-05-28" }, + }); + + expect(observed).toEqual([ + [ + "test-site", + "sign_up", + { from: "2026-05-22", to: "2026-05-28" }, + ], + ]); + expect(result).toEqual({ count: 10, source: "server_completions" }); + }); + + it("does not treat identity-linked events as evidence of a missing link", () => { + expect(UNLINKED_COMPLETIONS_QUERY).toContain( + "ifNull(profile_id, '') = ''" + ); + expect(UNLINKED_COMPLETIONS_QUERY).toContain( + "ifNull(anonymous_id, '') = ''" + ); + expect(UNLINKED_COMPLETIONS_QUERY).toContain( + "ifNull(session_id, '') = ''" + ); + }); + + it("does not confirm a filtered definition with unscoped event counts", async () => { + let confirmationQueries = 0; + const deps = defaultFunnelGoalDeps("test-site", TODAY.toDate(), { + confirmUnlinkedCompletions: async () => { + confirmationQueries += 1; + return 100; + }, + getTotalWebsiteUsers: async () => 0, + processGoalConversionCount: async () => 0, + }); + + const result = await deps.confirmCompletion?.({ + definitionId: GOAL.id, + definitionType: "goal", + expectation: { + currentCompletions: 0, + currentEntrants: 100, + definitionUpdatedAt: GOAL.updatedAt.toISOString(), + eventName: GOAL.target, + instruction: "Restore tracking", + kind: "tracking", + previousCompletions: 20, + }, + filters: [{ field: "country", operator: "equals", value: "PS" }], + range: { from: "2026-05-22", to: "2026-05-28" }, + }); + + expect(result).toBeUndefined(); + expect(confirmationQueries).toBe(0); + }); + + it("confirms the exact identity-link defect from one matching record", async () => { + const deps = defaultFunnelGoalDeps("test-site", TODAY.toDate(), { + confirmUnlinkedCompletions: async () => 1, + getTotalWebsiteUsers: async () => 0, + processGoalConversionCount: async () => 0, + }); + + const result = await deps.confirmCompletion?.({ + definitionId: GOAL.id, + definitionType: "goal", + expectation: { + currentCompletions: 0, + currentEntrants: 100, + definitionUpdatedAt: GOAL.updatedAt.toISOString(), + eventName: GOAL.target, + instruction: "Restore tracking", + kind: "tracking", + previousCompletions: 100, + }, + filters: [], + range: { from: "2026-05-22", to: "2026-05-28" }, + }); + + expect(result).toEqual({ count: 1, source: "server_completions" }); + }); + it("returns empty when nothing is configured", async () => { const signals = await detectFunnelGoalSignals(PARAMS, TODAY, makeDeps({})); expect(signals).toEqual([]); @@ -381,6 +503,7 @@ describe("detectFunnelGoalSignals", () => { definitionId: "f1", definitionType: "funnel", expectation: expect.objectContaining({ eventName: "purchase" }), + filters: [], range: { from: "2026-05-22", to: "2026-05-28" }, }, ]); @@ -388,16 +511,23 @@ describe("detectFunnelGoalSignals", () => { websiteId: PARAMS.websiteId, lookbackDays: PARAMS.lookbackDays, }); + expect( + terminalDecisionFromEvidence( + investigation.signal, + investigation.evidence + ) + ).toEqual({ disposition: "needs_context", gap: "expected_behavior" }); + const verifiedEvidence = withVerifiedRemediation(investigation); const decision = terminalDecisionFromEvidence( investigation.signal, - investigation.evidence + verifiedEvidence ); expect(decision).toMatchObject({ disposition: "action_ready" }); - expect( - validateInvestigationDecision({ - signal: investigation.signal, - evidence: investigation.evidence, - decision, + expect( + validateInvestigationDecision({ + signal: investigation.signal, + evidence: verifiedEvidence, + decision, }).insight ).toMatchObject({ remediationKind: "tracking", @@ -406,6 +536,68 @@ describe("detectFunnelGoalSignals", () => { expect(signals[0]?.definitionEvidence?.summary).not.toContain("2026-"); }); + it("turns identity-less exact-target records into a scoped repair", async () => { + let call = 0; + const [signal] = await detectFunnelGoalSignals( + PARAMS, + TODAY, + makeDeps({ + confirmCompletion: async () => ({ + count: 7, + source: "server_completions", + }), + fetchGoals: async () => [GOAL], + goalConversion: async () => { + call += 1; + return call === 1 + ? goalResult(0, 0, 100) + : goalResult(20, 20, 100); + }, + }) + ); + + expect(signal?.expectation).toMatchObject({ + confirmation: { + count: 7, + definitionId: GOAL.id, + definitionType: "goal", + source: "server_completions", + }, + instruction: + 'Link "sign_up" custom events to a Databuddy visitor or session so this goal can count them.', + }); + const investigation = prepareInvestigation(signal!, { + websiteId: PARAMS.websiteId, + lookbackDays: PARAMS.lookbackDays, + }); + expect( + terminalDecisionFromEvidence( + investigation.signal, + investigation.evidence + ) + ).toEqual({ disposition: "needs_context", gap: "expected_behavior" }); + const verifiedEvidence = withVerifiedRemediation(investigation); + const decision = terminalDecisionFromEvidence( + investigation.signal, + verifiedEvidence + ); + expect(decision).toMatchObject({ + disposition: "action_ready", + remediation: { + instruction: + 'Link "sign_up" custom events to a Databuddy visitor or session so this goal can count them.', + kind: "tracking", + }, + }); + const validation = validateInvestigationDecision({ + decision, + evidence: verifiedEvidence, + signal: investigation.signal, + }); + expect(validation.errors).toEqual([]); + expect(validation.insight).toMatchObject({ remediationKind: "tracking" }); + }); + it("keeps a missing purchase as needs-context when confirmation fails", async () => { let call = 0; const [signal] = await detectFunnelGoalSignals( @@ -488,7 +680,7 @@ describe("detectFunnelGoalSignals", () => { }); expect(investigation.signal.entity.label).toBe("Signup"); - expect(result.insight?.title).toBe("Signup needs context"); + expect(result.insight?.title).toBe("Signup conversion stopped"); expect(result.insight?.suggestion).toContain( "Did users complete Signup?" ); @@ -521,6 +713,42 @@ describe("detectFunnelGoalSignals", () => { expect(signals[0]?.kind).toBeUndefined(); }); + it("does not let recent definitions consume the bounded evaluation window", async () => { + const recent = Array.from({ length: 20 }, (_, index) => ({ + ...GOAL, + id: `recent-${index}`, + updatedAt: new Date("2026-05-20T00:00:00.000Z"), + })); + const evaluated = new Set(); + const diagnostics = { + activeDefinitionKeys: new Set(), + eligibleDefinitionKeys: new Set(), + failedDefinitions: 0, + truncatedDefinitions: 0, + }; + + await detectFunnelGoalSignals( + PARAMS, + TODAY, + makeDeps({ + fetchGoals: async () => [...recent, GOAL], + goalConversion: async (goal) => { + evaluated.add(goal.id); + return goalResult(20, 20, 100); + }, + }), + { diagnostics } + ); + + expect([...evaluated]).toEqual([GOAL.id]); + expect(diagnostics.activeDefinitionKeys).toEqual( + new Set([...recent.map((goal) => `goal:${goal.id}`), `goal:${GOAL.id}`]) + ); + expect(diagnostics.eligibleDefinitionKeys).toEqual( + new Set([`goal:${GOAL.id}`]) + ); + }); + it("evaluates definitions beyond the old ten-item cap", async () => { const goals = Array.from({ length: 11 }, (_, index) => ({ ...GOAL, @@ -545,10 +773,117 @@ describe("detectFunnelGoalSignals", () => { expect(signals.map((signal) => signal.metric)).toContain("goal:goal-11"); }); + it("bounds large definition sets and rotates coverage each week", async () => { + const goals = Array.from({ length: 100 }, (_, index) => ({ + ...GOAL, + id: `goal-${index.toString().padStart(3, "0")}`, + })); + const firstIds = new Set(); + const secondIds = new Set(); + const firstDiagnostics = { + failedDefinitions: 0, + truncatedDefinitions: 0, + }; + const secondDiagnostics = { + failedDefinitions: 0, + truncatedDefinitions: 0, + }; + + await detectFunnelGoalSignals( + PARAMS, + TODAY, + makeDeps({ + fetchGoals: async () => goals, + goalConversion: async (goal) => { + firstIds.add(goal.id); + return goalResult(20, 20, 100); + }, + }), + { diagnostics: firstDiagnostics } + ); + await detectFunnelGoalSignals( + PARAMS, + TODAY.add(7, "day"), + makeDeps({ + fetchGoals: async () => goals, + goalConversion: async (goal) => { + secondIds.add(goal.id); + return goalResult(20, 20, 100); + }, + }), + { diagnostics: secondDiagnostics } + ); + + expect(firstIds.size).toBe(16); + expect(secondIds.size).toBe(16); + expect([...firstIds].sort()).not.toEqual([...secondIds].sort()); + expect(firstDiagnostics).toEqual({ + failedDefinitions: 0, + truncatedDefinitions: 84, + }); + expect(secondDiagnostics).toEqual(firstDiagnostics); + }); + + it("uses a bounded definition window with exact total diagnostics", async () => { + const goals = Array.from({ length: 16 }, (_, index) => ({ + ...GOAL, + id: `selected-${index.toString().padStart(2, "0")}`, + })); + const eligibleKeys = [ + ...goals.map((goal) => `goal:${goal.id}`), + ...Array.from({ length: 84 }, (_, index) => `goal:other-${index}`), + ]; + const evaluated = new Set(); + const diagnostics = { + failedDefinitions: 0, + truncatedDefinitions: 0, + }; + let legacyFetches = 0; + let rotation: number | undefined; + + await detectFunnelGoalSignals( + PARAMS, + TODAY, + makeDeps({ + fetchDefinitionWindow: async (value) => { + rotation = value; + return { + activeKeys: eligibleKeys, + eligibleKeys, + funnels: [], + goals, + total: 100, + }; + }, + fetchFunnels: async () => { + legacyFetches += 1; + return []; + }, + fetchGoals: async () => { + legacyFetches += 1; + return []; + }, + goalConversion: async (goal) => { + evaluated.add(goal.id); + return goalResult(20, 20, 100); + }, + }), + { diagnostics } + ); + + expect(Number.isSafeInteger(rotation)).toBe(true); + expect(legacyFetches).toBe(0); + expect(evaluated.size).toBe(16); + expect(diagnostics).toEqual({ + failedDefinitions: 0, + truncatedDefinitions: 84, + }); + }); + it("isolates one failed definition and keeps a valid sibling", async () => { const failedGoal = { ...GOAL, id: "failed-goal" }; const validGoal = { ...GOAL, id: "valid-goal", name: "Purchase" }; - const diagnostics = { failedDefinitions: 0 }; + const diagnostics = { failedDefinitions: 0, truncatedDefinitions: 0 }; const calls = new Map(); const signals = await detectFunnelGoalSignals( PARAMS, @@ -617,9 +952,9 @@ describe("detectFunnelGoalSignals", () => { TODAY, makeDeps({ fetchGoals: async () => goals, - goalConversion: async (goal) => { + goalConversion: async () => { calls += 1; - if (goal.id === "goal-0") { + if (calls === 1) { throw abortError; } await blocked; diff --git a/apps/insights/src/funnel-detection.ts b/apps/insights/src/funnel-detection.ts index 78c82184c..c94a640cf 100644 --- a/apps/insights/src/funnel-detection.ts +++ b/apps/insights/src/funnel-detection.ts @@ -1,4 +1,5 @@ -import { and, db, eq, isNull, lte, sql } from "@databuddy/db"; +import { db, sql } from "@databuddy/db"; +import { chQuery } from "@databuddy/db/clickhouse"; import { type DataFilter, funnelDefinitions, @@ -8,9 +9,13 @@ import { import { type AnalyticsStep, getTotalWebsiteUsers, - processFunnelAnalytics, - processGoalAnalytics, + processFunnelConversionCounts, + processGoalConversionCount, } from "@databuddy/rpc/analytics-utils"; +import { + normalizeFunnelSteps, + toAnalyticsSteps, +} from "@databuddy/rpc/funnel-steps"; import type { InvestigationExpectation, WeekOverWeekPeriod, @@ -35,6 +40,8 @@ const MIN_ENTRANTS = 30; const MIN_COMPLETIONS = 10; const DEFINITION_QUERY_CONCURRENCY = 4; const DEFINITION_DETECTION_TIMEOUT_MS = 45_000; +const MAX_DEFINITIONS_PER_RUN = 16; +const DEFINITION_ROTATION_DAYS = 7; export interface FunnelDef { createdAt: Date; @@ -70,13 +77,25 @@ export interface GoalConversion { rate: number; } +export interface FunnelGoalDefinitionWindow { + activeKeys: string[]; + eligibleKeys: string[]; + funnels: FunnelDef[]; + goals: GoalDef[]; + total: number; +} + export interface FunnelGoalDeps { confirmCompletion?: ( request: CompletionConfirmationRequest, abortSignal?: AbortSignal ) => Promise; - fetchFunnels: () => Promise; - fetchGoals: () => Promise; + fetchDefinitionWindow?: ( + rotation: number, + comparisonStart: Date + ) => Promise; + fetchFunnels?: () => Promise; + fetchGoals?: () => Promise; funnelConversion: ( funnel: FunnelDef, range: PeriodRange, @@ -93,6 +112,7 @@ interface CompletionConfirmationRequest { definitionId: string; definitionType: "funnel" | "goal"; expectation: InvestigationExpectation; + filters: DataFilter[]; range: PeriodRange; } @@ -104,26 +124,376 @@ type CompletionConfirmation = | undefined; export interface FunnelGoalDetectionDiagnostics { + activeDefinitionKeys?: Set; + eligibleDefinitionKeys?: Set; + evaluatedDefinitionKeys?: Set; failedDefinitions: number; + failureMessages?: string[]; + truncatedDefinitions: number; +} + +function definitionRotation(asOf: dayjs.Dayjs): number { + const day = Math.floor(asOf.startOf("day").valueOf() / 86_400_000); + return Math.floor(day / DEFINITION_ROTATION_DAYS); +} + +function rotatingDefinitionWindow( + funnels: FunnelDef[], + goalDefs: GoalDef[], + asOf: dayjs.Dayjs, + comparisonStart: Date +): { + activeKeys: string[]; + eligibleKeys: string[]; + funnels: FunnelDef[]; + goals: GoalDef[]; + truncated: number; +} { + const activeKeys = [ + ...funnels.map((definition) => `funnel:${definition.id}`), + ...goalDefs.map((definition) => `goal:${definition.id}`), + ].sort((left, right) => left.localeCompare(right)); + const definitions = [ + ...funnels + .filter((definition) => + definitionPredatesComparison(definition, comparisonStart) + ) + .map((definition) => ({ + definition, + key: `funnel:${definition.id}`, + type: "funnel" as const, + })), + ...goalDefs + .filter((definition) => + definitionPredatesComparison(definition, comparisonStart) + ) + .map((definition) => ({ + definition, + key: `goal:${definition.id}`, + type: "goal" as const, + })), + ].sort((left, right) => left.key.localeCompare(right.key)); + let selected = definitions; + if (definitions.length > MAX_DEFINITIONS_PER_RUN) { + const rotation = definitionRotation(asOf); + const start = (rotation * MAX_DEFINITIONS_PER_RUN) % definitions.length; + selected = Array.from( + { length: MAX_DEFINITIONS_PER_RUN }, + (_, index) => definitions[(start + index) % definitions.length] + ); + } + return { + activeKeys, + eligibleKeys: definitions.map((item) => item.key), + funnels: selected + .filter((item) => item.type === "funnel") + .map((item) => item.definition as FunnelDef), + goals: selected + .filter((item) => item.type === "goal") + .map((item) => item.definition as GoalDef), + truncated: definitions.length - selected.length, + }; +} + +async function loadDefinitionWindow( + deps: FunnelGoalDeps, + asOf: dayjs.Dayjs, + comparisonStart: Date +): Promise<{ + activeKeys: string[]; + eligibleKeys: string[]; + funnels: FunnelDef[]; + goals: GoalDef[]; + truncated: number; +}> { + if (deps.fetchDefinitionWindow) { + const selected = await deps.fetchDefinitionWindow( + definitionRotation(asOf), + comparisonStart + ); + const evaluated = selected.funnels.length + selected.goals.length; + const activeKeySet = new Set(selected.activeKeys); + const eligibleKeySet = new Set(selected.eligibleKeys); + if ( + !Number.isSafeInteger(selected.total) || + selected.total < 0 || + activeKeySet.size !== selected.activeKeys.length || + selected.eligibleKeys.length !== selected.total || + eligibleKeySet.size !== selected.total || + !selected.eligibleKeys.every((key) => activeKeySet.has(key)) || + evaluated !== Math.min(selected.total, MAX_DEFINITIONS_PER_RUN) || + ![...selected.funnels, ...selected.goals].every((definition) => + eligibleKeySet.has( + `${"steps" in definition ? "funnel" : "goal"}:${definition.id}` + ) + ) || + ![...selected.funnels, ...selected.goals].every((definition) => + definitionPredatesComparison(definition, comparisonStart) + ) + ) { + throw new Error("Definition window returned an invalid selection"); + } + return { + activeKeys: selected.activeKeys, + eligibleKeys: selected.eligibleKeys, + funnels: selected.funnels, + goals: selected.goals, + truncated: selected.total - evaluated, + }; + } + if (!(deps.fetchFunnels && deps.fetchGoals)) { + throw new Error("Funnel and goal definition dependencies are missing"); + } + const [funnels, goals] = await Promise.all([ + deps.fetchFunnels(), + deps.fetchGoals(), + ]); + return rotatingDefinitionWindow(funnels, goals, asOf, comparisonStart); } interface GoalConversionDependencies { + confirmUnlinkedCompletions?: ( + websiteId: string, + eventName: string, + range: PeriodRange, + abortSignal?: AbortSignal + ) => Promise; getTotalWebsiteUsers: typeof getTotalWebsiteUsers; - processGoalAnalytics: typeof processGoalAnalytics; + processGoalConversionCount: typeof processGoalConversionCount; +} + +export const UNLINKED_COMPLETIONS_QUERY = `SELECT sum(count) AS count + FROM ( + SELECT count() AS count + FROM analytics.custom_events + WHERE owner_id = {websiteId:String} + AND event_name = {eventName:String} + AND ifNull(profile_id, '') = '' + AND ifNull(anonymous_id, '') = '' + AND ifNull(session_id, '') = '' + AND timestamp >= parseDateTimeBestEffort({from:String}) + AND timestamp < parseDateTimeBestEffort({toExclusive:String}) + UNION ALL + SELECT count() AS count + FROM analytics.custom_events + WHERE website_id = {websiteId:String} + AND owner_id != {websiteId:String} + AND event_name = {eventName:String} + AND ifNull(profile_id, '') = '' + AND ifNull(anonymous_id, '') = '' + AND ifNull(session_id, '') = '' + AND timestamp >= parseDateTimeBestEffort({from:String}) + AND timestamp < parseDateTimeBestEffort({toExclusive:String}) + )`; + +async function countUnlinkedCompletions( + websiteId: string, + eventName: string, + range: PeriodRange, + abortSignal?: AbortSignal +): Promise { + const [row] = await chQuery<{ count: number | string }>( + UNLINKED_COMPLETIONS_QUERY, + { + eventName, + from: range.from, + toExclusive: dayjs(range.to).add(1, "day").format("YYYY-MM-DD"), + websiteId, + }, + { abort_signal: abortSignal } + ); + const count = Number(row?.count ?? 0); + return Number.isSafeInteger(count) && count > 0 ? count : 0; } const DEFAULT_GOAL_CONVERSION_DEPENDENCIES: GoalConversionDependencies = { + confirmUnlinkedCompletions: countUnlinkedCompletions, getTotalWebsiteUsers, - processGoalAnalytics, + processGoalConversionCount, +}; + +type DefinitionWindowRow = Record & { + activeKeys: string[]; + createdAt: Date | null; + definitionType: "funnel" | "goal" | null; + eligibleKeys: string[]; + filters: DataFilter[] | null; + goalType: GoalDef["type"] | null; + id: string | null; + name: string | null; + steps: FunnelStep[] | null; + target: string | null; + totalCount: number | string; + updatedAt: Date | null; }; -function toAnalyticsSteps(steps: FunnelStep[]): AnalyticsStep[] { - return steps.map((step, index) => ({ - step_number: index + 1, - type: step.type === "PAGE_VIEW" ? "PAGE_VIEW" : "EVENT", - target: step.target, - name: step.name, - })); +async function fetchDefinitionWindow( + websiteId: string, + comparisonStart: Date, + rotation: number +): Promise { + const result = await db.execute(sql` + with active as ( + select + 'funnel'::text as definition_type, + ${funnelDefinitions.id} as id, + ('funnel:' || ${funnelDefinitions.id}) collate "C" as definition_key + from ${funnelDefinitions} + where ${funnelDefinitions.websiteId} = ${websiteId} + and ${funnelDefinitions.isActive} = true + and ${funnelDefinitions.deletedAt} is null + union all + select + 'goal'::text as definition_type, + ${goals.id} as id, + ('goal:' || ${goals.id}) collate "C" as definition_key + from ${goals} + where ${goals.websiteId} = ${websiteId} + and ${goals.isActive} = true + and ${goals.deletedAt} is null + ), eligible as ( + select + 'funnel'::text as definition_type, + ${funnelDefinitions.id} as id, + ('funnel:' || ${funnelDefinitions.id}) collate "C" as definition_key + from ${funnelDefinitions} + where ${funnelDefinitions.websiteId} = ${websiteId} + and ${funnelDefinitions.isActive} = true + and ${funnelDefinitions.deletedAt} is null + and ${funnelDefinitions.createdAt} <= ${comparisonStart} + and ${funnelDefinitions.updatedAt} <= ${comparisonStart} + and case + when jsonb_typeof(${funnelDefinitions.steps}) = 'array' + then jsonb_array_length(${funnelDefinitions.steps}) + else 0 + end > 1 + union all + select + 'goal'::text as definition_type, + ${goals.id} as id, + ('goal:' || ${goals.id}) collate "C" as definition_key + from ${goals} + where ${goals.websiteId} = ${websiteId} + and ${goals.isActive} = true + and ${goals.deletedAt} is null + and ${goals.createdAt} <= ${comparisonStart} + and ${goals.updatedAt} <= ${comparisonStart} + ), ranked as ( + select + definition_type, + id, + (row_number() over (order by definition_key) - 1)::bigint as position, + count(*) over ()::int as total_count + from eligible + ), rotated as ( + select + definition_type, + id, + total_count, + ( + position + - ((${rotation}::bigint * ${MAX_DEFINITIONS_PER_RUN}::bigint) % total_count::bigint) + + total_count::bigint + ) % total_count::bigint as distance + from ranked + ), selected as materialized ( + select definition_type, id, total_count, distance + from rotated + order by distance + limit ${MAX_DEFINITIONS_PER_RUN} + ), metadata as ( + select + coalesce( + (select array_agg(definition_key::text order by definition_key) from active), + array[]::text[] + ) as active_keys, + coalesce( + (select array_agg(definition_key::text order by definition_key) from eligible), + array[]::text[] + ) as eligible_keys, + (select count(*)::int from eligible) as total_count + ) + select + selected.definition_type as "definitionType", + selected.id, + metadata.total_count as "totalCount", + metadata.active_keys as "activeKeys", + metadata.eligible_keys as "eligibleKeys", + coalesce(${funnelDefinitions.name}, ${goals.name}) as name, + coalesce(${funnelDefinitions.filters}, ${goals.filters}) as filters, + coalesce(${funnelDefinitions.createdAt}, ${goals.createdAt}) as "createdAt", + coalesce(${funnelDefinitions.updatedAt}, ${goals.updatedAt}) as "updatedAt", + ${funnelDefinitions.steps} as steps, + ${goals.target} as target, + ${goals.type} as "goalType" + from metadata + left join selected on true + left join ${funnelDefinitions} + on selected.definition_type = 'funnel' + and ${funnelDefinitions.id} = selected.id + left join ${goals} + on selected.definition_type = 'goal' + and ${goals.id} = selected.id + order by selected.distance nulls last + `); + const funnels: FunnelDef[] = []; + const goalDefs: GoalDef[] = []; + for (const row of result.rows) { + if (!(row.definitionType && row.id)) { + continue; + } + if (row.definitionType === "funnel") { + if ( + row.createdAt === null || + row.name === null || + row.steps === null || + row.updatedAt === null + ) { + throw new Error(`Selected funnel ${row.id} has no steps`); + } + funnels.push({ + createdAt: row.createdAt, + filters: row.filters, + id: row.id, + name: row.name, + steps: normalizeFunnelSteps(row.steps), + updatedAt: row.updatedAt, + }); + continue; + } + if ( + row.createdAt === null || + row.goalType === null || + row.name === null || + row.target === null || + row.updatedAt === null + ) { + throw new Error(`Selected goal ${row.id} is incomplete`); + } + goalDefs.push({ + createdAt: row.createdAt, + filters: row.filters, + id: row.id, + name: row.name, + target: row.target, + type: row.goalType, + updatedAt: row.updatedAt, + }); + } + const total = Number(result.rows[0]?.totalCount ?? 0); + if ( + !Number.isSafeInteger(total) || + total < funnels.length + goalDefs.length + ) { + throw new Error("Definition selection returned an invalid total"); + } + return { + activeKeys: result.rows[0]?.activeKeys ?? [], + eligibleKeys: result.rows[0]?.eligibleKeys ?? [], + funnels, + goals: goalDefs, + total, + }; } export function defaultFunnelGoalDeps( @@ -132,52 +502,38 @@ export function defaultFunnelGoalDeps( goalDependencies: GoalConversionDependencies = DEFAULT_GOAL_CONVERSION_DEPENDENCIES ): FunnelGoalDeps { return { - fetchFunnels: () => - db - .select({ - createdAt: funnelDefinitions.createdAt, - filters: funnelDefinitions.filters, - id: funnelDefinitions.id, - name: funnelDefinitions.name, - steps: funnelDefinitions.steps, - updatedAt: funnelDefinitions.updatedAt, - }) - .from(funnelDefinitions) - .where( - and( - eq(funnelDefinitions.websiteId, websiteId), - eq(funnelDefinitions.isActive, true), - isNull(funnelDefinitions.deletedAt), - lte(funnelDefinitions.createdAt, asOf), - lte(funnelDefinitions.updatedAt, asOf), - sql`jsonb_array_length(${funnelDefinitions.steps}) > 1` - ) - ) - .orderBy(funnelDefinitions.createdAt), - fetchGoals: () => - db - .select({ - createdAt: goals.createdAt, - filters: goals.filters, - id: goals.id, - name: goals.name, - target: goals.target, - type: goals.type, - updatedAt: goals.updatedAt, - }) - .from(goals) - .where( - and( - eq(goals.websiteId, websiteId), - eq(goals.isActive, true), - isNull(goals.deletedAt), - lte(goals.createdAt, asOf), - lte(goals.updatedAt, asOf) - ) - ) - .orderBy(goals.createdAt), + ...(goalDependencies.confirmUnlinkedCompletions + ? { + confirmCompletion: async ( + request: CompletionConfirmationRequest, + abortSignal?: AbortSignal + ) => { + if (request.filters.length > 0) { + return; + } + const count = await goalDependencies.confirmUnlinkedCompletions?.( + websiteId, + request.expectation.eventName, + request.range, + abortSignal + ); + return count && count > 0 + ? { count, source: "server_completions" as const } + : undefined; + }, + } + : {}), + fetchDefinitionWindow: (rotation, comparisonStart) => + fetchDefinitionWindow( + websiteId, + comparisonStart < asOf ? comparisonStart : asOf, + rotation + ), funnelConversion: async (funnel, range, abortSignal) => { - const analytics = await processFunnelAnalytics( + if (funnel.steps.length < 2) { + throw new Error(`Funnel ${funnel.id} has fewer than two valid steps`); + } + const analytics = await processFunnelConversionCounts( toAnalyticsSteps(funnel.steps), funnel.filters ?? [], { @@ -185,29 +541,23 @@ export function defaultFunnelGoalDeps( startDate: range.from, endDate: `${range.to} 23:59:59`, }, - undefined, abortSignal ); return { - rate: analytics.overall_conversion_rate, - entrants: analytics.total_users_entered, - completions: analytics.total_users_completed, - steps: analytics.steps_analytics.map((step) => ({ - stepNumber: step.step_number, - users: step.users, - })), + rate: analytics.rate, + entrants: analytics.entrants, + completions: analytics.completions, + steps: analytics.steps, }; }, goalConversion: async (goal, range, abortSignal) => { const filters = goal.filters ?? []; - const steps: AnalyticsStep[] = [ - { - step_number: 1, - type: goal.type === "PAGE_VIEW" ? "PAGE_VIEW" : "EVENT", - target: goal.target, - name: goal.name, - }, - ]; + const step: AnalyticsStep = { + step_number: 1, + type: goal.type === "PAGE_VIEW" ? "PAGE_VIEW" : "EVENT", + target: goal.target, + name: goal.name, + }; const totalWebsiteUsers = await goalDependencies.getTotalWebsiteUsers( websiteId, range.from, @@ -215,21 +565,23 @@ export function defaultFunnelGoalDeps( filters, abortSignal ); - const analytics = await goalDependencies.processGoalAnalytics( - steps, + const completionCount = await goalDependencies.processGoalConversionCount( + step, filters, { websiteId, startDate: range.from, endDate: `${range.to} 23:59:59`, }, - totalWebsiteUsers, abortSignal ); return { - rate: analytics.overall_conversion_rate, - completions: analytics.total_users_completed, - entrants: analytics.total_users_entered, + rate: + totalWebsiteUsers > 0 + ? Math.round((completionCount / totalWebsiteUsers) * 10_000) / 100 + : 0, + completions: completionCount, + entrants: totalWebsiteUsers, }; }, }; @@ -287,13 +639,11 @@ async function withDetectionDeadline( function definitionPredatesComparison( definition: Pick, - previousFrom: string, - timezone: string + comparisonStart: Date ): boolean { - const comparisonStart = dayjs.tz(previousFrom, timezone).startOf("day"); return !( - dayjs(definition.createdAt).isAfter(comparisonStart) || - dayjs(definition.updatedAt).isAfter(comparisonStart) + definition.createdAt > comparisonStart || + definition.updatedAt > comparisonStart ); } @@ -319,6 +669,9 @@ function handleDefinitionFailure( } if (context.diagnostics) { context.diagnostics.failedDefinitions += 1; + context.diagnostics.failureMessages?.push( + (error instanceof Error ? error.message : String(error)).slice(0, 500) + ); } emitInsightsEvent("warn", "generation.detection.definition_failed", { website_id: context.websiteId, @@ -403,7 +756,11 @@ function missingFunnelExpectation( async function confirmExpectation( expectation: InvestigationExpectation, - definition: { id: string; type: "funnel" | "goal" }, + definition: { + filters: DataFilter[] | null; + id: string; + type: "funnel" | "goal"; + }, range: PeriodRange, deps: FunnelGoalDeps, abortSignal: AbortSignal, @@ -418,6 +775,7 @@ async function confirmExpectation( definitionId: definition.id, definitionType: definition.type, expectation, + filters: definition.filters ?? [], range, }, abortSignal @@ -425,6 +783,12 @@ async function confirmExpectation( return confirmation ? { ...expectation, + instruction: + confirmation.source === "server_completions" + ? instruction( + `Link "${expectation.eventName}" custom events to a Databuddy visitor or session so this ${definition.type} can count them.` + ) + : expectation.instruction, confirmation: { ...confirmation, definitionId: definition.id, @@ -459,7 +823,7 @@ function confirmationSummary( const scope = confirmation.definitionType === "funnel" ? "funnel" : "goal"; return confirmation.source === "revenue_transactions" ? ` Independent revenue tracking recorded ${confirmation.count} transactions for this ${scope}.` - : ` Independent server tracking recorded ${confirmation.count} completions for this ${scope}.`; + : ` Independent event tracking found ${confirmation.count} identity-less records matching this ${scope}'s exact event target.`; } export function detectFunnelGoalSignals( @@ -481,6 +845,10 @@ export function detectFunnelGoalSignals( from: window.previousFrom, to: window.previousTo, }; + const comparisonStart = dayjs + .tz(previous.from, params.timezone) + .startOf("day") + .toDate(); const activeDeps = deps ?? @@ -489,29 +857,42 @@ export function detectFunnelGoalSignals( today.toDate(), DEFAULT_GOAL_CONVERSION_DEPENDENCIES ); - const [funnels, goalDefs] = await Promise.all([ - activeDeps.fetchFunnels(), - activeDeps.fetchGoals(), - ]); + const selected = await loadDefinitionWindow( + activeDeps, + today, + comparisonStart + ); + const funnels = selected.funnels; + const goalDefs = selected.goals; + for (const key of selected.activeKeys) { + options.diagnostics?.activeDefinitionKeys?.add(key); + } + for (const key of selected.eligibleKeys) { + options.diagnostics?.eligibleDefinitionKeys?.add(key); + } + if (selected.truncated > 0) { + if (options.diagnostics) { + options.diagnostics.truncatedDefinitions += selected.truncated; + } + emitInsightsEvent("info", "generation.detection.definitions_rotated", { + website_id: params.websiteId, + evaluated_definitions: funnels.length + goalDefs.length, + truncated_definitions: selected.truncated, + }); + } const funnelSignals = await mapWithConcurrency( funnels, DEFINITION_QUERY_CONCURRENCY, async (funnel) => { try { - if ( - !definitionPredatesComparison( - funnel, - previous.from, - params.timezone - ) - ) { - return null; - } const [cur, prev] = await Promise.all([ activeDeps.funnelConversion(funnel, current, deadlineSignal), activeDeps.funnelConversion(funnel, previous, deadlineSignal), ]); + options.diagnostics?.evaluatedDefinitionKeys?.add( + `funnel:${funnel.id}` + ); if ( cur.entrants < MIN_ENTRANTS || prev.entrants < MIN_ENTRANTS || @@ -543,7 +924,11 @@ export function detectFunnelGoalSignals( const expectation = missingExpectation ? await confirmExpectation( missingExpectation, - { id: funnel.id, type: "funnel" }, + { + filters: funnel.filters, + id: funnel.id, + type: "funnel", + }, current, activeDeps, deadlineSignal, @@ -603,7 +988,7 @@ export function detectFunnelGoalSignals( expectation.confirmation.source === "revenue_transactions" ? "Flow revenue transactions" - : "Server completions", + : "Identity-less event records", current: expectation.confirmation.count, format: "number" as const, }, @@ -644,15 +1029,11 @@ export function detectFunnelGoalSignals( DEFINITION_QUERY_CONCURRENCY, async (goal) => { try { - if ( - !definitionPredatesComparison(goal, previous.from, params.timezone) - ) { - return null; - } const [cur, prev] = await Promise.all([ activeDeps.goalConversion(goal, current, deadlineSignal), activeDeps.goalConversion(goal, previous, deadlineSignal), ]); + options.diagnostics?.evaluatedDefinitionKeys?.add(`goal:${goal.id}`); if ( cur.entrants < MIN_ENTRANTS || prev.entrants < MIN_ENTRANTS || @@ -680,7 +1061,7 @@ export function detectFunnelGoalSignals( const expectation = missingExpectation ? await confirmExpectation( missingExpectation, - { id: goal.id, type: "goal" }, + { filters: goal.filters, id: goal.id, type: "goal" }, current, activeDeps, deadlineSignal, @@ -709,7 +1090,7 @@ export function detectFunnelGoalSignals( expectation.confirmation.source === "revenue_transactions" ? "Flow revenue transactions" - : "Server completions", + : "Identity-less event records", current: expectation.confirmation.count, format: "number" as const, }, diff --git a/apps/insights/src/generation-sources.test.ts b/apps/insights/src/generation-sources.test.ts index 48e6a5936..7231c7d42 100644 --- a/apps/insights/src/generation-sources.test.ts +++ b/apps/insights/src/generation-sources.test.ts @@ -18,6 +18,44 @@ const trafficDrop: DetectedSignal = { severity: "critical", }; +const confirmedGoalDrop: DetectedSignal = { + baseline: 20, + current: 0, + definitionEvidence: { + metrics: [ + { label: "Entrants", current: 100, format: "number" }, + { label: "Completions", current: 0, previous: 20, format: "number" }, + ], + queryType: "goals_summary", + summary: "Signup had 0 completions from 100 eligible visitors.", + }, + definitionUpdatedAt: "2026-06-01T00:00:00.000Z", + deltaPercent: -100, + detectedAt: "2026-07-11", + direction: "down", + entityLabel: "Signup", + expectation: { + confirmation: { + count: 12, + definitionId: "signup", + definitionType: "goal", + source: "revenue_transactions", + }, + currentCompletions: 0, + currentEntrants: 100, + definitionUpdatedAt: "2026-06-01T00:00:00.000Z", + eventName: "sign_up", + instruction: 'Restore the "sign_up" event when Signup completes.', + kind: "tracking", + previousCompletions: 20, + }, + kind: "missing_expected_data", + label: "Signup completion rate", + method: "wow", + metric: "goal:signup", + severity: "critical", +}; + describe("fixture investigation sources", () => { it("runs the production investigation path using only required sources", async () => { const calls: string[] = []; @@ -84,7 +122,7 @@ describe("fixture investigation sources", () => { engineId: "deterministic/v1", status: "completed", }); - expect(artifact.insight?.title).toBe("Visitors drop needs context"); + expect(artifact.insight?.title).toBe("Visitors fell 70%"); expect(calls.sort()).toEqual( [ "annotations", @@ -99,6 +137,100 @@ describe("fixture investigation sources", () => { ); }); + it("fails closed when a confirmed repair is not verified by the final definition read", async () => { + const calls: string[] = []; + const sources: InvestigationSources = { + hasTrackedData: async () => true, + detectMetricSignals: async () => [], + detectDefinitionSignals: async () => [confirmedGoalDrop], + loadObservations: async () => new Map(), + fetchAnnotations: async () => [], + createEvidenceReader: async (params) => { + calls.push("evidence reader"); + return async () => [ + { + evidenceId: "fixture:goal:stale-definition", + signalKey: params.signal.signalKey, + kind: "definition", + source: "product", + queryType: "goals_summary", + entity: params.signal.entity, + period: "current", + range: params.signal.period.current, + status: "ok", + rowCount: 1, + summary: "Signup no longer matches the detected definition.", + }, + ]; + }, + createServiceAuth: async () => undefined, + }; + + const artifact = await investigateWebsiteWithSources( + { + asOf: "2026-07-12", + domain: "example.com", + organizationId: "fixture-org", + timezone: "UTC", + websiteId: "fixture-site", + }, + sources + ); + + expect(calls).toEqual(["evidence reader"]); + expect(artifact.decision).toEqual({ + disposition: "needs_context", + gap: "expected_behavior", + }); + expect(artifact.insight).not.toHaveProperty("remediationKind"); + expect(artifact.evidence.every((item) => !item.remediation)).toBe(true); + }); + + it("carries bounded validation errors on an invalid evaluation artifact", async () => { + const sources: InvestigationSources = { + hasTrackedData: async () => true, + detectMetricSignals: async () => [trafficDrop], + detectDefinitionSignals: async () => [], + loadObservations: async () => new Map(), + fetchAnnotations: async () => [], + createEvidenceReader: async (params) => async () => [ + { + evidenceId: "fixture:failed-query", + signalKey: params.signal.signalKey, + kind: "breakdown", + source: "web", + queryType: "top_referrers", + period: "current", + range: params.signal.period.current, + status: "failed", + rowCount: 0, + error: "Warehouse unavailable", + }, + ], + createServiceAuth: async () => undefined, + }; + + const artifact = await investigateWebsiteWithSources( + { + asOf: "2026-07-12", + domain: "example.com", + organizationId: "fixture-org", + timezone: "UTC", + websiteId: "fixture-site", + }, + sources + ); + + expect(artifact.status).toBe("invalid_output"); + expect(artifact.validationErrors).toContain( + "A failed Databuddy query must be retried, not turned into a terminal decision." + ); + expect(artifact.validationErrors?.length).toBeLessThanOrEqual(5); + expect( + artifact.validationErrors?.every((error) => error.length <= 300) + ).toBe(true); + }); + it("does not fall through to downstream production reads after fixture preflight", async () => { const forbidden = () => { throw new Error("downstream source should not run"); @@ -177,6 +309,12 @@ describe("fixture investigation sources", () => { detectionComplete: false, detectedSignals: [], insight: null, + recoveryCoverage: { + definitionFailureMessages: [], + failedDefinitions: 0, + failedMetricFamilies: 1, + rotatedDefinitions: 0, + }, signal: null, status: "deferred", }); @@ -184,4 +322,47 @@ describe("fixture investigation sources", () => { ["definition detection", "metric detection"].sort() ); }); + + it("does not claim complete definition coverage after an active definition edit", async () => { + const forbidden = () => { + throw new Error("incomplete scan should stop before downstream reads"); + }; + const sources: InvestigationSources = { + createEvidenceReader: forbidden, + createServiceAuth: forbidden, + detectDefinitionSignals: async (_params, _today, _deps, options) => { + options?.diagnostics?.activeDefinitionKeys?.add("goal:edited"); + return []; + }, + detectMetricSignals: async () => [], + fetchAnnotations: forbidden, + hasTrackedData: async () => true, + loadObservations: forbidden, + }; + + const artifact = await investigateWebsiteWithSources( + { + asOf: "2026-07-12", + domain: "example.com", + organizationId: "fixture-org", + timezone: "UTC", + websiteId: "fixture-site", + }, + sources + ); + + expect(artifact).toMatchObject({ + detectionComplete: false, + recoveryCoverage: { + activeDefinitionKeys: ["goal:edited"], + definitionFailureMessages: [], + definitions: false, + eligibleDefinitionKeys: [], + failedDefinitions: 0, + failedMetricFamilies: 0, + rotatedDefinitions: 0, + }, + status: "deferred", + }); + }); }); diff --git a/apps/insights/src/generation.ts b/apps/insights/src/generation.ts index 084342875..f3fc9a0fb 100644 --- a/apps/insights/src/generation.ts +++ b/apps/insights/src/generation.ts @@ -101,6 +101,17 @@ export interface WebsiteInvestigationArtifact { engineId: string; evidence: InvestigationEvidence[]; insight: GeneratedInsight | null; + recoveryCoverage?: { + activeDefinitionKeys?: string[]; + failedDefinitions?: number; + definitionFailureMessages?: string[]; + failedMetricFamilies?: number; + definitionKeys: string[]; + definitions: boolean; + eligibleDefinitionKeys?: string[]; + metrics: boolean; + rotatedDefinitions?: number; + }; signal: InvestigationSignal | null; status: | "completed" @@ -108,6 +119,7 @@ export interface WebsiteInvestigationArtifact { | "invalid_output" | "no_data" | "no_signals"; + validationErrors?: string[]; } function getComparisonPeriod( @@ -161,6 +173,7 @@ export interface InvestigationSources { interface DeterministicInvestigationResult { decision: InvestigationDecision | null; insight: GeneratedInsight | null; + validationErrors: string[]; } function normalizeAsOf(asOf: Date | string, timezone: string): dayjs.Dayjs { @@ -179,6 +192,7 @@ function emptyInvestigationArtifact(params: { detectionComplete: boolean; detectedSignals: DetectedSignal[]; engineId: string; + recoveryCoverage?: WebsiteInvestigationArtifact["recoveryCoverage"]; status: "deferred" | "no_data" | "no_signals"; }): WebsiteInvestigationArtifact { return { @@ -189,6 +203,7 @@ function emptyInvestigationArtifact(params: { evidence: [], insight: null, engineId: params.engineId, + recoveryCoverage: params.recoveryCoverage, signal: null, status: params.status, }; @@ -276,6 +291,15 @@ async function investigateWebsiteCore( detectionComplete: false, detectedSignals: [], engineId, + recoveryCoverage: { + definitionFailureMessages: [], + failedDefinitions: 0, + failedMetricFamilies: 0, + definitionKeys: [], + definitions: false, + metrics: false, + rotatedDefinitions: 0, + }, status: "no_data", }); } @@ -288,7 +312,12 @@ async function investigateWebsiteCore( const detectionAbortSignal = AbortSignal.timeout(DETECTION_TIMEOUT_MS); const metricDiagnostics: DetectionDiagnostics = { failedFamilies: 0 }; const definitionDiagnostics: FunnelGoalDetectionDiagnostics = { + activeDefinitionKeys: new Set(), + eligibleDefinitionKeys: new Set(), + evaluatedDefinitionKeys: new Set(), + failureMessages: [], failedDefinitions: 0, + truncatedDefinitions: 0, }; const [metricSignals, funnelGoalSignals] = await Promise.all([ runtime.sources.detectMetricSignals( @@ -302,9 +331,34 @@ async function investigateWebsiteCore( diagnostics: definitionDiagnostics, }), ]); + const activeDefinitionKeys = [ + ...(definitionDiagnostics.activeDefinitionKeys ?? new Set()), + ]; + const eligibleDefinitionKeys = [ + ...(definitionDiagnostics.eligibleDefinitionKeys ?? new Set()), + ]; + const eligibleDefinitionKeySet = new Set(eligibleDefinitionKeys); + const allActiveDefinitionsAreComparable = activeDefinitionKeys.every((key) => + eligibleDefinitionKeySet.has(key) + ); + const recoveryCoverage = { + activeDefinitionKeys, + definitionFailureMessages: definitionDiagnostics.failureMessages ?? [], + failedDefinitions: definitionDiagnostics.failedDefinitions, + failedMetricFamilies: metricDiagnostics.failedFamilies, + definitionKeys: [ + ...(definitionDiagnostics.evaluatedDefinitionKeys ?? new Set()), + ], + definitions: + definitionDiagnostics.failedDefinitions === 0 && + definitionDiagnostics.truncatedDefinitions === 0 && + allActiveDefinitionsAreComparable, + eligibleDefinitionKeys, + metrics: metricDiagnostics.failedFamilies === 0, + rotatedDefinitions: definitionDiagnostics.truncatedDefinitions, + }; const detectionComplete = - metricDiagnostics.failedFamilies === 0 && - definitionDiagnostics.failedDefinitions === 0; + recoveryCoverage.metrics && recoveryCoverage.definitions; const detectedSignals = rankSignals([...metricSignals, ...funnelGoalSignals]); if (detectedSignals.length === 0) { @@ -325,6 +379,7 @@ async function investigateWebsiteCore( detectionComplete, detectedSignals, engineId, + recoveryCoverage, status: "deferred", }); } @@ -340,6 +395,7 @@ async function investigateWebsiteCore( detectionComplete, detectedSignals, engineId, + recoveryCoverage, status: "no_signals", }); } @@ -369,6 +425,7 @@ async function investigateWebsiteCore( detectionComplete, detectedSignals, engineId, + recoveryCoverage, status: "deferred", }); } @@ -417,6 +474,7 @@ async function investigateWebsiteCore( evidence: [...evidenceById.values()], insight: null, engineId, + recoveryCoverage, signal: investigation.signal, status: "completed", }; @@ -456,8 +514,10 @@ async function investigateWebsiteCore( evidence: [...evidenceById.values()], insight: investigationResult.insight, engineId, + recoveryCoverage, signal: investigation.signal, status: investigationResult.decision ? "completed" : "invalid_output", + validationErrors: investigationResult.validationErrors, }; } @@ -497,7 +557,10 @@ function evidenceReadRequest( input: { period: "current", queries: [{ type: "uptime_summary" }] }, }; } - if (signal.metric.key === "revenue") { + if ( + signal.metric.key === "revenue" || + signal.metric.key === "payment_failure_rate" + ) { return { name: "web_metrics", input: { period: "both", queries: [{ type: "revenue_overview" }] }, @@ -516,12 +579,7 @@ function evidenceReadRequest( }; } const type = - signal.metric.key === "pageviews" - ? "top_pages" - : signal.metric.key === "bounce_rate" || - signal.metric.key === "session_duration" - ? "entry_pages" - : "top_referrers"; + signal.metric.key === "pageviews" ? "top_pages" : "top_referrers"; return { name: "web_metrics", input: { @@ -588,7 +646,13 @@ async function runDeterministicInvestigation(params: { "Insights investigation stopped without valid supporting evidence" ); } - return { decision: null, insight: null }; + return { + decision: null, + insight: null, + validationErrors: validated.errors + .slice(0, 5) + .map((error) => error.slice(0, 300)), + }; } const decision = validated.decision; const insight = validated.insight; @@ -619,7 +683,7 @@ async function runDeterministicInvestigation(params: { generated_candidate_count: insight ? 1 : 0, }); } - return { decision, insight }; + return { decision, insight, validationErrors: [] }; } catch (error) { if (params.runtimeMode === "production") { captureInsightsError(error, "generation.investigation.failed", { @@ -752,8 +816,23 @@ export async function generateWebsiteInsights( websiteId: site.id, runId: input.runId, detectedSignals: analysis.detectedSignals, - canRecover: analysis.status !== "no_data" && analysis.detectionComplete, + canRecover: + analysis.status !== "no_data" && + (analysis.recoveryCoverage?.metrics ?? analysis.detectionComplete), + canRecoverConversion: + analysis.status !== "no_data" && + (analysis.recoveryCoverage?.definitions ?? analysis.detectionComplete), + recoverableConversionKeys: + analysis.recoveryCoverage?.definitionKeys ?? [], + activeConversionKeys: analysis.recoveryCoverage?.activeDefinitionKeys, retiredSignalKey: retiredSignalKeyForOutcome({ + coverage: { + definitions: + analysis.recoveryCoverage?.definitions ?? + analysis.detectionComplete, + metrics: + analysis.recoveryCoverage?.metrics ?? analysis.detectionComplete, + }, disposition: analysis.decision?.disposition, hasInsight: analysis.insight !== null, signalKey: analysis.signal?.signalKey, diff --git a/apps/insights/src/idempotency.integration.test.ts b/apps/insights/src/idempotency.integration.test.ts index 2f1c5e043..098d1c8b6 100644 --- a/apps/insights/src/idempotency.integration.test.ts +++ b/apps/insights/src/idempotency.integration.test.ts @@ -583,6 +583,7 @@ describeIntegration("insights idempotency integration", () => { totalItems: 1, }); await db().insert(insightRunItems).values({ + attempts: 2, id: itemId, queueJobId, runId, @@ -781,6 +782,59 @@ describeIntegration("insights idempotency integration", () => { expect(itemsAfter).toEqual(itemsBefore); }); + it("does not let a duplicate worker claim an already-running item", async () => { + const { processInsightsJob } = await import("./jobs"); + const org = await insertOrganization(); + const website = await insertWebsite({ organizationId: org.id }); + const runId = randomUUIDv7(); + const itemId = randomUUIDv7(); + const queueJobId = `job-${itemId}`; + await db().insert(insightRuns).values({ + id: runId, + organizationId: org.id, + status: "queued", + totalItems: 1, + }); + await db().insert(insightRunItems).values({ + id: itemId, + organizationId: org.id, + queueJobId, + runId, + status: "running", + websiteId: website.id, + }); + + await expect( + processInsightsJob({ + attemptsMade: 1, + data: { + itemId, + organizationId: org.id, + reason: "manual", + runId, + websiteId: website.id, + }, + id: queueJobId, + name: INSIGHTS_GENERATE_WEBSITE_JOB_NAME, + opts: { attempts: 3 }, + }) + ).rejects.toThrow("already running"); + + const [run] = await db() + .select({ status: insightRuns.status }) + .from(insightRuns) + .where(eq(insightRuns.id, runId)); + const [item] = await db() + .select({ + attempts: insightRunItems.attempts, + status: insightRunItems.status, + }) + .from(insightRunItems) + .where(eq(insightRunItems.id, itemId)); + expect(run.status).toBe("queued"); + expect(item).toEqual({ attempts: 0, status: "running" }); + }); + it("recovers a prepared item after its final worker dies before item success", async () => { const org = await insertOrganization(); const website = await insertWebsite({ organizationId: org.id }); @@ -908,7 +962,6 @@ describeIntegration("insights idempotency integration", () => { .where(eq(insightRuns.id, runId)); expect(result).toMatchObject({ - failedItems: 0, keptItems: 1, scannedItems: 1, syncedRuns: 1, @@ -977,6 +1030,37 @@ describeIntegration("insights idempotency integration", () => { expect(run.finishedAt).not.toEqual(staleAt); }); + it("keeps a settled run's original finish time when status is unchanged", async () => { + const org = await insertOrganization(); + const website = await insertWebsite({ organizationId: org.id }); + const runId = randomUUIDv7(); + const finishedAt = new Date("2025-01-01T00:00:00.000Z"); + await db().insert(insightRuns).values({ + completedItems: 1, + finishedAt, + id: runId, + organizationId: org.id, + status: "succeeded", + totalItems: 1, + }); + await db().insert(insightRunItems).values({ + finishedAt, + id: randomUUIDv7(), + organizationId: org.id, + runId, + status: "succeeded", + websiteId: website.id, + }); + + await syncRunStatus(runId); + + const [run] = await db() + .select({ finishedAt: insightRuns.finishedAt }) + .from(insightRuns) + .where(eq(insightRuns.id, runId)); + expect(run.finishedAt).toEqual(finishedAt); + }); + it("locks the run before deriving status from its items", async () => { const org = await insertOrganization(); const website = await insertWebsite({ organizationId: org.id }); @@ -1147,7 +1231,6 @@ describeIntegration("insights idempotency integration", () => { .where(eq(insightRuns.id, runId)); expect(result).toMatchObject({ - failedItems: 0, keptItems: 1, scannedItems: 1, }); @@ -1177,6 +1260,7 @@ describeIntegration("insights idempotency integration", () => { totalItems: 1, }); await db().insert(insightRunItems).values({ + attempts: 2, id: itemId, queueJobId, runId, diff --git a/apps/insights/src/index.ts b/apps/insights/src/index.ts index 98ad89534..adfa47246 100644 --- a/apps/insights/src/index.ts +++ b/apps/insights/src/index.ts @@ -1,7 +1,12 @@ import { setAiRequestLoggerProvider } from "@databuddy/ai/lib/request-logger"; import { db, shutdownPostgres, sql } from "@databuddy/db"; import { readBooleanEnv } from "@databuddy/env/boolean"; -import { closeInsightsQueue, getInsightsQueue } from "@databuddy/redis"; +import { + closeInsightsQueue, + getInsightsQueue, + INSIGHTS_DISPATCH_JOB_NAME, + INSIGHTS_MAINTENANCE_JOB_NAME, +} from "@databuddy/redis"; import { databuddyEvlogRedaction } from "@databuddy/shared/evlog-redaction"; import { Elysia } from "elysia"; import { initLogger } from "evlog"; @@ -15,6 +20,8 @@ import { import { ensureInsightsDispatchSchedule, ensureInsightsMaintenanceSchedule, + isScheduledDispatchActive, + removeInsightsSchedules, } from "./scheduler"; import { startInsightsWorker } from "./worker"; @@ -23,6 +30,7 @@ const environment = process.env.RAILWAY_ENVIRONMENT_NAME ?? (process.env.NODE_ENV === "development" ? "development" : "production"); const workerEnabled = readBooleanEnv("INSIGHTS_WORKER_ENABLED"); +const scheduledDispatchEnabled = isScheduledDispatchActive(workerEnabled); const DRAIN_TIMEOUT_MS = 10_000; initLogger({ @@ -124,18 +132,21 @@ async function shutdown(signal: string) { async function startRuntime() { emitInsightsEvent("info", "lifecycle.starting", { + scheduled_dispatch_enabled: scheduledDispatchEnabled, worker_enabled: workerEnabled, }); if (workerEnabled) { - insightsWorker = startInsightsWorker(); await Promise.all([ ensureInsightsDispatchSchedule(), ensureInsightsMaintenanceSchedule(), ]); + insightsWorker = startInsightsWorker(); emitInsightsEvent("info", "lifecycle.started", { + scheduled_dispatch_enabled: scheduledDispatchEnabled, worker_enabled: true, }); } else { + await removeInsightsSchedules(); emitInsightsEvent("info", "lifecycle.disabled", { worker_enabled: false, }); @@ -158,7 +169,7 @@ type ProbeResult = async function probe( name: string, - fn: () => Promise + fn: () => void | Promise ): Promise { const start = performance.now(); try { @@ -191,24 +202,46 @@ const app = new Elysia() ); }) .get("/health/status", async () => { - const [postgres, bullmqRedis] = await Promise.all([ + const [postgres, bullmqRedis, schedulers, worker] = await Promise.all([ probe("postgres", () => db.execute(sql`SELECT 1`).then(() => {})), probe("bullmqRedis", async () => { await getInsightsQueue().count(); }), + probe("schedulers", async () => { + const queue = getInsightsQueue(); + const [dispatch, maintenance] = await Promise.all([ + queue.getJobScheduler(INSIGHTS_DISPATCH_JOB_NAME), + queue.getJobScheduler(INSIGHTS_MAINTENANCE_JOB_NAME), + ]); + if (Boolean(dispatch) !== scheduledDispatchEnabled) { + throw new Error("Insight dispatch scheduler state is incorrect"); + } + if (Boolean(maintenance) !== workerEnabled) { + throw new Error("Insight maintenance scheduler state is incorrect"); + } + }), + probe("worker", () => { + if (workerEnabled && !insightsWorker?.isRunning()) { + throw new Error("Insights worker is not running"); + } + }), ]); - const services = { postgres, bullmqRedis }; + const services = { postgres, bullmqRedis, schedulers, worker }; const status = Object.values(services).every((s) => s.status === "ok") ? "ok" : "degraded"; return Response.json( - { status, workerEnabled, services }, + { status, workerEnabled, scheduledDispatchEnabled, services }, { status: status === "ok" ? 200 : 503 } ); }) - .get("/health", () => ({ status: "ok", workerEnabled })); + .get("/health", () => ({ + status: "ok", + workerEnabled, + scheduledDispatchEnabled, + })); export default { port: Number(process.env.PORT ?? 4002), diff --git a/apps/insights/src/investigation-flow.test.ts b/apps/insights/src/investigation-flow.test.ts index c790aebea..df4784a10 100644 --- a/apps/insights/src/investigation-flow.test.ts +++ b/apps/insights/src/investigation-flow.test.ts @@ -59,7 +59,7 @@ describe("scheduled investigation contract", () => { count: 12, definitionId: "signup", definitionType: "goal" as const, - source: "server_completions" as const, + source: "revenue_transactions" as const, }, definitionUpdatedAt: "2026-06-01T00:00:00.000Z", eventName: "sign_up", @@ -194,7 +194,7 @@ describe("scheduled investigation contract", () => { }); expect(result.errors).toEqual([]); expect(result.insight).toMatchObject({ - title: 'Goal "Signup" conversion needs context', + title: 'Goal "Signup" conversion stopped', }); expect(result.insight?.suggestion).toContain("Did users complete"); expect(result.insight?.suggestion).toContain( diff --git a/apps/insights/src/investigation.test.ts b/apps/insights/src/investigation.test.ts index cde659cca..3fd4b8f94 100644 --- a/apps/insights/src/investigation.test.ts +++ b/apps/insights/src/investigation.test.ts @@ -99,6 +99,59 @@ describe("prepareInvestigation", () => { }); }); + it("keeps revenue currencies in distinct signal identities", () => { + const usd = prepareInvestigation( + { ...baseSignal, metric: "revenue", label: "Revenue (USD)", currency: "USD" }, + { websiteId: "site-1", lookbackDays: 7 } + ).signal; + const eur = prepareInvestigation( + { ...baseSignal, metric: "revenue", label: "Revenue (EUR)", currency: "EUR" }, + { websiteId: "site-1", lookbackDays: 7 } + ).signal; + + expect(usd.currency).toBe("USD"); + expect(usd.signalKey).not.toBe(eur.signalKey); + }); + + it("uses the event name, not its metric label, as custom-event identity", () => { + const customEvent = prepareInvestigation( + { + ...baseSignal, + metric: "custom_event:checkout_completed", + label: "“checkout_completed” users", + }, + { websiteId: "site-1", lookbackDays: 7 } + ).signal; + + expect(customEvent.entity).toEqual({ + type: "event", + id: "checkout_completed", + label: "checkout_completed", + }); + }); + + it("keeps a long custom-event query target exact while bounding durable keys", () => { + const eventName = `checkout_${"step_".repeat(38)}`; + const customEvent = prepareInvestigation( + { + ...baseSignal, + metric: `custom_event:${eventName}`, + label: `“${eventName}” users`, + }, + { websiteId: "site-1", lookbackDays: 7 } + ).signal; + + expect(eventName.length).toBeGreaterThan(160); + expect(eventName.length).toBeLessThanOrEqual(256); + expect(customEvent.entity).toEqual({ + type: "event", + id: eventName, + label: eventName.slice(0, 120), + }); + expect(customEvent.metric.key.length).toBeLessThanOrEqual(160); + expect(customEvent.signalKey.length).toBeLessThanOrEqual(160); + }); + it("starts with only exact detector evidence", () => { const result = prepareInvestigation(baseSignal, { websiteId: "site-1", @@ -117,33 +170,62 @@ describe("prepareInvestigation", () => { ]); }); - it("reuses exact detector-owned goal evidence without another read", () => { + it("reuses detector goal context but re-reads a confirmed repair", () => { + const candidate = { + ...baseSignal, + definitionEvidence: { + metrics: [ + { + current: 0, + format: "number" as const, + label: "Completions", + previous: 20, + }, + ], + queryType: "goals_summary" as const, + summary: "Signup had 0 completions from 100 eligible visitors.", + }, + definitionUpdatedAt: "2026-06-01T00:00:00.000Z", + entityLabel: "Signup", + metric: "goal:goal-1", + }; const result = prepareInvestigation( + candidate, + { websiteId: "site-1", lookbackDays: 7 } + ); + + expect(needsAdditionalEvidence(result.signal, result.evidence)).toBe(false); + expect(needsAdditionalEvidence(result.signal, result.evidence.slice(0, 2))).toBe( + true + ); + + const confirmed = prepareInvestigation( { - ...baseSignal, - definitionEvidence: { - metrics: [ - { - current: 0, - format: "number", - label: "Completions", - previous: 20, - }, - ], - queryType: "goals_summary", - summary: "Signup had 0 completions from 100 eligible visitors.", + ...candidate, + expectation: { + confirmation: { + count: 12, + definitionId: "goal-1", + definitionType: "goal", + source: "revenue_transactions", + }, + definitionUpdatedAt: "2026-06-01T00:00:00.000Z", + eventName: "sign_up", + instruction: 'Restore the "sign_up" event when Signup completes.', + kind: "tracking", + previousCompletions: 20, + currentEntrants: 100, + currentCompletions: 0, }, - definitionUpdatedAt: "2026-06-01T00:00:00.000Z", - entityLabel: "Signup", - metric: "goal:goal-1", + kind: "missing_expected_data", }, { websiteId: "site-1", lookbackDays: 7 } ); - expect(needsAdditionalEvidence(result.signal, result.evidence)).toBe(false); - expect(needsAdditionalEvidence(result.signal, result.evidence.slice(0, 2))).toBe( + expect(needsAdditionalEvidence(confirmed.signal, confirmed.evidence)).toBe( true ); + expect(confirmed.evidence.some((item) => item.remediation)).toBe(false); }); it("ignores unscoped annotations", () => { diff --git a/apps/insights/src/investigation.ts b/apps/insights/src/investigation.ts index 783eb0686..eae3835db 100644 --- a/apps/insights/src/investigation.ts +++ b/apps/insights/src/investigation.ts @@ -35,6 +35,9 @@ export function needsAdditionalEvidence( if (signal.entity.type !== "goal" && signal.entity.type !== "funnel") { return true; } + if (signal.expectation?.confirmation) { + return true; + } return !evidence.some( (item) => item.kind === "definition" && @@ -133,7 +136,7 @@ export function signalKeyForMetric(metric: string): string { export function signalKeyForDetectedSignal( signal: Pick< DetectedSignal, - "definitionUpdatedAt" | "expectation" | "kind" | "metric" + "currency" | "definitionUpdatedAt" | "expectation" | "kind" | "metric" > ): string { const definitionVersion = @@ -141,22 +144,22 @@ export function signalKeyForDetectedSignal( (signal.kind === "missing_expected_data" ? signal.expectation?.definitionUpdatedAt : undefined); + const metric = signal.currency + ? `${signal.metric}:${signal.currency.toLowerCase()}` + : signal.metric; return signalKeyForMetric( - definitionVersion ? `${signal.metric}@${definitionVersion}` : signal.metric + definitionVersion ? `${metric}@${definitionVersion}` : metric ); } function metricFormat(metric: string): InsightMetric["format"] { if ( - metric === "bounce_rate" || + metric === "payment_failure_rate" || metric.startsWith("funnel:") || metric.startsWith("goal:") ) { return "percent"; } - if (metric === "session_duration") { - return "duration_s"; - } if (metric === "lcp" || metric === "inp") { return "duration_ms"; } @@ -164,7 +167,7 @@ function metricFormat(metric: string): InsightMetric["format"] { } function isLowerBetter(metric: string): boolean { - return ["bounce_rate", "error_count", "lcp", "inp"].includes(metric); + return ["error_count", "lcp", "inp", "payment_failure_rate"].includes(metric); } const SEVERITY_RANK = { critical: 2, warning: 1, info: 0 } as const; @@ -172,6 +175,7 @@ const SEVERITY_RANK = { critical: 2, warning: 1, info: 0 } as const; function isDirectSignal(signal: DetectedSignal): boolean { return ( signal.metric === "revenue" || + signal.metric === "payment_failure_rate" || signal.metric === "error_count" || signal.metric === "lcp" || signal.metric === "inp" || @@ -240,11 +244,8 @@ function insightType( ? "vitals_degraded" : "performance_improved"; } - if (signal.metric === "bounce_rate") { - return "bounce_rate_change"; - } - if (signal.metric === "session_duration") { - return "engagement_change"; + if (signal.metric === "payment_failure_rate") { + return signal.direction === "up" ? "quality_shift" : "positive_trend"; } if (signal.metric.startsWith("funnel:")) { return signal.direction === "down" ? "funnel_regression" : "positive_trend"; @@ -265,7 +266,8 @@ function insightType( function entity(signal: DetectedSignal): InvestigationSignal["entity"] { const [prefix, ...idParts] = signal.metric.split(":"); - const id = boundedKey(idParts.join(":").trim()); + const rawId = idParts.join(":").trim(); + const id = boundedKey(rawId); if (prefix === "funnel" || prefix === "goal") { return { type: prefix, @@ -274,7 +276,7 @@ function entity(signal: DetectedSignal): InvestigationSignal["entity"] { }; } if (prefix === "custom_event") { - return { type: "event", id, label: signal.label.slice(0, 120) }; + return { type: "event", id: rawId, label: rawId.slice(0, 120) }; } if (signal.metric === "error_count") { return { type: "error", id: signal.metric, label: signal.label }; @@ -296,7 +298,7 @@ function sourceForMetric(metric: string): InvestigationEvidence["source"] { ) { return "product"; } - if (metric === "revenue") { + if (metric === "revenue" || metric === "payment_failure_rate") { return "business"; } return "web"; @@ -381,6 +383,7 @@ export function prepareInvestigation( format: metricFormat(candidate.metric), }, changePercent: candidate.deltaPercent, + ...(candidate.currency ? { currency: candidate.currency } : {}), direction: candidate.direction, severity: candidate.severity, sentiment: improved ? "positive" : "negative", @@ -428,7 +431,6 @@ export function prepareInvestigation( rowCount: 1, summary: candidate.definitionEvidence.summary, metrics: candidate.definitionEvidence.metrics, - ...(candidate.expectation ? { remediation: candidate.expectation } : {}), }); } const dismissalAnnotations = annotations.filter( diff --git a/apps/insights/src/jobs.ts b/apps/insights/src/jobs.ts index bb2fbd4cc..0af7c2ee9 100644 --- a/apps/insights/src/jobs.ts +++ b/apps/insights/src/jobs.ts @@ -1,4 +1,13 @@ -import { and, db, eq, isNull, notInArray, sql } from "@databuddy/db"; +import { + and, + db, + eq, + isNull, + notInArray, + or, + sql, + withTransaction, +} from "@databuddy/db"; import { insightRunItems, insightRuns } from "@databuddy/db/schema"; import { INSIGHTS_DISPATCH_JOB_NAME, @@ -33,7 +42,10 @@ import { withInsightsLogContext, } from "./lib/evlog-insights"; import { processRollupJob } from "./rollup"; -import { dispatchDueInsightRuns } from "./scheduler"; +import { + dispatchDueInsightRuns, + scheduledDispatchOrganizations, +} from "./scheduler"; const SUCCESS_CHECKPOINT_ATTEMPTS = 3; const SUCCESSFUL_ITEM_STATUSES: ("skipped" | "succeeded")[] = [ @@ -52,6 +64,7 @@ type InsightsJob = Pick< >; interface CanonicalGenerateItem extends InsightRunIdentity { + attempts: number; errorMessage: string | null; queueJobId: string; reason: InsightsGenerateWebsiteJobData["reason"]; @@ -130,6 +143,7 @@ async function loadCanonicalGenerateItem( ): Promise { const [item] = await db .select({ + attempts: insightRunItems.attempts, errorMessage: insightRunItems.errorMessage, itemId: insightRunItems.id, organizationId: insightRunItems.organizationId, @@ -303,21 +317,8 @@ async function processGenerateWebsiteJob( } const now = new Date(); - const [, started] = await Promise.all([ - db - .update(insightRuns) - .set({ - status: "running", - startedAt: sql`coalesce(${insightRuns.startedAt}, ${now})`, - updatedAt: now, - }) - .where( - and( - eq(insightRuns.id, data.runId), - eq(insightRuns.organizationId, data.organizationId) - ) - ), - db + const started = await withTransaction(async (tx) => { + const claimed = await tx .update(insightRunItems) .set({ attempts: job.attemptsMade + 1, @@ -330,11 +331,34 @@ async function processGenerateWebsiteJob( .where( and( itemIdentityCondition(data), - notInArray(insightRunItems.status, SUCCESSFUL_ITEM_STATUSES) + or( + eq(insightRunItems.status, "queued"), + and( + eq(insightRunItems.status, "running"), + eq(insightRunItems.attempts, job.attemptsMade) + ) + ) ) ) - .returning({ id: insightRunItems.id }), - ]); + .returning({ id: insightRunItems.id }); + if (claimed.length === 0) { + return claimed; + } + await tx + .update(insightRuns) + .set({ + status: "running", + startedAt: sql`coalesce(${insightRuns.startedAt}, ${now})`, + updatedAt: now, + }) + .where( + and( + eq(insightRuns.id, data.runId), + eq(insightRuns.organizationId, data.organizationId) + ) + ); + return claimed; + }); if (started.length === 0) { const concurrentlyCompleted = await loadSuccessfulItem(data); if (concurrentlyCompleted) { @@ -344,9 +368,8 @@ async function processGenerateWebsiteJob( status: concurrentlyCompleted.status, }; } - throw new Error("Insight run item not found"); + throw new Error("Insight run item is already running or unavailable"); } - let result: GenerateWebsiteInsightsResult; try { result = await generateWebsiteInsights({ @@ -383,7 +406,16 @@ export async function processInsightsJob(job: InsightsJob) { try { let result: unknown; if (job.name === INSIGHTS_DISPATCH_JOB_NAME) { - result = await dispatchDueInsightRuns(); + const organizations = scheduledDispatchOrganizations(); + if (organizations === null || organizations.length > 0) { + result = await dispatchDueInsightRuns( + new Date(), + organizations ?? undefined + ); + } else { + result = { status: "disabled" }; + emitInsightsEvent("info", "scheduler.dispatch_job_ignored", context); + } } else if (job.name === INSIGHTS_MAINTENANCE_JOB_NAME) { result = await recoverStaleInsightRuns(); } else if (job.name === INSIGHTS_GENERATE_WEBSITE_JOB_NAME) { diff --git a/apps/insights/src/recovery.ts b/apps/insights/src/recovery.ts index cf76b59d6..d3350b130 100644 --- a/apps/insights/src/recovery.ts +++ b/apps/insights/src/recovery.ts @@ -5,14 +5,17 @@ import { eq, inArray, isNotNull, + isNull, lt, ne, notExists, or, + sql, } from "@databuddy/db"; import { insightRunEffects, insightRunItems, + insightRollups, insightRuns, type InsightRun, type InsightRunItem, @@ -20,10 +23,13 @@ import { } from "@databuddy/db/schema"; import { getInsightsQueue, + INSIGHTS_GENERATE_WEBSITE_JOB_NAME, INSIGHTS_JOB_TIMEOUT_MS, INSIGHTS_ROLLUP_JOB_NAME, + insightsWebsiteJobId, insightsRollupJobId, } from "@databuddy/redis"; +import { randomUUIDv7 } from "bun"; import { captureInsightsError, emitInsightsEvent, @@ -40,6 +46,7 @@ const DEFAULT_STALE_ITEM_MS = Math.max( const MIN_STALE_ITEM_MS = INSIGHTS_JOB_TIMEOUT_MS * 2; const MAX_STALE_ITEMS_PER_SWEEP = 100; const MAX_STALE_RUNS_PER_SWEEP = 100; +const ROLLUP_RANGE_COUNT = 3; const ACTIVE_QUEUE_STATES = new Set([ "active", @@ -51,14 +58,26 @@ const ACTIVE_QUEUE_STATES = new Set([ type RecoverableItem = Pick< InsightRunItem, - "id" | "queueJobId" | "runId" | "status" | "updatedAt" + | "id" + | "organizationId" + | "queueJobId" + | "runId" + | "status" + | "updatedAt" + | "websiteId" +> & + Pick; + +type RollupRun = Pick< + InsightRun, + "id" | "organizationId" | "reason" | "timezone" >; interface RunStatusSummary { completedItems: number; failedItems: number; queuedItems: number; - run: InsightRun | null; + run: RollupRun | null; runningItems: number; settled: boolean; skippedItems: number; @@ -67,13 +86,19 @@ interface RunStatusSummary { } export interface InsightRecoveryResult { - failedItems: number; keptItems: number; + requeuedItems: number; + requeuedRollups: number; scannedItems: number; scannedRuns: number; syncedRuns: number; } +type MissingRollupRun = Pick< + InsightRun, + "completedItems" | "id" | "organizationId" | "reason" | "status" | "timezone" +>; + function parseDurationMs( value: string | undefined, fallback: number, @@ -105,35 +130,128 @@ export function getInsightsStaleItemMs( return parseDurationMs(value, DEFAULT_STALE_ITEM_MS, MIN_STALE_ITEM_MS); } -async function staleItemFailureReason( +type StaleQueueState = + | { kind: "active" } + | { kind: "missing"; reason: string } + | { kind: "terminal"; reason: string }; + +async function staleItemQueueState( item: RecoverableItem -): Promise { +): Promise { if (!item.queueJobId) { - return "Insight queue job id is missing after stale timeout"; + return { + kind: "missing", + reason: "Insight queue job id is missing after stale timeout", + }; } const job = await getInsightsQueue().getJob(item.queueJobId); if (!job) { - return "Insight queue job is missing after stale timeout"; + return { + kind: "missing", + reason: "Insight queue job is missing after stale timeout", + }; } const state = await job.getState(); if (ACTIVE_QUEUE_STATES.has(state)) { - return null; + return { kind: "active" }; } - return `Insight queue job is ${state} but the database item is still ${item.status}`; + return { + kind: "terminal", + reason: `Insight queue job is ${state} but the database item is still ${item.status}`, + }; +} + +async function requeueStaleItem( + item: RecoverableItem, + now: Date, + rotateJobId: boolean +): Promise { + const queueJobId = + rotateJobId || !item.queueJobId + ? `${insightsWebsiteJobId(item.runId, item.websiteId)}-recovery-${randomUUIDv7()}` + : item.queueJobId; + const claimed = await db + .update(insightRunItems) + .set({ + errorMessage: null, + finishedAt: null, + queueJobId, + startedAt: null, + status: "queued", + updatedAt: now, + }) + .where( + and( + eq(insightRunItems.id, item.id), + eq(insightRunItems.status, item.status), + eq(insightRunItems.updatedAt, item.updatedAt), + item.queueJobId + ? eq(insightRunItems.queueJobId, item.queueJobId) + : isNull(insightRunItems.queueJobId) + ) + ) + .returning({ id: insightRunItems.id }); + if (claimed.length === 0) { + return false; + } + + try { + await getInsightsQueue().add( + INSIGHTS_GENERATE_WEBSITE_JOB_NAME, + { + itemId: item.id, + organizationId: item.organizationId, + reason: item.reason, + requestedByUserId: item.requestedByUserId, + runId: item.runId, + websiteId: item.websiteId, + }, + { jobId: queueJobId } + ); + } catch (error) { + await db + .update(insightRunItems) + .set({ updatedAt: item.updatedAt }) + .where( + and( + eq(insightRunItems.id, item.id), + eq(insightRunItems.status, "queued"), + eq(insightRunItems.queueJobId, queueJobId), + eq(insightRunItems.updatedAt, now) + ) + ); + throw error; + } + + emitInsightsEvent("warn", "recovery.stale_job_requeued", { + item_id: item.id, + organization_id: item.organizationId, + queue_job_id: queueJobId, + run_id: item.runId, + previous_status: item.status, + rotated_job_id: queueJobId !== item.queueJobId, + website_id: item.websiteId, + }); + return true; } async function staleItems(cutoff: Date): Promise { return await db .select({ id: insightRunItems.id, + organizationId: insightRunItems.organizationId, queueJobId: insightRunItems.queueJobId, + reason: insightRuns.reason, + requestedByUserId: insightRuns.requestedByUserId, runId: insightRunItems.runId, status: insightRunItems.status, updatedAt: insightRunItems.updatedAt, + websiteId: insightRunItems.websiteId, }) .from(insightRunItems) + .innerJoin(insightRuns, eq(insightRuns.id, insightRunItems.runId)) .where( and( or( @@ -285,6 +403,11 @@ export async function syncRunStatus(runId: string): Promise { } const now = new Date(); + const finishedAt = settled + ? run?.status === status && run.finishedAt + ? run.finishedAt + : now + : null; await tx .update(insightRuns) .set({ @@ -292,7 +415,7 @@ export async function syncRunStatus(runId: string): Promise { errorMessage: settled && failedItems > 0 ? summarizeItemErrors(items) : null, failedItems, - finishedAt: settled ? now : null, + finishedAt, skippedItems, status, updatedAt: now, @@ -326,20 +449,47 @@ export async function syncRunStatus(runId: string): Promise { } export async function queueRollupIfSettled( - summary: RunStatusSummary -): Promise { + summary: RunStatusSummary, + options: { repairCompleted?: boolean } = {} +): Promise { if (!(summary.run && summary.settled && summary.completedItems > 0)) { - return; + return false; } if ( summary.status !== "succeeded" && summary.status !== "partially_succeeded" ) { - return; + return false; } try { - await getInsightsQueue().add( + const queue = getInsightsQueue(); + const jobId = insightsRollupJobId(summary.run.id); + const existing = await queue.getJob(jobId); + if (existing) { + const state = await existing.getState(); + if (ACTIVE_QUEUE_STATES.has(state)) { + return false; + } + if ( + state === "failed" || + (state === "completed" && options.repairCompleted) + ) { + await existing.retry(state, { + resetAttemptsMade: true, + resetAttemptsStarted: true, + }); + emitInsightsEvent("warn", "recovery.rollup_retried", { + run_id: summary.run.id, + organization_id: summary.run.organizationId, + previous_job_state: state, + }); + return true; + } + return false; + } + + await queue.add( INSIGHTS_ROLLUP_JOB_NAME, { organizationId: summary.run.organizationId, @@ -347,7 +497,7 @@ export async function queueRollupIfSettled( runId: summary.run.id, timezone: summary.run.timezone, }, - { jobId: insightsRollupJobId(summary.run.id) } + { jobId } ); emitInsightsEvent("info", "recovery.rollup_queued", { run_id: summary.run.id, @@ -355,14 +505,54 @@ export async function queueRollupIfSettled( run_status: summary.status, completed_items: summary.completedItems, }); + return true; } catch (error) { captureInsightsError(error, "recovery.rollup_queue_failed", { run_id: summary.run.id, organization_id: summary.run.organizationId, }); + return false; } } +async function latestSettledRunsMissingRollup(): Promise { + const result = await db.execute(sql` + with latest_settled as ( + select + ${insightRuns.id} as id, + ${insightRuns.organizationId} as "organizationId", + ${insightRuns.reason} as reason, + ${insightRuns.status} as status, + ${insightRuns.timezone} as timezone, + ${insightRuns.completedItems} as "completedItems", + row_number() over ( + partition by ${insightRuns.organizationId} + order by ${insightRuns.createdAt} desc, ${insightRuns.id} desc + ) as position + from ${insightRuns} + where ${insightRuns.status} in ('succeeded', 'partially_succeeded') + and ${insightRuns.completedItems} > 0 + and ${insightRuns.finishedAt} is not null + ) + select + latest_settled.id, + latest_settled."organizationId", + latest_settled.reason, + latest_settled.status, + latest_settled.timezone, + latest_settled."completedItems" + from latest_settled + where latest_settled.position = 1 + and ( + select count(*) + from ${insightRollups} + where ${insightRollups.runId} = latest_settled.id + ) < ${ROLLUP_RANGE_COUNT} + limit ${MAX_STALE_RUNS_PER_SWEEP} + `); + return result.rows; +} + export async function recoverStaleInsightRuns( now = new Date() ): Promise { @@ -370,8 +560,9 @@ export async function recoverStaleInsightRuns( const cutoff = new Date(now.getTime() - getInsightsStaleItemMs()); const items = await staleItems(cutoff); const affectedRunIds = new Set(); - let failedItems = 0; let keptItems = 0; + let requeuedItems = 0; + let requeuedRollups = 0; for (const item of items) { if (item.status === "failed") { @@ -388,8 +579,8 @@ export async function recoverStaleInsightRuns( continue; } - const reason = await staleItemFailureReason(item); - if (!reason) { + const queueState = await staleItemQueueState(item); + if (queueState.kind === "active") { keptItems += 1; continue; } @@ -403,43 +594,30 @@ export async function recoverStaleInsightRuns( }); continue; } - - const updated = await db - .update(insightRunItems) - .set({ - errorMessage: reason, - finishedAt: now, - status: "failed", - updatedAt: now, - }) - .where( - and( - eq(insightRunItems.id, item.id), - eq(insightRunItems.status, item.status), - eq(insightRunItems.updatedAt, item.updatedAt) - ) - ) - .returning({ id: insightRunItems.id }); - if (updated.length === 0) { + try { affectedRunIds.add(item.runId); + if ( + await requeueStaleItem( + item, + now, + item.status === "running" || queueState.kind === "terminal" + ) + ) { + requeuedItems += 1; + } else { + keptItems += 1; + } + } catch (error) { keptItems += 1; - emitInsightsEvent("info", "recovery.stale_item_changed", { + captureInsightsError(error, "recovery.stale_job_requeue_failed", { item_id: item.id, + organization_id: item.organizationId, queue_job_id: item.queueJobId, + queue_state: queueState.kind, run_id: item.runId, - previous_status: item.status, + website_id: item.websiteId, }); - continue; } - affectedRunIds.add(item.runId); - failedItems += 1; - emitInsightsEvent("warn", "recovery.stale_item_failed", { - item_id: item.id, - queue_job_id: item.queueJobId, - run_id: item.runId, - previous_status: item.status, - reason, - }); } const runIds = new Set([...affectedRunIds, ...(await staleRunIds(cutoff))]); @@ -449,17 +627,40 @@ export async function recoverStaleInsightRuns( await queueRollupIfSettled(summary); } + for (const run of await latestSettledRunsMissingRollup()) { + if ( + await queueRollupIfSettled( + { + completedItems: run.completedItems, + failedItems: 0, + queuedItems: 0, + run, + runningItems: 0, + settled: true, + skippedItems: 0, + status: run.status, + totalItems: run.completedItems, + }, + { repairCompleted: true } + ) + ) { + requeuedRollups += 1; + } + } + emitInsightsEvent("info", "recovery.sweep_completed", { duration_ms: Math.round(performance.now() - startedAt), - failed_items: failedItems, kept_items: keptItems, + requeued_items: requeuedItems, + requeued_rollups: requeuedRollups, scanned_items: items.length, synced_runs: runIds.size, }); return { - failedItems, keptItems, + requeuedItems, + requeuedRollups, scannedItems: items.length, scannedRuns: runIds.size, syncedRuns: runIds.size, diff --git a/apps/insights/src/resolution.test.ts b/apps/insights/src/resolution.test.ts index 988fad63b..aa921a01a 100644 --- a/apps/insights/src/resolution.test.ts +++ b/apps/insights/src/resolution.test.ts @@ -13,14 +13,16 @@ describe("retiredSignalKeyForOutcome", () => { it("retires an old finding only when the new decision is intentionally silent", () => { expect( retiredSignalKeyForOutcome({ - disposition: "monitor", + coverage: { definitions: true, metrics: true }, + disposition: "monitor", hasInsight: false, signalKey: "goal:signup", }) ).toBe("goal:signup"); expect( retiredSignalKeyForOutcome({ - disposition: "not_a_problem", + coverage: { definitions: true, metrics: true }, + disposition: "not_a_problem", hasInsight: false, signalKey: "goal:signup", }) @@ -30,12 +32,32 @@ describe("retiredSignalKeyForOutcome", () => { it("keeps a surfaced unresolved monitor open", () => { expect( retiredSignalKeyForOutcome({ - disposition: "monitor", + coverage: { definitions: true, metrics: true }, + disposition: "monitor", hasInsight: true, signalKey: "error_count", }) ).toBeUndefined(); }); + + it("uses the selected signal family instead of unrelated scan failures", () => { + expect( + retiredSignalKeyForOutcome({ + coverage: { definitions: false, metrics: true }, + disposition: "not_a_problem", + hasInsight: false, + signalKey: "visitors", + }) + ).toBe("visitors"); + expect( + retiredSignalKeyForOutcome({ + coverage: { definitions: false, metrics: true }, + disposition: "not_a_problem", + hasInsight: false, + signalKey: "goal:signup", + }) + ).toBeUndefined(); + }); }); function signal(metric: string, direction: "up" | "down"): DetectedSignal { @@ -182,6 +204,123 @@ describe("computeResolutions", () => { ]); }); + it("resolves currency-specific revenue findings independently", () => { + const decisions = computeResolutions({ + canRecover: true, + detectedSignals: [ + { ...signal("revenue", "down"), currency: "USD" }, + ], + now: NOW, + openInsights: [ + openInsight({ + id: "usd", + type: "quality_shift", + changePercent: -42, + sentiment: "negative", + subjectKey: "revenue:usd", + }), + openInsight({ + id: "eur", + type: "quality_shift", + changePercent: -42, + sentiment: "negative", + subjectKey: "revenue:eur", + }), + ], + }); + + expect(decisions).toEqual([{ id: "eur", reason: "recovered" }]); + }); + + it("recovers a currency-specific payment failure after a confirmed drop", () => { + const decisions = computeResolutions({ + canRecover: true, + detectedSignals: [ + { ...signal("payment_failure_rate", "down"), currency: "USD" }, + ], + now: NOW, + openInsights: [ + openInsight({ + id: "payment-failures", + type: "quality_shift", + changePercent: 60, + sentiment: "negative", + subjectKey: "payment_failure_rate:usd", + }), + ], + }); + + expect(decisions).toEqual([ + { id: "payment-failures", reason: "recovered" }, + ]); + }); + + it("keeps an unconfirmed payment recovery open inside the TTL", () => { + const decisions = computeResolutions({ + canRecover: true, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + id: "payment-failures", + type: "quality_shift", + changePercent: 60, + sentiment: "negative", + subjectKey: "payment_failure_rate:usd", + }), + ], + }); + + expect(decisions).toEqual([]); + }); + + it("expires an unconfirmed payment recovery as stale after the TTL", () => { + const old = new Date(NOW.getTime() - 80 * 60 * 60 * 1000); + const decisions = computeResolutions({ + canRecover: true, + detectedSignals: [ + { ...signal("payment_failure_rate", "down"), currency: "EUR" }, + ], + now: NOW, + openInsights: [ + openInsight({ + createdAt: old, + id: "payment-failures", + type: "quality_shift", + changePercent: 60, + sentiment: "negative", + subjectKey: "payment_failure_rate:usd", + }), + ], + }); + + expect(decisions).toEqual([ + { id: "payment-failures", reason: "stale" }, + ]); + }); + + it("keeps an old payment finding open while the exact regression fires", () => { + const decisions = computeResolutions({ + canRecover: true, + detectedSignals: [ + { ...signal("payment_failure_rate", "up"), currency: "USD" }, + ], + now: NOW, + openInsights: [ + openInsight({ + createdAt: new Date(NOW.getTime() - 80 * 60 * 60 * 1000), + id: "payment-failures", + type: "quality_shift", + changePercent: 60, + sentiment: "negative", + subjectKey: "payment_failure_rate:usd", + }), + ], + }); + + expect(decisions).toEqual([]); + }); + it("resolves an exact goal when only a sibling conversion fires", () => { const decisions = computeResolutions({ canRecover: true, @@ -290,6 +429,206 @@ describe("computeResolutions", () => { expect(decisions).toEqual([]); }); + it("recovers complete metric families while a definition rotation is partial", () => { + const decisions = computeResolutions({ + canRecover: true, + canRecoverConversion: false, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + id: "traffic", + type: "traffic_drop", + changePercent: -40, + sentiment: "negative", + subjectKey: "visitors", + }), + openInsight({ + id: "goal", + type: "conversion_leak", + changePercent: -100, + sentiment: "negative", + subjectKey: "goal:signup", + }), + ], + }); + + expect(decisions).toEqual([{ id: "traffic", reason: "recovered" }]); + }); + + it("recovers only conversion definitions evaluated in a partial rotation", () => { + const decisions = computeResolutions({ + canRecover: true, + canRecoverConversion: false, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + id: "evaluated", + type: "conversion_leak", + changePercent: -100, + sentiment: "negative", + subjectKey: "goal:evaluated", + }), + openInsight({ + id: "not-evaluated", + type: "conversion_leak", + changePercent: -100, + sentiment: "negative", + subjectKey: "goal:not-evaluated", + }), + ], + recoverableConversionKeys: new Set(["goal:evaluated"]), + }); + + expect(decisions).toEqual([{ id: "evaluated", reason: "recovered" }]); + }); + + it("eventually expires transient insights when detection stays incomplete", () => { + const old = new Date(NOW.getTime() - 80 * 60 * 60 * 1000); + const decisions = computeResolutions({ + canRecover: false, + canRecoverConversion: false, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + createdAt: old, + id: "goal", + type: "conversion_leak", + }), + ], + }); + + expect(decisions).toEqual([{ id: "goal", reason: "stale" }]); + }); + + it("keeps an exact conversion open until its rotated definition is evaluated", () => { + const weekOld = new Date(NOW.getTime() - 8 * 24 * 60 * 60 * 1000); + const decisions = computeResolutions({ + activeConversionKeys: new Set(["goal:waiting"]), + canRecover: true, + canRecoverConversion: false, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + createdAt: weekOld, + id: "waiting", + subjectKey: "goal:waiting", + type: "conversion_leak", + }), + ], + recoverableConversionKeys: new Set(["goal:another"]), + }); + + expect(decisions).toEqual([]); + }); + + it("keeps a versioned conversion open until its active definition is evaluated", () => { + const decisions = computeResolutions({ + activeConversionKeys: new Set(["goal:waiting"]), + canRecover: true, + canRecoverConversion: false, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + id: "waiting-versioned", + subjectKey: "goal:waiting@2026-05-20T00:00:00.000Z", + type: "conversion_leak", + }), + ], + recoverableConversionKeys: new Set(["goal:another"]), + }); + + expect(decisions).toEqual([]); + }); + + it("recovers a versioned conversion after that definition is evaluated", () => { + const decisions = computeResolutions({ + activeConversionKeys: new Set(["goal:evaluated"]), + canRecover: true, + canRecoverConversion: false, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + id: "evaluated-versioned", + subjectKey: "goal:evaluated@2026-05-01T00:00:00.000Z", + type: "conversion_leak", + }), + ], + recoverableConversionKeys: new Set(["goal:evaluated"]), + }); + + expect(decisions).toEqual([ + { id: "evaluated-versioned", reason: "recovered" }, + ]); + }); + + it("resolves an exact conversion whose definition was removed", () => { + const decisions = computeResolutions({ + activeConversionKeys: new Set(["goal:active"]), + canRecover: true, + canRecoverConversion: false, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + id: "removed", + subjectKey: "goal:removed", + type: "conversion_leak", + }), + ], + recoverableConversionKeys: new Set(), + }); + + expect(decisions).toEqual([{ id: "removed", reason: "recovered" }]); + }); + + it("resolves a versioned conversion whose definition was removed", () => { + const decisions = computeResolutions({ + activeConversionKeys: new Set(["goal:active"]), + canRecover: true, + canRecoverConversion: false, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + id: "removed-versioned", + subjectKey: "goal:removed@2026-05-01T00:00:00.000Z", + type: "conversion_leak", + }), + ], + recoverableConversionKeys: new Set(), + }); + + expect(decisions).toEqual([ + { id: "removed-versioned", reason: "recovered" }, + ]); + }); + + it("keeps an exact conversion open when its active definition was edited", () => { + const decisions = computeResolutions({ + activeConversionKeys: new Set(["goal:edited"]), + canRecover: true, + canRecoverConversion: false, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + id: "edited", + subjectKey: "goal:edited", + type: "conversion_leak", + }), + ], + recoverableConversionKeys: new Set(), + }); + + expect(decisions).toEqual([]); + }); + it("matches legacy null-change regressions without guessing direction", () => { const decisions = computeResolutions({ canRecover: true, @@ -335,10 +674,27 @@ describe("computeResolutions", () => { type: "custom_event_spike", changePercent: 60, sentiment: "positive", + subjectKey: "custom_event:signup", }), ], }); expect(stillFiring).toEqual([]); + + const recovered = computeResolutions({ + canRecover: true, + detectedSignals: [], + now: NOW, + openInsights: [ + openInsight({ + id: "i1", + type: "custom_event_spike", + changePercent: 60, + sentiment: "positive", + subjectKey: "custom_event:signup", + }), + ], + }); + expect(recovered).toEqual([{ id: "i1", reason: "recovered" }]); }); it("maps funnel and goal signals to the conversion family", () => { diff --git a/apps/insights/src/resolution.ts b/apps/insights/src/resolution.ts index 534c71b58..d08bf0371 100644 --- a/apps/insights/src/resolution.ts +++ b/apps/insights/src/resolution.ts @@ -14,13 +14,17 @@ import { signalKeyForDetectedSignal } from "./investigation"; import { emitInsightsEvent } from "./lib/evlog-insights"; const DEFAULT_STALE_TTL_MS = 72 * 60 * 60 * 1000; +const ISO_DEFINITION_VERSION_PATTERN = + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; type InsightFamily = | "errors" | "vitals" | "traffic" | "engagement" - | "conversion"; + | "custom_event" + | "conversion" + | "revenue"; const TRANSIENT_TYPE_FAMILY: Record = { error_spike: "errors", @@ -29,14 +33,21 @@ const TRANSIENT_TYPE_FAMILY: Record = { traffic_spike: "traffic", bounce_rate_change: "engagement", engagement_change: "engagement", + custom_event_spike: "custom_event", conversion_leak: "conversion", funnel_regression: "conversion", }; function signalFamily(metric: string): InsightFamily | null { + if (metric.startsWith("custom_event:")) { + return "custom_event"; + } if (metric.startsWith("funnel:") || metric.startsWith("goal:")) { return "conversion"; } + if (metric === "revenue" || metric === "payment_failure_rate") { + return "revenue"; + } switch (metric) { case "visitors": case "sessions": @@ -55,6 +66,21 @@ function signalFamily(metric: string): InsightFamily | null { } } +function isRevenueSubject(subjectKey: string): boolean { + return ( + subjectKey === "revenue" || + subjectKey.startsWith("revenue:") || + isPaymentFailureSubject(subjectKey) + ); +} + +function isPaymentFailureSubject(subjectKey: string): boolean { + return ( + subjectKey === "payment_failure_rate" || + subjectKey.startsWith("payment_failure_rate:") + ); +} + export interface OpenInsightRow { changePercent: number | null; createdAt: Date; @@ -70,11 +96,18 @@ export interface ResolutionDecision { } export function retiredSignalKeyForOutcome(params: { + coverage: { definitions: boolean; metrics: boolean }; disposition: InvestigationDecision["disposition"] | undefined; hasInsight: boolean; signalKey: string | undefined; }): string | undefined { - if (params.hasInsight) { + const isConversion = + params.signalKey?.startsWith("goal:") || + params.signalKey?.startsWith("funnel:"); + const familyComplete = isConversion + ? params.coverage.definitions + : params.coverage.metrics; + if (params.hasInsight || !familyComplete) { return; } return params.disposition === "monitor" || @@ -84,6 +117,12 @@ export function retiredSignalKeyForOutcome(params: { } function hasExactSubject(insight: OpenInsightRow): boolean { + if (isRevenueSubject(insight.subjectKey)) { + return true; + } + if (insight.subjectKey.startsWith("custom_event:")) { + return true; + } if (insight.type === "traffic_drop" || insight.type === "traffic_spike") { return ( insight.subjectKey === "visitors" || @@ -112,11 +151,36 @@ function hasExactSubject(insight: OpenInsightRow): boolean { return false; } +function familyForInsight(insight: OpenInsightRow): InsightFamily | undefined { + if (isRevenueSubject(insight.subjectKey)) { + return "revenue"; + } + if (insight.subjectKey.startsWith("custom_event:")) { + return "custom_event"; + } + return TRANSIENT_TYPE_FAMILY[insight.type]; +} + +function conversionDefinitionKey(subjectKey: string): string { + const versionSeparator = subjectKey.lastIndexOf("@"); + if (versionSeparator < 0) { + return subjectKey; + } + const version = subjectKey.slice(versionSeparator + 1); + return !ISO_DEFINITION_VERSION_PATTERN.test(version) || + Number.isNaN(Date.parse(version)) + ? subjectKey + : subjectKey.slice(0, versionSeparator); +} + export function computeResolutions(params: { + activeConversionKeys?: ReadonlySet; canRecover: boolean; + canRecoverConversion?: boolean; detectedSignals: DetectedSignal[]; now: Date; openInsights: OpenInsightRow[]; + recoverableConversionKeys?: ReadonlySet; retiredSignalKey?: string; staleTtlMs?: number; }): ResolutionDecision[] { @@ -143,9 +207,30 @@ export function computeResolutions(params: { decisions.push({ id: insight.id, reason: "stale" }); continue; } - const family = TRANSIENT_TYPE_FAMILY[insight.type]; + const family = familyForInsight(insight); if (family) { - if (!params.canRecover) { + const exactConversionSubject = + family === "conversion" && hasExactSubject(insight); + const definitionKey = exactConversionSubject + ? conversionDefinitionKey(insight.subjectKey) + : insight.subjectKey; + const definitionWasRemoved = + exactConversionSubject && + params.activeConversionKeys !== undefined && + !params.activeConversionKeys.has(definitionKey); + const canRecover = + family === "conversion" + ? (params.canRecoverConversion ?? params.canRecover) || + (params.recoverableConversionKeys?.has(definitionKey) ?? false) || + definitionWasRemoved + : params.canRecover; + if (!canRecover) { + if (exactConversionSubject) { + continue; + } + if (params.now.getTime() - insight.createdAt.getTime() >= staleTtlMs) { + decisions.push({ id: insight.id, reason: "stale" }); + } continue; } const direction = @@ -160,7 +245,17 @@ export function computeResolutions(params: { ? activeFamilies.has(family) : activeKeys.has(`${family}:${direction}`); if (!stillFiring) { - decisions.push({ id: insight.id, reason: "recovered" }); + const paymentRecoveryConfirmed = + !isPaymentFailureSubject(insight.subjectKey) || + activeExactKeys.has(`${insight.subjectKey}:down`); + if (paymentRecoveryConfirmed) { + decisions.push({ id: insight.id, reason: "recovered" }); + } else if ( + params.now.getTime() - insight.createdAt.getTime() >= + staleTtlMs + ) { + decisions.push({ id: insight.id, reason: "stale" }); + } } continue; } @@ -172,10 +267,13 @@ export function computeResolutions(params: { } export async function resolveInsightsForWebsite(params: { + activeConversionKeys?: string[]; canRecover: boolean; + canRecoverConversion?: boolean; detectedSignals: DetectedSignal[]; now?: Date; organizationId: string; + recoverableConversionKeys?: string[]; retiredSignalKey?: string; runId: string; websiteId: string; @@ -200,10 +298,16 @@ export async function resolveInsightsForWebsite(params: { ); const decisions = computeResolutions({ + activeConversionKeys: + params.activeConversionKeys === undefined + ? undefined + : new Set(params.activeConversionKeys), canRecover: params.canRecover, + canRecoverConversion: params.canRecoverConversion, detectedSignals: params.detectedSignals, now, openInsights, + recoverableConversionKeys: new Set(params.recoverableConversionKeys ?? []), retiredSignalKey: params.retiredSignalKey, }); @@ -255,6 +359,7 @@ export async function resolveInsightsForWebsite(params: { recovered_count: recoveredIds.length, stale_count: staleIds.length, can_recover: params.canRecover, + can_recover_conversion: params.canRecoverConversion ?? params.canRecover, }); return decisions; diff --git a/apps/insights/src/rollup.ts b/apps/insights/src/rollup.ts index a057457b3..e7e8a0d0f 100644 --- a/apps/insights/src/rollup.ts +++ b/apps/insights/src/rollup.ts @@ -2,6 +2,7 @@ import { and, db, desc, eq, gte, isNull, sql } from "@databuddy/db"; import { analyticsInsights, insightRollups, + insightRuns, type InsightRollupRange, websites, } from "@databuddy/db/schema"; @@ -120,6 +121,18 @@ async function persistRollup(input: { generatedAt: input.generatedAt, updatedAt: input.generatedAt, }, + setWhere: sql` + ${insightRollups.runId} is null + or ( + select row(${insightRuns.createdAt}, ${insightRuns.id}) + from ${insightRuns} + where ${insightRuns.id} = excluded.run_id + ) >= ( + select row(${insightRuns.createdAt}, ${insightRuns.id}) + from ${insightRuns} + where ${insightRuns.id} = ${insightRollups.runId} + ) + `, }); } diff --git a/apps/insights/src/scheduler.integration.test.ts b/apps/insights/src/scheduler.integration.test.ts index d3576e0fd..8ec23d16e 100644 --- a/apps/insights/src/scheduler.integration.test.ts +++ b/apps/insights/src/scheduler.integration.test.ts @@ -3,12 +3,18 @@ import { afterAll, afterEach, beforeEach, describe, expect, it } from "bun:test" import { db as appDb, shutdownPostgres } from "@databuddy/db"; import { insightGenerationConfigs, + insightRollups, insightRunItems, insightRuns, } from "@databuddy/db/schema"; import { closeInsightsQueue, + getBullMQWorkerConnectionOptions, getInsightsQueue, + INSIGHTS_QUEUE_ENV_PREFIX, + INSIGHTS_QUEUE_NAME, + INSIGHTS_ROLLUP_JOB_NAME, + insightsRollupJobId, type InsightsGenerateWebsiteJobData, } from "@databuddy/redis"; import { @@ -25,7 +31,18 @@ import { } from "@databuddy/test"; import { asc, eq } from "drizzle-orm"; import { randomUUIDv7 } from "bun"; -import { dispatchDueInsightRuns, retryConfigSoon } from "./scheduler"; +import { Worker } from "bullmq"; +import { processRollupJob } from "./rollup"; +import { + claimDueConfig, + dispatchDueInsightRuns, + retryConfigSoon, +} from "./scheduler"; +import { + getInsightsStaleItemMs, + recoverStaleInsightRuns, +} from "./recovery"; +import { loadPreparedInsightRun, prepareInsightRun } from "./effects"; const runIntegration = process.env.INSIGHTS_INTEGRATION_TESTS === "true" && hasTestDb; @@ -95,6 +112,11 @@ describeIntegration("insights scheduler integration", () => { expect(items.map((item) => item.websiteId).sort()).toEqual( [included.id, second.id].sort() ); + expect( + items.every( + (item) => item.configSnapshot.timezone === runs[0]?.timezone + ) + ).toBe(true); const jobs = await queueJobsForOrg(org.id); expect(jobs).toHaveLength(2); @@ -105,7 +127,6 @@ describeIntegration("insights scheduler integration", () => { [included.id, second.id].sort() ); expect(jobs.every((job) => job.data.runId === runs[0].id)).toBe(true); - const [config] = await db() .select({ lastRunAt: insightGenerationConfigs.lastRunAt, @@ -121,6 +142,195 @@ describeIntegration("insights scheduler integration", () => { ); }); + it("requeues a durable run item when the process dies before BullMQ add", async () => { + const org = await insertOrganization(); + organizationIds.add(org.id); + await insertWebsite({ organizationId: org.id }); + await db().insert(insightGenerationConfigs).values({ + id: randomUUIDv7(), + organizationId: org.id, + enabled: true, + }); + const queued = await queueInsightGenerationRun({ + organizationId: org.id, + reason: "scheduled", + }); + const [item] = await itemsForRun(queued.runId!); + const job = item?.queueJobId + ? await getInsightsQueue().getJob(item.queueJobId) + : undefined; + expect(job).toBeDefined(); + await job?.remove(); + + const staleAt = new Date("2026-01-01T00:00:00.000Z"); + await appDb + .update(insightRunItems) + .set({ updatedAt: staleAt }) + .where(eq(insightRunItems.id, item!.id)); + const result = await recoverStaleInsightRuns( + new Date(staleAt.getTime() + getInsightsStaleItemMs() + 1000) + ); + + expect(result.requeuedItems).toBe(1); + const recovered = await getInsightsQueue().getJob(item!.queueJobId!); + expect(recovered?.data).toMatchObject({ + itemId: item!.id, + organizationId: org.id, + runId: queued.runId, + }); + }); + + it("rotates the job identity after a worker dies before preparing", async () => { + const org = await insertOrganization(); + organizationIds.add(org.id); + const website = await insertWebsite({ organizationId: org.id }); + const runId = randomUUIDv7(); + const itemId = randomUUIDv7(); + const oldJobId = `dead-worker-${itemId}`; + const staleAt = new Date("2026-01-01T00:00:00.000Z"); + await db().insert(insightRuns).values({ + id: runId, + organizationId: org.id, + status: "running", + totalItems: 1, + updatedAt: staleAt, + }); + await db().insert(insightRunItems).values({ + id: itemId, + organizationId: org.id, + queueJobId: oldJobId, + runId, + startedAt: staleAt, + status: "running", + updatedAt: staleAt, + websiteId: website.id, + }); + + const result = await recoverStaleInsightRuns( + new Date(staleAt.getTime() + getInsightsStaleItemMs() + 1000) + ); + const [item] = await itemsForRun(runId); + + expect(result.requeuedItems).toBe(1); + expect(item).toMatchObject({ status: "queued" }); + expect(item.queueJobId).not.toBe(oldJobId); + expect(item.startedAt).toBeNull(); + expect(await getInsightsQueue().getJob(item.queueJobId!)).toBeDefined(); + }); + + it("reuses prepared effects while fencing the dead worker", async () => { + const org = await insertOrganization(); + organizationIds.add(org.id); + const website = await insertWebsite({ organizationId: org.id }); + const runId = randomUUIDv7(); + const itemId = randomUUIDv7(); + const oldJobId = `dead-worker-${itemId}`; + const staleAt = new Date("2026-01-01T00:00:00.000Z"); + await db().insert(insightRuns).values({ + id: runId, + organizationId: org.id, + status: "running", + totalItems: 1, + updatedAt: staleAt, + }); + await db().insert(insightRunItems).values({ + id: itemId, + organizationId: org.id, + queueJobId: oldJobId, + runId, + startedAt: staleAt, + status: "running", + websiteId: website.id, + }); + const preparedInput = { + effects: [ + { + effectKey: "pending-channel", + payload: { + blocks: [], + channelId: "pending-channel", + organizationId: org.id, + text: "A bounded finding", + websiteId: website.id, + }, + }, + ], + itemId, + organizationId: org.id, + queueJobId: oldJobId, + result: { insightIds: [], resultCount: 0, status: "succeeded" as const }, + runId, + websiteId: website.id, + }; + await prepareInsightRun(preparedInput); + await db() + .update(insightRunItems) + .set({ updatedAt: staleAt }) + .where(eq(insightRunItems.id, itemId)); + + const result = await recoverStaleInsightRuns( + new Date(staleAt.getTime() + getInsightsStaleItemMs() + 1000) + ); + const [item] = await itemsForRun(runId); + const newIdentity = { + itemId, + organizationId: org.id, + queueJobId: item.queueJobId, + runId, + websiteId: website.id, + }; + + expect(result.requeuedItems).toBe(1); + expect(item.queueJobId).not.toBe(oldJobId); + expect(await loadPreparedInsightRun(newIdentity)).toMatchObject({ + resultCount: 0, + status: "succeeded", + }); + await expect(prepareInsightRun(preparedInput)).rejects.toThrow( + "not found while preparing effects" + ); + }); + + it("dispatches only organizations selected for a canary", async () => { + const selected = await insertOrganization(); + const heldBack = await insertOrganization(); + organizationIds.add(selected.id); + organizationIds.add(heldBack.id); + await insertWebsite({ organizationId: selected.id }); + await insertWebsite({ organizationId: heldBack.id }); + const now = new Date(); + const dueAt = new Date(now.getTime() - 1000); + await db().insert(insightGenerationConfigs).values([ + { + id: randomUUIDv7(), + organizationId: selected.id, + enabled: true, + nextRunAt: dueAt, + }, + { + id: randomUUIDv7(), + organizationId: heldBack.id, + enabled: true, + nextRunAt: dueAt, + }, + ]); + + const result = await dispatchDueInsightRuns(now, [selected.id]); + + expect(result).toMatchObject({ + scannedConfigs: 1, + claimedConfigs: 1, + dispatchedRuns: 1, + }); + expect(await runsForOrg(selected.id)).toHaveLength(1); + expect(await runsForOrg(heldBack.id)).toHaveLength(0); + const [heldBackConfig] = await db() + .select({ nextRunAt: insightGenerationConfigs.nextRunAt }) + .from(insightGenerationConfigs) + .where(eq(insightGenerationConfigs.organizationId, heldBack.id)); + expect(heldBackConfig?.nextRunAt).toEqual(dueAt); + }); + it("advances a due config when the organization has no websites", async () => { const org = await insertOrganization(); organizationIds.add(org.id); @@ -226,20 +436,32 @@ describeIntegration("insights scheduler integration", () => { const org = await insertOrganization(); organizationIds.add(org.id); const configId = randomUUIDv7(); - const claimedNextRunAt = new Date("2026-01-22T09:00:00.000Z"); + const dueAt = new Date("2026-01-22T09:00:00.000Z"); const changedNextRunAt = new Date("2026-01-23T09:00:00.000Z"); await db().insert(insightGenerationConfigs).values({ id: configId, organizationId: org.id, enabled: true, - nextRunAt: claimedNextRunAt, + nextRunAt: dueAt, }); + const [dueConfig] = await db() + .select() + .from(insightGenerationConfigs) + .where(eq(insightGenerationConfigs.id, configId)); + const claimed = await claimDueConfig( + dueConfig, + new Date("2026-01-22T09:01:00.000Z") + ); + expect(claimed).not.toBeNull(); + if (!claimed) { + throw new Error("Expected config claim"); + } await db() .update(insightGenerationConfigs) .set({ nextRunAt: changedNextRunAt }) .where(eq(insightGenerationConfigs.id, configId)); - await retryConfigSoon(configId, claimedNextRunAt, new Date()); + await retryConfigSoon(claimed, new Date()); const [config] = await db() .select() @@ -249,17 +471,320 @@ describeIntegration("insights scheduler integration", () => { await db() .update(insightGenerationConfigs) - .set({ enabled: false, nextRunAt: claimedNextRunAt }) + .set({ enabled: false, nextRunAt: claimed.nextRunAt }) .where(eq(insightGenerationConfigs.id, configId)); - await retryConfigSoon(configId, claimedNextRunAt, new Date()); + await retryConfigSoon(claimed, new Date()); const [disabled] = await db() .select() .from(insightGenerationConfigs) .where(eq(insightGenerationConfigs.id, configId)); expect(disabled).toMatchObject({ enabled: false, - nextRunAt: claimedNextRunAt, + nextRunAt: claimed.nextRunAt, + }); + }); + + it("does not claim a stale config snapshot after an admin edit", async () => { + const org = await insertOrganization(); + organizationIds.add(org.id); + const now = new Date("2026-01-22T09:00:00.000Z"); + await db().insert(insightGenerationConfigs).values({ + id: randomUUIDv7(), + organizationId: org.id, + enabled: true, + frequency: "daily", + nextRunAt: new Date(now.getTime() - 1000), + }); + const [stale] = await db() + .select() + .from(insightGenerationConfigs) + .where(eq(insightGenerationConfigs.organizationId, org.id)); + await db() + .update(insightGenerationConfigs) + .set({ + frequency: "weekly", + updatedAt: new Date(stale.updatedAt.getTime() + 1000), + }) + .where(eq(insightGenerationConfigs.id, stale.id)); + + expect(await claimDueConfig(stale, now)).toBeNull(); + const [current] = await db() + .select() + .from(insightGenerationConfigs) + .where(eq(insightGenerationConfigs.id, stale.id)); + expect(current.frequency).toBe("weekly"); + expect(current.nextRunAt).toEqual(stale.nextRunAt); + }); + + it("recovers a durable scheduled run after a claim-process crash", async () => { + const org = await insertOrganization(); + organizationIds.add(org.id); + await insertWebsite({ organizationId: org.id }); + const now = new Date(); + const dueAt = new Date(now.getTime() - 1000); + await db().insert(insightGenerationConfigs).values({ + id: randomUUIDv7(), + organizationId: org.id, + enabled: true, + nextRunAt: dueAt, + }); + const queued = await queueInsightGenerationRun({ + organizationId: org.id, + reason: "scheduled", + }); + + const result = await dispatchDueInsightRuns(now); + + expect(result).toMatchObject({ + claimedConfigs: 1, + dispatchedRuns: 0, + skippedConfigs: 1, + }); + expect(await runsForOrg(org.id)).toHaveLength(1); + const [config] = await db() + .select() + .from(insightGenerationConfigs) + .where(eq(insightGenerationConfigs.organizationId, org.id)); + expect(config.lastRunAt).toBeInstanceOf(Date); + expect(config.nextRunAt && config.nextRunAt > now).toBe(true); + expect((await runsForOrg(org.id))[0]?.id).toBe(queued.runId); + }); + + it("preserves the original due time across an expired dispatch lease", async () => { + const org = await insertOrganization(); + organizationIds.add(org.id); + await insertWebsite({ organizationId: org.id }); + const now = new Date(); + const dueAt = new Date(now.getTime() - 1000); + const configId = randomUUIDv7(); + await db().insert(insightGenerationConfigs).values({ + id: configId, + organizationId: org.id, + enabled: true, + nextRunAt: dueAt, + }); + const [dueConfig] = await db() + .select() + .from(insightGenerationConfigs) + .where(eq(insightGenerationConfigs.id, configId)); + const claimed = await claimDueConfig(dueConfig, now); + expect(claimed?.dispatchDueAt).toEqual(dueAt); + if (!claimed?.nextRunAt) { + throw new Error("Expected a leased config"); + } + + const queued = await queueInsightGenerationRun({ + organizationId: org.id, + reason: "scheduled", }); + await db() + .update(insightRuns) + .set({ finishedAt: new Date(), status: "succeeded" }) + .where(eq(insightRuns.id, queued.runId!)); + + const result = await dispatchDueInsightRuns( + new Date(claimed.nextRunAt.getTime() + 1) + ); + + expect(result).toMatchObject({ + claimedConfigs: 1, + dispatchedRuns: 0, + skippedConfigs: 1, + }); + expect(await runsForOrg(org.id)).toHaveLength(1); + const [config] = await db() + .select() + .from(insightGenerationConfigs) + .where(eq(insightGenerationConfigs.id, configId)); + expect(config.dispatchDueAt).toBeNull(); + expect(config.lastRunAt).toBeInstanceOf(Date); + }); + + it("requeues a missing rollup for the latest settled run", async () => { + const org = await insertOrganization(); + organizationIds.add(org.id); + const runId = randomUUIDv7(); + const finishedAt = new Date("2026-01-22T09:00:00.000Z"); + await db().insert(insightRuns).values({ + completedItems: 1, + finishedAt, + id: runId, + organizationId: org.id, + reason: "scheduled", + status: "succeeded", + totalItems: 1, + }); + + const result = await recoverStaleInsightRuns( + new Date(finishedAt.getTime() + getInsightsStaleItemMs() + 1000) + ); + + expect(result.requeuedRollups).toBe(1); + const job = await getInsightsQueue().getJob(insightsRollupJobId(runId)); + expect(job?.data).toMatchObject({ + organizationId: org.id, + runId, + }); + }); + + it("uses run creation order when repairing the latest missing rollup", async () => { + const org = await insertOrganization(); + organizationIds.add(org.id); + const olderRunId = randomUUIDv7(); + const newerRunId = randomUUIDv7(); + const olderCreatedAt = new Date("2026-01-20T09:00:00.000Z"); + const newerCreatedAt = new Date("2026-01-21T09:00:00.000Z"); + await db().insert(insightRuns).values([ + { + completedItems: 1, + createdAt: olderCreatedAt, + finishedAt: new Date("2026-01-23T09:00:00.000Z"), + id: olderRunId, + organizationId: org.id, + reason: "scheduled", + status: "succeeded", + totalItems: 1, + }, + { + completedItems: 1, + createdAt: newerCreatedAt, + finishedAt: new Date("2026-01-22T09:00:00.000Z"), + id: newerRunId, + organizationId: org.id, + reason: "scheduled", + status: "succeeded", + totalItems: 1, + }, + ]); + await db().insert(insightRollups).values( + (["7d", "30d", "90d"] as const).map((range) => ({ + id: randomUUIDv7(), + narrative: `older ${range}`, + organizationId: org.id, + range, + runId: olderRunId, + })) + ); + + const result = await recoverStaleInsightRuns( + new Date("2026-02-01T00:00:00.000Z") + ); + + expect(result.requeuedRollups).toBe(1); + expect( + await getInsightsQueue().getJob(insightsRollupJobId(newerRunId)) + ).toBeDefined(); + expect( + await getInsightsQueue().getJob(insightsRollupJobId(olderRunId)) + ).toBeUndefined(); + }); + + it("retries a retained failed rollup job", async () => { + const org = await insertOrganization(); + organizationIds.add(org.id); + const runId = randomUUIDv7(); + const finishedAt = new Date(); + await db().insert(insightRuns).values({ + completedItems: 1, + finishedAt, + id: runId, + organizationId: org.id, + reason: "scheduled", + status: "succeeded", + totalItems: 1, + }); + + const jobId = insightsRollupJobId(runId); + const worker = new Worker( + INSIGHTS_QUEUE_NAME, + async (job) => { + if (job.id === jobId) { + throw new Error("simulated retained rollup failure"); + } + return null; + }, + { + connection: getBullMQWorkerConnectionOptions({ + envPrefix: INSIGHTS_QUEUE_ENV_PREFIX, + }), + } + ); + worker.on("error", () => undefined); + await worker.waitUntilReady(); + const failedJob = await getInsightsQueue().add( + INSIGHTS_ROLLUP_JOB_NAME, + { + organizationId: org.id, + reason: "scheduled", + runId, + timezone: "UTC", + }, + { attempts: 1, jobId, removeOnFail: false } + ); + for (let attempt = 0; attempt < 100; attempt += 1) { + if ((await failedJob.getState()) === "failed") { + break; + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } + expect(await failedJob.getState()).toBe("failed"); + await worker.close(); + + const result = await recoverStaleInsightRuns( + new Date(finishedAt.getTime() + getInsightsStaleItemMs() + 1000) + ); + + expect(result.requeuedRollups).toBe(1); + expect(await failedJob.getState()).toBe("waiting"); + }); + + it("does not let a delayed older rollup overwrite a newer run", async () => { + const org = await insertOrganization(); + organizationIds.add(org.id); + const olderRunId = randomUUIDv7(); + const newerRunId = randomUUIDv7(); + const olderAt = new Date("2026-01-01T00:00:00.000Z"); + const newerAt = new Date("2026-01-02T00:00:00.000Z"); + await db().insert(insightRuns).values([ + { + createdAt: olderAt, + finishedAt: olderAt, + id: olderRunId, + organizationId: org.id, + status: "succeeded", + }, + { + createdAt: newerAt, + finishedAt: newerAt, + id: newerRunId, + organizationId: org.id, + status: "succeeded", + }, + ]); + await db().insert(insightRollups).values( + (["7d", "30d", "90d"] as const).map((range) => ({ + id: randomUUIDv7(), + organizationId: org.id, + range, + runId: newerRunId, + narrative: `newer ${range}`, + })) + ); + + await processRollupJob({ + organizationId: org.id, + reason: "scheduled", + runId: olderRunId, + timezone: "UTC", + }); + + const rows = await db() + .select({ narrative: insightRollups.narrative, runId: insightRollups.runId }) + .from(insightRollups) + .where(eq(insightRollups.organizationId, org.id)); + expect(rows).toHaveLength(3); + expect(rows.every((row) => row.runId === newerRunId)).toBe(true); + expect(rows.every((row) => row.narrative.startsWith("newer"))).toBe(true); }); it("coalesces concurrent queue requests into one active organization run", async () => { diff --git a/apps/insights/src/scheduler.test.ts b/apps/insights/src/scheduler.test.ts new file mode 100644 index 000000000..3bae11e6b --- /dev/null +++ b/apps/insights/src/scheduler.test.ts @@ -0,0 +1,54 @@ +import "@databuddy/test/env"; +import { afterEach, describe, expect, it } from "bun:test"; +import { + isScheduledDispatchActive, + isScheduledDispatchEnabled, + scheduledDispatchOrganizations, +} from "./scheduler"; + +const original = process.env.INSIGHTS_SCHEDULED_DISPATCH_ENABLED; +const originalOrganizations = + process.env.INSIGHTS_SCHEDULED_ORGANIZATION_IDS; + +afterEach(() => { + if (original === undefined) { + delete process.env.INSIGHTS_SCHEDULED_DISPATCH_ENABLED; + } else { + process.env.INSIGHTS_SCHEDULED_DISPATCH_ENABLED = original; + } + if (originalOrganizations === undefined) { + delete process.env.INSIGHTS_SCHEDULED_ORGANIZATION_IDS; + } else { + process.env.INSIGHTS_SCHEDULED_ORGANIZATION_IDS = originalOrganizations; + } +}); + +describe("scheduled insights dispatch rollout", () => { + it("reports the effective flag and canary state", () => { + delete process.env.INSIGHTS_SCHEDULED_DISPATCH_ENABLED; + delete process.env.INSIGHTS_SCHEDULED_ORGANIZATION_IDS; + expect(isScheduledDispatchEnabled()).toBe(false); + + process.env.INSIGHTS_SCHEDULED_DISPATCH_ENABLED = "false"; + expect(isScheduledDispatchEnabled()).toBe(false); + + process.env.INSIGHTS_SCHEDULED_DISPATCH_ENABLED = "true"; + expect(isScheduledDispatchEnabled()).toBe(false); + + process.env.INSIGHTS_SCHEDULED_ORGANIZATION_IDS = "org-a, org-b, org-a"; + expect(isScheduledDispatchEnabled()).toBe(true); + expect(scheduledDispatchOrganizations()).toEqual(["org-a", "org-b"]); + + process.env.INSIGHTS_SCHEDULED_ORGANIZATION_IDS = "*"; + expect(isScheduledDispatchEnabled()).toBe(true); + expect(scheduledDispatchOrganizations()).toBeNull(); + }); + + it("is inactive when the worker is disabled", () => { + process.env.INSIGHTS_SCHEDULED_DISPATCH_ENABLED = "true"; + process.env.INSIGHTS_SCHEDULED_ORGANIZATION_IDS = "*"; + + expect(isScheduledDispatchActive(false)).toBe(false); + expect(isScheduledDispatchActive(true)).toBe(true); + }); +}); diff --git a/apps/insights/src/scheduler.ts b/apps/insights/src/scheduler.ts index c66fdfa35..3d4bf7dd3 100644 --- a/apps/insights/src/scheduler.ts +++ b/apps/insights/src/scheduler.ts @@ -1,5 +1,16 @@ -import { and, asc, db, eq, lte } from "@databuddy/db"; -import { insightGenerationConfigs } from "@databuddy/db/schema"; +import { + and, + asc, + db, + desc, + eq, + gte, + inArray, + isNull, + lte, +} from "@databuddy/db"; +import { insightGenerationConfigs, insightRuns } from "@databuddy/db/schema"; +import { readBooleanEnv } from "@databuddy/env/boolean"; import { queueInsightGenerationRun } from "@databuddy/rpc/insight-generation"; import { getNextInsightRunAt, @@ -17,6 +28,7 @@ const DEFAULT_DISPATCH_INTERVAL_MS = 5 * 60 * 1000; const MIN_DISPATCH_INTERVAL_MS = 60 * 1000; const MAX_DUE_CONFIGS_PER_TICK = 100; const FAILED_DISPATCH_RETRY_MS = 60 * 1000; +const DISPATCH_CLAIM_LEASE_MS = 5 * 60 * 1000; type DueConfig = typeof insightGenerationConfigs.$inferSelect; @@ -40,6 +52,34 @@ function dispatchIntervalMs(): number { return parsed; } +export function isScheduledDispatchEnabled(): boolean { + const organizations = scheduledDispatchOrganizations(); + return organizations === null || organizations.length > 0; +} + +export function isScheduledDispatchActive(workerEnabled: boolean): boolean { + return workerEnabled && isScheduledDispatchEnabled(); +} + +/** `null` means every enabled organization; an empty list means disabled. */ +export function scheduledDispatchOrganizations(): string[] | null { + if (!readBooleanEnv("INSIGHTS_SCHEDULED_DISPATCH_ENABLED")) { + return []; + } + const raw = process.env.INSIGHTS_SCHEDULED_ORGANIZATION_IDS?.trim(); + if (raw === "*") { + return null; + } + return [ + ...new Set( + (raw ?? "") + .split(",") + .map((value) => value.trim()) + .filter(Boolean) + ), + ]; +} + function nextRunAtFor(config: DueConfig, from: Date): Date | null { return getNextInsightRunAt( { @@ -51,36 +91,50 @@ function nextRunAtFor(config: DueConfig, from: Date): Date | null { ); } -async function dueConfigs(now: Date): Promise { +async function dueConfigs( + now: Date, + organizationIds?: readonly string[] +): Promise { + if (organizationIds && organizationIds.length === 0) { + return []; + } + const conditions = [ + eq(insightGenerationConfigs.enabled, true), + lte(insightGenerationConfigs.nextRunAt, now), + ]; + if (organizationIds) { + conditions.push( + inArray(insightGenerationConfigs.organizationId, [...organizationIds]) + ); + } return await db .select() .from(insightGenerationConfigs) - .where( - and( - eq(insightGenerationConfigs.enabled, true), - lte(insightGenerationConfigs.nextRunAt, now) - ) - ) + .where(and(...conditions)) .orderBy(asc(insightGenerationConfigs.nextRunAt)) .limit(MAX_DUE_CONFIGS_PER_TICK); } -async function claimConfig( +export async function claimDueConfig( config: DueConfig, now: Date ): Promise { + if (!config.nextRunAt) { + return null; + } const [claimed] = await db .update(insightGenerationConfigs) .set({ - frequency: normalizeInsightScheduleFrequency(config.frequency), - nextRunAt: nextRunAtFor(config, now), + dispatchDueAt: config.dispatchDueAt ?? config.nextRunAt, + nextRunAt: new Date(now.getTime() + DISPATCH_CLAIM_LEASE_MS), updatedAt: now, }) .where( and( eq(insightGenerationConfigs.id, config.id), eq(insightGenerationConfigs.enabled, true), - lte(insightGenerationConfigs.nextRunAt, now) + eq(insightGenerationConfigs.nextRunAt, config.nextRunAt), + eq(insightGenerationConfigs.updatedAt, config.updatedAt) ) ) .returning(); @@ -89,20 +143,69 @@ async function claimConfig( } async function markConfigDispatched( - configId: string, + config: DueConfig, + lastRunAt: Date, now: Date -): Promise { - await db +): Promise { + if (!config.nextRunAt) { + return false; + } + const updated = await db .update(insightGenerationConfigs) - .set({ lastRunAt: now, updatedAt: now }) - .where(eq(insightGenerationConfigs.id, configId)); + .set({ + dispatchDueAt: null, + lastRunAt, + nextRunAt: nextRunAtFor(config, now), + updatedAt: now, + }) + .where( + and( + eq(insightGenerationConfigs.id, config.id), + eq(insightGenerationConfigs.enabled, true), + eq(insightGenerationConfigs.nextRunAt, config.nextRunAt), + eq(insightGenerationConfigs.updatedAt, config.updatedAt), + config.dispatchDueAt + ? eq(insightGenerationConfigs.dispatchDueAt, config.dispatchDueAt) + : isNull(insightGenerationConfigs.dispatchDueAt) + ) + ) + .returning({ id: insightGenerationConfigs.id }); + return updated.length === 1; +} + +async function durableScheduledRunAfter( + organizationId: string, + dueAt: Date +): Promise<{ createdAt: Date; id: string } | null> { + const [run] = await db + .select({ createdAt: insightRuns.createdAt, id: insightRuns.id }) + .from(insightRuns) + .where( + and( + eq(insightRuns.organizationId, organizationId), + eq(insightRuns.reason, "scheduled"), + gte(insightRuns.createdAt, dueAt), + inArray(insightRuns.status, [ + "queued", + "running", + "succeeded", + "partially_succeeded", + "skipped", + ]) + ) + ) + .orderBy(desc(insightRuns.createdAt)) + .limit(1); + return run ?? null; } export async function retryConfigSoon( - configId: string, - claimedNextRunAt: Date, + config: DueConfig, now: Date ): Promise { + if (!config.nextRunAt) { + return; + } await db .update(insightGenerationConfigs) .set({ @@ -111,14 +214,25 @@ export async function retryConfigSoon( }) .where( and( - eq(insightGenerationConfigs.id, configId), + eq(insightGenerationConfigs.id, config.id), eq(insightGenerationConfigs.enabled, true), - eq(insightGenerationConfigs.nextRunAt, claimedNextRunAt) + eq(insightGenerationConfigs.nextRunAt, config.nextRunAt), + eq(insightGenerationConfigs.updatedAt, config.updatedAt) ) ); } export async function ensureInsightsDispatchSchedule(): Promise { + if (!isScheduledDispatchEnabled()) { + const removed = await getInsightsQueue().removeJobScheduler( + INSIGHTS_DISPATCH_JOB_NAME + ); + emitInsightsEvent("info", "scheduler.dispatch_disabled", { + removed_existing_schedule: removed, + }); + return; + } + const intervalMs = dispatchIntervalMs(); await getInsightsQueue().upsertJobScheduler( INSIGHTS_DISPATCH_JOB_NAME, @@ -156,11 +270,24 @@ export async function ensureInsightsMaintenanceSchedule(): Promise { }); } +export async function removeInsightsSchedules(): Promise { + const queue = getInsightsQueue(); + const [dispatchRemoved, maintenanceRemoved] = await Promise.all([ + queue.removeJobScheduler(INSIGHTS_DISPATCH_JOB_NAME), + queue.removeJobScheduler(INSIGHTS_MAINTENANCE_JOB_NAME), + ]); + emitInsightsEvent("info", "scheduler.schedules_removed", { + dispatch_removed: dispatchRemoved, + maintenance_removed: maintenanceRemoved, + }); +} + export async function dispatchDueInsightRuns( - now = new Date() + now = new Date(), + organizationIds?: readonly string[] ): Promise { const startedAt = performance.now(); - const configs = await dueConfigs(now); + const configs = await dueConfigs(now, organizationIds); const result: DispatchDueInsightRunsResult = { scannedConfigs: configs.length, claimedConfigs: 0, @@ -170,7 +297,7 @@ export async function dispatchDueInsightRuns( }; for (const config of configs) { - const claimed = await claimConfig(config, now); + const claimed = await claimDueConfig(config, now); if (!claimed) { result.skippedConfigs += 1; continue; @@ -178,13 +305,41 @@ export async function dispatchDueInsightRuns( result.claimedConfigs += 1; try { + const durableRun = claimed.dispatchDueAt + ? await durableScheduledRunAfter( + claimed.organizationId, + claimed.dispatchDueAt + ) + : null; + if (durableRun) { + await markConfigDispatched(claimed, durableRun.createdAt, now); + result.skippedConfigs += 1; + emitInsightsEvent("warn", "scheduler.dispatch_receipt_recovered", { + config_id: claimed.id, + organization_id: claimed.organizationId, + run_id: durableRun.id, + }); + continue; + } const queued = await queueInsightGenerationRun({ organizationId: claimed.organizationId, reason: "scheduled", }); if (queued.reusedRun) { - if (claimed.nextRunAt) { - await retryConfigSoon(claimed.id, claimed.nextRunAt, now); + const reusedScheduledRun = queued.runId + ? await durableScheduledRunAfter( + claimed.organizationId, + claimed.dispatchDueAt ?? now + ) + : null; + if (reusedScheduledRun) { + await markConfigDispatched( + claimed, + reusedScheduledRun.createdAt, + now + ); + } else { + await retryConfigSoon(claimed, now); } result.skippedConfigs += 1; emitInsightsEvent("warn", "scheduler.config_skipped_active_run", { @@ -194,7 +349,9 @@ export async function dispatchDueInsightRuns( }); continue; } - await markConfigDispatched(claimed.id, now); + if (queued.status !== "disabled") { + await markConfigDispatched(claimed, now, now); + } if (queued.status !== "queued") { result.skippedConfigs += 1; emitInsightsEvent("warn", "scheduler.config_skipped", { @@ -213,9 +370,7 @@ export async function dispatchDueInsightRuns( run_id: queued.runId, }); } catch (error) { - if (claimed.nextRunAt) { - await retryConfigSoon(claimed.id, claimed.nextRunAt, now); - } + await retryConfigSoon(claimed, now); result.skippedConfigs += 1; captureInsightsError(error, "scheduler.config_dispatch_failed", { config_id: claimed.id, diff --git a/apps/insights/src/terminal-decision.ts b/apps/insights/src/terminal-decision.ts index f809c5620..429c8098d 100644 --- a/apps/insights/src/terminal-decision.ts +++ b/apps/insights/src/terminal-decision.ts @@ -23,16 +23,14 @@ export function terminalDecisionFromEvidence( item.entity.id === signal.entity.id && item.remediation ); - if ( - signal.kind === "missing_expected_data" && - signal.expectation && - remediationEvidence?.remediation - ) { + if (signal.kind === "missing_expected_data" && signal.expectation) { const confirmation = signal.expectation.confirmation; if ( - !confirmation || + !(remediationEvidence?.remediation && confirmation) || confirmation.definitionId !== signal.entity.id || - confirmation.definitionType !== signal.entity.type + confirmation.definitionType !== signal.entity.type || + (confirmation.source !== "revenue_transactions" && + confirmation.source !== "server_completions") ) { return { disposition: "needs_context", gap: "expected_behavior" }; } @@ -49,6 +47,7 @@ export function terminalDecisionFromEvidence( signal.severity !== "info" && signal.sentiment === "negative" && (signal.metric.key === "revenue" || + signal.metric.key === "payment_failure_rate" || (signal.severity === "critical" && ["pageviews", "sessions", "visitors"].includes(signal.metric.key))) ) { diff --git a/apps/status/app/error.tsx b/apps/status/app/error.tsx index 27eb51975..85c1153e3 100644 --- a/apps/status/app/error.tsx +++ b/apps/status/app/error.tsx @@ -20,10 +20,10 @@ export default function ErrorPage({ return ( } - code="503" - description="Current monitor data could not be loaded. Try again in a moment." + code="500" + description="An unexpected error prevented this page from loading. Try again in a moment." detail={error.digest} - title="Status unavailable" + title="Something went wrong" /> ); } diff --git a/bun.lock b/bun.lock index c996faea3..4210f5bf3 100644 --- a/bun.lock +++ b/bun.lock @@ -315,6 +315,7 @@ }, "devDependencies": { "@biomejs/biome": "2.5.1", + "@databuddy/tracker": "workspace:*", "@tailwindcss/postcss": "^4.2.2", "@types/d3": "^7.4.3", "@types/d3-geo": "^3.1.0", @@ -569,6 +570,7 @@ "name": "@databuddy/email", "version": "0.0.0", "dependencies": { + "@databuddy/shared": "workspace:*", "@types/node": "^26.0.0", "react": "catalog:", "react-dom": "catalog:", diff --git a/docker-compose.selfhost.yml b/docker-compose.selfhost.yml index ee1f7066e..16405b1a0 100644 --- a/docker-compose.selfhost.yml +++ b/docker-compose.selfhost.yml @@ -161,9 +161,11 @@ services: BULLMQ_REDIS_URL: ${BULLMQ_REDIS_URL:?Set BULLMQ_REDIS_URL in your environment} INSIGHTS_BULLMQ_REDIS_URL: ${INSIGHTS_BULLMQ_REDIS_URL:-} INSIGHTS_DISPATCH_INTERVAL_MS: ${INSIGHTS_DISPATCH_INTERVAL_MS:-300000} + INSIGHTS_SCHEDULED_DISPATCH_ENABLED: ${INSIGHTS_SCHEDULED_DISPATCH_ENABLED:-false} + INSIGHTS_SCHEDULED_ORGANIZATION_IDS: ${INSIGHTS_SCHEDULED_ORGANIZATION_IDS:-} INSIGHTS_MAINTENANCE_INTERVAL_MS: ${INSIGHTS_MAINTENANCE_INTERVAL_MS:-300000} INSIGHTS_STALE_ITEM_MS: ${INSIGHTS_STALE_ITEM_MS:-900000} - INSIGHTS_WORKER_CONCURRENCY: ${INSIGHTS_WORKER_CONCURRENCY:-5} + INSIGHTS_WORKER_CONCURRENCY: ${INSIGHTS_WORKER_CONCURRENCY:-2} INSIGHTS_WORKER_ENABLED: ${INSIGHTS_WORKER_ENABLED:-false} CLICKHOUSE_URL: ${CLICKHOUSE_URL:?Set CLICKHOUSE_URL in your environment} healthcheck: diff --git a/lefthook.yml b/lefthook.yml index 60dd5cc5b..ab8adae4b 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -7,7 +7,7 @@ output: - execution_out pre-commit: - parallel: true + parallel: false commands: no-secrets: glob: @@ -24,15 +24,15 @@ pre-commit: run: | blocked_files="$(echo "{staged_files}" | tr ' ' '\n' | grep -vE '(^|/)\.env\.example$' || true)" [ -z "$blocked_files" ] || (echo "Blocked: staged file(s) may contain secrets:" && echo "$blocked_files" | sed 's/^/ - /' && exit 1) - check-types: - glob: "**/*.{ts,tsx}" - run: bun run check-types ch-types: glob: "packages/db/src/clickhouse/schema/**/*.sql" run: | set -e bun run generate-db git add packages/db/src/clickhouse/schema/tables.generated.ts + check-types: + glob: "**/*.{ts,tsx}" + run: bun run check-types lockfile-check: glob: - "**/package.json" diff --git a/package.json b/package.json index 53119557b..74af5d69c 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "dev:status": "dotenv -- turbo dev --filter @databuddy/status --filter @databuddy/api", "git:setup-aliases": "git config --local include.path ../gitconfig", "eval": "dotenv -- bun run --cwd packages/evals src/cli.ts", - "eval:insights": "bun run --cwd packages/evals src/insight-history.ts", + "eval:insights": "dotenv -- bun run --cwd packages/evals src/insight-history.ts", "eval:insights:production-shadow": "dotenv -- bun run --cwd packages/evals src/insight-production-shadow.ts", "eval:slack": "dotenv -- bun run --cwd packages/evals src/cli.ts slack-harness", "eval:ui": "dotenv -- bun run --cwd packages/evals ui/serve.ts", diff --git a/packages/ai/package.json b/packages/ai/package.json index aa73274b1..2b828751f 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -18,7 +18,7 @@ "./agents/types": "./src/ai/agents/types.ts", "./config/context": "./src/ai/config/context.ts", "./config/models": "./src/ai/config/models.ts", - "./insights/evidence-reader": "./src/ai/tools/insights-agent-tools.ts", + "./insights/evidence-reader": "./src/ai/insights/evidence-reader.ts", "./insights/fetch-context": "./src/ai/insights/fetch-context.ts", "./insights/validate": "./src/ai/insights/validate.ts", "./lib/accessible-websites": "./src/lib/accessible-websites.ts", diff --git a/packages/ai/src/ai/agents/cache.test.ts b/packages/ai/src/ai/agents/cache.test.ts index d7fabedf8..65b504081 100644 --- a/packages/ai/src/ai/agents/cache.test.ts +++ b/packages/ai/src/ai/agents/cache.test.ts @@ -65,7 +65,7 @@ vi.mock("@databuddy/redis", () => ({ INSIGHTS_JOB_OPTIONS: {}, INSIGHTS_JOB_TIMEOUT_MS: 120_000, INSIGHTS_QUEUE_ENV_PREFIX: "INSIGHTS", - INSIGHTS_QUEUE_NAME: "insights-generation", + INSIGHTS_QUEUE_NAME: "insights-investigations-v1", INSIGHTS_ROLLUP_JOB_NAME: "insights-rollup", activeStreamKey: (id: string) => `active:${id}`, appendStreamChunk: vi.fn(async () => undefined), diff --git a/packages/ai/src/ai/tools/insights-agent-tools.test.ts b/packages/ai/src/ai/insights/evidence-reader.test.ts similarity index 74% rename from packages/ai/src/ai/tools/insights-agent-tools.test.ts rename to packages/ai/src/ai/insights/evidence-reader.test.ts index a4d26293f..4451acc15 100644 --- a/packages/ai/src/ai/tools/insights-agent-tools.test.ts +++ b/packages/ai/src/ai/insights/evidence-reader.test.ts @@ -1,13 +1,16 @@ import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test"; import * as actualQuery from "../../query"; -import * as actualProductContext from "../insights/product-context"; +import * as actualProductContext from "./product-context"; import type { InvestigationEvidence, InvestigationSignal, } from "@databuddy/shared/insights"; import type { AppContext } from "../config/context"; -import type { ProductInsightTarget } from "../insights/product-context"; -import type { InsightEvidenceReadRequest } from "./insights-agent-tools"; +import type { + ProductInsightTarget, + ProductMetricsFetcher, +} from "./product-context"; +import type { InsightEvidenceReadRequest } from "./evidence-reader"; type WebQueryMode = | "changed" @@ -16,6 +19,9 @@ type WebQueryMode = | "failed" | "large" | "normal" + | "payment-failure" + | "payment-unavailable" + | "payment-unverified" | "revenue-reconcile" | "uptime-empty" | "vital-pages"; @@ -60,6 +66,55 @@ mock.module("../../query", () => ({ if (webQueryMode === "changed") { return [{ sessions: 121 }]; } + if ( + webQueryMode === "payment-unavailable" && + request.type === "revenue_overview" + ) { + return [ + { + currency: "USD", + failed_payment_attempts: null, + observed_failure_event_types: null, + payment_diagnostics_available: 0, + payment_failure_rate: null, + successful_payment_attempts: null, + total_revenue: 75, + total_transactions: 1, + }, + ]; + } + if ( + webQueryMode === "payment-unverified" && + request.type === "revenue_overview" + ) { + return [ + { + currency: "USD", + failed_payment_attempts: 0, + observed_failure_event_types: 0, + payment_failure_rate: 0, + required_failure_event_types: 2, + successful_payment_attempts: 20, + }, + ]; + } + if ( + webQueryMode === "payment-failure" && + request.type === "revenue_overview" + ) { + return [ + { + currency: "USD", + failed_payment_attempts: 8, + observed_failure_event_types: 1, + payment_failure_rate: 20, + recovered_payment_attempts: 2, + required_failure_event_types: 2, + successful_payment_attempts: 32, + top_payment_failure_reason: "insufficient_funds", + }, + ]; + } if ( webQueryMode === "revenue-reconcile" && request.type === "revenue_overview" @@ -175,7 +230,7 @@ mock.module("../../query", () => ({ }, })); -mock.module("../insights/product-context", () => ({ +mock.module("./product-context", () => ({ ...actualProductContext, fetchProductMetrics: async ( _context: unknown, @@ -225,7 +280,7 @@ mock.module("../insights/product-context", () => ({ })); const { countEvidenceRows, createInsightEvidenceReader } = await import( - "./insights-agent-tools" + "./evidence-reader" ); const signal: InvestigationSignal = { @@ -282,10 +337,14 @@ const appContext: AppContext = { function createReader( onEvidence?: (evidence: InvestigationEvidence) => void, - selectedSignal: InvestigationSignal = signal + selectedSignal: InvestigationSignal = signal, + productMetricsFetcher?: ProductMetricsFetcher ) { return createInsightEvidenceReader({ domain: "example.com", + ...(productMetricsFetcher + ? { fetchProductMetrics: productMetricsFetcher } + : {}), onEvidence, signal: selectedSignal, timezone: "UTC", @@ -306,7 +365,7 @@ describe("insight evidence reader", () => { afterAll(() => { mock.module("../../query", () => actualQuery); - mock.module("../insights/product-context", () => actualProductContext); + mock.module("./product-context", () => actualProductContext); }); test("returns stable, signal-scoped success and empty evidence", async () => { @@ -696,6 +755,86 @@ describe("insight evidence reader", () => { expect(result.every((item) => item.evidenceId.length > 0)).toBe(true); }); + test("queries a long custom-event name without replacing it with a bounded key", async () => { + const eventName = `checkout_${"step_".repeat(38)}`; + const eventSignal: InvestigationSignal = { + ...signal, + signalKey: "custom_event:bounded", + entity: { + type: "event", + id: eventName, + label: eventName.slice(0, 120), + }, + metric: { + ...signal.metric, + key: "custom_event:bounded", + label: "Long checkout event", + }, + }; + const reader = createReader(undefined, eventSignal); + + await reader( + { name: "product_metrics", input: { period: "current" } }, + appContext + ); + + expect(productCalls).toEqual([ + { + range: { from: "2026-07-01", to: "2026-07-07" }, + target: { id: eventName, type: "event" }, + }, + ]); + }); + + test("uses injected product evidence without RPC-backed default reads or service auth", async () => { + const goalSignal: InvestigationSignal = { + ...signal, + signalKey: "goal:private", + entity: { type: "goal", id: "goal_private", label: "Private goal" }, + metric: { ...signal.metric, key: "goal:private" }, + }; + const injectedCalls: ProductInsightTarget[] = []; + const injected: ProductMetricsFetcher = async ( + context, + _range, + target + ) => { + expect(context.serviceAuth).toBeUndefined(); + injectedCalls.push(target); + return { + results: [ + { + type: "goals_summary", + count: 1, + goals: [ + { + id: target.id, + is_active: true, + name: "Private goal", + total_users_entered: 40, + total_users_completed: 10, + overall_conversion_rate: 25, + }, + ], + }, + ], + }; + }; + const reader = createReader(undefined, goalSignal, injected); + + const result = await reader( + { name: "product_metrics", input: { period: "current" } }, + appContext + ); + + expect(injectedCalls).toEqual([{ id: "goal_private", type: "goal" }]); + expect(productCalls).toEqual([]); + expect(result[0]).toMatchObject({ + queryType: "goals_summary", + status: "ok", + }); + }); + test("attaches only an exact backend-owned tracking repair", async () => { const expectation = { definitionUpdatedAt: "2026-06-01T00:00:00.000Z", @@ -795,6 +934,148 @@ describe("insight evidence reader", () => { }); }); + test("includes safe payment failure causes and observed occurrences", async () => { + webQueryMode = "payment-failure"; + const paymentSignal: InvestigationSignal = { + ...revenueSignal, + signalKey: "payment-failure-rate:usd", + metric: { + ...revenueSignal.metric, + current: 20, + key: "payment_failure_rate", + label: "Payment failure rate (USD)", + previous: 10, + }, + }; + const reader = createReader(undefined, paymentSignal); + const result = await reader( + { + name: "web_metrics", + input: { + period: "both", + queries: [{ type: "revenue_overview" }], + }, + }, + appContext + ); + + expect(result[0]?.summary).toContain( + "Most common failure: insufficient funds." + ); + expect(result[0]?.summary).toContain( + "1 distinct Stripe failure event type was observed" + ); + expect(result[0]?.summary).not.toContain("coverage"); + expect(result[0]?.metrics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + current: 1, + label: "Observed failure event types", + }), + ]) + ); + expect( + result[0]?.metrics.some( + (metric) => metric.label === "Required failure event types" + ) + ).toBe(false); + }); + + test("reports a tracked zero without inferring webhook coverage", async () => { + webQueryMode = "payment-unverified"; + const paymentSignal: InvestigationSignal = { + ...revenueSignal, + signalKey: "payment-failure-rate:usd", + metric: { + ...revenueSignal.metric, + current: 0, + key: "payment_failure_rate", + label: "Payment failure rate (USD)", + previous: 0, + }, + }; + const reader = createReader(undefined, paymentSignal); + const result = await reader( + { + name: "web_metrics", + input: { + period: "both", + queries: [{ type: "revenue_overview" }], + }, + }, + appContext + ); + + expect(result[0]?.summary).toContain("Tracked USD payment failure rate was 0%"); + expect(result[0]?.summary).toContain( + "No recognized Stripe failure event types were observed" + ); + expect(result[0]?.summary).not.toContain("coverage"); + }); + + test("does not present unavailable payment diagnostics as measured zeros", async () => { + webQueryMode = "payment-unavailable"; + const paymentSignal: InvestigationSignal = { + ...revenueSignal, + signalKey: "payment-failure-rate:usd", + metric: { + ...revenueSignal.metric, + key: "payment_failure_rate", + label: "Payment failure rate (USD)", + }, + }; + const reader = createReader(undefined, paymentSignal); + const result = await reader( + { + name: "web_metrics", + input: { + period: "both", + queries: [{ type: "revenue_overview" }], + }, + }, + appContext + ); + + expect(result[0]?.summary).toBe( + "Stripe payment diagnostics are unavailable for the selected filters." + ); + expect(result[0]?.summary).not.toContain("failed payment attempts 0"); + expect( + result[0]?.metrics.some((metric) => + metric.label.toLowerCase().includes("payment") + ) + ).toBe(false); + }); + + test("scopes revenue evidence to the detected currency", async () => { + const reader = createReader(undefined, { + ...revenueSignal, + currency: "USD", + metric: { ...revenueSignal.metric, label: "Revenue (USD)" }, + }); + + await reader( + { + name: "web_metrics", + input: { + period: "both", + queries: [{ type: "revenue_overview" }], + }, + }, + appContext + ); + + expect(queryRequests.length).toBeGreaterThanOrEqual(2); + expect(queryRequests.every((request) => + request.filters?.some( + (filter) => + filter.field === "currency" && + filter.op === "eq" && + filter.value === "USD" + ) + )).toBe(true); + }); + test("re-reads revenue once when query totals conflict with the detector", async () => { webQueryMode = "revenue-reconcile"; const observed: InvestigationEvidence[] = []; diff --git a/packages/ai/src/ai/tools/insights-agent-tools.ts b/packages/ai/src/ai/insights/evidence-reader.ts similarity index 85% rename from packages/ai/src/ai/tools/insights-agent-tools.ts rename to packages/ai/src/ai/insights/evidence-reader.ts index 7008609a2..025fbd475 100644 --- a/packages/ai/src/ai/tools/insights-agent-tools.ts +++ b/packages/ai/src/ai/insights/evidence-reader.ts @@ -11,11 +11,12 @@ import { fetchOpsMetrics, OPS_INSIGHT_QUERY_TYPES, type OpsInsightQuery, -} from "../insights/ops-context"; +} from "./ops-context"; import { fetchProductMetrics, type ProductInsightTarget, -} from "../insights/product-context"; + type ProductMetricsFetcher, +} from "./product-context"; import { executeQuery } from "../../query"; import type { QueryRequest } from "../../query/types"; @@ -239,13 +240,26 @@ function evidenceRows( return [record]; } +function finiteValue(value: unknown): number | null { + if ( + value === null || + value === undefined || + value === "" || + typeof value === "boolean" + ) { + return null; + } + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + function finiteNumber( row: EvidenceRow | undefined, ...keys: string[] ): number | null { for (const key of keys) { - const value = Number(row?.[key]); - if (Number.isFinite(value)) { + const value = finiteValue(row?.[key]); + if (value !== null) { return value; } } @@ -278,9 +292,20 @@ function compactNumber(value: number): string { ); } -function revenueMatchesDetector(data: unknown, expected: number): boolean { +function isRevenueMetric(metricKey: string): boolean { + return metricKey === "revenue" || metricKey === "payment_failure_rate"; +} + +function revenueMetricMatchesDetector( + data: unknown, + metricKey: string, + expected: number +): boolean { const row = evidenceRows(data)[0]; - const actual = finiteNumber(row, "total_revenue", "revenue"); + const actual = + metricKey === "payment_failure_rate" + ? finiteNumber(row, "payment_failure_rate") + : finiteNumber(row, "total_revenue", "revenue"); return ( actual !== null && Math.abs(actual - expected) <= Math.max(0.01, Math.abs(expected) * 0.01) @@ -368,18 +393,48 @@ function summarizePages(rows: EvidenceRow[]): string | null { : null; } -function summarizeRevenue(rows: EvidenceRow[]): string | null { +function summarizeRevenue( + rows: EvidenceRow[], + metricKey: string +): string | null { const row = rows[0]; if (!row) { return null; } + const currency = textValue(row, "currency"); + const currencyLabel = currency ? `${currency} ` : ""; + if (metricKey === "payment_failure_rate") { + if (finiteNumber(row, "payment_diagnostics_available") === 0) { + return "Stripe payment diagnostics are unavailable for the selected filters."; + } + const failureRate = finiteNumber(row, "payment_failure_rate"); + const failed = finiteNumber(row, "failed_payment_attempts"); + const successful = finiteNumber(row, "successful_payment_attempts"); + const recovered = finiteNumber(row, "recovered_payment_attempts"); + const observedTypes = finiteNumber(row, "observed_failure_event_types"); + const topReason = textValue(row, "top_payment_failure_reason")?.replaceAll( + "_", + " " + ); + if (failureRate === null && failed === null && successful === null) { + return null; + } + const cause = topReason ? ` Most common failure: ${topReason}.` : ""; + const observations = + observedTypes === null + ? "" + : observedTypes === 0 + ? " No recognized Stripe failure event types were observed in this range." + : ` ${compactNumber(observedTypes)} distinct Stripe failure event ${observedTypes === 1 ? "type was" : "types were"} observed in this range.`; + return `Tracked ${currencyLabel}payment failure rate was ${compactNumber(failureRate ?? 0)}%: ${compactNumber(failed ?? 0)} failed and ${compactNumber(successful ?? 0)} successful attempts${recovered === null ? "" : `, with ${compactNumber(recovered)} later recovered`}.${cause}${observations}`; + } const revenue = finiteNumber(row, "total_revenue", "revenue"); const transactions = finiteNumber(row, "total_transactions", "transactions"); const customers = finiteNumber(row, "unique_customers", "customers"); if (revenue === null && transactions === null && customers === null) { return null; } - return `Tracked revenue was ${compactNumber(revenue ?? 0)} from ${compactNumber(transactions ?? 0)} transactions across ${compactNumber(customers ?? 0)} customers.`; + return `Tracked ${currencyLabel}revenue was ${compactNumber(revenue ?? 0)} from ${compactNumber(transactions ?? 0)} transactions across ${compactNumber(customers ?? 0)} customers.`; } function summarizeProduct( @@ -449,13 +504,11 @@ function summarizeNamedRows(rows: EvidenceRow[]): string | null { } const numeric = Object.entries(row).find( ([key, value]) => - key !== "name" && - key !== "path" && - typeof value !== "boolean" && - Number.isFinite(Number(value)) + key !== "name" && key !== "path" && finiteValue(value) !== null ); + const value = finiteValue(numeric?.[1]); return [ - `${shortText(name, 90)}${numeric ? ` — ${numeric[0].replaceAll("_", " ")} ${compactNumber(Number(numeric[1]))}` : ""}`, + `${shortText(name, 90)}${numeric && value !== null ? ` — ${numeric[0].replaceAll("_", " ")} ${compactNumber(value)}` : ""}`, ]; }); return summaries.length > 0 ? `Top results: ${summaries.join("; ")}.` : null; @@ -467,11 +520,13 @@ function summarizeAggregate(rows: EvidenceRow[]): string | null { return null; } const facts = Object.entries(row) - .filter(([, value]) => Number.isFinite(Number(value))) + .flatMap(([key, value]) => { + const number = finiteValue(value); + return number === null ? [] : [[key, number] as const]; + }) .slice(0, 4) .map( - ([key, value]) => - `${key.replaceAll("_", " ")} ${compactNumber(Number(value))}` + ([key, value]) => `${key.replaceAll("_", " ")} ${compactNumber(value)}` ); return facts.length > 0 ? `${facts.join("; ")}.` : null; } @@ -503,7 +558,7 @@ function summarizeEvidenceData( } else if (queryType === "errors_by_page") { summary = summarizePages(rows); } else if (queryType === "revenue_overview") { - summary = summarizeRevenue(rows); + summary = summarizeRevenue(rows, signal.metric.key); } else if (queryType === "goals_summary" || queryType === "funnels_summary") { summary = summarizeProduct(rows, signal.entity.label, signal.expectation); } else if (queryType.startsWith("web_vitals_by_")) { @@ -572,6 +627,32 @@ function evidenceMetrics( ]; } if (evidence.queryType === "revenue_overview") { + const paymentMetrics = + finiteNumber(row, "payment_diagnostics_available") === 0 + ? [] + : [ + ...metric( + "Payment failure rate", + finiteNumber(row, "payment_failure_rate"), + "percent" + ), + ...metric( + "Failed payment attempts", + finiteNumber(row, "failed_payment_attempts") + ), + ...metric( + "Successful payment attempts", + finiteNumber(row, "successful_payment_attempts") + ), + ...metric( + "Recovered payment attempts", + finiteNumber(row, "recovered_payment_attempts") + ), + ...metric( + "Observed failure event types", + finiteNumber(row, "observed_failure_event_types") + ), + ]; return [ ...metric( "Queried revenue", @@ -585,6 +666,7 @@ function evidenceMetrics( "Customers", finiteNumber(row, "unique_customers", "customers") ), + ...paymentMetrics, ]; } if ( @@ -709,6 +791,8 @@ export function countEvidenceRows(data: unknown): number { export interface CreateInsightEvidenceReaderParams { domain: string; + /** Overrides product reads for snapshot/evaluation runtimes. */ + fetchProductMetrics?: ProductMetricsFetcher; onEvidence?: (evidence: InvestigationEvidence) => void; signal: InvestigationSignal; timezone: string; @@ -943,21 +1027,20 @@ export function createInsightEvidenceReader( } } - const webQueryTypeSchema = - params.signal.metric.key === "revenue" - ? z.literal("revenue_overview") - : params.signal.entity.type === "vital" - ? z.literal("web_vitals_by_page") - : params.signal.entity.type === "campaign" - ? z.literal("utm_campaigns") - : z.enum([ - "country", - "device_types", - "entry_pages", - "top_pages", - "top_referrers", - "utm_campaigns", - ]); + const webQueryTypeSchema = isRevenueMetric(params.signal.metric.key) + ? z.literal("revenue_overview") + : params.signal.entity.type === "vital" + ? z.literal("web_vitals_by_page") + : params.signal.entity.type === "campaign" + ? z.literal("utm_campaigns") + : z.enum([ + "country", + "device_types", + "entry_pages", + "top_pages", + "top_referrers", + "utm_campaigns", + ]); const querySchema = z .object({ type: webQueryTypeSchema, @@ -965,10 +1048,9 @@ export function createInsightEvidenceReader( .strict(); const webInputSchema = z .object({ - period: - params.signal.metric.key === "revenue" - ? z.literal("both") - : z.literal("current"), + period: isRevenueMetric(params.signal.metric.key) + ? z.literal("both") + : z.literal("current"), queries: z.array(querySchema).min(1).max(MAX_QUERIES), }) .strict(); @@ -977,12 +1059,12 @@ export function createInsightEvidenceReader( { period, queries }: z.infer, abortSignal?: AbortSignal ): Promise { - if (params.signal.metric.key !== "revenue" && period !== "current") { + if (!isRevenueMetric(params.signal.metric.key) && period !== "current") { throw new Error( "Investigations use the detector baseline and query only the current period" ); } - if (params.signal.metric.key === "revenue" && period !== "both") { + if (isRevenueMetric(params.signal.metric.key) && period !== "both") { throw new Error( "Revenue investigations must reconcile both detector periods" ); @@ -1007,8 +1089,15 @@ export function createInsightEvidenceReader( to: p.range.to, timezone: params.timezone, limit: isVitalPageQuery ? 50 : requestedLimit, - filters: - params.signal.entity.type === "campaign" + filters: params.signal.currency + ? [ + { + field: "currency", + op: "eq" as const, + value: params.signal.currency, + }, + ] + : params.signal.entity.type === "campaign" ? [ { field: "utm_campaign", @@ -1028,7 +1117,13 @@ export function createInsightEvidenceReader( p.label === "current" ? params.signal.metric.current : (params.signal.metric.previous ?? 0); - if (!revenueMatchesDetector(data, expected)) { + if ( + !revenueMetricMatchesDetector( + data, + params.signal.metric.key, + expected + ) + ) { data = await read(); } } @@ -1102,7 +1197,12 @@ export function createInsightEvidenceReader( abortSignal, entity: scope.entity, fetch: () => - fetchProductMetrics(appContext, p.range, scope.target, abortSignal), + (params.fetchProductMetrics ?? fetchProductMetrics)( + appContext, + p.range, + scope.target, + abortSignal + ), period: p.label, queries: [{ type: scope.queryType }], range: p.range, @@ -1189,3 +1289,5 @@ export function createInsightEvidenceReader( } }; } + +export type { ProductMetricsFetcher } from "./product-context"; diff --git a/packages/ai/src/ai/insights/product-context.test.ts b/packages/ai/src/ai/insights/product-context.test.ts new file mode 100644 index 000000000..0b0bdb21d --- /dev/null +++ b/packages/ai/src/ai/insights/product-context.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "bun:test"; +import { summarizeFunnelSteps } from "./product-context"; + +describe("summarizeFunnelSteps", () => { + it("serializes the validated step sequence with users from matching step numbers", () => { + expect( + summarizeFunnelSteps( + [ + { name: "Signup", target: "/signup", type: "PAGE_VIEW" }, + { name: "Paid", target: "purchase", type: "CUSTOM" }, + ], + [ + { step_number: 2, users: "5" }, + { step_number: 1, users: 12 }, + ] + ) + ).toEqual([ + { + name: "Signup", + step_number: 1, + target: "/signup", + type: "PAGE_VIEW", + users: 12, + }, + { + name: "Paid", + step_number: 2, + target: "purchase", + type: "CUSTOM", + users: 5, + }, + ]); + }); + + it("does not reindex a funnel around a malformed middle step", () => { + expect( + summarizeFunnelSteps( + [ + { name: "Signup", target: "/signup", type: "PAGE_VIEW" }, + { name: "Broken", type: "EVENT" }, + { name: "Paid", target: "purchase", type: "CUSTOM" }, + ], + [ + { step_number: 1, users: 12 }, + { step_number: 3, users: 5 }, + ] + ) + ).toEqual([]); + }); +}); diff --git a/packages/ai/src/ai/insights/product-context.ts b/packages/ai/src/ai/insights/product-context.ts index c78fe0e83..ada13dbfb 100644 --- a/packages/ai/src/ai/insights/product-context.ts +++ b/packages/ai/src/ai/insights/product-context.ts @@ -2,12 +2,47 @@ import { type AppContext, requireWebsiteId } from "../config/context"; import { callRPCProcedure } from "../tools/utils"; import { executeQuery } from "../../query"; import type { QueryRequest } from "../../query/types"; +import { normalizeFunnelSteps } from "@databuddy/rpc/funnel-steps"; export interface ProductInsightTarget { id: string; type: "event" | "funnel" | "goal"; } +export interface ProductMetricsResult { + results: Record[]; +} + +export type ProductMetricsFetcher = ( + appContext: AppContext, + range: { from: string; to: string }, + target: ProductInsightTarget, + abortSignal?: AbortSignal +) => Promise; + +export function summarizeFunnelSteps( + steps: unknown, + analyticsSteps: Record[] +) { + const normalized = normalizeFunnelSteps(steps); + const analyticsByStep = new Map( + analyticsSteps.flatMap((step) => { + const stepNumber = toNumber(step.step_number); + return Number.isInteger(stepNumber) && stepNumber > 0 + ? [[stepNumber, step] as const] + : []; + }) + ); + + return normalized.map((step, index) => ({ + step_number: index + 1, + name: step.name, + target: step.target, + type: step.type, + users: toNumber(analyticsByStep.get(index + 1)?.users), + })); +} + function toNumber(value: unknown): number { if (typeof value === "number" && Number.isFinite(value)) { return value; @@ -79,6 +114,7 @@ async function getGoalsSummary( overall_conversion_rate: toNumber(analytics.overall_conversion_rate), total_users_entered: toNumber(analytics.total_users_entered), total_users_completed: toNumber(analytics.total_users_completed), + error_context_available: primaryStep.error_context_available === true, error_rate: toNumber(primaryStep.error_rate), }, ], @@ -128,9 +164,6 @@ async function getFunnelsSummary( const analyticsSteps = Array.isArray(analytics.steps_analytics) ? (analytics.steps_analytics as Record[]) : []; - const definitionSteps = Array.isArray(funnel.steps) - ? (funnel.steps as Record[]) - : []; return { count: 1, @@ -140,18 +173,13 @@ async function getFunnelsSummary( is_active: funnel.isActive, name: funnel.name, definition_updated_at: isoDate(funnel.updatedAt), - steps: definitionSteps.map((step, index) => ({ - step_number: index + 1, - name: step.name, - target: step.target, - type: step.type, - users: toNumber(analyticsSteps[index]?.users), - })), + steps: summarizeFunnelSteps(funnel.steps, analyticsSteps), overall_conversion_rate: toNumber(analytics.overall_conversion_rate), total_users_entered: toNumber(analytics.total_users_entered), total_users_completed: toNumber(analytics.total_users_completed), biggest_dropoff_step: toNumber(analytics.biggest_dropoff_step), biggest_dropoff_rate: toNumber(analytics.biggest_dropoff_rate), + error_context_available: errorInsights.available === true, error_correlation_rate: toNumber(errorInsights.error_correlation_rate), }, ], @@ -205,7 +233,7 @@ export async function fetchProductMetrics( range: { from: string; to: string }, target: ProductInsightTarget, abortSignal?: AbortSignal -) { +): Promise { let result: Record; switch (target.type) { case "goal": diff --git a/packages/ai/src/ai/insights/validate.test.ts b/packages/ai/src/ai/insights/validate.test.ts index 6b80a4149..9a819ac18 100644 --- a/packages/ai/src/ai/insights/validate.test.ts +++ b/packages/ai/src/ai/insights/validate.test.ts @@ -75,7 +75,7 @@ const goalExpectation = { count: 12, definitionId: "signup", definitionType: "goal" as const, - source: "server_completions" as const, + source: "revenue_transactions" as const, }, definitionUpdatedAt: "2026-06-01T00:00:00.000Z", eventName: "sign_up", @@ -154,7 +154,7 @@ describe("validateInvestigationDecision", () => { expect(result.decision).toEqual(missingGoalAction); expect(result.insight).toMatchObject({ title: "Fix tracking for Signup", - description: missingGoalSignal.detection.reason, + description: "Signup completion rate fell from 20% to 0%.", suggestion: missingGoalAction.remediation.instruction, severity: missingGoalSignal.severity, sentiment: missingGoalSignal.sentiment, @@ -210,12 +210,6 @@ describe("validateInvestigationDecision", () => { it("only permits actions for exact negative entities", () => { expect(canRecommendAction(signal)).toBe(false); - expect( - canRecommendAction({ - ...signal, - metric: { ...signal.metric, key: "bounce_rate" }, - }) - ).toBe(false); expect( canRecommendAction(missingGoalSignal) ).toBe(true); @@ -492,113 +486,11 @@ describe("validateInvestigationDecision", () => { }); expect(needsContext.errors).toEqual([]); expect(needsContext.insight).toMatchObject({ - title: "Visitors drop needs context", + title: "Visitors fell 40%", severity: "critical", }); }); - it("keeps generic website rate monitors silent until evidence localizes the same metric", () => { - for (const metric of [ - { - key: "bounce_rate", - label: "Bounce rate", - current: 70, - previous: 40, - format: "percent" as const, - }, - { - key: "session_duration", - label: "Session duration", - current: 30, - previous: 90, - format: "duration_s" as const, - }, - ]) { - const rateSignal: InvestigationSignal = { - ...signal, - signalKey: metric.key, - insightType: - metric.key === "bounce_rate" - ? "bounce_rate_change" - : "engagement_change", - entity: { type: "website", id: "website", label: metric.label }, - metric, - changePercent: 75, - direction: metric.key === "bounce_rate" ? "up" : "down", - severity: "critical", - }; - const genericBreakdown: InvestigationEvidence = { - ...evidence, - evidenceId: `evidence:${metric.key}:generic`, - signalKey: rateSignal.signalKey, - queryType: "entry_pages", - summary: "Entry pages were ranked by visitor count.", - metrics: [], - }; - const emptyBreakdown: InvestigationEvidence = { - evidenceId: `evidence:${metric.key}:empty`, - signalKey: rateSignal.signalKey, - kind: "breakdown", - source: "web", - queryType: "entry_pages", - period: "current", - range: rateSignal.period.current, - status: "empty", - rowCount: 0, - summary: "entry_pages returned no rows.", - }; - const unrelatedPageMetric: InvestigationEvidence = { - ...genericBreakdown, - evidenceId: `evidence:${metric.key}:traffic-only`, - entity: { type: "page", id: "page:pricing", label: "/pricing" }, - metrics: [{ label: "Visitors", current: 200, format: "number" }], - }; - - for (const item of [ - emptyBreakdown, - genericBreakdown, - unrelatedPageMetric, - ]) { - const result = validateInvestigationDecision({ - signal: rateSignal, - evidence: [item], - decision: { disposition: "monitor" }, - }); - expect(result.errors).toEqual([]); - expect(result.insight).toBeNull(); - } - - const localizedBreakdown: InvestigationEvidence = { - ...unrelatedPageMetric, - evidenceId: `evidence:${metric.key}:localized`, - summary: `${metric.label} is concentrated on /pricing.`, - metrics: [ - { - label: metric.label, - current: metric.current, - format: metric.format, - }, - ], - }; - const localized = validateInvestigationDecision({ - signal: rateSignal, - evidence: [localizedBreakdown], - decision: { disposition: "monitor" }, - }); - - expect(localized.errors).toEqual([]); - expect(localized.insight).toMatchObject({ - title: "Investigate /pricing", - evidence: [ - { - type: "segment", - description: localizedBreakdown.summary, - }, - ], - }); - } - }); - it("surfaces a critical exact error as unresolved when the repair is unknown", () => { const errorSignal: InvestigationSignal = { ...signal, @@ -876,10 +768,10 @@ describe("validateInvestigationDecision", () => { expect(result.errors).toEqual([]); expect(result.insight).toMatchObject({ - title: "Visitors drop needs context", - description: signal.detection.reason, + title: "Visitors fell 40%", + description: "Visitors fell from 1,000 to 600.", suggestion: - "Visitors fell from 1000 to 600. Was this expected? If not, check source traffic against recorded events on the first affected day.", + "Was this traffic drop expected? If not, check source traffic against recorded events on the first affected day.", priority: 6, sources: ["web"], }); @@ -899,6 +791,7 @@ describe("validateInvestigationDecision", () => { kind: "impact", source: "business", queryType: "revenue_overview", + summary: "Tracked revenue was 600 from 12 transactions across 8 customers.", metrics: [ { label: "Queried revenue", current: 600, format: "number" }, ], @@ -908,6 +801,7 @@ describe("validateInvestigationDecision", () => { evidenceId: "evidence:revenue:previous", period: "previous", range: revenueSignal.period.previous, + summary: "Tracked revenue was 1,000 from 20 transactions across 13 customers.", metrics: [ { label: "Queried revenue", current: 1000, format: "number" }, ], @@ -956,11 +850,12 @@ describe("validateInvestigationDecision", () => { }, }); expect(surfaced.insight?.suggestion).toBe( - "Revenue changed from 1000 to 600. Was this expected? If not, reconcile billing events with revenue tracking." + "Was this revenue change expected? If not, reconcile billing events with revenue tracking." ); expect(surfaced.insight?.impactSummary).toBe( - "Tracked revenue decreased by 400 compared with the previous period (1,000 → 600)." + "Tracked revenue was 600 from 12 transactions across 8 customers." ); + expect(surfaced.insight?.evidence).toEqual([]); expect(surfaced.insight?.metrics).toEqual([ { label: "Revenue", @@ -987,6 +882,112 @@ describe("validateInvestigationDecision", () => { ); }); + it("treats an empty revenue overview as a confirmed zero", () => { + const revenueSignal: InvestigationSignal = { + ...signal, + signalKey: "revenue:USD", + currency: "USD", + severity: "critical", + metric: { + ...signal.metric, + key: "revenue", + label: "Revenue (USD)", + current: 0, + previous: 1000, + }, + }; + const currentRevenueEvidence: InvestigationEvidence = { + evidenceId: "evidence:revenue:current-empty", + signalKey: revenueSignal.signalKey, + kind: "impact", + source: "business", + queryType: "revenue_overview", + period: "current", + range: revenueSignal.period.current, + status: "empty", + rowCount: 0, + summary: "revenue_overview returned no rows.", + }; + const previousRevenueEvidence: InvestigationEvidence = { + evidenceId: "evidence:revenue:previous", + signalKey: revenueSignal.signalKey, + kind: "impact", + source: "business", + queryType: "revenue_overview", + period: "previous", + range: revenueSignal.period.previous, + status: "ok", + rowCount: 1, + summary: "Tracked USD revenue was 1,000.", + metrics: [ + { label: "Queried revenue", current: 1000, format: "number" }, + ], + }; + + const result = validateInvestigationDecision({ + signal: revenueSignal, + evidence: [currentRevenueEvidence, previousRevenueEvidence], + decision: { + disposition: "needs_context", + gap: "planned_external_change", + }, + }); + + expect(result.errors).toEqual([]); + expect(result.insight).toMatchObject({ + title: "Revenue (USD) fell 100%", + suggestion: + "Was this revenue change expected? If not, reconcile billing events with revenue tracking.", + }); + }); + + it("gives a vanished custom event a concrete instrumentation check", () => { + const eventSignal: InvestigationSignal = { + ...signal, + signalKey: "custom_event:checkout_completed", + kind: "missing_expected_data", + entity: { + type: "event", + id: "checkout_completed", + label: "checkout_completed", + }, + metric: { + key: "custom_event:checkout_completed", + label: "“checkout_completed” users", + current: 0, + previous: 24, + format: "number", + }, + severity: "critical", + }; + const eventEvidence: InvestigationEvidence = { + evidenceId: "evidence:event:checkout-completed", + signalKey: eventSignal.signalKey, + kind: "definition", + source: "product", + queryType: "custom_events_summary", + entity: eventSignal.entity, + period: "current", + range: eventSignal.period.current, + status: "ok", + rowCount: 1, + summary: "checkout_completed recorded no users in the current period.", + }; + + const result = validateInvestigationDecision({ + signal: eventSignal, + evidence: [eventEvidence], + decision: { disposition: "monitor" }, + }); + + expect(result.errors).toEqual([]); + expect(result.insight).toMatchObject({ + title: "Investigate checkout_completed", + suggestion: + "Verify that checkout_completed still fires at its expected trigger in Databuddy DevTools. If it does, compare the last nonzero day with the first zero day for ingestion or filtering changes.", + }); + }); + it("does not turn an internal query failure into a terminal outcome", () => { for (const decision of [ actionReady, diff --git a/packages/ai/src/ai/insights/validate.ts b/packages/ai/src/ai/insights/validate.ts index 1934d062d..31a4bb5dd 100644 --- a/packages/ai/src/ai/insights/validate.ts +++ b/packages/ai/src/ai/insights/validate.ts @@ -43,12 +43,6 @@ const ACTION_TITLE_PREFIX: Record = { operations: "Address", }; -const GENERIC_WEBSITE_RATE_METRICS = new Set([ - "bounce_rate", - "session_duration", -]); -const LOCALIZED_RATE_ENTITY_TYPES = new Set(["page", "channel", "campaign"]); - const ERROR_DISPLAY_LABEL_MAX_CHARS = 60; const ERROR_HELP_SUFFIX_MARKERS = [ ". for more information", @@ -147,6 +141,7 @@ const REPAIR_VERBS = new Set([ "fix", "guard", "handle", + "link", "pause", "reduce", "remove", @@ -195,16 +190,29 @@ function evidenceDescription(evidence: UsableInvestigationEvidence): string { } function needsContextTitle(signal: InvestigationSignal): string { - if (["event", "funnel", "goal"].includes(signal.entity.type)) { - return `${signal.entity.label} needs context`; - } - const change = - signal.direction === "down" - ? "drop" - : signal.direction === "up" - ? "increase" - : "change"; - return `${signal.metric.label} ${change} needs context`; + if (signal.entity.type === "funnel" || signal.entity.type === "goal") { + const label = signal.entity.label.toLowerCase().includes("conversion") + ? signal.entity.label + : `${signal.entity.label} conversion`; + return signal.metric.current === 0 && (signal.metric.previous ?? 0) > 0 + ? `${label} stopped` + : `${label} ${signal.direction === "down" ? "fell" : "rose"}`; + } + if (signal.metric.key === "payment_failure_rate") { + return `${signal.metric.label} rose to ${compactNumber(signal.metric.current)}%`; + } + const derivedChange = + signal.metric.previous === undefined + ? signal.changePercent + : signal.metric.previous === 0 + ? signal.changePercent + : ((signal.metric.current - signal.metric.previous) / + signal.metric.previous) * + 100; + const change = Number.isFinite(derivedChange) + ? ` ${compactNumber(Math.abs(derivedChange ?? 0))}%` + : ""; + return `${signal.metric.label} ${signal.direction === "down" ? "fell" : "rose"}${change}`; } function storedEvidenceType( @@ -316,17 +324,34 @@ function valuesAgree(expected: number, actual: number): boolean { ); } -function queriedRevenue( +function isRevenueMetric(metricKey: string): boolean { + return metricKey === "revenue" || metricKey === "payment_failure_rate"; +} + +function queriedRevenueMetric( evidence: InvestigationEvidence[], - period: "current" | "previous" + period: "current" | "previous", + metricKey: string ): number | null { const item = evidence.find( (candidate) => candidate.queryType === "revenue_overview" && candidate.period === period && - (candidate.status === "ok" || candidate.status === "truncated") + (candidate.status === "ok" || + candidate.status === "truncated" || + candidate.status === "empty") ); - return item ? evidenceMetricValue(item, "Queried revenue") : null; + if (item?.status === "empty") { + return 0; + } + return item + ? evidenceMetricValue( + item, + metricKey === "payment_failure_rate" + ? "Payment failure rate" + : "Queried revenue" + ) + : null; } export function isRelevantInvestigationEvidence( @@ -356,7 +381,7 @@ export function isRelevantInvestigationEvidence( if (signal.entity.type === "uptime_monitor") { return evidence.source === "ops" && evidence.queryType.includes("uptime"); } - if (signal.metric.key === "revenue") { + if (isRevenueMetric(signal.metric.key)) { return ( evidence.source === "business" && evidence.queryType.startsWith("revenue") ); @@ -384,37 +409,9 @@ export function canRecommendAction(signal: InvestigationSignal): boolean { Boolean(signal.expectation) && (signal.entity.type === "funnel" || signal.entity.type === "goal") && confirmation?.definitionId === signal.entity.id && - confirmation.definitionType === signal.entity.type - ); -} - -function requiresLocalizedRateEvidence(signal: InvestigationSignal): boolean { - return ( - signal.entity.type === "website" && - GENERIC_WEBSITE_RATE_METRICS.has(signal.metric.key) - ); -} - -function localizesWebsiteRate( - signal: InvestigationSignal, - evidence: InvestigationEvidence -): evidence is UsableInvestigationEvidence { - return ( - requiresLocalizedRateEvidence(signal) && - isUsableEvidence(evidence) && - isRelevantInvestigationEvidence(signal, evidence) && - evidence.kind === "breakdown" && - evidence.period === "current" && - Boolean( - evidence.entity && LOCALIZED_RATE_ENTITY_TYPES.has(evidence.entity.type) - ) && - Boolean( - evidence.metrics?.some( - (metric) => - metric.label === signal.metric.label && - metric.format === signal.metric.format - ) - ) + confirmation.definitionType === signal.entity.type && + (confirmation.source === "revenue_transactions" || + confirmation.source === "server_completions") ); } @@ -471,10 +468,13 @@ function contextQuestion( signal: InvestigationSignal ): string { if (signal.metric.key === "revenue") { - return `Revenue changed from ${signal.metric.previous ?? "its prior level"} to ${signal.metric.current}. Was this expected? If not, reconcile billing events with revenue tracking.`; + return "Was this revenue change expected? If not, reconcile billing events with revenue tracking."; + } + if (signal.metric.key === "payment_failure_rate") { + return "Was this payment failure increase expected? If not, check the failing payment method or checkout path before retrying customers."; } if (["visitors", "sessions", "pageviews"].includes(signal.metric.key)) { - return `${signal.metric.label} fell from ${signal.metric.previous ?? "its prior level"} to ${signal.metric.current}. Was this expected? If not, check source traffic against recorded events on the first affected day.`; + return "Was this traffic drop expected? If not, check source traffic against recorded events on the first affected day."; } if (signal.entity.type === "goal" || signal.entity.type === "funnel") { if (signal.expectation) { @@ -507,6 +507,11 @@ function investigationSuggestion( if (signal.entity.type === "goal" || signal.entity.type === "funnel") { return `The failing step is not established yet. Replay ${label} in Databuddy DevTools and verify its entry and completion events.`; } + if (signal.entity.type === "event") { + return signal.metric.current === 0 && (signal.metric.previous ?? 0) > 0 + ? `Verify that ${label} still fires at its expected trigger in Databuddy DevTools. If it does, compare the last nonzero day with the first zero day for ingestion or filtering changes.` + : `Compare ${label} between the last healthy and first affected day by page and device, then verify its expected trigger in Databuddy DevTools.`; + } return `Break down ${label} by its largest affected segment and verify the change in the next complete window.`; } @@ -532,6 +537,14 @@ function insightDescription(signal: InvestigationSignal): string { const movement = signal.direction === "down" ? "improved" : "worsened"; return `${signal.metric.label} ${movement} to ${signal.metric.current} ms but remains above the ${healthyMaximum} ms healthy threshold.`; } + if (signal.metric.previous !== undefined) { + if (signal.metric.current === signal.metric.previous) { + return `${signal.metric.label} held at ${formatMetricValue(signal.metric.current, signal.metric.format)}.`; + } + const movement = + signal.metric.current > signal.metric.previous ? "rose" : "fell"; + return `${signal.metric.label} ${movement} from ${formatMetricValue(signal.metric.previous, signal.metric.format)} to ${formatMetricValue(signal.metric.current, signal.metric.format)}.`; + } return signal.detection.reason; } @@ -541,16 +554,21 @@ function compactNumber(value: number): string { ); } -function impactSummary( - signal: InvestigationSignal, - evidence: UsableInvestigationEvidence +function formatMetricValue( + value: number, + format: InsightMetric["format"] ): string { - if (signal.metric.key !== "revenue" || signal.metric.previous === undefined) { - return evidence.summary; + const number = compactNumber(value); + if (format === "percent") { + return `${number}%`; + } + if (format === "duration_ms") { + return `${number} ms`; + } + if (format === "duration_s") { + return `${number} s`; } - const difference = signal.metric.current - signal.metric.previous; - const direction = difference < 0 ? "decreased" : "increased"; - return `Tracked revenue ${direction} by ${compactNumber(Math.abs(difference))} compared with the previous period (${compactNumber(signal.metric.previous)} → ${compactNumber(signal.metric.current)}).`; + return number; } function backendConfidence( @@ -616,10 +634,6 @@ function toGeneratedInsight( const hasExactUnresolvedTarget = evidence.some((item) => explainsExactEntity(signal, item) ); - const localizedRateEvidence = evidence.find((item) => - localizesWebsiteRate(signal, item) - ); - const needsLocalizedRateEvidence = requiresLocalizedRateEvidence(signal); const requiresExactUnresolvedTarget = [ "error", "event", @@ -632,11 +646,8 @@ function toGeneratedInsight( decision.disposition === "monitor" && signal.sentiment === "negative" && ((signal.severity === "critical" && - (needsLocalizedRateEvidence - ? Boolean(localizedRateEvidence) - : !requiresExactUnresolvedTarget || hasExactUnresolvedTarget)) || - (signal.severity === "warning" && - (Boolean(localizedRateEvidence) || hasExactUnresolvedTarget))); + (!requiresExactUnresolvedTarget || hasExactUnresolvedTarget)) || + (signal.severity === "warning" && hasExactUnresolvedTarget)); if ( decision.disposition === "not_a_problem" || (decision.disposition === "monitor" && !unresolvedMonitor) @@ -659,6 +670,8 @@ function toGeneratedInsight( for (const metric of usableEvidence.flatMap((item) => item.metrics ?? [])) { const duplicatesPrimaryLabel = (signal.metric.key === "revenue" && metric.label === "Queried revenue") || + (signal.metric.key === "payment_failure_rate" && + metric.label === "Payment failure rate") || (signal.metric.key === "lcp" && metric.label === "p75 LCP") || (signal.metric.key === "inp" && metric.label === "p75 INP"); const duplicatesPrimaryValue = @@ -692,13 +705,12 @@ function toGeneratedInsight( ) : undefined; const investigationEvidence = unresolvedMonitor - ? (localizedRateEvidence ?? - usableEvidence.find( + ? usableEvidence.find( (item) => isDiagnosticEvidence(item) && item.entity && explainsExactEntity(signal, item) - )) + ) : undefined; const title = actionReady ? `${ACTION_TITLE_PREFIX[decision.remediation.kind]} ${displayEntityLabel(citedEvidence?.entity ?? signal.entity)}` @@ -712,7 +724,7 @@ function toGeneratedInsight( ? "goals_summary" : signal.entity.type === "funnel" ? "funnels_summary" - : signal.metric.key === "revenue" + : isRevenueMetric(signal.metric.key) ? "revenue_overview" : signal.entity.type === "vital" ? "web_vitals_by_page:qualified" @@ -721,7 +733,12 @@ function toGeneratedInsight( ? usableEvidence.find((item) => item.queryType === impactQueryType) : undefined; const displayEvidence = [ - ...(actionReady && citedEvidence ? [citedEvidence] : []), + ...(actionReady && + citedEvidence && + (citedEvidence.queryType !== impactQueryType || + citedEvidence.evidenceId === impactEvidence?.evidenceId) + ? [citedEvidence] + : []), ...(unresolvedMonitor && investigationEvidence ? [investigationEvidence] : []), @@ -730,7 +747,7 @@ function toGeneratedInsight( : usableEvidence.filter( (item) => !item.queryType.startsWith("detector:") && - item.evidenceId !== impactEvidence?.evidenceId + item.queryType !== impactQueryType )), ]; const insight: GeneratedInsight = { @@ -759,7 +776,7 @@ function toGeneratedInsight( ...(impactEvidence && impactEvidence.evidenceId !== citedEvidence?.evidenceId && impactEvidence.evidenceId !== investigationEvidence?.evidenceId - ? { impactSummary: impactSummary(signal, impactEvidence) } + ? { impactSummary: impactEvidence.summary } : {}), evidence: [ ...displayEvidence.slice(0, 3).map((item) => ({ @@ -840,25 +857,35 @@ export function validateInvestigationDecision( "Investigate at least one relevant Databuddy query before submitting a terminal decision." ); } - if (signal.metric.key === "revenue" && signal.sentiment === "negative") { - const currentRevenue = queriedRevenue(evidence, "current"); - const previousRevenue = queriedRevenue(evidence, "previous"); - if (currentRevenue === null || previousRevenue === null) { + if (isRevenueMetric(signal.metric.key) && signal.sentiment === "negative") { + const currentValue = queriedRevenueMetric( + evidence, + "current", + signal.metric.key + ); + const previousValue = queriedRevenueMetric( + evidence, + "previous", + signal.metric.key + ); + if (currentValue === null || previousValue === null) { errors.push( - "Revenue decisions require revenue_overview evidence for both detector periods." + "Revenue and payment decisions require revenue_overview evidence for both detector periods." ); } else if ( - !valuesAgree(signal.metric.current, currentRevenue) || + !valuesAgree(signal.metric.current, currentValue) || (signal.metric.previous !== undefined && - !valuesAgree(signal.metric.previous, previousRevenue)) + !valuesAgree(signal.metric.previous, previousValue)) ) { errors.push( - "Revenue query totals conflict with the detector. Retry the query instead of producing customer advice." + signal.metric.key === "revenue" + ? "Revenue query totals conflict with the detector. Retry the query instead of producing customer advice." + : "Payment failure rate conflicts with the revenue query. Retry the query instead of producing customer advice." ); } } if ( - signal.metric.key === "revenue" && + isRevenueMetric(signal.metric.key) && signal.sentiment === "negative" && signal.severity === "critical" && decision.disposition === "monitor" @@ -878,7 +905,7 @@ export function validateInvestigationDecision( ); } if ( - signal.metric.key === "revenue" && + isRevenueMetric(signal.metric.key) && decision.disposition === "needs_context" && decision.gap === "business_priority" ) { diff --git a/packages/ai/src/ai/mcp/conversation-store.test.ts b/packages/ai/src/ai/mcp/conversation-store.test.ts index ff0a2d219..59b1b0fa5 100644 --- a/packages/ai/src/ai/mcp/conversation-store.test.ts +++ b/packages/ai/src/ai/mcp/conversation-store.test.ts @@ -32,7 +32,7 @@ vi.mock("@databuddy/redis", () => ({ INSIGHTS_JOB_OPTIONS: {}, INSIGHTS_JOB_TIMEOUT_MS: 120_000, INSIGHTS_QUEUE_ENV_PREFIX: "INSIGHTS", - INSIGHTS_QUEUE_NAME: "insights-generation", + INSIGHTS_QUEUE_NAME: "insights-investigations-v1", INSIGHTS_ROLLUP_JOB_NAME: "insights-rollup", activeStreamKey: (id: string) => `active:${id}`, appendStreamChunk: vi.fn(async () => undefined), diff --git a/packages/ai/src/ai/mcp/tools.ts b/packages/ai/src/ai/mcp/tools.ts index fa393472f..d63b3c781 100644 --- a/packages/ai/src/ai/mcp/tools.ts +++ b/packages/ai/src/ai/mcp/tools.ts @@ -17,10 +17,12 @@ import { LinkFolderSelectorSchema, LinkFolderWithUsageSchema, LinkRowOutputSchema, + getLinkSummary, listLinkFolders, listLinks, parseLinkRow, resolveLinkFolder, + searchLinks, summarizeLink, summarizeLinkFolder, summarizeLinkFoldersWithUsage, @@ -943,15 +945,15 @@ const listLinkFoldersTool = defineMcpTool( } const rpcContext = buildRpcContext(ctx); - const [folders, links] = await Promise.all([ + const [folders, summary] = await Promise.all([ listLinkFolders(rpcContext, orgId), - listLinks(rpcContext, orgId), + getLinkSummary(rpcContext, orgId), ]); return { - folders: summarizeLinkFoldersWithUsage(folders, links), + folders: summarizeLinkFoldersWithUsage(folders), count: folders.length, - unfiledCount: links.filter((link) => !link.folderId).length, + unfiledCount: summary.unfiledTotal, hint: folders.length === 0 ? "No link folders exist yet. Leave links unfiled unless the user creates a folder in Databuddy." @@ -964,7 +966,7 @@ const listLinksTool = defineMcpTool( { name: "list_links", description: - "List short links and existing folders for the website's organization. Use to enumerate all links before referencing one or choosing where a new link should go.", + "List the newest short links and existing folders for the website's organization. The count covers the full catalog; use search_links to find a specific older link.", inputSchema: z.object({ ...WebsiteSelectorSchema, }), @@ -984,24 +986,29 @@ const listLinksTool = defineMcpTool( throw new McpToolError("not_found", orgId.message); } const rpcContext = buildRpcContext(ctx); - const [links, folders] = await Promise.all([ + const [page, folders, summary] = await Promise.all([ listLinks(rpcContext, orgId), listLinkFolders(rpcContext, orgId), + getLinkSummary(rpcContext, orgId), ]); - if (links.length === 0) { + if (page.items.length === 0) { return { - links, + links: page.items, count: 0, - folders: summarizeLinkFoldersWithUsage(folders, links), - unfiledCount: 0, + folders: summarizeLinkFoldersWithUsage(folders), + unfiledCount: summary.unfiledTotal, hint: "No links yet for this organization.", }; } return { - links: links.map((link) => summarizeLink(link, folders)), - count: links.length, - folders: summarizeLinkFoldersWithUsage(folders, links), - unfiledCount: links.filter((link) => !link.folderId).length, + links: page.items.map((link) => summarizeLink(link, folders)), + count: summary.total, + folders: summarizeLinkFoldersWithUsage(folders, page.items), + unfiledCount: summary.unfiledTotal, + hint: + page.hasMore || summary.total > page.items.length + ? `Showing the ${page.items.length} newest of ${summary.total} links. Use search_links for older links.` + : undefined, }; } ); @@ -1010,12 +1017,14 @@ const searchLinksTool = defineMcpTool( { name: "search_links", description: - "Find short links matching a substring on name, slug, target URL, or external ID. Use when you know part of a link identifier.", + "Find short links across the full catalog matching a substring on name, slug, target URL, or external ID. Returns at most the newest 50 matches.", inputSchema: z.object({ ...WebsiteSelectorSchema, query: z .string() + .trim() .min(1) + .max(255) .describe("Search query (matches name, slug, URL, or external ID)"), }), outputSchema: z.object({ @@ -1031,9 +1040,10 @@ const searchLinksTool = defineMcpTool( }) ), count: z.number(), + hasMore: z.boolean(), }), resolveWebsite: true, - ratelimit: { limit: 60, windowSec: 60 }, + ratelimit: { limit: 20, windowSec: 60 }, }, async (input, ctx) => { const orgId = await getOrganizationId(getResolvedWebsiteId(ctx)); @@ -1041,20 +1051,12 @@ const searchLinksTool = defineMcpTool( throw new McpToolError("not_found", orgId.message); } const rpcContext = buildRpcContext(ctx); - const [allLinks, folders] = await Promise.all([ - listLinks(rpcContext, orgId), + const [page, folders] = await Promise.all([ + searchLinks(rpcContext, orgId, input.query), listLinkFolders(rpcContext, orgId), ]); - const queryLower = input.query.toLowerCase(); - const matches = allLinks.filter( - (link) => - link.name.toLowerCase().includes(queryLower) || - link.slug.toLowerCase().includes(queryLower) || - link.targetUrl.toLowerCase().includes(queryLower) || - link.externalId?.toLowerCase().includes(queryLower) - ); return { - links: matches.map((link) => ({ + links: page.items.map((link) => ({ id: link.id, name: link.name, slug: link.slug, @@ -1063,7 +1065,8 @@ const searchLinksTool = defineMcpTool( folder: summarizeLink(link, folders).folder, externalId: link.externalId, })), - count: matches.length, + count: page.items.length, + hasMore: page.hasMore, }; } ); diff --git a/packages/ai/src/ai/prompts/clickhouse-schema.ts b/packages/ai/src/ai/prompts/clickhouse-schema.ts index 13b32b49a..49f805d21 100644 --- a/packages/ai/src/ai/prompts/clickhouse-schema.ts +++ b/packages/ai/src/ai/prompts/clickhouse-schema.ts @@ -1,6 +1,7 @@ import { AGENT_TABLE_COLUMNS, AGENT_TENANT_COLUMN_BY_TABLE, + CUSTOM_EVENTS_VISITOR_KEY, EVENTS_VISITOR_KEY, validateAgentSQL, } from "@databuddy/db/clickhouse"; @@ -191,7 +192,7 @@ const GUIDELINES = `## Query Guidelines - Geographic data (country, region, city) exists only on analytics.events, NOT on web_vitals_spans or error_spans. Join via session_id if needed. - All timestamps are in UTC. - analytics.custom_events.properties contains JSON strings — use JSONExtractString(properties, 'key') to parse. -- Identity: \`anonymous_id\` is a per-device id; \`profile_id\` is the customer-assigned user id ('' when anonymous, on analytics.events, analytics.custom_events, and analytics.revenue). To count people, dedupe identified users across devices with \`uniq(${EVENTS_VISITOR_KEY})\` (on custom_events/revenue wrap the fallback: \`if(profile_id != '', profile_id, ifNull(anonymous_id, ''))\` since anonymous_id is Nullable there). To count only identified users: \`uniqIf(profile_id, profile_id != '')\`. error_spans/web_vitals_spans/outgoing_links have no profile_id — resolve an identified user's rows there via their anonymous_ids from analytics.events. +- Identity: \`anonymous_id\` is a per-device id; \`profile_id\` is the customer-assigned user id ('' when anonymous, on analytics.events, analytics.custom_events, and analytics.revenue). To count people, dedupe identified users across devices with \`uniq(${EVENTS_VISITOR_KEY})\`; on custom_events/revenue use \`uniq(${CUSTOM_EVENTS_VISITOR_KEY})\` because anonymous_id is Nullable and rows missing both identifiers must not count as a person. To count only identified users: \`uniqIf(profile_id, profile_id != '')\`. error_spans/web_vitals_spans/outgoing_links have no profile_id — resolve an identified user's rows there via their anonymous_ids from analytics.events. ## Aggregate function preferences - Percentiles: use \`quantileTDigest(p)(col)\` for p50/p75/p95/p99. Plain \`quantile(p)\` uses reservoir sampling and is noisy at the tails (~10% error at p99). \`quantileTDigest\` is within 0.1% of exact at the same memory cost. diff --git a/packages/ai/src/ai/tools/link-catalog.test.ts b/packages/ai/src/ai/tools/link-catalog.test.ts index 7ef8dfbd7..cf7cc2250 100644 --- a/packages/ai/src/ai/tools/link-catalog.test.ts +++ b/packages/ai/src/ai/tools/link-catalog.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "bun:test"; import { type LinkFolder, + fetchLinkCatalogPage, + fetchLinkSummary, resolveLinkFolderFromList, } from "./link-catalog"; @@ -80,3 +82,45 @@ describe("resolveLinkFolderFromList", () => { } }); }); + +describe("paginated link catalog", () => { + it("uses one bounded server-side search beyond the legacy 1,000-row cap", async () => { + const source = Array.from({ length: 1001 }, (_, index) => ({ + id: `link-${index}`, + name: `Example ${index}`, + slug: `example-${index}`, + targetUrl: `https://example.com/${index}`, + })); + const calls: Array<{ limit: number; offset: number; search?: string }> = []; + + const page = await fetchLinkCatalogPage(async (input) => { + calls.push(input); + const matches = source.filter((link) => + link.name.toLowerCase().includes(input.search?.toLowerCase() ?? "") + ); + const items = matches.slice(input.offset, input.offset + input.limit); + return { + hasMore: input.offset + items.length < matches.length, + items, + }; + }, { search: "Example 1000" }); + + expect(page.items).toHaveLength(1); + expect(page.items[0]?.id).toBe("link-1000"); + expect(page.total).toBeUndefined(); + expect(calls).toEqual([ + { limit: 50, offset: 0, search: "Example 1000" }, + ]); + }); + + it("loads exact catalog and unfiled totals with one summary call", async () => { + const calls: Array<{ search?: string }> = []; + const summary = await fetchLinkSummary(async (input) => { + calls.push(input); + return { total: 7, unfiledTotal: 3 }; + }, "campaign"); + + expect(summary).toEqual({ total: 7, unfiledTotal: 3 }); + expect(calls).toEqual([{ search: "campaign" }]); + }); +}); diff --git a/packages/ai/src/ai/tools/link-catalog.ts b/packages/ai/src/ai/tools/link-catalog.ts index 8a4d41865..6797f57e5 100644 --- a/packages/ai/src/ai/tools/link-catalog.ts +++ b/packages/ai/src/ai/tools/link-catalog.ts @@ -41,6 +41,7 @@ const LinkFolderSchema = z.object({ organizationId: z.string(), createdBy: z.string().optional(), deletedAt: DateStringSchema.nullable().optional(), + linkCount: z.number().int().nonnegative().optional(), }); const LinkRowSchema = z.object({ @@ -58,6 +59,19 @@ const LinkRowSchema = z.object({ organizationId: z.string().optional(), }); +const LinkPageSchema = z.object({ + hasMore: z.boolean(), + items: z.array(LinkRowSchema), + total: z.number().int().nonnegative().optional(), +}); + +const LinkSummarySchema = z.object({ + total: z.number().int().nonnegative(), + unfiledTotal: z.number().int().nonnegative(), +}); + +const MODEL_LINK_LIMIT = 50; + export const LinkFolderSelectorSchema = z.object({ folderId: z .string() @@ -81,6 +95,18 @@ export type LinkFolder = z.infer; export type LinkFolderSelector = z.infer; export type LinkRow = z.infer; +type LinkPageFetcher = (input: { + limit: number; + offset: number; + folderId?: string | null; + search?: string; +}) => Promise; + +type LinkSummaryFetcher = (input: { search?: string }) => Promise; + +export type LinkPage = z.infer; +export type LinkSummary = z.infer; + export type LinkFolderResolution = | { folder: LinkFolder | null; @@ -94,11 +120,6 @@ export type LinkFolderResolution = ok: false; }; -export function parseLinkRows(value: unknown): LinkRow[] { - const result = z.array(LinkRowSchema).safeParse(value); - return result.success ? result.data : []; -} - export function parseLinkRow(value: unknown): LinkRow { const result = LinkRowSchema.safeParse(value); if (!result.success) { @@ -112,13 +133,87 @@ export function parseLinkFolders(value: unknown): LinkFolder[] { return result.success ? result.data : []; } -export async function listLinks( +export async function fetchLinkCatalogPage( + fetchPage: LinkPageFetcher, + filters: { folderId?: string | null; search?: string } = {} +): Promise { + const result = LinkPageSchema.safeParse( + await fetchPage({ + ...filters, + limit: MODEL_LINK_LIMIT, + offset: 0, + }) + ); + if (!result.success) { + throw new Error("Received an invalid paginated link response."); + } + return result.data; +} + +export async function fetchLinkSummary( + fetchSummary: LinkSummaryFetcher, + search?: string +): Promise { + const result = LinkSummarySchema.safeParse(await fetchSummary({ search })); + if (!result.success) { + throw new Error("Received an invalid link summary response."); + } + return result.data; +} + +async function loadLinks( + context: AppContext, + organizationId: string, + filters: { folderId?: string | null; search?: string } = {} +): Promise { + const { callRPCProcedure } = await import("./utils"); + return fetchLinkCatalogPage( + (input) => + callRPCProcedure( + "links", + "paginated", + { + ...input, + organizationId, + sort: "newest", + type: "all", + }, + context + ), + filters + ); +} + +export function listLinks( context: AppContext, organizationId: string -): Promise { +): Promise { + return loadLinks(context, organizationId); +} + +export function searchLinks( + context: AppContext, + organizationId: string, + query: string +): Promise { + return loadLinks(context, organizationId, { search: query }); +} + +export async function getLinkSummary( + context: AppContext, + organizationId: string, + search?: string +): Promise { const { callRPCProcedure } = await import("./utils"); - return parseLinkRows( - await callRPCProcedure("links", "list", { organizationId }, context) + return fetchLinkSummary( + (input) => + callRPCProcedure( + "links", + "summary", + { ...input, organizationId }, + context + ), + search ); } @@ -144,9 +239,9 @@ export function summarizeLinkFolder(folder: LinkFolder) { export function summarizeLinkFoldersWithUsage( folders: LinkFolder[], - links: LinkRow[] + visibleLinks: LinkRow[] = [] ) { - const counts = links.reduce( + const visibleCounts = visibleLinks.reduce( (map, link) => map.set(link.folderId ?? null, (map.get(link.folderId ?? null) ?? 0) + 1), new Map() @@ -154,7 +249,7 @@ export function summarizeLinkFoldersWithUsage( return folders.map((folder) => ({ ...summarizeLinkFolder(folder), - linkCount: counts.get(folder.id) ?? 0, + linkCount: folder.linkCount ?? visibleCounts.get(folder.id) ?? 0, })); } diff --git a/packages/ai/src/ai/tools/links.ts b/packages/ai/src/ai/tools/links.ts index 2bdc4ccd8..3acac6c78 100644 --- a/packages/ai/src/ai/tools/links.ts +++ b/packages/ai/src/ai/tools/links.ts @@ -4,12 +4,14 @@ import { z } from "zod"; import { getCachedWebsite } from "../../lib/website-utils"; import { LinkFolderSelectorSchema, + getLinkSummary, hasLinkFolderSelector, listLinkFolders, listLinks, parseLinkRow, resolveLinkFolder, resolveLinkFolderFromList, + searchLinks, summarizeLink, summarizeLinkFolder, summarizeLinkFoldersWithUsage, @@ -44,15 +46,15 @@ export function createLinksTools() { const context = getAppContext(options); try { const organizationId = await getOrganizationIdFromWebsite(websiteId); - const [folders, links] = await Promise.all([ + const [folders, summary] = await Promise.all([ listLinkFolders(context, organizationId), - listLinks(context, organizationId), + getLinkSummary(context, organizationId), ]); return { - folders: summarizeLinkFoldersWithUsage(folders, links), + folders: summarizeLinkFoldersWithUsage(folders), count: folders.length, - unfiledCount: links.filter((link) => !link.folderId).length, + unfiledCount: summary.unfiledTotal, hint: folders.length === 0 ? "No link folders exist yet. Leave links unfiled unless the user creates a folder in Databuddy." @@ -69,21 +71,38 @@ export function createLinksTools() { const listLinksTool = tool({ description: - "List short links and existing folders for the website org (slug, target URL, folder, metadata).", - inputSchema: z.object({ websiteId: z.string() }), - execute: async ({ websiteId }, options) => { + "List the newest short links and existing folders for the website org. Set search to find a specific link across the full catalog.", + inputSchema: z.object({ + search: z.string().trim().min(1).max(255).optional(), + websiteId: z.string(), + }), + execute: async ({ search, websiteId }, options) => { const context = getAppContext(options); try { const organizationId = await getOrganizationIdFromWebsite(websiteId); - const [links, folders] = await Promise.all([ - listLinks(context, organizationId), + const [page, folders, summary] = await Promise.all([ + search + ? searchLinks(context, organizationId, search) + : listLinks(context, organizationId), listLinkFolders(context, organizationId), + search + ? Promise.resolve(null) + : getLinkSummary(context, organizationId), ]); + const count = summary?.total ?? page.items.length; return { - links: links.map((link) => summarizeLink(link, folders)), - count: links.length, - folders: summarizeLinkFoldersWithUsage(folders, links), - unfiledCount: links.filter((link) => !link.folderId).length, + links: page.items.map((link) => summarizeLink(link, folders)), + count, + folders: summarizeLinkFoldersWithUsage(folders, page.items), + unfiledCount: + summary?.unfiledTotal ?? + page.items.filter((link) => !link.folderId).length, + hint: + page.hasMore || count > page.items.length + ? search + ? `Showing the ${page.items.length} newest matching links; more matches exist.` + : `Showing the ${page.items.length} newest of ${count} links.` + : undefined, }; } catch (error) { logger.error("Failed to list links", { websiteId, error }); diff --git a/packages/ai/src/ai/tools/profiles.ts b/packages/ai/src/ai/tools/profiles.ts index 52eacd0b8..f4f05050c 100644 --- a/packages/ai/src/ai/tools/profiles.ts +++ b/packages/ai/src/ai/tools/profiles.ts @@ -163,7 +163,7 @@ export function buildProfileTools(opts: ProfileToolsOptions): ToolSet { }), list_profile_traits: tool({ - description: `Distribution of identified-user traits: every trait key, its values, and how many profiles carry each value, plus the total identified profile count. Call this before segmenting to learn which keys and values exist and to quantify identified-vs-anonymous coverage. To measure sessions or behavior per segment, follow up with get_data using a trait: filter (e.g. session_metrics filtered by trait:plan eq pro); that is how profiles link to sessions, and prefer aggregate query types like session_metrics/summary_metrics for totals instead of summing time series rows by hand.${suffix}`, + description: `Bounded distribution of identified-user traits: up to 200 highest-coverage trait keys, top values for every returned key, profile counts, and explicit key/value truncation metadata. Check hasMoreKeys and hasMoreValues before treating the result as exhaustive. Call this before segmenting to learn which common keys and values exist and to quantify identified-vs-anonymous coverage. To measure sessions or behavior per segment, follow up with get_data using a trait: filter (e.g. session_metrics filtered by trait:plan eq pro); that is how profiles link to sessions, and prefer aggregate query types like session_metrics/summary_metrics for totals instead of summing time series rows by hand.${suffix}`, inputSchema: z.object({ websiteId: opts.websiteIdSchema, }), @@ -171,8 +171,12 @@ export function buildProfileTools(opts: ProfileToolsOptions): ToolSet { const site = await opts.resolveSite(websiteId, options); const distribution = await getTraitDistribution(site.websiteId); logger.info("Fetched trait distribution", { + hasMoreKeys: distribution.hasMoreKeys, + hasMoreValues: distribution.hasMoreValues, websiteId: site.websiteId, identifiedProfiles: distribution.identifiedProfiles, + returnedTraitKeys: distribution.returnedTraitKeys, + totalTraitKeys: distribution.totalTraitKeys, traitRows: distribution.traits.length, }); return distribution; diff --git a/packages/ai/src/query/batch-executor.test.ts b/packages/ai/src/query/batch-executor.test.ts index 7ae461600..2fff1a09a 100644 --- a/packages/ai/src/query/batch-executor.test.ts +++ b/packages/ai/src/query/batch-executor.test.ts @@ -1,29 +1,27 @@ -import { chQuery } from "@databuddy/db/clickhouse"; +import * as actualClickHouse from "@databuddy/db/clickhouse"; +import { afterAll, beforeEach, describe, expect, it, mock } from "bun:test"; import type { RequestLogger } from "evlog"; -import { beforeEach, describe, expect, it, vi } from "vitest"; import { setAiRequestLoggerProvider } from "../lib/request-logger"; -import { +import { QueryBuilders } from "./builders"; +import { SimpleQueryBuilder } from "./simple-builder"; + +const realClickHouseModule = { ...actualClickHouse }; +const realChQuery = realClickHouseModule.chQuery; +const mockChQuery = mock(realChQuery); +mock.module("@databuddy/db/clickhouse", () => ({ + ...realClickHouseModule, + chQuery: mockChQuery, +})); + +const { areQueriesCompatible, buildUnionQuery, executeBatch, extractOuterSelectColumns, getCompatibleQueries, getSchemaGroups, -} from "./batch-executor"; -import { QueryBuilders } from "./builders"; -import { SimpleQueryBuilder } from "./simple-builder"; +} = await import("./batch-executor"); -vi.mock("@databuddy/db/clickhouse", () => ({ - chQuery: vi.fn(), -})); - -type MockChQuery = { - mockRejectedValueOnce: (value: unknown) => MockChQuery; - mockReset: () => void; - mockResolvedValueOnce: (value: Record[]) => MockChQuery; -}; - -const mockChQuery = chQuery as unknown as MockChQuery; const lastSelectColumns = extractOuterSelectColumns; function compileSql(type: string): string { @@ -57,9 +55,14 @@ function transientClickHouseError(): Error { beforeEach(() => { mockChQuery.mockReset(); + mockChQuery.mockImplementation(realChQuery); setAiRequestLoggerProvider(null); }); +afterAll(() => { + mock.module("@databuddy/db/clickhouse", () => realClickHouseModule); +}); + describe("batch-executor schema signatures", () => { const builderEntries = Object.entries(QueryBuilders); const builderCases = builderEntries @@ -128,7 +131,7 @@ describe("executeBatch single query retry", () => { const [result] = await executeBatch([singleQueryRequest]); - expect(chQuery).toHaveBeenCalledTimes(2); + expect(mockChQuery).toHaveBeenCalledTimes(2); expect(result).toEqual({ type: "top_pages", data: [{ name: "/", pageviews: 2, visitors: 1 }], @@ -146,7 +149,7 @@ describe("executeBatch single query retry", () => { const [result] = await executeBatch([singleQueryRequest]); - expect(chQuery).toHaveBeenCalledTimes(2); + expect(mockChQuery).toHaveBeenCalledTimes(2); expect(result).toEqual({ type: "top_pages", data: [], @@ -159,7 +162,7 @@ describe("executeBatch single query retry", () => { const [result] = await executeBatch([singleQueryRequest]); - expect(chQuery).toHaveBeenCalledTimes(1); + expect(mockChQuery).toHaveBeenCalledTimes(1); expect(result).toEqual({ type: "top_pages", data: [], @@ -176,7 +179,7 @@ describe("executeBatch single query retry", () => { const [result] = await executeBatch([singleQueryRequest]); - expect(chQuery).toHaveBeenCalledTimes(1); + expect(mockChQuery).toHaveBeenCalledTimes(1); expect(result).toEqual({ type: "top_pages", data: [], @@ -188,9 +191,9 @@ describe("executeBatch single query retry", () => { describe("executeBatch union fallback logging", () => { it("logs batch union fallback as a warning when single queries recover", async () => { const mockLogger = { - error: vi.fn(), - set: vi.fn(), - warn: vi.fn(), + error: mock(), + set: mock(), + warn: mock(), } as unknown as RequestLogger; setAiRequestLoggerProvider(() => mockLogger); mockChQuery @@ -203,7 +206,7 @@ describe("executeBatch union fallback logging", () => { { ...singleQueryRequest, type: "top_pages" }, ]); - expect(chQuery).toHaveBeenCalledTimes(3); + expect(mockChQuery).toHaveBeenCalledTimes(3); expect(mockLogger.warn).toHaveBeenCalledWith( "Union query failed", expect.objectContaining({ diff --git a/packages/ai/src/query/builder-execution.test.ts b/packages/ai/src/query/builder-execution.test.ts index c48263021..afe36c993 100644 --- a/packages/ai/src/query/builder-execution.test.ts +++ b/packages/ai/src/query/builder-execution.test.ts @@ -27,7 +27,12 @@ if (!isClickHouseUp) { } afterAll(async () => { - if (isClickHouseUp) { + // The explicit integration run has more ClickHouse suites after this file. + // Isolated builder runs still close their client normally. + if ( + isClickHouseUp && + process.env.CLICKHOUSE_INTEGRATION_TESTS !== "true" + ) { await clickHouse.close(); } }); diff --git a/packages/ai/src/query/builders/custom-events.integration.test.ts b/packages/ai/src/query/builders/custom-events.integration.test.ts new file mode 100644 index 000000000..5c3b0b017 --- /dev/null +++ b/packages/ai/src/query/builders/custom-events.integration.test.ts @@ -0,0 +1,65 @@ +import { randomUUIDv7 } from "bun"; +import { describe, expect, it } from "bun:test"; +import { chQuery, clickHouse } from "@databuddy/db/clickhouse"; +import { CustomEventsBuilders } from "./custom-events"; + +const describeIntegration = + process.env.CLICKHOUSE_INTEGRATION_TESTS === "true" ? describe : describe.skip; + +describeIntegration("custom event identity against ClickHouse", () => { + it("excludes unidentified rows while deduplicating anonymous and profile identities", async () => { + const websiteId = `custom-event-identity-${randomUUIDv7()}`; + const timestamp = "2026-08-02 12:00:00"; + const row = ( + eventName: string, + anonymousId: string | null, + profileId = "" + ) => ({ + anonymous_id: anonymousId, + event_name: eventName, + namespace: null, + owner_id: websiteId, + path: null, + profile_id: profileId, + properties: "{}", + session_id: null, + source: "integration-test", + timestamp, + website_id: websiteId, + }); + + await clickHouse.insert({ + table: "analytics.custom_events", + format: "JSONEachRow", + values: [ + row("unidentified", null), + row("unidentified", null), + row("anonymous", "anon-1"), + row("anonymous", "anon-1"), + row("profile", "anon-2", "profile-1"), + row("profile", "anon-3", "profile-1"), + ], + }); + + const query = CustomEventsBuilders.custom_events?.customSql?.({ + endDate: "2026-08-03", + startDate: "2026-08-01", + websiteId, + }); + if (!query || typeof query === "string") { + throw new Error("custom_events did not compile"); + } + + const rows = await chQuery<{ + name: string; + total_events: number | string; + unique_users: number | string; + }>(query.sql, query.params); + const byName = new Map(rows.map((result) => [result.name, result])); + + expect(Number(byName.get("unidentified")?.total_events)).toBe(2); + expect(Number(byName.get("unidentified")?.unique_users)).toBe(0); + expect(Number(byName.get("anonymous")?.unique_users)).toBe(1); + expect(Number(byName.get("profile")?.unique_users)).toBe(1); + }); +}); diff --git a/packages/ai/src/query/builders/custom-events.ts b/packages/ai/src/query/builders/custom-events.ts index 3c7c086de..461975b62 100644 --- a/packages/ai/src/query/builders/custom-events.ts +++ b/packages/ai/src/query/builders/custom-events.ts @@ -1,3 +1,4 @@ +import { CUSTOM_EVENTS_VISITOR_KEY } from "@databuddy/db/clickhouse"; import { Analytics } from "../../types/tables"; import { appendFilterClause } from "../simple-builder"; import type { Filter, SimpleQueryConfig } from "../types"; @@ -60,12 +61,12 @@ export const CustomEventsBuilders: Record = { SELECT event_name as name, COUNT(*) as total_events, - uniq(anonymous_id) as unique_users, + uniq(${CUSTOM_EVENTS_VISITOR_KEY}) as unique_users, uniq(session_id) as unique_sessions, MAX(timestamp) as last_occurrence, MIN(timestamp) as first_occurrence, countIf(properties != '{}' AND isValidJSON(properties)) as events_with_properties, - ROUND((uniq(anonymous_id) / SUM(uniq(anonymous_id)) OVER()) * 100, 2) as percentage + ROUND((uniq(${CUSTOM_EVENTS_VISITOR_KEY}) / SUM(uniq(${CUSTOM_EVENTS_VISITOR_KEY})) OVER()) * 100, 2) as percentage FROM ${Analytics.custom_events} WHERE ${projectWhereClause(filterParams)} @@ -196,7 +197,7 @@ export const CustomEventsBuilders: Record = { path as name, COUNT(*) as total_events, uniq(event_name) as unique_event_types, - uniq(anonymous_id) as unique_users + uniq(${CUSTOM_EVENTS_VISITOR_KEY}) as unique_users FROM ${Analytics.custom_events} WHERE ${projectWhereClause(filterParams)} @@ -246,7 +247,7 @@ export const CustomEventsBuilders: Record = { toDate(timestamp) as date, COUNT(*) as total_events, uniq(event_name) as unique_event_types, - uniq(anonymous_id) as unique_users, + uniq(${CUSTOM_EVENTS_VISITOR_KEY}) as unique_users, uniq(session_id) as unique_sessions, uniq(path) as unique_pages FROM ${Analytics.custom_events} @@ -342,7 +343,7 @@ export const CustomEventsBuilders: Record = { SELECT COUNT(*) as total_events, uniq(event_name) as unique_event_types, - uniq(anonymous_id) as unique_users, + uniq(${CUSTOM_EVENTS_VISITOR_KEY}) as unique_users, uniq(session_id) as unique_sessions, uniq(path) as unique_pages FROM ${Analytics.custom_events} @@ -863,7 +864,7 @@ export const CustomEventsBuilders: Record = { SELECT event_name, COUNT(*) as total_events, - uniq(anonymous_id) as unique_users, + uniq(${CUSTOM_EVENTS_VISITOR_KEY}) as unique_users, uniq(session_id) as unique_sessions FROM ${Analytics.custom_events} WHERE diff --git a/packages/ai/src/query/builders/pages.ts b/packages/ai/src/query/builders/pages.ts index a950c255e..1a93e46a1 100644 --- a/packages/ai/src/query/builders/pages.ts +++ b/packages/ai/src/query/builders/pages.ts @@ -102,6 +102,7 @@ export const PagesBuilders: Record = { "utm_medium", "utm_campaign", "profile_id", + "anonymous_id", ], customizable: true, plugins: { @@ -130,7 +131,7 @@ export const PagesBuilders: Record = { SELECT e.session_id, argMin(CASE WHEN trimRight(path(e.path), '/') = '' THEN '/' ELSE trimRight(path(e.path), '/') END, e.time) as entry_page, - argMin(e.anonymous_id, e.time) as anonymous_id, + argMin(e.anonymous_id, e.time) as visitor_id, any(sa.session_referrer) as referrer, any(sa.session_utm_source) as utm_source, any(sa.session_utm_medium) as utm_medium, @@ -153,7 +154,7 @@ export const PagesBuilders: Record = { SELECT session_id, argMin(CASE WHEN trimRight(path(path), '/') = '' THEN '/' ELSE trimRight(path(path), '/') END, time) as entry_page, - argMin(anonymous_id, time) as anonymous_id + argMin(anonymous_id, time) as visitor_id FROM analytics.events WHERE client_id = {websiteId:String} AND time >= toDateTime({startDate:String}) @@ -179,7 +180,7 @@ export const PagesBuilders: Record = { SELECT entry_page as name, COUNT(*) as pageviews, - uniq(anonymous_id) as visitors + uniq(visitor_id) as visitors FROM session_entry GROUP BY entry_page ) @@ -216,6 +217,7 @@ export const PagesBuilders: Record = { "utm_medium", "utm_campaign", "profile_id", + "anonymous_id", ], customizable: true, plugins: { @@ -244,7 +246,7 @@ export const PagesBuilders: Record = { SELECT e.session_id, argMax(CASE WHEN trimRight(path(e.path), '/') = '' THEN '/' ELSE trimRight(path(e.path), '/') END, e.time) as exit_page, - argMax(e.anonymous_id, e.time) as anonymous_id, + argMax(e.anonymous_id, e.time) as visitor_id, any(sa.session_referrer) as referrer, any(sa.session_utm_source) as utm_source, any(sa.session_utm_medium) as utm_medium, @@ -267,7 +269,7 @@ export const PagesBuilders: Record = { SELECT session_id, argMax(CASE WHEN trimRight(path(path), '/') = '' THEN '/' ELSE trimRight(path(path), '/') END, time) as exit_page, - argMax(anonymous_id, time) as anonymous_id + argMax(anonymous_id, time) as visitor_id FROM analytics.events WHERE client_id = {websiteId:String} AND time >= toDateTime({startDate:String}) @@ -290,7 +292,7 @@ export const PagesBuilders: Record = { SELECT exit_page as name, uniq(session_id) as pageviews, - uniq(anonymous_id) as visitors + uniq(visitor_id) as visitors FROM session_exit GROUP BY exit_page ) diff --git a/packages/ai/src/query/builders/profiles.ts b/packages/ai/src/query/builders/profiles.ts index 25f382745..eff56a600 100644 --- a/packages/ai/src/query/builders/profiles.ts +++ b/packages/ai/src/query/builders/profiles.ts @@ -1,6 +1,7 @@ import { CUSTOM_EVENTS_VISITOR_KEY, EVENTS_VISITOR_KEY, + revenueLatestCte, visitorMatch, } from "@databuddy/db/clickhouse"; import { Analytics } from "../../types/tables"; @@ -241,6 +242,61 @@ const PROFILE_ACTIVITY_CTE = ` AND timestamp <= toDateTime({endDate:String}) )`; +function stripePaymentIntentId(alias = ""): string { + const prefix = alias ? `${alias}.` : ""; + return `if( + JSONExtractString(${prefix}metadata, 'stripe_payment_intent_id') != '', + JSONExtractString(${prefix}metadata, 'stripe_payment_intent_id'), + if(${prefix}provider = 'stripe' AND startsWith(${prefix}transaction_id, 'pi_'), ${prefix}transaction_id, '') +)`; +} + +const ATTRIBUTED_REVENUE_VISITOR_KEY = + "if(attributed_profile_id != '', attributed_profile_id, ifNull(attributed_anonymous_id, ''))"; + +function stripeProfileContextCtes(visitorPredicate: string): string { + const paymentIntentId = stripePaymentIntentId(); + return ` + profile_payment_intents AS ( + SELECT DISTINCT ${paymentIntentId} AS payment_intent_id + FROM ${Analytics.revenue} + WHERE (owner_id = {websiteId:String} OR website_id = {websiteId:String}) + AND provider = 'stripe' + AND ${visitorPredicate} + AND ${paymentIntentId} != '' + ), + profile_payment_context AS ( + SELECT + owner_id, + ${paymentIntentId} AS payment_intent_id, + argMaxIf(profile_id, synced_at, profile_id != '') AS profile_id, + argMaxIf(ifNull(anonymous_id, ''), synced_at, ifNull(anonymous_id, '') != '') AS anonymous_id, + argMaxIf(ifNull(session_id, ''), synced_at, ifNull(session_id, '') != '') AS session_id + FROM ${Analytics.revenue} + WHERE (owner_id = {websiteId:String} OR website_id = {websiteId:String}) + AND provider = 'stripe' + AND ${paymentIntentId} IN ( + SELECT payment_intent_id FROM profile_payment_intents + ) + GROUP BY owner_id, payment_intent_id + )`; +} + +function attributedProfileRevenueCte(latestCte: string): string { + return ` + profile_revenue_attributed AS ( + SELECT + r.*, + coalesce(nullIf(r.profile_id, ''), nullIf(context.profile_id, ''), '') AS attributed_profile_id, + coalesce(r.anonymous_id, nullIf(context.anonymous_id, '')) AS attributed_anonymous_id, + coalesce(r.session_id, nullIf(context.session_id, '')) AS attributed_session_id + FROM ${latestCte} r + LEFT JOIN profile_payment_context context + ON context.owner_id = r.owner_id + AND context.payment_intent_id = ${stripePaymentIntentId("r")} + )`; +} + export const ProfilesBuilders: Record = { profile_list: { meta: { @@ -333,14 +389,28 @@ export const ProfilesBuilders: Record = { AND ${CUSTOM_EVENTS_VISITOR_KEY} IN (SELECT visitor_id FROM visitor_profiles) GROUP BY visitor_id ), + ${stripeProfileContextCtes( + `${CUSTOM_EVENTS_VISITOR_KEY} IN (SELECT visitor_id FROM visitor_profiles)` + )}, + ${revenueLatestCte({ + candidateWhere: `( + ${CUSTOM_EVENTS_VISITOR_KEY} IN (SELECT visitor_id FROM visitor_profiles) + OR ${stripePaymentIntentId()} IN ( + SELECT payment_intent_id FROM profile_payment_intents + ) + )`, + name: "profile_revenue_latest", + scope: + "(owner_id = {websiteId:String} OR website_id = {websiteId:String})", + })}, + ${attributedProfileRevenueCte("profile_revenue_latest")}, visitor_revenue AS ( SELECT - ${CUSTOM_EVENTS_VISITOR_KEY} as visitor_id, - toFloat64(sumIf(amount, status IN ('completed', 'refunded') AND type != 'subscription_event')) as ltv - FROM ${Analytics.revenue} - WHERE (owner_id = {websiteId:String} OR website_id = {websiteId:String}) - AND ${CUSTOM_EVENTS_VISITOR_KEY} IN (SELECT visitor_id FROM visitor_profiles) - GROUP BY visitor_id + ${ATTRIBUTED_REVENUE_VISITOR_KEY} as visitor_id, + toFloat64(sumIf(amount, status IN ('completed', 'refunded') AND type != 'subscription_event')) as ltv + FROM profile_revenue_attributed + WHERE ${ATTRIBUTED_REVENUE_VISITOR_KEY} IN (SELECT visitor_id FROM visitor_profiles) + GROUP BY visitor_id ), visitor_sessions AS ( SELECT @@ -564,7 +634,19 @@ export const ProfilesBuilders: Record = { AND time >= toDateTime({startDate:String}) AND time <= toDateTime({endDate:String}) AND session_id != '' - ) + ), + ${stripeProfileContextCtes(`( + ${VISITOR_MATCH} + OR anonymous_id IN (SELECT anonymous_id FROM visitor_ids) + OR session_id IN (SELECT session_id FROM visitor_sessions) + )`)}, + ${revenueLatestCte({ + candidateWhere: `created >= toDateTime({startDate:String}) + AND created <= toDateTime({endDate:String})`, + name: "profile_revenue_latest", + scope: "(owner_id = {websiteId:String} OR website_id = {websiteId:String})", + })}, + ${attributedProfileRevenueCte("profile_revenue_latest")} SELECT transaction_id, provider, @@ -574,15 +656,14 @@ export const ProfilesBuilders: Record = { currency, ifNull(product_name, '') as product_name, created - FROM ${Analytics.revenue} - WHERE (owner_id = {websiteId:String} OR website_id = {websiteId:String}) - AND type != 'subscription_event' + FROM profile_revenue_attributed + WHERE type != 'subscription_event' AND created >= toDateTime({startDate:String}) AND created <= toDateTime({endDate:String}) AND ( - ${VISITOR_MATCH} - OR anonymous_id IN (SELECT anonymous_id FROM visitor_ids) - OR session_id IN (SELECT session_id FROM visitor_sessions) + attributed_profile_id = {visitorId:String} + OR attributed_anonymous_id IN (SELECT anonymous_id FROM visitor_ids) + OR attributed_session_id IN (SELECT session_id FROM visitor_sessions) ) ORDER BY created DESC LIMIT {limit:Int32} OFFSET {offset:Int32} diff --git a/packages/ai/src/query/builders/revenue.integration.test.ts b/packages/ai/src/query/builders/revenue.integration.test.ts new file mode 100644 index 000000000..b558b90e7 --- /dev/null +++ b/packages/ai/src/query/builders/revenue.integration.test.ts @@ -0,0 +1,1148 @@ +import { describe, expect, it } from "bun:test"; +import { chQuery, clickHouse } from "@databuddy/db/clickhouse"; +import { randomUUIDv7 } from "bun"; +import type { Filter } from "../types"; +import { ProfilesBuilders } from "./profiles"; +import { RevenueBuilders } from "./revenue"; + +const describeIntegration = + process.env.CLICKHOUSE_INTEGRATION_TESTS === "true" ? describe : describe.skip; + +function stripeMetadata( + recordKind: "attempt" | "link" | "money", + extra: Record = {} +): string { + return JSON.stringify({ + databuddy_revenue_model: "stripe_events_v1", + stripe_record_kind: recordKind, + ...extra, + }); +} + +function revenueRow( + websiteId: string, + transactionId: string, + amount: number, + type: string, + status: string, + metadata = "{}", + created = "2026-08-02 12:00:00", + overrides: Record = {} +): Record { + return { + owner_id: websiteId, + website_id: websiteId, + customer_id: "cus_shared", + transaction_id: transactionId, + provider: "stripe", + type, + status, + amount, + original_amount: amount, + original_currency: "USD", + currency: "USD", + metadata, + created, + synced_at: created, + ...overrides, + }; +} + +async function revenueOverview( + websiteId: string, + startDate = "2026-07-01", + endDate = "2026-08-03", + filters?: Filter[] +): Promise[]> { + const query = RevenueBuilders.revenue_overview?.customSql?.({ + endDate, + startDate, + websiteId, + ...(filters?.length ? { filters } : {}), + }); + if (!query || typeof query === "string") { + throw new Error("Revenue overview did not compile"); + } + return chQuery>(query.sql, query.params); +} + +async function recentTransactions( + websiteId: string, + startDate: string, + endDate: string +): Promise[]> { + const query = RevenueBuilders.recent_transactions?.customSql?.({ + endDate, + startDate, + websiteId, + }); + if (!query || typeof query === "string") { + throw new Error("Recent transactions did not compile"); + } + return chQuery>(query.sql, query.params); +} + +function attributionEvent( + websiteId: string, + sessionId: string, + time: string, + utmCampaign: string | null +): Record { + return { + id: randomUUIDv7(), + client_id: websiteId, + event_name: "screen_view", + anonymous_id: `anon-${sessionId}`, + session_id: sessionId, + time, + url: "https://example.com/checkout", + path: "/checkout", + ip: "127.0.0.1", + user_agent: "integration-test", + utm_campaign: utmCampaign, + properties: "{}", + created_at: time, + }; +} + +describeIntegration("revenue query builders against ClickHouse", () => { + for (const [name, builder] of Object.entries(RevenueBuilders)) { + it(`executes ${name}`, async () => { + const query = builder.customSql?.({ + endDate: "2026-01-02", + startDate: "2026-01-01", + websiteId: "__revenue_builder_integration__", + }); + expect(query).toBeDefined(); + expect(typeof query).not.toBe("string"); + if (!query || typeof query === "string") { + throw new Error(`${name} did not compile to a parameterized query`); + } + + await chQuery(query.sql, query.params, { + clickhouse_settings: { + max_execution_time: 15, + max_result_rows: 100, + }, + readonly: true, + }); + }); + } + + it("preserves legacy totals and reconciles immutable Stripe event records", async () => { + const websiteId = `revenue-cutover-${randomUUIDv7()}`; + const row = ( + transactionId: string, + amount: number, + type: string, + status: string, + rowMetadata = "{}", + created = "2026-08-02 12:00:00", + overrides: Record = {} + ) => + revenueRow( + websiteId, + transactionId, + amount, + type, + status, + rowMetadata, + created, + overrides + ); + + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + // Legacy invoice and PaymentIntent rows are still reconciled by the + // bounded amount/time fallback. + row("pi_legacy", 100, "sale", "completed", "{}", "2026-07-01 12:00:00"), + row( + "in_legacy", + 100, + "subscription", + "completed", + "{}", + "2026-07-01 12:00:00" + ), + // A standalone PaymentIntent must not suppress itself. + row( + "pi_standalone", + 40, + "sale", + "completed", + stripeMetadata("money", { + stripe_money_kind: "standalone_candidate", + stripe_payment_intent_id: "pi_standalone", + }) + ), + // InvoicePayment allocations replace their linked PaymentIntent + // candidates exactly, including partial/multi-payment invoices. + row( + "pi_invoice_a", + 120, + "sale", + "completed", + stripeMetadata("money", { + stripe_money_kind: "standalone_candidate", + stripe_payment_intent_id: "pi_invoice_a", + }) + ), + row( + "pi_invoice_b", + 80, + "sale", + "completed", + stripeMetadata("money", { + stripe_money_kind: "standalone_candidate", + stripe_payment_intent_id: "pi_invoice_b", + }) + ), + row( + "inpay_a", + 120, + "subscription", + "completed", + stripeMetadata("money", { + stripe_invoice_id: "in_modern", + stripe_invoice_payment_id: "inpay_a", + stripe_money_kind: "invoice_payment", + stripe_payment_intent_id: "pi_invoice_a", + }) + ), + row( + "inpay_b", + 80, + "subscription", + "completed", + stripeMetadata("money", { + stripe_invoice_id: "in_modern", + stripe_invoice_payment_id: "inpay_b", + stripe_money_kind: "invoice_payment", + stripe_payment_intent_id: "pi_invoice_b", + }) + ), + row( + "evt_failed", + 50, + "subscription_event", + "failed", + stripeMetadata("attempt", { + stripe_event_type: "payment_intent.payment_failed", + stripe_failure_code: "card_declined", + stripe_failure_decline_code: "insufficient_funds", + stripe_failure_type: "card_error", + stripe_invoice_id: "in_recovered", + stripe_payment_intent_id: "pi_recovered", + }), + "2026-08-01 12:00:00", + { synced_at: "2026-08-03 12:00:00" } + ), + // The invoice event wins duplicate-attempt counting, while the richer + // PaymentIntent event still supplies its safe decline code. + row( + "evt_invoice_failed", + 50, + "subscription_event", + "failed", + stripeMetadata("attempt", { + stripe_event_type: "invoice.payment_failed", + stripe_failure_type: "card_error", + stripe_invoice_id: "in_recovered", + stripe_payment_intent_id: "pi_recovered", + }), + "2026-08-01 12:00:00", + { synced_at: "2026-08-03 12:00:01" } + ), + row( + "evt_canceled", + 25, + "subscription_event", + "canceled", + stripeMetadata("attempt", { + stripe_cancellation_reason: "requested_by_customer", + stripe_event_type: "payment_intent.canceled", + stripe_payment_intent_id: "pi_canceled", + }) + ), + row( + "pi_recovered", + 50, + "sale", + "completed", + stripeMetadata("money", { + stripe_money_kind: "standalone_candidate", + stripe_payment_intent_id: "pi_recovered", + }) + ), + row( + "re_recovered_partial", + -10, + "refund", + "refunded", + stripeMetadata("money", { + stripe_money_kind: "refund", + stripe_payment_intent_id: "pi_recovered", + }), + "2026-08-03 13:00:00" + ), + ], + }); + const [overview] = (await revenueOverview(websiteId)) as { + canceled_payment_attempts: number | string; + failed_payment_amount: number | string; + failed_payment_attempts: number | string; + payment_failure_rate: number | string; + refund_amount: number | string; + refund_count: number | string; + recovered_payment_attempts: number | string; + observed_failure_event_types: number | string; + required_failure_event_types: number | string; + successful_payment_attempts: number | string; + top_payment_cancellation_reason: string; + top_payment_failure_reason: string; + total_revenue: number | string; + total_transactions: number | string; + }[]; + + expect(Number(overview?.total_revenue)).toBe(390); + expect(Number(overview?.total_transactions)).toBe(5); + expect(Number(overview?.failed_payment_attempts)).toBe(1); + expect(Number(overview?.canceled_payment_attempts)).toBe(1); + expect(Number(overview?.failed_payment_amount)).toBe(50); + expect(Number(overview?.recovered_payment_attempts)).toBe(1); + expect(Number(overview?.successful_payment_attempts)).toBe(4); + expect(Number(overview?.payment_failure_rate)).toBe(20); + expect(Number(overview?.refund_amount)).toBe(-10); + expect(Number(overview?.refund_count)).toBe(1); + expect(Number(overview?.observed_failure_event_types)).toBe(2); + expect(Number(overview?.required_failure_event_types)).toBe(2); + expect(overview?.top_payment_failure_reason).toBe("insufficient_funds"); + expect(overview?.top_payment_cancellation_reason).toBe( + "requested_by_customer" + ); + }); + + it("does not leak Stripe payment diagnostics into another provider", async () => { + const websiteId = `revenue-provider-scope-${randomUUIDv7()}`; + + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + websiteId, + "paddle-sale", + 75, + "sale", + "completed", + "{}", + "2026-08-02 12:00:00", + { customer_id: "paddle-customer", provider: "paddle" } + ), + revenueRow( + websiteId, + "stripe-failure", + 45, + "subscription_event", + "failed", + stripeMetadata("attempt", { + stripe_event_type: "payment_intent.payment_failed", + stripe_failure_code: "do_not_honor", + stripe_payment_intent_id: "pi_provider_scope", + }) + ), + ], + }); + + const [paddle] = await revenueOverview( + websiteId, + "2026-07-01", + "2026-08-03", + [{ field: "provider", op: "eq", value: "paddle" }] + ); + expect(Number(paddle?.total_revenue)).toBe(75); + expect(Number(paddle?.total_transactions)).toBe(1); + expect(Number(paddle?.payment_diagnostics_available)).toBe(0); + expect(paddle?.failed_payment_attempts).toBeNull(); + expect(paddle?.successful_payment_attempts).toBeNull(); + expect(paddle?.observed_failure_event_types).toBeNull(); + expect(paddle?.required_failure_event_types).toBeNull(); + + const [stripe] = await revenueOverview( + websiteId, + "2026-07-01", + "2026-08-03", + [{ field: "provider", op: "eq", value: "stripe" }] + ); + expect(Number(stripe?.total_revenue)).toBe(0); + expect(Number(stripe?.payment_diagnostics_available)).toBe(1); + expect(Number(stripe?.failed_payment_attempts)).toBe(1); + expect(Number(stripe?.failed_payment_amount)).toBe(45); + expect(Number(stripe?.observed_failure_event_types)).toBe(1); + expect(Number(stripe?.required_failure_event_types)).toBe(2); + expect(stripe?.top_payment_failure_reason).toBe("do_not_honor"); + }, 10_000); + + it("reconciles invoice fallbacks as exact allocation events arrive", async () => { + const websiteId = `revenue-invoice-fallback-${randomUUIDv7()}`; + const invoiceId = `in_${randomUUIDv7()}`; + const firstPaymentId = `inpay_${randomUUIDv7()}`; + const secondPaymentId = `inpay_${randomUUIDv7()}`; + const paymentIntentId = `pi_${randomUUIDv7()}`; + const at = "2026-08-02 12:00:00"; + + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + websiteId, + paymentIntentId, + 60, + "sale", + "completed", + stripeMetadata("money", { + stripe_money_kind: "standalone_candidate", + stripe_payment_intent_id: paymentIntentId, + }), + at + ), + revenueRow( + websiteId, + `evt_invoice:${firstPaymentId}`, + 0, + "subscription_event", + "linked", + stripeMetadata("link", { + stripe_invoice_id: invoiceId, + stripe_invoice_payment_id: firstPaymentId, + stripe_payment_intent_id: paymentIntentId, + }), + at + ), + revenueRow( + websiteId, + invoiceId, + 100, + "subscription_event", + "completed", + stripeMetadata("money", { + stripe_invoice_id: invoiceId, + stripe_money_kind: "invoice_fallback", + }), + at + ), + ], + }); + + let [overview] = await revenueOverview(websiteId); + expect(Number(overview?.total_revenue)).toBe(100); + expect(Number(overview?.total_transactions)).toBe(1); + + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + websiteId, + firstPaymentId, + 60, + "subscription", + "completed", + stripeMetadata("money", { + stripe_invoice_id: invoiceId, + stripe_invoice_payment_id: firstPaymentId, + stripe_money_kind: "invoice_payment", + stripe_payment_intent_id: paymentIntentId, + }), + "2026-08-02 12:01:00" + ), + ], + }); + + [overview] = await revenueOverview(websiteId); + expect(Number(overview?.total_revenue)).toBe(100); + expect(Number(overview?.total_transactions)).toBe(2); + + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + websiteId, + secondPaymentId, + 40, + "subscription", + "completed", + stripeMetadata("money", { + stripe_invoice_id: invoiceId, + stripe_invoice_payment_id: secondPaymentId, + stripe_money_kind: "invoice_payment", + }), + "2026-08-02 12:02:00" + ), + ], + }); + + await clickHouse.command({ + query: "OPTIMIZE TABLE analytics.revenue PARTITION 202608 FINAL", + }); + [overview] = await revenueOverview(websiteId); + expect(Number(overview?.total_revenue)).toBe(100); + expect(Number(overview?.total_transactions)).toBe(2); + }, 10_000); + + it("keeps currencies separate and counts retry events without collapsing them", async () => { + const websiteId = `revenue-currency-${randomUUIDv7()}`; + const at = "2026-08-02 12:00:00"; + const invoiceSuccessMetadata = stripeMetadata("money", { + stripe_invoice_id: "in_recovered", + stripe_invoice_payment_id: "inpay_recovered", + stripe_money_kind: "invoice_payment", + stripe_payment_intent_id: "pi_invoice_recovered", + }); + const retrySuccessMetadata = stripeMetadata("money", { + stripe_money_kind: "standalone_candidate", + stripe_payment_intent_id: "pi_retry", + }); + const retryMetadata = stripeMetadata("attempt", { + stripe_payment_intent_id: "pi_retry", + }); + + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow(websiteId, "pi_legacy", 100, "sale", "completed", "{}", at), + revenueRow( + websiteId, + "in_legacy_same_customer", + 100, + "subscription", + "completed", + "{}", + at + ), + revenueRow( + websiteId, + "in_legacy_other_customer", + 100, + "subscription", + "completed", + "{}", + at, + { customer_id: "cus_other" } + ), + revenueRow( + websiteId, + "inpay_recovered", + 60, + "subscription", + "completed", + invoiceSuccessMetadata, + at + ), + revenueRow( + websiteId, + "evt_invoice_failed", + 60, + "subscription_event", + "failed", + stripeMetadata("attempt", { + stripe_invoice_id: "in_recovered", + }), + at + ), + revenueRow( + websiteId, + "evt_invoice_failure_family", + 30, + "subscription_event", + "failed", + stripeMetadata("attempt", { + stripe_event_type: "invoice.payment_failed", + stripe_invoice_id: "in_failure_family", + stripe_payment_intent_id: "pi_failure_family", + }), + at + ), + revenueRow( + websiteId, + "evt_pi_failure_family", + 30, + "subscription_event", + "failed", + stripeMetadata("attempt", { + stripe_event_type: "payment_intent.payment_failed", + stripe_payment_intent_id: "pi_failure_family", + }), + at + ), + revenueRow( + websiteId, + "evt_retry_1", + 40, + "subscription_event", + "failed", + retryMetadata, + at + ), + revenueRow( + websiteId, + "evt_retry_2", + 40, + "subscription_event", + "failed", + retryMetadata, + at + ), + // An immutable redelivery of the same Stripe event is still one attempt. + revenueRow( + websiteId, + "evt_retry_2", + 40, + "subscription_event", + "failed", + retryMetadata, + at + ), + revenueRow( + websiteId, + "pi_retry", + 40, + "sale", + "completed", + retrySuccessMetadata, + at + ), + revenueRow( + websiteId, + "pi_eur", + 70, + "sale", + "completed", + stripeMetadata("money", { + stripe_money_kind: "standalone_candidate", + stripe_payment_intent_id: "pi_eur", + }), + at, + { + currency: "EUR", + original_currency: "EUR", + } + ), + revenueRow( + websiteId, + "evt_gbp_failed_only", + 25, + "subscription_event", + "failed", + stripeMetadata("attempt", { + stripe_event_type: "payment_intent.payment_failed", + stripe_payment_intent_id: "pi_gbp_failed_only", + }), + at, + { + currency: "GBP", + original_currency: "GBP", + } + ), + ], + }); + + const overview = await revenueOverview(websiteId); + const usd = overview.find((row) => row.currency === "USD"); + const eur = overview.find((row) => row.currency === "EUR"); + const gbp = overview.find((row) => row.currency === "GBP"); + + expect(overview).toHaveLength(3); + expect(Number(usd?.total_revenue)).toBe(300); + expect(Number(usd?.total_transactions)).toBe(4); + expect(Number(usd?.failed_payment_attempts)).toBe(4); + expect(Number(usd?.failed_payment_amount)).toBe(170); + expect(Number(usd?.recovered_payment_attempts)).toBe(2); + expect(Number(usd?.successful_payment_attempts)).toBe(2); + expect(Number(usd?.payment_failure_rate)).toBeCloseTo(66.67, 2); + expect(Number(eur?.total_revenue)).toBe(70); + expect(Number(eur?.successful_payment_attempts)).toBe(1); + expect(Number(gbp?.total_revenue)).toBe(0); + expect(Number(gbp?.failed_payment_attempts)).toBe(1); + expect(Number(gbp?.payment_failure_rate)).toBe(100); + expect( + await revenueOverview(websiteId, "2026-07-01", "2026-08-03", [ + { field: "currency", op: "eq", value: "EUR" }, + ]) + ).toEqual([ + expect.objectContaining({ currency: "EUR", total_revenue: 70 }), + ]); + }); + + it("preserves org-owned invoice attribution after sparse delivery and FINAL merge", async () => { + const ownerId = `revenue-owner-${randomUUIDv7()}`; + const websiteId = `revenue-site-${randomUUIDv7()}`; + const invoiceId = `in_${randomUUIDv7()}`; + const paymentId = `inpay_${randomUUIDv7()}`; + const at = "2026-08-02 12:00:00"; + + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + ownerId, + `evt_invoice_${randomUUIDv7()}`, + 0, + "subscription_event", + "linked", + stripeMetadata("link", { stripe_invoice_id: invoiceId }), + at, + { + customer_id: "cus_rich", + product_name: "Pro plan", + synced_at: "2026-08-02 12:01:00", + website_id: websiteId, + } + ), + ], + }); + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + ownerId, + paymentId, + 125, + "subscription", + "completed", + stripeMetadata("money", { + stripe_invoice_id: invoiceId, + stripe_invoice_payment_id: paymentId, + stripe_money_kind: "invoice_payment", + stripe_payment_intent_id: `pi_${randomUUIDv7()}`, + }), + at, + { + customer_id: "", + product_name: null, + synced_at: "2026-08-02 12:05:00", + website_id: null, + } + ), + ], + }); + + await clickHouse.command({ + query: "OPTIMIZE TABLE analytics.revenue PARTITION 202608 FINAL", + }); + + const [overview] = await revenueOverview( + websiteId, + "2026-08-01", + "2026-08-03" + ); + expect(Number(overview?.total_revenue)).toBe(125); + expect(Number(overview?.total_transactions)).toBe(1); + expect( + await revenueOverview( + `unrelated-site-${randomUUIDv7()}`, + "2026-08-01", + "2026-08-03" + ) + ).toEqual([]); + + const productQuery = RevenueBuilders.revenue_by_product?.customSql?.({ + endDate: "2026-08-03", + startDate: "2026-08-01", + websiteId, + }); + if (!productQuery || typeof productQuery === "string") { + throw new Error("Revenue by product did not compile"); + } + const products = await chQuery<{ + customers: number | string; + name: string; + revenue: number | string; + transactions: number | string; + }>(productQuery.sql, productQuery.params); + expect(products).toEqual([ + expect.objectContaining({ + customers: 1, + name: "Pro plan", + revenue: 125, + transactions: 1, + }), + ]); + }); + + it("does not attribute an earlier transaction to a future customer session", async () => { + const websiteId = `revenue-as-of-${randomUUIDv7()}`; + const customerId = `customer-${randomUUIDv7()}`; + const sessionId = `session-${randomUUIDv7()}`; + const earlierTransactionId = `txn-earlier-${randomUUIDv7()}`; + const directFutureTransactionId = `txn-direct-future-${randomUUIDv7()}`; + const laterTransactionId = `txn-later-${randomUUIDv7()}`; + + await clickHouse.insert({ + table: "analytics.events", + format: "JSONEachRow", + values: [ + attributionEvent( + websiteId, + sessionId, + "2026-08-02 12:00:00", + "future-campaign" + ), + ], + }); + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + websiteId, + earlierTransactionId, + 50, + "sale", + "completed", + "{}", + "2026-08-01 12:00:00", + { customer_id: customerId, provider: "paddle" } + ), + revenueRow( + websiteId, + directFutureTransactionId, + 60, + "sale", + "completed", + "{}", + "2026-08-01 13:00:00", + { + customer_id: customerId, + provider: "paddle", + session_id: sessionId, + } + ), + revenueRow( + websiteId, + laterTransactionId, + 75, + "sale", + "completed", + "{}", + "2026-08-02 12:00:00", + { + customer_id: customerId, + provider: "paddle", + session_id: sessionId, + } + ), + ], + }); + + const rows = await recentTransactions( + websiteId, + "2026-08-01", + "2026-08-03" + ); + const earlier = rows.find( + (row) => row.transaction_id === earlierTransactionId + ); + const directFuture = rows.find( + (row) => row.transaction_id === directFutureTransactionId + ); + const later = rows.find((row) => row.transaction_id === laterTransactionId); + + expect(Number(earlier?.is_attributed)).toBe(0); + expect(earlier?.utm_campaign).toBe("Unattributed"); + expect(Number(directFuture?.is_attributed)).toBe(0); + expect(directFuture?.utm_campaign).toBe("Unattributed"); + expect(Number(later?.is_attributed)).toBe(1); + expect(later?.utm_campaign).toBe("future-campaign"); + }); + + it("resolves an exact session event more than 90 days before revenue", async () => { + const websiteId = `revenue-old-session-${randomUUIDv7()}`; + const sessionId = `session-${randomUUIDv7()}`; + const transactionId = `txn-${randomUUIDv7()}`; + + await clickHouse.insert({ + table: "analytics.events", + format: "JSONEachRow", + values: [ + attributionEvent( + websiteId, + sessionId, + "2025-12-01 12:00:00", + "original-campaign" + ), + ], + }); + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + websiteId, + transactionId, + 100, + "sale", + "completed", + "{}", + "2026-08-02 12:00:00", + { provider: "paddle", session_id: sessionId } + ), + ], + }); + + const rows = await recentTransactions( + websiteId, + "2026-08-01", + "2026-08-03" + ); + const transaction = rows.find( + (row) => row.transaction_id === transactionId + ); + + expect(Number(transaction?.is_attributed)).toBe(1); + expect(transaction?.utm_campaign).toBe("original-campaign"); + }); + + it("does not fill first-touch dimensions from a later session event", async () => { + const websiteId = `revenue-first-touch-${randomUUIDv7()}`; + const sessionId = `session-${randomUUIDv7()}`; + const transactionId = `txn-${randomUUIDv7()}`; + + await clickHouse.insert({ + table: "analytics.events", + format: "JSONEachRow", + values: [ + attributionEvent( + websiteId, + sessionId, + "2026-08-01 11:00:00", + null + ), + attributionEvent( + websiteId, + sessionId, + "2026-08-02 12:00:00", + "future-campaign" + ), + ], + }); + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + websiteId, + transactionId, + 100, + "sale", + "completed", + "{}", + "2026-08-01 12:00:00", + { provider: "paddle", session_id: sessionId } + ), + ], + }); + + const rows = await recentTransactions( + websiteId, + "2026-08-01", + "2026-08-03" + ); + const transaction = rows.find( + (row) => row.transaction_id === transactionId + ); + + expect(Number(transaction?.is_attributed)).toBe(1); + expect(transaction?.utm_campaign).toBe("None"); + }); + + it("attributes invoice-only money from its relation context", async () => { + const websiteId = `revenue-invoice-context-${randomUUIDv7()}`; + const sessionId = `session-${randomUUIDv7()}`; + const anonymousId = `anon-${randomUUIDv7()}`; + const at = "2026-08-02 12:00:00"; + + await clickHouse.insert({ + table: "analytics.events", + format: "JSONEachRow", + values: [ + { + id: randomUUIDv7(), + client_id: websiteId, + event_name: "screen_view", + anonymous_id: anonymousId, + session_id: sessionId, + time: at, + url: "https://example.com/checkout", + path: "/checkout", + ip: "127.0.0.1", + user_agent: "integration-test", + properties: "{}", + created_at: at, + }, + ], + }); + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + websiteId, + "evt_invoice_only_context", + 0, + "subscription_event", + "linked", + stripeMetadata("link", { + stripe_invoice_id: "in_invoice_only", + }), + at, + { + anonymous_id: anonymousId, + customer_id: "", + session_id: sessionId, + } + ), + revenueRow( + websiteId, + "in_invoice_only", + 125, + "subscription", + "completed", + stripeMetadata("money", { + stripe_invoice_id: "in_invoice_only", + stripe_money_kind: "invoice", + }), + at, + { customer_id: "" } + ), + ], + }); + + const query = RevenueBuilders.revenue_attribution_overview?.customSql?.({ + endDate: "2026-08-03", + startDate: "2026-08-01", + websiteId, + }); + if (!query || typeof query === "string") { + throw new Error("Revenue attribution overview did not compile"); + } + const rows = await chQuery<{ + name: string; + revenue: number | string; + transactions: number | string; + }>(query.sql, query.params); + + const attributed = rows.find((row) => row.name === "Attributed"); + expect(Number(attributed?.revenue)).toBe(125); + expect(Number(attributed?.transactions)).toBe(1); + }); + + it("attributes identity-free refunds back to the paying profile", async () => { + const websiteId = `revenue-refund-${randomUUIDv7()}`; + const profileId = `profile-${randomUUIDv7()}`; + const anonymousId = `anon-${randomUUIDv7()}`; + const sessionId = `session-${randomUUIDv7()}`; + const at = "2026-08-02 12:00:00"; + const paymentMetadata = stripeMetadata("money", { + stripe_money_kind: "standalone_candidate", + stripe_payment_intent_id: "pi_profile_refund", + }); + const refundMetadata = stripeMetadata("money", { + stripe_money_kind: "refund", + stripe_payment_intent_id: "pi_profile_refund", + }); + + await clickHouse.insert({ + table: "analytics.events", + format: "JSONEachRow", + values: [ + { + id: randomUUIDv7(), + client_id: websiteId, + event_name: "screen_view", + anonymous_id: anonymousId, + profile_id: profileId, + session_id: sessionId, + time: at, + url: "https://example.com/checkout", + path: "/checkout", + ip: "127.0.0.1", + user_agent: "integration-test", + properties: "{}", + created_at: at, + }, + ], + }); + await clickHouse.insert({ + table: "analytics.revenue", + format: "JSONEachRow", + values: [ + revenueRow( + websiteId, + "pi_profile_refund", + 100, + "sale", + "completed", + paymentMetadata, + at, + { + anonymous_id: anonymousId, + profile_id: profileId, + session_id: sessionId, + } + ), + revenueRow( + websiteId, + "re_profile_refund", + -20, + "refund", + "refunded", + refundMetadata, + at + ), + ], + }); + + const listQuery = ProfilesBuilders.profile_list?.customSql?.({ + endDate: "2026-08-03", + startDate: "2026-08-01", + websiteId, + limit: 10, + offset: 0, + }); + if (!listQuery || typeof listQuery === "string") { + throw new Error("Profile list did not compile"); + } + const profiles = await chQuery<{ ltv: number | string; profile_id: string }>( + listQuery.sql, + listQuery.params + ); + + const detailQuery = ProfilesBuilders.profile_revenue?.customSql?.({ + endDate: "2026-08-03", + startDate: "2026-08-01", + websiteId, + filters: [{ field: "anonymous_id", op: "eq", value: profileId }], + }); + if (!detailQuery || typeof detailQuery === "string") { + throw new Error("Profile revenue did not compile"); + } + const transactions = await chQuery<{ + amount: number | string; + transaction_id: string; + }>(detailQuery.sql, detailQuery.params); + + expect(Number(profiles.find((profile) => profile.profile_id === profileId)?.ltv)).toBe(80); + expect(transactions.map((transaction) => transaction.transaction_id).sort()).toEqual([ + "pi_profile_refund", + "re_profile_refund", + ]); + }); +}); diff --git a/packages/ai/src/query/builders/revenue.ts b/packages/ai/src/query/builders/revenue.ts index cd335d11a..dc1a0e547 100644 --- a/packages/ai/src/query/builders/revenue.ts +++ b/packages/ai/src/query/builders/revenue.ts @@ -1,7 +1,13 @@ +import { revenueLatestCte } from "@databuddy/db/clickhouse"; +import { STRIPE_FAILURE_WEBHOOK_EVENTS } from "@databuddy/shared/stripe-webhooks"; import { Analytics } from "../../types/tables"; import { escapeLikePattern } from "../simple-builder"; import type { CustomSqlFn, Filter, SimpleQueryConfig } from "../types"; +const STRIPE_FAILURE_EVENT_SQL = STRIPE_FAILURE_WEBHOOK_EVENTS.map( + ({ event }) => `'${event}'` +).join(",\n\t\t\t\t\t\t"); + const REVENUE_FILTER_COLUMNS: Record = { country: "country", region: "region", @@ -14,10 +20,58 @@ const REVENUE_FILTER_COLUMNS: Record = { utm_campaign: "utm_campaign", referrer: "referrer_domain", path: "entry_path", - provider: "provider", + provider: "revenue_provider", type: "type", + currency: "currency", }; +const REVENUE_ALLOWED_FILTERS = ["currency", "provider", "type"]; +const REVENUE_OVERVIEW_ALLOWED_FILTERS = ["currency", "provider"]; + +function fixedValueMatchesFilter(value: string, filter: Filter): boolean { + const values = ( + Array.isArray(filter.value) ? filter.value : [filter.value] + ).map((item) => String(item)); + if ((filter.op === "in" || filter.op === "not_in") && values.length === 0) { + return true; + } + const expected = String(filter.value); + + switch (filter.op) { + case "eq": + return value === expected; + case "ne": + return value !== expected; + case "in": + return values.includes(value); + case "not_in": + return !values.includes(value); + case "contains": + return value.includes(expected); + case "not_contains": + return !value.includes(expected); + case "starts_with": + return value.startsWith(expected); + default: + return false; + } +} + +function stripePaymentMetricsInScope(filters?: Filter[]): boolean { + return (filters ?? []).every((filter) => { + if (!filter || filter.having) { + return true; + } + if (filter.field === "currency") { + return true; + } + if (filter.field === "provider") { + return fixedValueMatchesFilter("stripe", filter); + } + return false; + }); +} + function buildRevenueWhereClause( filters: Filter[] | undefined, extraConditions: string[] = [] @@ -75,6 +129,52 @@ function buildRevenueWhereClause( }; } +function buildCurrencyWhereClause(filters?: Filter[]): string { + const conditions: string[] = []; + filters?.forEach((filter, index) => { + if (!filter || filter.having || filter.field !== "currency") { + return; + } + const key = `rf${index}`; + if (filter.op === "in" || filter.op === "not_in") { + const values = Array.isArray(filter.value) + ? filter.value + : [filter.value]; + if (values.length > 0) { + conditions.push( + `currency ${filter.op === "in" ? "IN" : "NOT IN"} {${key}:Array(String)}` + ); + } + return; + } + if (filter.op === "contains" || filter.op === "not_contains") { + conditions.push( + `currency ${filter.op === "contains" ? "LIKE" : "NOT LIKE"} {${key}:String}` + ); + return; + } + if (filter.op === "starts_with") { + conditions.push(`currency LIKE {${key}:String}`); + return; + } + conditions.push( + `currency ${filter.op === "ne" ? "!=" : "="} {${key}:String}` + ); + }); + return conditions.length > 0 ? ` WHERE ${conditions.join(" AND ")}` : ""; +} + +function buildStripePaymentWhereClause( + filters: Filter[] | undefined, + paymentMetricsInScope: boolean +): string { + const currencyWhereClause = buildCurrencyWhereClause(filters); + if (paymentMetricsInScope) { + return currencyWhereClause; + } + return currencyWhereClause ? `${currencyWhereClause} AND 0` : " WHERE 0"; +} + function isOrgScope(filterParams?: Record): boolean { return filterParams?.__orgLevel === "true"; } @@ -86,67 +186,452 @@ function buildAttributionCte( const directScope = orgScope ? "(owner_id = {organizationId:String} OR website_id IN {websiteIds:Array(String)})" : "(owner_id = {websiteId:String} OR website_id = {websiteId:String})"; - const aliasedScope = orgScope - ? "(r.owner_id = {organizationId:String} OR r.website_id IN {websiteIds:Array(String)})" - : "(r.owner_id = {websiteId:String} OR r.website_id = {websiteId:String})"; + const eventScope = orgScope + ? "client_id IN {websiteIds:Array(String)}" + : "client_id = {websiteId:String}"; + const relatedInvoiceScope = `( + ${directScope} + OR ( + provider = 'stripe' + AND JSONExtractString(metadata, 'stripe_invoice_id') != '' + AND (owner_id, JSONExtractString(metadata, 'stripe_invoice_id')) IN ( + SELECT owner_id, invoice_id FROM scoped_stripe_invoice_keys + ) + ) + )`; + const attributedWebsiteScope = orgScope + ? "1" + : `(r.owner_id = {websiteId:String} + OR r.website_id = {websiteId:String} + OR r.linked_website_id = {websiteId:String})`; return ` - pi_dedup AS ( - SELECT amount, toUnixTimestamp(created) as ts + legacy_pi_dedup AS ( + SELECT amount, toUnixTimestamp(created) as ts, customer_id FROM ${Analytics.revenue} WHERE ${directScope} AND created >= toDateTime({startDate:String}) AND created <= toDateTime(concat({endDate:String}, ' 23:59:59')) + AND provider = 'stripe' AND startsWith(transaction_id, 'pi_') + AND type != 'subscription_event' + AND status = 'completed' AND amount > 0 + AND customer_id != '' + ), + scoped_stripe_invoice_keys AS ( + SELECT DISTINCT + owner_id, + if( + JSONExtractString(metadata, 'stripe_invoice_id') != '', + JSONExtractString(metadata, 'stripe_invoice_id'), + transaction_id + ) AS invoice_id + FROM ${Analytics.revenue} + WHERE ${directScope} + AND created >= toDateTime({startDate:String}) - INTERVAL 90 DAY + AND created <= toDateTime(concat({endDate:String}, ' 23:59:59')) + AND provider = 'stripe' + AND ( + JSONExtractString(metadata, 'stripe_invoice_id') != '' + OR startsWith(transaction_id, 'in_') + ) + AND ( + JSONExtractString(metadata, 'stripe_record_kind') IN ('link', 'money') + OR JSONExtractString(metadata, 'databuddy_revenue_model') = 'stripe_invoice_v2' + ) + ), + ${revenueLatestCte({ + candidateWhere: `created >= toDateTime({startDate:String}) - INTERVAL 90 DAY + AND created <= toDateTime(concat({endDate:String}, ' 23:59:59'))`, + name: "revenue_latest_range", + scope: relatedInvoiceScope, + })}, + stripe_relation_rows AS ( + SELECT + owner_id, + if( + JSONExtractString(metadata, 'stripe_invoice_id') != '', + JSONExtractString(metadata, 'stripe_invoice_id'), + transaction_id + ) AS invoice_id, + JSONExtractString(metadata, 'stripe_payment_intent_id') AS payment_intent_id, + ifNull(website_id, '') AS context_website_id, + ifNull(anonymous_id, '') AS context_anonymous_id, + ifNull(session_id, '') AS context_session_id, + customer_id AS context_customer_id, + ifNull(product_name, '') AS context_product_name, + synced_at + FROM ${Analytics.revenue} + WHERE ${relatedInvoiceScope} + AND created >= toDateTime({startDate:String}) - INTERVAL 90 DAY + AND created <= toDateTime(concat({endDate:String}, ' 23:59:59')) + AND provider = 'stripe' + AND ( + JSONExtractString(metadata, 'stripe_invoice_id') != '' + OR startsWith(transaction_id, 'in_') + ) + AND ( + JSONExtractString(metadata, 'stripe_record_kind') IN ('link', 'money') + OR JSONExtractString(metadata, 'databuddy_revenue_model') = 'stripe_invoice_v2' + ) + ), + linked_payment_intents AS ( + SELECT DISTINCT + payment_intent_id + FROM stripe_relation_rows + WHERE payment_intent_id != '' + ), + stripe_payment_context_rows AS ( + SELECT + owner_id, + if( + JSONExtractString(metadata, 'stripe_payment_intent_id') != '', + JSONExtractString(metadata, 'stripe_payment_intent_id'), + if(startsWith(transaction_id, 'pi_'), transaction_id, '') + ) AS payment_intent_id, + ifNull(website_id, '') AS context_website_id, + ifNull(anonymous_id, '') AS context_anonymous_id, + ifNull(session_id, '') AS context_session_id, + customer_id AS context_customer_id, + ifNull(product_name, '') AS context_product_name, + synced_at + FROM ${Analytics.revenue} + WHERE ${relatedInvoiceScope} + AND created >= toDateTime({startDate:String}) - INTERVAL 90 DAY + AND created <= toDateTime(concat({endDate:String}, ' 23:59:59')) + AND provider = 'stripe' + AND ( + JSONExtractString(metadata, 'stripe_payment_intent_id') != '' + OR startsWith(transaction_id, 'pi_') + ) + ), + stripe_payment_context AS ( + SELECT + owner_id, + payment_intent_id, + argMaxIf(context_website_id, synced_at, context_website_id != '') AS website_id, + argMaxIf(context_anonymous_id, synced_at, context_anonymous_id != '') AS anonymous_id, + argMaxIf(context_session_id, synced_at, context_session_id != '') AS session_id, + argMaxIf(context_customer_id, synced_at, context_customer_id != '') AS customer_id, + argMaxIf(context_product_name, synced_at, context_product_name != '') AS product_name + FROM stripe_payment_context_rows + WHERE payment_intent_id != '' + GROUP BY owner_id, payment_intent_id + ), + stripe_invoice_context_rows AS ( + SELECT + owner_id, + invoice_id, + context_website_id, + context_anonymous_id, + context_session_id, + context_customer_id, + context_product_name, + synced_at + FROM stripe_relation_rows + UNION ALL + SELECT + relation.owner_id, + relation.invoice_id, + payment.website_id AS context_website_id, + payment.anonymous_id AS context_anonymous_id, + payment.session_id AS context_session_id, + payment.customer_id AS context_customer_id, + payment.product_name AS context_product_name, + relation.synced_at + FROM stripe_relation_rows relation + INNER JOIN stripe_payment_context payment + ON payment.owner_id = relation.owner_id + AND payment.payment_intent_id = relation.payment_intent_id + ), + stripe_invoice_context AS ( + SELECT + owner_id, + invoice_id, + argMaxIf(context_website_id, synced_at, context_website_id != '') AS website_id, + argMaxIf(context_anonymous_id, synced_at, context_anonymous_id != '') AS anonymous_id, + argMaxIf(context_session_id, synced_at, context_session_id != '') AS session_id, + argMaxIf(context_customer_id, synced_at, context_customer_id != '') AS customer_id, + argMaxIf(context_product_name, synced_at, context_product_name != '') AS product_name + FROM stripe_invoice_context_rows + WHERE invoice_id != '' + GROUP BY owner_id, invoice_id + ), + stripe_payment_attempt_rows AS ( + SELECT + transaction_id AS attempt_id, + multiIf( + JSONExtractString(metadata, 'stripe_payment_intent_id') != '', + concat('pi:', JSONExtractString(metadata, 'stripe_payment_intent_id')), + JSONExtractString(metadata, 'stripe_invoice_id') != '', + concat('in:', JSONExtractString(metadata, 'stripe_invoice_id')), + concat('event:', transaction_id) + ) AS attempt_key, + JSONExtractString(metadata, 'stripe_event_type') AS event_type, + JSONExtractString(metadata, 'stripe_invoice_id') AS invoice_id, + coalesce( + nullIf(JSONExtractString(metadata, 'stripe_failure_decline_code'), ''), + nullIf(JSONExtractString(metadata, 'stripe_failure_code'), ''), + nullIf(JSONExtractString(metadata, 'stripe_failure_type'), ''), + '' + ) AS failure_reason, + multiIf( + JSONExtractString(metadata, 'stripe_failure_decline_code') != '', 3, + JSONExtractString(metadata, 'stripe_failure_code') != '', 2, + JSONExtractString(metadata, 'stripe_failure_type') != '', 1, + 0 + ) AS failure_reason_rank, + JSONExtractString(metadata, 'stripe_cancellation_reason') AS cancellation_reason, + status, + amount, + currency, + created, + synced_at + FROM ${Analytics.revenue} + WHERE ${directScope} + AND provider = 'stripe' + AND type = 'subscription_event' + AND JSONExtractString(metadata, 'stripe_record_kind') = 'attempt' + AND created >= toDateTime({startDate:String}) + AND created <= toDateTime(concat({endDate:String}, ' 23:59:59')) + ), + stripe_payment_attempt_candidates AS ( + SELECT + attempt_id, + latest.1 AS attempt_key, + latest.2 AS event_type, + latest.3 AS invoice_id, + latest.4 AS status, + latest.5 AS currency, + latest.6 AS amount, + latest.7 AS failure_reason, + latest.8 AS failure_reason_rank, + latest.9 AS cancellation_reason + FROM ( + SELECT + attempt_id, + argMax( + tuple( + attempt_key, + event_type, + invoice_id, + status, + currency, + amount, + failure_reason, + failure_reason_rank, + cancellation_reason + ), + tuple( + toUnixTimestamp(synced_at), + cityHash64(toString(tuple( + attempt_key, + event_type, + invoice_id, + status, + currency, + amount, + failure_reason, + failure_reason_rank, + cancellation_reason + ))) + ) + ) AS latest + FROM stripe_payment_attempt_rows + GROUP BY attempt_id + ) + ), + stripe_invoice_failure_keys AS ( + SELECT DISTINCT invoice_id + FROM stripe_payment_attempt_candidates + WHERE event_type = 'invoice.payment_failed' AND invoice_id != '' + ), + stripe_invoice_failure_attempt_keys AS ( + SELECT DISTINCT attempt_key + FROM stripe_payment_attempt_candidates + WHERE event_type = 'invoice.payment_failed' + ), + stripe_payment_failure_reasons AS ( + SELECT + attempt_key, + argMax( + failure_reason, + tuple(failure_reason_rank, attempt_id) + ) AS failure_reason + FROM stripe_payment_attempt_candidates + GROUP BY attempt_key + ), + stripe_failure_observations AS ( + SELECT + currency, + uniqExactIf( + event_type, + status = 'failed' + AND event_type IN ( + ${STRIPE_FAILURE_EVENT_SQL} + ) + ) AS observed_failure_event_types + FROM stripe_payment_attempt_candidates + GROUP BY currency + ), + stripe_payment_attempts AS ( + SELECT + attempt.attempt_id, + attempt.attempt_key, + attempt.event_type, + family.failure_reason AS failure_reason, + attempt.cancellation_reason, + attempt.status, + attempt.currency, + attempt.amount + FROM stripe_payment_attempt_candidates attempt + LEFT JOIN stripe_payment_failure_reasons family USING (attempt_key) + WHERE NOT ( + attempt.event_type = 'payment_intent.payment_failed' + AND ( + attempt.attempt_key IN (SELECT attempt_key FROM stripe_invoice_failure_attempt_keys) + OR attempt.invoice_id IN (SELECT invoice_id FROM stripe_invoice_failure_keys) + ) + ) + ), + revenue_with_invoice_context AS ( + SELECT + r.*, + r.owner_id AS reconciliation_owner_id, + coalesce( + nullIf(invoice_context.website_id, ''), + nullIf(payment_context.website_id, '') + ) AS linked_website_id, + coalesce( + nullIf(invoice_context.anonymous_id, ''), + nullIf(payment_context.anonymous_id, '') + ) AS linked_anonymous_id, + coalesce( + nullIf(invoice_context.session_id, ''), + nullIf(payment_context.session_id, '') + ) AS linked_session_id, + coalesce( + nullIf(invoice_context.customer_id, ''), + nullIf(payment_context.customer_id, '') + ) AS linked_customer_id, + coalesce( + nullIf(invoice_context.product_name, ''), + nullIf(payment_context.product_name, '') + ) AS linked_product_name + FROM revenue_latest_range r + LEFT JOIN stripe_invoice_context invoice_context + ON invoice_context.owner_id = r.owner_id + AND invoice_context.invoice_id = if( + JSONExtractString(r.metadata, 'stripe_invoice_id') != '', + JSONExtractString(r.metadata, 'stripe_invoice_id'), + if(startsWith(r.transaction_id, 'in_'), r.transaction_id, '') + ) + LEFT JOIN stripe_payment_context payment_context + ON payment_context.owner_id = r.owner_id + AND payment_context.payment_intent_id = if( + JSONExtractString(r.metadata, 'stripe_payment_intent_id') != '', + JSONExtractString(r.metadata, 'stripe_payment_intent_id'), + if(startsWith(r.transaction_id, 'pi_'), r.transaction_id, '') + ) + ), + stripe_invoice_payment_totals AS ( + SELECT + owner_id, + JSONExtractString(metadata, 'stripe_invoice_id') AS invoice_id, + currency, + sum(amount) AS amount + FROM revenue_latest_range + WHERE provider = 'stripe' + AND status = 'completed' + AND JSONExtractString(metadata, 'stripe_money_kind') = 'invoice_payment' + AND JSONExtractString(metadata, 'stripe_invoice_id') != '' + GROUP BY owner_id, invoice_id, currency ), revenue_base AS ( SELECT r.transaction_id, - r.amount, - r.type, - r.anonymous_id as r_anonymous_id, - r.session_id as r_session_id, - r.customer_id as r_customer_id, + if( + JSONExtractString(r.metadata, 'stripe_money_kind') = 'invoice_fallback', + r.amount - ifNull(invoice_payments.amount, 0), + r.amount + ) AS amount, + if( + JSONExtractString(r.metadata, 'stripe_money_kind') = 'invoice_fallback', + 'subscription', + r.type + ) AS type, + coalesce(r.anonymous_id, nullIf(r.linked_anonymous_id, '')) as r_anonymous_id, + coalesce(r.session_id, nullIf(r.linked_session_id, '')) as r_session_id, + coalesce(nullIf(r.customer_id, ''), nullIf(r.linked_customer_id, '')) as r_customer_id, r.product_id, - r.product_name, + coalesce(r.product_name, nullIf(r.linked_product_name, '')) as product_name, r.provider, + r.currency, + r.metadata, r.created - FROM ${Analytics.revenue} r - WHERE - ${aliasedScope} - AND r.created >= toDateTime({startDate:String}) + FROM revenue_with_invoice_context r + LEFT JOIN stripe_invoice_payment_totals invoice_payments + ON invoice_payments.owner_id = r.reconciliation_owner_id + AND invoice_payments.invoice_id = JSONExtractString(r.metadata, 'stripe_invoice_id') + AND invoice_payments.currency = r.currency + WHERE + ${attributedWebsiteScope} + AND r.created >= toDateTime({startDate:String}) AND r.created <= toDateTime(concat({endDate:String}, ' 23:59:59')) - AND r.type != 'subscription_event' + AND ( + r.type != 'subscription_event' + OR JSONExtractString(r.metadata, 'stripe_money_kind') = 'invoice_fallback' + ) + AND ( + (r.type = 'refund' AND r.status = 'refunded') + OR (r.type != 'refund' AND r.status = 'completed') + ) + AND NOT ( + r.provider = 'stripe' + AND startsWith(r.transaction_id, 'pi_') + AND r.transaction_id IN ( + SELECT payment_intent_id FROM linked_payment_intents + ) + ) AND NOT ( - startsWith(r.transaction_id, 'in_') + r.provider = 'stripe' + AND startsWith(r.transaction_id, 'in_') + AND JSONExtractString(r.metadata, 'databuddy_revenue_model') = '' AND ( - (r.amount, toUnixTimestamp(r.created)) IN (SELECT amount, ts FROM pi_dedup) - OR (r.amount, toUnixTimestamp(r.created) + 1) IN (SELECT amount, ts FROM pi_dedup) - OR (r.amount, toUnixTimestamp(r.created) - 1) IN (SELECT amount, ts FROM pi_dedup) + (r.amount, toUnixTimestamp(r.created), r.customer_id) IN (SELECT amount, ts, customer_id FROM legacy_pi_dedup) + OR (r.amount, toUnixTimestamp(r.created) + 1, r.customer_id) IN (SELECT amount, ts, customer_id FROM legacy_pi_dedup) + OR (r.amount, toUnixTimestamp(r.created) - 1, r.customer_id) IN (SELECT amount, ts, customer_id FROM legacy_pi_dedup) ) ) - ), - active_customers AS ( - SELECT DISTINCT r_customer_id as customer_id - FROM revenue_base - WHERE r_customer_id IS NOT NULL AND r_customer_id != '' + AND NOT ( + r.provider = 'stripe' + AND JSONExtractString(r.metadata, 'stripe_money_kind') = 'invoice_fallback' + AND r.amount <= ifNull(invoice_payments.amount, 0) + ) ), customer_session_map AS ( SELECT - r.customer_id as customer_id, - argMin(r.session_id, r.created) as mapped_session_id - FROM ${Analytics.revenue} r - INNER JOIN active_customers ac ON r.customer_id = ac.customer_id - WHERE ${aliasedScope} - AND r.customer_id IS NOT NULL AND r.customer_id != '' - AND r.session_id IS NOT NULL AND r.session_id != '' - GROUP BY r.customer_id + provider, + customer_id, + argMin(session_id, created) as mapped_session_id, + min(created) as mapped_session_created + FROM ${Analytics.revenue} + WHERE ${directScope} + AND created >= toDateTime({startDate:String}) - INTERVAL 90 DAY + AND created <= toDateTime(concat({endDate:String}, ' 23:59:59')) + AND customer_id != '' + AND session_id IS NOT NULL AND session_id != '' + GROUP BY provider, customer_id ), attributed_sessions AS ( - SELECT r_session_id AS session_id FROM revenue_base - WHERE r_session_id IS NOT NULL AND r_session_id != '' + SELECT DISTINCT session_id + FROM ${Analytics.revenue} + WHERE ${directScope} + AND created >= toDateTime({startDate:String}) - INTERVAL 90 DAY + AND created <= toDateTime(concat({endDate:String}, ' 23:59:59')) + AND session_id IS NOT NULL AND session_id != '' UNION DISTINCT SELECT mapped_session_id AS session_id FROM customer_session_map WHERE mapped_session_id IS NOT NULL AND mapped_session_id != '' @@ -154,21 +639,21 @@ function buildAttributionCte( first_touch_by_session AS ( SELECT session_id, - argMin(country, time) as first_country, - argMin(region, time) as first_region, - argMin(city, time) as first_city, - argMin(browser_name, time) as first_browser, - argMin(device_type, time) as first_device, - argMin(os_name, time) as first_os, - argMin(domain(referrer), time) as first_referrer, - argMin(utm_source, time) as first_utm_source, - argMin(utm_medium, time) as first_utm_medium, - argMin(utm_campaign, time) as first_utm_campaign, + min(time) as first_touch_time, + argMin(ifNull(country, ''), time) as first_country, + argMin(ifNull(region, ''), time) as first_region, + argMin(ifNull(city, ''), time) as first_city, + argMin(ifNull(browser_name, ''), time) as first_browser, + argMin(ifNull(device_type, ''), time) as first_device, + argMin(ifNull(os_name, ''), time) as first_os, + argMin(domain(ifNull(referrer, '')), time) as first_referrer, + argMin(ifNull(utm_source, ''), time) as first_utm_source, + argMin(ifNull(utm_medium, ''), time) as first_utm_medium, + argMin(ifNull(utm_campaign, ''), time) as first_utm_campaign, argMin(path, time) as first_path FROM ${Analytics.events} - WHERE client_id = {websiteId:String} + WHERE ${eventScope} AND session_id IN (SELECT session_id FROM attributed_sessions) - AND time >= toDateTime({startDate:String}) - INTERVAL 90 DAY AND time <= toDateTime(concat({endDate:String}, ' 23:59:59')) GROUP BY session_id ), @@ -182,7 +667,9 @@ function buildAttributionCte( rb.r_customer_id, rb.product_id, rb.product_name, - rb.provider, + rb.provider as revenue_provider, + rb.currency, + rb.metadata, rb.created, CASE WHEN ft_direct.session_id != '' THEN 1 @@ -205,15 +692,19 @@ function buildAttributionCte( ON rb.r_session_id = ft_direct.session_id AND rb.r_session_id IS NOT NULL AND rb.r_session_id != '' + AND ft_direct.first_touch_time <= rb.created LEFT JOIN customer_session_map csm - ON rb.r_customer_id = csm.customer_id + ON rb.provider = csm.provider + AND rb.r_customer_id = csm.customer_id AND rb.r_customer_id IS NOT NULL AND rb.r_customer_id != '' + AND csm.mapped_session_created <= rb.created AND ft_direct.session_id = '' LEFT JOIN first_touch_by_session ft_customer ON csm.mapped_session_id = ft_customer.session_id AND csm.mapped_session_id IS NOT NULL AND csm.mapped_session_id != '' + AND ft_customer.first_touch_time <= rb.created ) `; } @@ -227,6 +718,13 @@ function buildScopeParams( interface RevenueQueryConfig { extraConditions?: string[]; + from?: ( + defaultSource: string, + context: { + paymentMetricsInScope: boolean; + stripePaymentWhereClause: string; + } + ) => string; groupBy?: string; innerCte?: { name: string; body: (filteredSource: string) => string }; limit?: number; @@ -252,7 +750,16 @@ function buildRevenueQuery( const withClause = config.innerCte ? `WITH ${baseCte},\n\t\t${config.innerCte.name} AS (${config.innerCte.body(filteredSource)})` : `WITH ${baseCte}`; - const fromExpr = config.innerCte ? config.innerCte.name : filteredSource; + const defaultSource = config.innerCte ? config.innerCte.name : filteredSource; + const paymentMetricsInScope = stripePaymentMetricsInScope(filters); + const fromExpr = + config.from?.(defaultSource, { + paymentMetricsInScope, + stripePaymentWhereClause: buildStripePaymentWhereClause( + filters, + paymentMetricsInScope + ), + }) ?? defaultSource; const parts = [withClause, config.select, `FROM ${fromExpr}`]; if (config.groupBy) { @@ -296,10 +803,11 @@ function makeRevenueBuilder( } const REVENUE_METRICS = ` + currency, sumIf(amount, type != 'refund') as revenue, countIf(type != 'refund') as transactions, uniq(r_customer_id) as customers, - ROUND((sumIf(amount, type != 'refund') / nullIf(SUM(sumIf(amount, type != 'refund')) OVER(), 0)) * 100, 2) as percentage`; + ROUND((sumIf(amount, type != 'refund') / nullIf(SUM(sumIf(amount, type != 'refund')) OVER (PARTITION BY currency), 0)) * 100, 2) as percentage`; function dimensionCase(column: string, fallback: string): string { return `CASE @@ -319,6 +827,7 @@ function recentTransactionDimension( const REVENUE_BREAKDOWN_FIELDS = [ { name: "name", type: "string" as const, label: "Name" }, + { name: "currency", type: "string" as const, label: "Currency" }, { name: "revenue", type: "number" as const, label: "Revenue" }, { name: "transactions", type: "number" as const, label: "Transactions" }, { name: "customers", type: "number" as const, label: "Customers" }, @@ -328,13 +837,14 @@ const REVENUE_BREAKDOWN_FIELDS = [ const REVENUE_GEO_BREAKDOWN_FIELDS = [ { name: "name", type: "string" as const, label: "Name" }, { name: "country", type: "string" as const, label: "Country" }, + { name: "currency", type: "string" as const, label: "Currency" }, { name: "revenue", type: "number" as const, label: "Revenue" }, { name: "transactions", type: "number" as const, label: "Transactions" }, { name: "customers", type: "number" as const, label: "Customers" }, { name: "percentage", type: "number" as const, label: "Share", unit: "%" }, ]; -export const RevenueBuilders: Record = { +const revenueBuilderDefinitions: Record = { revenue_overview: { meta: { title: "Revenue Overview", @@ -343,6 +853,7 @@ export const RevenueBuilders: Record = { category: "Revenue", tags: ["revenue", "overview", "summary"], output_fields: [ + { name: "currency", type: "string", label: "Currency" }, { name: "total_revenue", type: "number", label: "Total Revenue" }, { name: "total_transactions", @@ -374,23 +885,238 @@ export const RevenueBuilders: Record = { type: "number", label: "Attributed Revenue", }, + { + name: "payment_diagnostics_available", + type: "number", + label: "Payment Diagnostics Available", + description: + "1 when Stripe payment diagnostics support the selected filters; otherwise 0.", + }, + { + name: "failed_payment_attempts", + type: "number", + label: "Failed Payment Attempts", + }, + { + name: "canceled_payment_attempts", + type: "number", + label: "Canceled Payment Attempts", + }, + { + name: "failed_payment_amount", + type: "number", + label: "Failed Payment Amount", + }, + { + name: "recovered_payment_attempts", + type: "number", + label: "Recovered Payment Attempts", + }, + { + name: "successful_payment_attempts", + type: "number", + label: "Successful Payment Attempts", + }, + { + name: "observed_failure_event_types", + type: "number", + label: "Observed Failure Event Types", + }, + { + name: "required_failure_event_types", + type: "number", + label: "Required Failure Event Types", + }, + { + name: "top_payment_failure_reason", + type: "string", + label: "Top Payment Failure Reason", + }, + { + name: "top_payment_cancellation_reason", + type: "string", + label: "Top Payment Cancellation Reason", + }, + { + name: "payment_failure_rate", + type: "number", + label: "Payment Failure Rate", + unit: "%", + }, ], default_visualization: "metric", version: "1.0", }, customSql: makeRevenueBuilder(() => ({ + innerCte: { + name: "revenue_summary", + body: (source) => ` + SELECT + currency, + sumIf(amount, type != 'refund') as total_revenue, + countIf(type != 'refund') as total_transactions, + sumIf(amount, type = 'refund') as refund_amount, + countIf(type = 'refund') as refund_count, + sumIf(amount, type = 'subscription') as subscription_revenue, + countIf(type = 'subscription') as subscription_count, + sumIf(amount, type = 'sale') as sale_revenue, + countIf(type = 'sale') as sale_count, + uniq(r_customer_id) as unique_customers, + countIf(is_attributed = 1 AND type != 'refund') as attributed_transactions, + sumIf(amount, is_attributed = 1 AND type != 'refund') as attributed_revenue, + uniqExactIf( + transaction_id, + revenue_provider = 'stripe' + AND type != 'refund' + AND JSONExtractString(metadata, 'databuddy_revenue_model') = 'stripe_events_v1' + ) as successful_payment_attempts, + arrayDistinct(arrayFlatten(groupArrayIf( + arrayFilter(key -> key != '', [ + if( + JSONExtractString(metadata, 'stripe_payment_intent_id') != '', + concat('pi:', JSONExtractString(metadata, 'stripe_payment_intent_id')), + if(startsWith(transaction_id, 'pi_'), concat('pi:', transaction_id), '') + ), + if( + JSONExtractString(metadata, 'stripe_invoice_id') != '', + concat('in:', JSONExtractString(metadata, 'stripe_invoice_id')), + if(startsWith(transaction_id, 'in_'), concat('in:', transaction_id), '') + ), + if( + JSONExtractString(metadata, 'stripe_payment_intent_id') = '' + AND JSONExtractString(metadata, 'stripe_invoice_id') = '' + AND NOT startsWith(transaction_id, 'pi_') + AND NOT startsWith(transaction_id, 'in_'), + concat('event:', transaction_id), + '' + ) + ]), + revenue_provider = 'stripe' + AND type != 'refund' + AND JSONExtractString(metadata, 'databuddy_revenue_model') = 'stripe_events_v1' + ))) as successful_payment_keys + FROM ${source} + GROUP BY currency + `, + }, + from: (source, { paymentMetricsInScope, stripePaymentWhereClause }) => `( + SELECT + if(summary.currency = '', attempts.currency, summary.currency) AS currency, + summary.total_revenue, + summary.total_transactions, + summary.refund_amount, + summary.refund_count, + summary.subscription_revenue, + summary.subscription_count, + summary.sale_revenue, + summary.sale_count, + summary.unique_customers, + summary.attributed_transactions, + summary.attributed_revenue, + summary.successful_payment_attempts, + summary.successful_payment_keys, + attempts.attempt_key, + attempts.event_type AS attempt_event_type, + attempts.failure_reason AS attempt_failure_reason, + attempts.cancellation_reason AS attempt_cancellation_reason, + attempts.status AS attempt_status, + attempts.amount AS attempt_amount, + toUInt8(${paymentMetricsInScope ? 1 : 0}) AS payment_metrics_in_scope, + failure_observations.observed_failure_event_types, + has(summary.successful_payment_keys, attempts.attempt_key) AS attempt_recovered + FROM ${source} summary + FULL OUTER JOIN ( + SELECT * FROM stripe_payment_attempts${stripePaymentWhereClause} + ) attempts USING (currency) + LEFT JOIN ( + SELECT * FROM stripe_failure_observations${stripePaymentWhereClause} + ) failure_observations + ON failure_observations.currency = if(summary.currency = '', attempts.currency, summary.currency) + )`, select: `SELECT - sumIf(amount, type != 'refund') as total_revenue, - countIf(type != 'refund') as total_transactions, - sumIf(amount, type = 'refund') as refund_amount, - countIf(type = 'refund') as refund_count, - sumIf(amount, type = 'subscription') as subscription_revenue, - countIf(type = 'subscription') as subscription_count, - sumIf(amount, type = 'sale') as sale_revenue, - countIf(type = 'sale') as sale_count, - uniq(r_customer_id) as unique_customers, - countIf(is_attributed = 1 AND type != 'refund') as attributed_transactions, - sumIf(amount, is_attributed = 1 AND type != 'refund') as attributed_revenue`, + currency, + any(total_revenue) as total_revenue, + any(total_transactions) as total_transactions, + any(refund_amount) as refund_amount, + any(refund_count) as refund_count, + any(subscription_revenue) as subscription_revenue, + any(subscription_count) as subscription_count, + any(sale_revenue) as sale_revenue, + any(sale_count) as sale_count, + any(unique_customers) as unique_customers, + any(attributed_transactions) as attributed_transactions, + any(attributed_revenue) as attributed_revenue, + any(payment_metrics_in_scope) as payment_diagnostics_available, + if( + any(payment_metrics_in_scope) = 1, + countIf(attempt_status = 'failed'), + NULL + ) as failed_payment_attempts, + if( + any(payment_metrics_in_scope) = 1, + countIf(attempt_status = 'canceled'), + NULL + ) as canceled_payment_attempts, + if( + any(payment_metrics_in_scope) = 1, + sumIf(attempt_amount, attempt_status = 'failed'), + NULL + ) as failed_payment_amount, + if( + any(payment_metrics_in_scope) = 1, + uniqExactIf( + attempt_key, + attempt_status = 'failed' AND attempt_recovered + ), + NULL + ) as recovered_payment_attempts, + if( + any(payment_metrics_in_scope) = 1, + any(successful_payment_attempts), + NULL + ) as successful_payment_attempts, + if( + any(payment_metrics_in_scope) = 1, + ifNull(any(observed_failure_event_types), 0), + NULL + ) as observed_failure_event_types, + if( + any(payment_metrics_in_scope) = 1, + toUInt8(${STRIPE_FAILURE_WEBHOOK_EVENTS.length}), + NULL + ) as required_failure_event_types, + if( + any(payment_metrics_in_scope) = 1, + arrayElement( + topKIf(1)( + attempt_failure_reason, + attempt_status = 'failed' AND attempt_failure_reason != '' + ), + 1 + ), + NULL + ) as top_payment_failure_reason, + if( + any(payment_metrics_in_scope) = 1, + arrayElement( + topKIf(1)( + attempt_cancellation_reason, + attempt_status = 'canceled' AND attempt_cancellation_reason != '' + ), + 1 + ), + NULL + ) as top_payment_cancellation_reason, + if( + any(payment_metrics_in_scope) = 1, + round( + 100 * failed_payment_attempts / + nullIf(failed_payment_attempts + successful_payment_attempts, 0), + 2 + ), + NULL + ) as payment_failure_rate`, + groupBy: "currency", })), timeField: "created", customizable: false, @@ -405,6 +1131,7 @@ export const RevenueBuilders: Record = { tags: ["revenue", "time-series", "trends"], output_fields: [ { name: "date", type: "string", label: "Date" }, + { name: "currency", type: "string", label: "Currency" }, { name: "revenue", type: "number", label: "Revenue" }, { name: "transactions", type: "number", label: "Transactions" }, { name: "customers", type: "number", label: "Customers" }, @@ -428,6 +1155,7 @@ export const RevenueBuilders: Record = { customSql: makeRevenueBuilder(() => ({ select: `SELECT toDate(created) as date, + currency, sumIf(amount, type != 'refund') as revenue, countIf(type != 'refund') as transactions, uniq(r_customer_id) as customers, @@ -435,7 +1163,7 @@ export const RevenueBuilders: Record = { countIf(type = 'refund') as refund_count, sumIf(amount, is_attributed = 1 AND type != 'refund') as attributed_revenue, countIf(is_attributed = 1 AND type != 'refund') as attributed_transactions`, - groupBy: "date", + groupBy: "date, currency", orderBy: "date ASC", })), timeField: "created", @@ -454,8 +1182,8 @@ export const RevenueBuilders: Record = { }, customSql: makeRevenueBuilder(() => ({ select: `SELECT - provider as name,${REVENUE_METRICS}`, - groupBy: "provider", + revenue_provider as name,${REVENUE_METRICS}`, + groupBy: "revenue_provider, currency", orderBy: "revenue DESC", })), timeField: "created", @@ -471,6 +1199,7 @@ export const RevenueBuilders: Record = { output_fields: [ { name: "name", type: "string", label: "Product" }, { name: "product_id", type: "string", label: "Product ID" }, + { name: "currency", type: "string", label: "Currency" }, { name: "revenue", type: "number", label: "Revenue" }, { name: "transactions", type: "number", label: "Transactions" }, { name: "customers", type: "number", label: "Customers" }, @@ -484,7 +1213,7 @@ export const RevenueBuilders: Record = { select: `SELECT coalesce(product_name, 'Unknown') as name, product_id,${REVENUE_METRICS}`, - groupBy: "product_name, product_id", + groupBy: "product_name, product_id, currency", orderBy: "revenue DESC", limit, }), @@ -507,7 +1236,7 @@ export const RevenueBuilders: Record = { customSql: makeRevenueBuilder(() => ({ select: `SELECT CASE WHEN is_attributed = 1 THEN 'Attributed' ELSE 'Unattributed' END as name,${REVENUE_METRICS}`, - groupBy: "is_attributed", + groupBy: "is_attributed, currency", orderBy: "revenue DESC", })), timeField: "created", @@ -528,7 +1257,7 @@ export const RevenueBuilders: Record = { (limit) => ({ select: `SELECT ${dimensionCase("country", "Unknown")} as name,${REVENUE_METRICS}`, - groupBy: "name", + groupBy: "name, currency", orderBy: "revenue DESC", limit, }), @@ -554,7 +1283,7 @@ export const RevenueBuilders: Record = { select: `SELECT ${dimensionCase("region", "Unknown")} as name, country,${REVENUE_METRICS}`, - groupBy: "name, country", + groupBy: "name, country, currency", orderBy: "revenue DESC", limit, }), @@ -580,7 +1309,7 @@ export const RevenueBuilders: Record = { select: `SELECT ${dimensionCase("city", "Unknown")} as name, country,${REVENUE_METRICS}`, - groupBy: "name, country", + groupBy: "name, country, currency", orderBy: "revenue DESC", limit, }), @@ -605,7 +1334,7 @@ export const RevenueBuilders: Record = { (limit) => ({ select: `SELECT ${dimensionCase("browser_name", "Unknown")} as name,${REVENUE_METRICS}`, - groupBy: "name", + groupBy: "name, currency", orderBy: "revenue DESC", limit, }), @@ -629,7 +1358,7 @@ export const RevenueBuilders: Record = { (limit) => ({ select: `SELECT ${dimensionCase("device_type", "Unknown")} as name,${REVENUE_METRICS}`, - groupBy: "name", + groupBy: "name, currency", orderBy: "revenue DESC", limit, }), @@ -653,7 +1382,7 @@ export const RevenueBuilders: Record = { (limit) => ({ select: `SELECT ${dimensionCase("os_name", "Unknown")} as name,${REVENUE_METRICS}`, - groupBy: "name", + groupBy: "name, currency", orderBy: "revenue DESC", limit, }), @@ -677,7 +1406,7 @@ export const RevenueBuilders: Record = { (limit) => ({ select: `SELECT referrer_name as name,${REVENUE_METRICS}`, - groupBy: "referrer_name", + groupBy: "referrer_name, currency", orderBy: "revenue DESC", limit, innerCte: { @@ -685,6 +1414,7 @@ export const RevenueBuilders: Record = { body: (source) => ` SELECT ${dimensionCase("referrer_domain", "Direct")} as referrer_name, + currency, amount, type, r_customer_id @@ -712,7 +1442,7 @@ export const RevenueBuilders: Record = { (limit) => ({ select: `SELECT ${dimensionCase("utm_source", "None")} as name,${REVENUE_METRICS}`, - groupBy: "name", + groupBy: "name, currency", orderBy: "revenue DESC", limit, }), @@ -736,7 +1466,7 @@ export const RevenueBuilders: Record = { (limit) => ({ select: `SELECT ${dimensionCase("utm_medium", "None")} as name,${REVENUE_METRICS}`, - groupBy: "name", + groupBy: "name, currency", orderBy: "revenue DESC", limit, }), @@ -760,7 +1490,7 @@ export const RevenueBuilders: Record = { (limit) => ({ select: `SELECT ${dimensionCase("utm_campaign", "None")} as name,${REVENUE_METRICS}`, - groupBy: "name", + groupBy: "name, currency", orderBy: "revenue DESC", limit, }), @@ -784,7 +1514,7 @@ export const RevenueBuilders: Record = { (limit) => ({ select: `SELECT ${dimensionCase("entry_path", "Unknown")} as name,${REVENUE_METRICS}`, - groupBy: "name", + groupBy: "name, currency", orderBy: "revenue DESC", limit, }), @@ -806,6 +1536,7 @@ export const RevenueBuilders: Record = { { name: "provider", type: "string", label: "Provider" }, { name: "type", type: "string", label: "Type" }, { name: "amount", type: "number", label: "Amount" }, + { name: "currency", type: "string", label: "Currency" }, { name: "anonymous_id", type: "string", label: "Anonymous ID" }, { name: "product_name", type: "string", label: "Product" }, { name: "created", type: "datetime", label: "Created" }, @@ -824,9 +1555,10 @@ export const RevenueBuilders: Record = { (limit) => ({ select: `SELECT transaction_id, - provider, + revenue_provider as provider, type, amount, + currency, r_anonymous_id as anonymous_id, product_name, created, @@ -848,3 +1580,17 @@ export const RevenueBuilders: Record = { plugins: { normalizeGeo: true }, }, }; + +export const RevenueBuilders: Record = + Object.fromEntries( + Object.entries(revenueBuilderDefinitions).map(([name, config]) => [ + name, + { + ...config, + allowedFilters: + name === "revenue_overview" + ? REVENUE_OVERVIEW_ALLOWED_FILTERS + : REVENUE_ALLOWED_FILTERS, + }, + ]) + ); diff --git a/packages/ai/src/query/builders/vitals.test.ts b/packages/ai/src/query/builders/vitals.test.ts index 6140c4a09..0c5fd9f68 100644 --- a/packages/ai/src/query/builders/vitals.test.ts +++ b/packages/ai/src/query/builders/vitals.test.ts @@ -19,7 +19,7 @@ describe("vitals_overview", () => { "quantilesDeterministic(0.50, 0.75, 0.90, 0.95, 0.99)" ); expect(normalizedSql).toContain( - "cityHash64(tuple(timestamp, metric_value))" + "cityHash64(tuple( timestamp, metric_value, session_id, anonymous_id, path ))" ); expect(normalizedSql).not.toContain("TDigest"); expect(params).toMatchObject({ diff --git a/packages/ai/src/query/builders/vitals.ts b/packages/ai/src/query/builders/vitals.ts index 1267c4332..9416d7487 100644 --- a/packages/ai/src/query/builders/vitals.ts +++ b/packages/ai/src/query/builders/vitals.ts @@ -139,7 +139,13 @@ export const VitalsBuilders: Record = { metric_name, quantilesDeterministic(0.50, 0.75, 0.90, 0.95, 0.99)( metric_value, - cityHash64(tuple(timestamp, metric_value)) + cityHash64(tuple( + timestamp, + metric_value, + session_id, + anonymous_id, + path + )) ) as _q, avg(metric_value) as avg_value, count() as samples diff --git a/packages/ai/src/query/simple-builder.test.ts b/packages/ai/src/query/simple-builder.test.ts index 6a9a275b1..ae675818e 100644 --- a/packages/ai/src/query/simple-builder.test.ts +++ b/packages/ai/src/query/simple-builder.test.ts @@ -527,7 +527,7 @@ describe("SimpleQueryBuilder.compile", () => { "AND if(profile_id != '', profile_id, anonymous_id) IN (" ); expect(sql).toContain( - "SELECT DISTINCT if(profile_id != '', profile_id, ifNull(anonymous_id, ''))" + "SELECT DISTINCT coalesce(nullIf(profile_id, ''), nullIf(anonymous_id, ''))" ); expect(sql).toContain("FROM analytics.custom_events"); expect(sql).toContain("AND event_name = {f0:String}"); @@ -535,6 +535,27 @@ describe("SimpleQueryBuilder.compile", () => { expect(params.f0).toBe("signup"); }); + it.each([ + "custom_events", + "custom_events_by_path", + "custom_events_trends", + "custom_events_summary", + "custom_events_discovery", + ])("counts users by canonical profile-first identity in %s", (type) => { + const config = QueryBuilders[type]; + if (!config) { + throw new Error(`${type} builder is missing`); + } + + const { sql } = compileBuilder(type, config); + + expect(sql).toContain( + "uniq(coalesce(nullIf(profile_id, ''), nullIf(anonymous_id, ''))) as unique_users" + ); + expect(sql).not.toContain("ifNull(anonymous_id, '')"); + expect(sql).not.toContain("uniq(anonymous_id)"); + }); + it("keeps profile_list event_name operator semantics in the custom-events subquery", () => { const config = QueryBuilders.profile_list; if (!config) { @@ -842,6 +863,31 @@ describe("SimpleQueryBuilder.compile", () => { expect(params.f0).toBe("US"); }); + it.each(["entry_pages", "exit_pages"])( + "filters %s by anonymous visitor id", + (type) => { + const config = QueryBuilders[type]; + if (!config) { + throw new Error(`${type} builder is missing`); + } + + const { params, sql } = new SimpleQueryBuilder( + config, + makeRequest({ + filters: [ + { field: "anonymous_id", op: "eq", value: "visitor-1" }, + ], + type, + }) + ).compile(); + + expect(sql).toContain("anonymous_id = {f0:String}"); + expect(sql).toContain("as visitor_id"); + expect(sql).not.toMatch(/arg(?:Min|Max)\([^\n]+\) as anonymous_id/); + expect(params.f0).toBe("visitor-1"); + } + ); + it("normalizes standard session attribution queries", () => { const { sql, params } = compile( { @@ -1001,6 +1047,175 @@ describe("SimpleQueryBuilder.compile", () => { expect(sql).not.toContain("event_name = 'pageview'"); }); + it("collapses immutable revenue versions in revenue and profile reads", () => { + for (const type of [ + "revenue_overview", + "profile_list", + "profile_revenue", + ]) { + const config = QueryBuilders[type]; + if (!config) { + throw new Error(`${type} builder is missing`); + } + const { sql } = compileBuilder(type, config); + + expect(sql, type).toContain( + "GROUP BY owner_id, provider, transaction_id" + ); + expect(sql, type).toContain("FROM analytics.revenue"); + expect(sql, type).not.toContain("analytics.revenue FINAL"); + } + }); + + it("keeps profile LTV lifetime while activity stays window-scoped", () => { + const config = QueryBuilders.profile_list; + if (!config) { + throw new Error("profile_list builder is missing"); + } + const { sql } = compileBuilder("profile_list", config); + + expect(sql).toContain("time >= {startDate:DateTime}"); + expect(sql).toContain("timestamp >= {startDate:DateTime}"); + expect(sql).not.toContain("created >= {startDate:DateTime}"); + expect(sql).not.toContain("created <= {endDate:DateTime}"); + }); + + it("counts only terminal successful states in revenue totals", () => { + const config = QueryBuilders.revenue_overview; + if (!config) { + throw new Error("revenue_overview builder is missing"); + } + const { sql } = compileBuilder("revenue_overview", config); + + expect(sql).toContain("r.type = 'refund' AND r.status = 'refunded'"); + expect(sql).toContain("r.type != 'refund' AND r.status = 'completed'"); + expect(sql).toContain("customer_session_map AS"); + expect(sql).toContain( + "created >= {startDate:DateTime} - INTERVAL 90 DAY" + ); + expect(sql).toContain("ON rb.provider = csm.provider"); + expect(sql).toContain("legacy_pi_dedup AS"); + expect(sql).toContain("scoped_stripe_invoice_keys AS"); + expect(sql).toContain("stripe_relation_rows AS"); + expect(sql).toContain("linked_payment_intents AS"); + expect(sql).toContain("stripe_invoice_payment_totals AS"); + expect(sql).toContain( + "JSONExtractString(r.metadata, 'stripe_money_kind') = 'invoice_fallback'" + ); + expect(sql).toContain( + "r.amount - ifNull(invoice_payments.amount, 0)" + ); + expect(sql).toContain( + "OR JSONExtractString(r.metadata, 'stripe_money_kind') = 'invoice_fallback'" + ); + expect(sql).toContain("OR r.linked_website_id = {websiteId:String}"); + expect(sql).toContain( + "coalesce(r.product_name, nullIf(r.linked_product_name, ''))" + ); + expect(sql).toContain("stripe_invoice_failure_attempt_keys AS"); + expect(sql).toContain("stripe_payment_attempts AS"); + expect(sql).toContain("startsWith(r.transaction_id, 'in_')"); + expect(sql).toContain( + "JSONExtractString(r.metadata, 'databuddy_revenue_model') = ''" + ); + expect(sql).toContain( + "SELECT payment_intent_id FROM linked_payment_intents" + ); + expect(sql).toContain("stripe_record_kind') = 'attempt'"); + expect(sql).toContain("recovered_payment_attempts"); + }); + + it("keeps revenue attribution transaction-as-of without truncating exact sessions", () => { + const config = QueryBuilders.recent_transactions; + if (!config) { + throw new Error("recent_transactions builder is missing"); + } + const { sql } = compileBuilder("recent_transactions", config); + const firstTouchStart = sql.indexOf("first_touch_by_session AS"); + const firstTouchEnd = sql.indexOf("revenue_attributed AS"); + const firstTouchSql = sql.slice(firstTouchStart, firstTouchEnd); + + expect(firstTouchStart).toBeGreaterThan(-1); + expect(firstTouchEnd).toBeGreaterThan(firstTouchStart); + expect(firstTouchSql).not.toContain("INTERVAL 90 DAY"); + expect(sql).toContain("min(created) as mapped_session_created"); + expect(sql).toContain("csm.mapped_session_created <= rb.created"); + expect(sql).toContain("min(time) as first_touch_time"); + expect(sql).toContain("ft_direct.first_touch_time <= rb.created"); + expect(sql).toContain("ft_customer.first_touch_time <= rb.created"); + expect(sql).toContain("argMin(ifNull(utm_campaign, ''), time)"); + }); + + it("keeps revenue currencies separate and accepts an exact currency filter", () => { + const config = QueryBuilders.revenue_overview; + if (!config) { + throw new Error("revenue_overview builder is missing"); + } + const { params, sql } = compileBuilder("revenue_overview", config, { + filters: [{ field: "currency", op: "eq", value: "EUR" }], + }); + + expect(sql).toContain("GROUP BY currency"); + expect(sql).toContain("revenue_attributed WHERE currency = {rf0:String}"); + expect(params.rf0).toBe("EUR"); + }); + + it("keeps Stripe payment diagnostics inside the provider scope", () => { + const config = QueryBuilders.revenue_overview; + if (!config) { + throw new Error("revenue_overview builder is missing"); + } + + const paddle = compileBuilder("revenue_overview", config, { + filters: [{ field: "provider", op: "eq", value: "paddle" }], + }); + const stripe = compileBuilder("revenue_overview", config, { + filters: [{ field: "provider", op: "eq", value: "stripe" }], + }); + const mixed = compileBuilder("revenue_overview", config, { + filters: [ + { field: "provider", op: "in", value: ["paddle", "stripe"] }, + ], + }); + const withoutStripe = compileBuilder("revenue_overview", config, { + filters: [{ field: "provider", op: "ne", value: "stripe" }], + }); + const attributedSlice = compileBuilder("revenue_overview", config, { + filters: [{ field: "country", op: "eq", value: "US" }], + }); + + expect(config.allowedFilters).toEqual(["currency", "provider"]); + expect(paddle.sql).toContain("SELECT * FROM stripe_payment_attempts WHERE 0"); + expect(paddle.sql).toContain( + "SELECT * FROM stripe_failure_observations WHERE 0" + ); + expect(paddle.sql).toContain("toUInt8(0) AS payment_metrics_in_scope"); + expect(paddle.sql).toContain( + "any(payment_metrics_in_scope) as payment_diagnostics_available" + ); + expect(paddle.sql).toContain( + "countIf(attempt_status = 'failed'),\n\t\t\t\t\tNULL" + ); + expect(paddle.params.rf0).toBe("paddle"); + expect(stripe.sql).not.toContain("stripe_payment_attempts WHERE 0"); + expect(stripe.sql).toContain("toUInt8(1) AS payment_metrics_in_scope"); + expect(stripe.params.rf0).toBe("stripe"); + expect(mixed.sql).not.toContain("stripe_payment_attempts WHERE 0"); + expect(mixed.sql).toContain("toUInt8(1) AS payment_metrics_in_scope"); + expect(withoutStripe.sql).toContain( + "SELECT * FROM stripe_payment_attempts WHERE 0" + ); + expect(withoutStripe.sql).toContain( + "toUInt8(0) AS payment_metrics_in_scope" + ); + expect(attributedSlice.sql).toContain( + "SELECT * FROM stripe_payment_attempts WHERE 0" + ); + expect(attributedSlice.sql).toContain( + "toUInt8(0) AS payment_metrics_in_scope" + ); + }); + it("builds bounded error fingerprints ranked by affected people", () => { const config = QueryBuilders.error_fingerprints; if (!config) { diff --git a/packages/ai/src/types/tables.ts b/packages/ai/src/types/tables.ts index acb7ea985..f914ba81a 100644 --- a/packages/ai/src/types/tables.ts +++ b/packages/ai/src/types/tables.ts @@ -6,7 +6,6 @@ import type { OutgoingLinksRow, RevenueRow, UptimeMonitorRow, - WebVitalsHourlyRow, WebVitalsSpansRow, } from "@databuddy/db/clickhouse/tables"; @@ -14,7 +13,6 @@ export const Analytics = { events: "analytics.events", error_spans: "analytics.error_spans", web_vitals_spans: "analytics.web_vitals_spans", - web_vitals_hourly: "analytics.web_vitals_hourly", custom_events: "analytics.custom_events", blocked_traffic: "analytics.blocked_traffic", outgoing_links: "analytics.outgoing_links", @@ -32,7 +30,6 @@ export interface TableFieldsMap { "analytics.events": keyof EventsRow; "analytics.outgoing_links": keyof OutgoingLinksRow; "analytics.revenue": keyof RevenueRow; - "analytics.web_vitals_hourly": keyof WebVitalsHourlyRow; "analytics.web_vitals_spans": keyof WebVitalsSpansRow; "uptime.uptime_monitor": keyof UptimeMonitorRow; } diff --git a/packages/db/src/clickhouse/apply.ts b/packages/db/src/clickhouse/apply.ts index f73487fb3..86d9e7e8c 100644 --- a/packages/db/src/clickhouse/apply.ts +++ b/packages/db/src/clickhouse/apply.ts @@ -6,13 +6,15 @@ import { readSql, sqlFiles } from "./schema-parse"; const SCHEMA_DIR = join(dirname(fileURLToPath(import.meta.url)), "schema"); const SINGLE_NODE = process.env.CLICKHOUSE_CLUSTER == null; +const CREATE_DATABASE_PATTERN = + /CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)\./i; function databaseOf(sql: string): string { - const m = sql.match( - /CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)\./i - ); + const m = sql.match(CREATE_DATABASE_PATTERN); if (!m) { - throw new Error(`Could not determine database for statement: ${sql.slice(0, 80)}`); + throw new Error( + `Could not determine database for statement: ${sql.slice(0, 80)}` + ); } return m[1]; } @@ -33,7 +35,9 @@ export async function applyClickHouseSchema(): Promise<{ const tables = files.filter((f) => !f.endsWith("_mv.sql")); const views = files.filter((f) => f.endsWith("_mv.sql")); - const databases = [...new Set(files.map((f) => databaseOf(readSql(f))))].sort(); + const databases = [ + ...new Set(files.map((f) => databaseOf(readSql(f)))), + ].sort(); for (const db of databases) { await clickHouse.command({ query: `CREATE DATABASE IF NOT EXISTS ${db}` }); } diff --git a/packages/db/src/clickhouse/codegen.ts b/packages/db/src/clickhouse/codegen.ts index 33c1d67a2..3c27dfb99 100644 --- a/packages/db/src/clickhouse/codegen.ts +++ b/packages/db/src/clickhouse/codegen.ts @@ -13,27 +13,34 @@ import { const SCHEMA_DIR = join(dirname(fileURLToPath(import.meta.url)), "schema"); const OUT_FILE = join(SCHEMA_DIR, "tables.generated.ts"); +const ARRAY_PATTERN = /^Array\((.*)\)$/i; +const BOOLEAN_PATTERN = /^Bool/i; +const DATE_PATTERN = /^(?:Date|DateTime)/i; +const FLOAT_PATTERN = /^Float/i; +const INTEGER_PATTERN = /Int/i; +const LOW_CARDINALITY_PATTERN = /^LowCardinality\((.*)\)$/i; +const NULLABLE_PATTERN = /^Nullable\((.*)\)$/i; function tsType(rawType: string): string { let t = rawType.trim(); let nullable = false; - const lc = t.match(/^LowCardinality\((.*)\)$/i); + const lc = t.match(LOW_CARDINALITY_PATTERN); if (lc) { t = lc[1].trim(); } - const nul = t.match(/^Nullable\((.*)\)$/i); + const nul = t.match(NULLABLE_PATTERN); if (nul) { nullable = true; t = nul[1].trim(); } - const arr = t.match(/^Array\((.*)\)$/i); + const arr = t.match(ARRAY_PATTERN); if (arr) { return `${tsType(arr[1])}[]`; } let base: string; - if (/Int/i.test(t) || /^Float/i.test(t)) { + if (INTEGER_PATTERN.test(t) || FLOAT_PATTERN.test(t)) { base = "number"; - } else if (/^Bool/i.test(t)) { + } else if (BOOLEAN_PATTERN.test(t)) { base = "boolean"; } else { base = "string"; @@ -49,12 +56,12 @@ function pascal(name: string): string { } function insertTsType(rawType: string): string { - let t = rawType.replace(/^LowCardinality\((.*)\)$/i, "$1").trim(); - const nul = t.match(/^Nullable\((.*)\)$/i); + let t = rawType.replace(LOW_CARDINALITY_PATTERN, "$1").trim(); + const nul = t.match(NULLABLE_PATTERN); if (nul) { t = nul[1].trim(); } - if (/^(?:Date|DateTime)/i.test(t)) { + if (DATE_PATTERN.test(t)) { const base = "number | string"; return isNullable(rawType) ? `${base} | null` : base; } @@ -113,5 +120,8 @@ const columnsConst = `export const TABLE_COLUMNS = {\n${tables ) .join("\n")}\n} as const satisfies Record;`; -writeFileSync(OUT_FILE, `${header}\n${interfaces}\n\n${registry}\n\n${columnsConst}\n`); +writeFileSync( + OUT_FILE, + `${header}\n${interfaces}\n\n${registry}\n\n${columnsConst}\n` +); console.info(`Generated ${OUT_FILE} (${tables.length} tables)`); diff --git a/packages/db/src/clickhouse/identity.test.ts b/packages/db/src/clickhouse/identity.test.ts index b459ea8ec..3c119a77f 100644 --- a/packages/db/src/clickhouse/identity.test.ts +++ b/packages/db/src/clickhouse/identity.test.ts @@ -51,4 +51,10 @@ describe("identity sql expressions", () => { expect(expression).toContain("anonymous_id"); } }); + + test("nullable visitor keys do not invent an unidentified visitor", () => { + expect(CUSTOM_EVENTS_VISITOR_KEY).toContain("nullIf(profile_id, '')"); + expect(CUSTOM_EVENTS_VISITOR_KEY).toContain("nullIf(anonymous_id, '')"); + expect(CUSTOM_EVENTS_VISITOR_KEY).not.toContain("ifNull(anonymous_id, '')"); + }); }); diff --git a/packages/db/src/clickhouse/identity.ts b/packages/db/src/clickhouse/identity.ts index dab25b3b4..83086d5f7 100644 --- a/packages/db/src/clickhouse/identity.ts +++ b/packages/db/src/clickhouse/identity.ts @@ -2,7 +2,7 @@ export const EVENTS_VISITOR_KEY = "if(profile_id != '', profile_id, anonymous_id)"; export const CUSTOM_EVENTS_VISITOR_KEY = - "if(profile_id != '', profile_id, ifNull(anonymous_id, ''))"; + "coalesce(nullIf(profile_id, ''), nullIf(anonymous_id, ''))"; export function visitorMatch(param = "visitorId"): string { return `(anonymous_id = {${param}:String} OR profile_id = {${param}:String})`; diff --git a/packages/db/src/clickhouse/index.ts b/packages/db/src/clickhouse/index.ts index 318fad1d1..952524b69 100644 --- a/packages/db/src/clickhouse/index.ts +++ b/packages/db/src/clickhouse/index.ts @@ -1,5 +1,6 @@ export * from "./client"; export * from "./identity"; +export * from "./revenue"; export * from "./schema"; export * from "./schema/tables.generated"; export * from "./sql-validation"; diff --git a/packages/db/src/clickhouse/revenue.integration.test.ts b/packages/db/src/clickhouse/revenue.integration.test.ts new file mode 100644 index 000000000..6c2c14591 --- /dev/null +++ b/packages/db/src/clickhouse/revenue.integration.test.ts @@ -0,0 +1,265 @@ +import { randomUUIDv7 } from "bun"; +import { describe, expect, test } from "bun:test"; +import { chQuery, clickHouse } from "./client"; +import { revenueLatestCte } from "./revenue"; + +const describeIntegration = + process.env.CLICKHOUSE_INTEGRATION_TESTS === "true" ? describe : describe.skip; + +interface CollapsedRevenueRow { + amount: number | string; + created?: string; + status: string; + transaction_id: string; + type: string; +} + +interface RevenueHealthSummary { + failed_attempts: number | string; + refunds: number | string; + successful_payments: number | string; + total_revenue: number | string; +} + +function metadata( + eventId: string, + eventCreated: number, + paymentIntentId: string, + recordKind: "attempt" | "money" +): string { + return JSON.stringify({ + databuddy_revenue_model: "stripe_events_v1", + stripe_event_created: eventCreated, + stripe_event_id: eventId, + stripe_payment_intent_id: paymentIntentId, + stripe_record_kind: recordKind, + }); +} + +function revenueRow( + ownerId: string, + transactionId: string, + status: string, + syncedAt: string, + options: { + amount?: string; + created?: string; + eventCreated?: number; + eventId?: string; + paymentIntentId?: string; + recordKind?: "attempt" | "money"; + type?: string; + } = {} +) { + const created = options.created ?? "2026-08-02 12:00:00"; + return { + amount: options.amount ?? "100.0000", + created, + currency: "USD", + metadata: metadata( + options.eventId ?? `evt-${transactionId}-${status}`, + options.eventCreated ?? 1_785_672_000, + options.paymentIntentId ?? transactionId, + options.recordKind ?? "money" + ), + original_amount: options.amount ?? "100.0000", + original_currency: "USD", + owner_id: ownerId, + provider: "stripe", + status, + synced_at: syncedAt, + transaction_id: transactionId, + type: options.type ?? "sale", + website_id: ownerId, + }; +} + +describeIntegration("revenue lifecycle collapse against ClickHouse", () => { + test("keeps immutable attempts and money across out-of-order delivery", async () => { + const ownerId = `revenue-lifecycle-${randomUUIDv7()}`; + + await clickHouse.insert({ + format: "JSONEachRow", + table: "analytics.revenue", + values: [ + // A late-delivered, older attempt has its immutable Stripe event ID, + // so it cannot replace the successful PaymentIntent row. + revenueRow( + ownerId, + "pi_late_failure", + "completed", + "2026-08-02 12:01:00", + { eventCreated: 1_785_672_200, eventId: "evt_success" } + ), + revenueRow( + ownerId, + "evt_old_failure", + "failed", + "2026-08-02 12:05:00", + { + eventCreated: 1_785_672_100, + eventId: "evt_old_failure", + paymentIntentId: "pi_late_failure", + recordKind: "attempt", + type: "subscription_event", + } + ), + // A legitimate recovery preserves both the failed attempt and money. + revenueRow( + ownerId, + "evt_initial_failure", + "failed", + "2026-08-02 12:02:00", + { + eventCreated: 1_785_672_300, + eventId: "evt_initial_failure", + paymentIntentId: "pi_recovered", + recordKind: "attempt", + type: "subscription_event", + } + ), + revenueRow( + ownerId, + "pi_recovered", + "completed", + "2026-08-02 12:03:00", + { eventCreated: 1_785_672_400, eventId: "evt_recovery" } + ), + // Refunds keep their own immutable refund IDs and relation metadata. + revenueRow( + ownerId, + "pi_refunded_payment", + "completed", + "2026-08-02 12:01:00", + { eventCreated: 1_785_672_500, eventId: "evt_charge" } + ), + revenueRow( + ownerId, + "re_refund", + "refunded", + "2026-08-02 12:04:00", + { + amount: "-100.0000", + eventCreated: 1_785_672_600, + eventId: "evt_refund", + paymentIntentId: "pi_refunded_payment", + type: "refund", + } + ), + ], + }); + + const rows = await chQuery( + `WITH ${revenueLatestCte({ + scope: "owner_id = {ownerId:String}", + })} + SELECT transaction_id, type, status, amount + FROM revenue_latest + ORDER BY transaction_id`, + { ownerId } + ); + const byId = new Map(rows.map((row) => [row.transaction_id, row])); + + expect(byId.get("pi_late_failure")?.status).toBe("completed"); + expect(byId.get("evt_old_failure")?.status).toBe("failed"); + expect(byId.get("pi_recovered")?.status).toBe("completed"); + expect(byId.get("evt_initial_failure")?.status).toBe("failed"); + expect(byId.get("pi_refunded_payment")?.status).toBe("completed"); + expect(byId.get("re_refund")).toMatchObject({ + status: "refunded", + type: "refund", + }); + expect(Number(byId.get("re_refund")?.amount)).toBe(-100); + expect(rows).toHaveLength(6); + + const [summary] = await chQuery( + `WITH ${revenueLatestCte({ + scope: "owner_id = {ownerId:String}", + })} + SELECT + toFloat64(sumIf( + amount, + type != 'subscription_event' + AND ((type = 'refund' AND status = 'refunded') OR status = 'completed') + )) AS total_revenue, + countIf(type = 'subscription_event' AND status = 'failed') AS failed_attempts, + countIf(type != 'subscription_event' AND type != 'refund' AND status = 'completed') AS successful_payments, + countIf(type = 'refund' AND status = 'refunded') AS refunds + FROM revenue_latest`, + { ownerId } + ); + + expect(Number(summary?.total_revenue)).toBe(200); + expect(Number(summary?.failed_attempts)).toBe(2); + expect(Number(summary?.successful_payments)).toBe(3); + expect(Number(summary?.refunds)).toBe(1); + }); + + test("uses the winning lifecycle date for legacy retained versions", async () => { + const ownerId = `revenue-window-${randomUUIDv7()}`; + const table = `analytics.revenue_versions_${randomUUIDv7().replaceAll("-", "")}`; + const transactionId = "pi_legacy_recovery"; + + await clickHouse.command({ + query: `CREATE TABLE ${table} AS analytics.revenue ENGINE = Memory`, + }); + try { + await clickHouse.insert({ + format: "JSONEachRow", + table, + values: [ + revenueRow( + ownerId, + transactionId, + "failed", + "2026-07-01 12:01:00", + { + created: "2026-07-01 12:00:00", + eventCreated: 1_783_000_000, + eventId: "evt_legacy_failure", + recordKind: "attempt", + type: "subscription_event", + } + ), + revenueRow( + ownerId, + transactionId, + "completed", + "2026-08-02 12:01:00", + { + created: "2026-08-02 12:00:00", + eventCreated: 1_785_672_000, + eventId: "evt_legacy_recovery", + } + ), + ], + }); + + const rowsInWindow = (startDate: string, endDate: string) => + chQuery( + `WITH ${revenueLatestCte({ + candidateWhere: `created >= toDateTime({startDate:String}) + AND created <= toDateTime(concat({endDate:String}, ' 23:59:59'))`, + scope: "owner_id = {ownerId:String}", + source: table, + })} + SELECT transaction_id, status, created + FROM revenue_latest + WHERE created >= toDateTime({startDate:String}) + AND created <= toDateTime(concat({endDate:String}, ' 23:59:59'))`, + { endDate, ownerId, startDate } + ); + + expect(await rowsInWindow("2026-07-01", "2026-07-02")).toEqual([]); + expect(await rowsInWindow("2026-08-01", "2026-08-03")).toEqual([ + expect.objectContaining({ + created: "2026-08-02 12:00:00", + status: "completed", + transaction_id: transactionId, + }), + ]); + } finally { + await clickHouse.command({ query: `DROP TABLE IF EXISTS ${table}` }); + } + }); +}); diff --git a/packages/db/src/clickhouse/revenue.test.ts b/packages/db/src/clickhouse/revenue.test.ts new file mode 100644 index 000000000..54e4cb773 --- /dev/null +++ b/packages/db/src/clickhouse/revenue.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, test } from "bun:test"; +import { revenueLatestCte } from "./revenue"; + +describe("revenueLatestCte", () => { + test("selects one deterministic lifecycle state per provider transaction", () => { + const sql = revenueLatestCte({ + name: "scoped_revenue", + scope: "owner_id = {ownerId:String}", + }); + + expect(sql).toContain( + "GROUP BY owner_id, provider, transaction_id" + ); + expect(sql).toContain( + "tuple(_revenue_state_rank, _revenue_event_unix, _revenue_source_unix, _revenue_identity_richness, _revenue_tiebreaker)" + ); + expect(sql).toContain( + "toUInt64(toUnixTimestamp(synced_at)) AS _revenue_source_unix" + ); + expect(sql).toContain("status = 'completed', 4"); + expect(sql).toContain("status = 'refunded', 5"); + expect(sql).toContain("status = 'canceled', 3"); + expect(sql).toContain("status = 'failed', 2"); + expect(sql).toContain( + "JSONExtractUInt(metadata, 'stripe_event_created')" + ); + expect(sql).toContain(") AS _revenue_event_unix"); + expect(sql).toContain("cityHash64(toString(tuple("); + expect(sql).toContain("toUInt8(profile_id != '')"); + expect(sql).toContain(") AS _revenue_tiebreaker"); + expect(sql).not.toContain("% 33554432"); + expect(sql).toContain("WHERE owner_id = {ownerId:String}"); + expect(sql).not.toContain(" FINAL"); + }); + + test("carries non-empty identity without changing the latest lifecycle row", () => { + const sql = revenueLatestCte({ scope: "owner_id = 'org_1'" }); + + expect(sql).toContain("tuple(profile_id != '', _revenue_state_rank"); + expect(sql).toContain( + "tuple(ifNull(anonymous_id, '') != '', _revenue_state_rank" + ); + expect(sql).toContain("tuple(customer_id != '', _revenue_state_rank"); + expect(sql).toContain("tuple(ifNull(product_name, '') != ''"); + expect(sql).toContain("tuple(lengthUTF8(metadata), _revenue_state_rank"); + expect(sql).toContain("latest_customer_id AS customer_id"); + expect(sql).toContain("latest_metadata AS metadata"); + expect(sql).toContain("latest.11 AS created"); + expect(sql).toContain("latest.12 AS synced_at"); + expect(sql).toContain("latest.2 AS status"); + expect(sql).not.toContain("min(created)"); + expect(sql).not.toContain("canonical_created"); + }); + + test("uses indexed range candidates without excluding another version of a key", () => { + const sql = revenueLatestCte({ + candidateWhere: "created >= {from:DateTime}", + scope: "owner_id = {ownerId:String}", + }); + + expect(sql).toContain( + "(owner_id, provider, transaction_id) IN (" + ); + expect(sql).toContain("AND created >= {from:DateTime}"); + expect(sql.match(/FROM analytics\.revenue/g)).toHaveLength(2); + expect(sql.match(/owner_id = \{ownerId:String\}/g)).toHaveLength(2); + }); + + test("can exercise retained versions from a trusted test source", () => { + const sql = revenueLatestCte({ + scope: "owner_id = {ownerId:String}", + source: "analytics.revenue_retained_versions_test", + }); + + expect(sql).toContain("FROM analytics.revenue_retained_versions_test"); + expect(sql).not.toContain("FROM analytics.revenue\n"); + }); +}); diff --git a/packages/db/src/clickhouse/revenue.ts b/packages/db/src/clickhouse/revenue.ts new file mode 100644 index 000000000..a84dfccc2 --- /dev/null +++ b/packages/db/src/clickhouse/revenue.ts @@ -0,0 +1,169 @@ +const REVENUE_STATE_RANK = `multiIf( + status = 'refunded', 5, + status = 'completed', 4, + status = 'canceled', 3, + status = 'failed', 2, + 1 +)`; + +const REVENUE_EVENT_UNIX = `greatest( + toUInt64(toUnixTimestamp(created)), + if( + isValidJSON(metadata), + JSONExtractUInt(metadata, 'stripe_event_created'), + toUInt64(0) + ) +)`; + +interface RevenueLatestCteOptions { + candidateWhere?: string; + name?: string; + scope: string; + source?: string; +} + +/** + * Collapses immutable MergeTree versions to one deterministic row per + * provider transaction. Terminal state precedence is intentional: local + * delivery order cannot regress a completed payment to failed/canceled, while + * a refund still supersedes completion. Immutable provider event time then + * orders versions within the same state, with synced_at used only as a + * deterministic delivery-order fallback. + * + * This is a legacy defense for versions that still exist. ReplacingMergeTree + * may already have discarded older same-key rows; current ingestion avoids + * that loss by giving attempts, payments, and refunds immutable provider IDs. + * + * `scope` must identify the tenant (and may narrow to a transaction); apply + * lifecycle fields such as `created` after this CTE so an older pending version + * cannot leak back into a date range. `candidateWhere` can cheaply identify + * relevant keys (for example with the created minmax index); every version of + * those keys is still considered before filtering. `source` is an internal + * test override and must always be a trusted table expression. + */ +export function revenueLatestCte({ + candidateWhere, + name = "revenue_latest", + scope, + source = "analytics.revenue", +}: RevenueLatestCteOptions): string { + const candidateKeys = candidateWhere + ? ` + AND (owner_id, provider, transaction_id) IN ( + SELECT owner_id, provider, transaction_id + FROM ${source} + WHERE ${scope} + AND ${candidateWhere} + GROUP BY owner_id, provider, transaction_id + )` + : ""; + return `${name} AS ( + SELECT + owner_id, + nullIf(latest_website_id, '') AS website_id, + transaction_id, + provider, + latest.1 AS type, + latest.2 AS status, + latest.3 AS amount, + latest.4 AS original_amount, + latest.5 AS original_currency, + latest.6 AS currency, + nullIf(latest_anonymous_id, '') AS anonymous_id, + nullIf(latest_session_id, '') AS session_id, + latest_customer_id AS customer_id, + nullIf(latest_product_id, '') AS product_id, + nullIf(latest_product_name, '') AS product_name, + latest_metadata AS metadata, + latest.11 AS created, + latest.12 AS synced_at, + latest_profile_id AS profile_id + FROM ( + SELECT + owner_id, + provider, + transaction_id, + argMax( + tuple( + type, + status, + amount, + original_amount, + original_currency, + currency, + customer_id, + product_id, + product_name, + metadata, + created, + synced_at + ), + tuple(_revenue_state_rank, _revenue_event_unix, _revenue_source_unix, _revenue_identity_richness, _revenue_tiebreaker) + ) AS latest, + argMax( + ifNull(website_id, ''), + tuple(ifNull(website_id, '') != '', _revenue_state_rank, _revenue_event_unix, _revenue_source_unix, _revenue_identity_richness, _revenue_tiebreaker) + ) AS latest_website_id, + argMax( + ifNull(anonymous_id, ''), + tuple(ifNull(anonymous_id, '') != '', _revenue_state_rank, _revenue_event_unix, _revenue_source_unix, _revenue_identity_richness, _revenue_tiebreaker) + ) AS latest_anonymous_id, + argMax( + ifNull(session_id, ''), + tuple(ifNull(session_id, '') != '', _revenue_state_rank, _revenue_event_unix, _revenue_source_unix, _revenue_identity_richness, _revenue_tiebreaker) + ) AS latest_session_id, + argMax( + customer_id, + tuple(customer_id != '', _revenue_state_rank, _revenue_event_unix, _revenue_source_unix, _revenue_identity_richness, _revenue_tiebreaker) + ) AS latest_customer_id, + argMax( + profile_id, + tuple(profile_id != '', _revenue_state_rank, _revenue_event_unix, _revenue_source_unix, _revenue_identity_richness, _revenue_tiebreaker) + ) AS latest_profile_id, + argMax( + ifNull(product_id, ''), + tuple(ifNull(product_id, '') != '', _revenue_state_rank, _revenue_event_unix, _revenue_source_unix, _revenue_identity_richness, _revenue_tiebreaker) + ) AS latest_product_id, + argMax( + ifNull(product_name, ''), + tuple(ifNull(product_name, '') != '', _revenue_state_rank, _revenue_event_unix, _revenue_source_unix, _revenue_identity_richness, _revenue_tiebreaker) + ) AS latest_product_name, + argMax( + metadata, + tuple(lengthUTF8(metadata), _revenue_state_rank, _revenue_event_unix, _revenue_source_unix, _revenue_identity_richness, _revenue_tiebreaker) + ) AS latest_metadata + FROM ( + SELECT + *, + toUInt64(toUnixTimestamp(synced_at)) AS _revenue_source_unix, + ${REVENUE_STATE_RANK} AS _revenue_state_rank, + ${REVENUE_EVENT_UNIX} AS _revenue_event_unix, + toUInt8(ifNull(website_id, '') != '') + + toUInt8(ifNull(anonymous_id, '') != '') + + toUInt8(ifNull(session_id, '') != '') + + toUInt8(customer_id != '') + + toUInt8(profile_id != '') AS _revenue_identity_richness, + cityHash64(toString(tuple( + website_id, + type, + status, + amount, + original_amount, + original_currency, + currency, + anonymous_id, + session_id, + customer_id, + product_id, + product_name, + metadata, + created, + profile_id + ))) AS _revenue_tiebreaker + FROM ${source} + WHERE ${scope}${candidateKeys} + ) + GROUP BY owner_id, provider, transaction_id + ) +)`; +} diff --git a/packages/db/src/clickhouse/schema-parse.test.ts b/packages/db/src/clickhouse/schema-parse.test.ts new file mode 100644 index 000000000..c548c4e0f --- /dev/null +++ b/packages/db/src/clickhouse/schema-parse.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "bun:test"; +import { parseTable } from "./schema-parse"; + +describe("parseTable indexes", () => { + it("captures normalized secondary index definitions", () => { + const table = parseTable(` + CREATE TABLE IF NOT EXISTS analytics.example + ( + \`client_id\` String, + \`timestamp\` DateTime64(3, 'UTC'), + INDEX \`idx_client_id\` + client_id TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_timestamp timestamp TYPE minmax GRANULARITY 2 + ) + ENGINE = MergeTree + ORDER BY (client_id, timestamp) + `); + + expect(table.indexes).toEqual([ + { + name: "idx_client_id", + definition: "client_id TYPE bloom_filter(0.01) GRANULARITY 1", + }, + { + name: "idx_timestamp", + definition: "timestamp TYPE minmax GRANULARITY 2", + }, + ]); + }); + + it("does not treat indexes as columns", () => { + const table = parseTable(` + CREATE TABLE analytics.example + ( + client_id String, + INDEX idx_client_id client_id TYPE set(100) GRANULARITY 1 + ) + ENGINE = MergeTree + ORDER BY client_id + `); + + expect(table.columns.map((column) => column.name)).toEqual(["client_id"]); + expect(table.indexes).toHaveLength(1); + }); +}); diff --git a/packages/db/src/clickhouse/schema-parse.ts b/packages/db/src/clickhouse/schema-parse.ts index 3d84103b6..56fa012bb 100644 --- a/packages/db/src/clickhouse/schema-parse.ts +++ b/packages/db/src/clickhouse/schema-parse.ts @@ -1,26 +1,44 @@ import { readdirSync, readFileSync, statSync } from "node:fs"; import { join } from "node:path"; +const COLUMN_DEFAULT_PATTERN = /\b(?:DEFAULT|EPHEMERAL)\b/i; +const COLUMN_NAME_PATTERN = /^`?(\w+)`?\s+/; +const COMPUTED_COLUMN_PATTERN = /\b(?:MATERIALIZED|ALIAS)\b/i; +const INDEX_NAME_PATTERN = /^INDEX\s+`?(\w+)`?\s+/i; +const LOW_CARDINALITY_PATTERN = /^LowCardinality\((.*)\)$/i; +const MATERIALIZED_VIEW_PATTERN = /MATERIALIZED\s+VIEW/i; +const NULLABLE_PATTERN = /^Nullable\(/i; +const QUALIFIED_NAME_PATTERN = + /CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:`([^`]+)`|(\w+))\.(?:`([^`]+)`|(\w+))/i; +const TABLE_NAME_PATTERN = + /CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?[\w.]*\.(\w+)/i; + export interface ParsedColumn { + computed: boolean; + hasDefault: boolean; name: string; - type: string; nullable: boolean; - hasDefault: boolean; - computed: boolean; + type: string; +} + +export interface ParsedIndex { + definition: string; + name: string; } export function isNullable(type: string): boolean { - const inner = type.replace(/^LowCardinality\((.*)\)$/i, "$1").trim(); - return /^Nullable\(/i.test(inner); + const inner = type.replace(LOW_CARDINALITY_PATTERN, "$1").trim(); + return NULLABLE_PATTERN.test(inner); } export interface ParsedTable { - name: string; - isView: boolean; columns: ParsedColumn[]; engine: string; - partitionBy: string; + indexes: ParsedIndex[]; + isView: boolean; + name: string; orderBy: string; + partitionBy: string; settings: string; } @@ -85,9 +103,7 @@ const COLUMN_MODIFIERS = const NON_COLUMN = /^(?:INDEX|CONSTRAINT|PROJECTION|PRIMARY\s+KEY)\b/i; export function tableNameOf(sql: string): string { - const m = sql.match( - /CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?[\w.]*\.(\w+)/i - ); + const m = sql.match(TABLE_NAME_PATTERN); if (!m) { throw new Error("Could not parse table name"); } @@ -95,9 +111,7 @@ export function tableNameOf(sql: string): string { } export function qualifiedNameOf(sql: string): string { - const m = sql.match( - /CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:`([^`]+)`|(\w+))\.(?:`([^`]+)`|(\w+))/i - ); + const m = sql.match(QUALIFIED_NAME_PATTERN); if (!m) { throw new Error("Could not parse qualified table name"); } @@ -112,7 +126,7 @@ export function parseColumns(sql: string): ParsedColumn[] { if (NON_COLUMN.test(item)) { continue; } - const nameMatch = item.match(/^`?(\w+)`?\s+/); + const nameMatch = item.match(COLUMN_NAME_PATTERN); if (!nameMatch) { continue; } @@ -123,17 +137,34 @@ export function parseColumns(sql: string): ParsedColumn[] { name: nameMatch[1], type, nullable: isNullable(type), - hasDefault: /\b(?:DEFAULT|EPHEMERAL)\b/i.test(afterName), - computed: /\b(?:MATERIALIZED|ALIAS)\b/i.test(afterName), + hasDefault: COLUMN_DEFAULT_PATTERN.test(afterName), + computed: COMPUTED_COLUMN_PATTERN.test(afterName), }); } return cols; } +function normalizeDefinition(definition: string): string { + return definition.replaceAll("`", "").replace(/\s+/g, " ").trim(); +} + +export function parseIndexes(sql: string): ParsedIndex[] { + const indexes: ParsedIndex[] = []; + for (const item of splitTopLevel(firstParenGroup(sql).body)) { + const match = item.match(INDEX_NAME_PATTERN); + if (!match) { + continue; + } + indexes.push({ + name: match[1], + definition: normalizeDefinition(item.slice(match[0].length)), + }); + } + return indexes; +} + function clause(tail: string, keyword: string, stops: string[]): string { - const lookahead = stops.length - ? `(?=(?:${stops.join("|")})\\b|$)` - : "(?=$)"; + const lookahead = stops.length ? `(?=(?:${stops.join("|")})\\b|$)` : "(?=$)"; const re = new RegExp(`${keyword}\\s+([\\s\\S]*?)\\s*${lookahead}`, "i"); const m = tail.match(re); return m ? m[1].trim() : ""; @@ -141,13 +172,18 @@ function clause(tail: string, keyword: string, stops: string[]): string { export function parseTable(sql: string): ParsedTable { const name = tableNameOf(sql); - const isView = /MATERIALIZED\s+VIEW/i.test(sql); + const isView = MATERIALIZED_VIEW_PATTERN.test(sql); const columns = parseColumns(sql); - const tail = sql.slice(firstParenGroup(sql).end + 1).replace(/\s+/g, " ").trim(); + const indexes = parseIndexes(sql); + const tail = sql + .slice(firstParenGroup(sql).end + 1) + .replace(/\s+/g, " ") + .trim(); return { name, isView, columns, + indexes, engine: clause(tail, "ENGINE\\s*=", [ "PARTITION BY", "PRIMARY KEY", diff --git a/packages/db/src/clickhouse/schema/README.md b/packages/db/src/clickhouse/schema/README.md index b4edf1d6a..b32ade725 100644 --- a/packages/db/src/clickhouse/schema/README.md +++ b/packages/db/src/clickhouse/schema/README.md @@ -1,9 +1,9 @@ # ClickHouse schema -The `.sql` files in this directory are the source of truth for our ClickHouse -schema. They mirror what is deployed in production (`SHOW CREATE TABLE`), so a -file here should always be byte-comparable to the live table definition. One -file per object, named exactly after the table/view it defines. +The `.sql` files in this directory are reference definitions for the deployed +ClickHouse schema. `ch:verify` compares their columns, secondary indexes, +engine (including the Keeper path), partition and sorting keys, and settings +with the live cluster. One file exists per object and is named after it. ## Layout @@ -14,7 +14,7 @@ schema/ analytics/ core/ events, custom_events errors/ error_spans - web-vitals/ web_vitals_spans, web_vitals_hourly, web_vitals_hourly_mv + web-vitals/ web_vitals_spans pageviews/ daily_pageviews, daily_pageviews_mv traffic/ blocked_traffic, ai_traffic_spans links/ outgoing_links, link_visits @@ -25,12 +25,37 @@ schema/ Materialized views keep the `_mv` suffix and live in the same domain as the table they feed. Every statement is `CREATE ... IF NOT EXISTS`, so applying the -whole tree is idempotent and safe to re-run. +whole tree is idempotent. It only bootstraps missing objects; it does not update +an existing object's indexes, sorting key, engine, or Keeper path. All tables use the `Replicated*MergeTree` engine family with `{shard}` / `{replica}` macros, matching the production cluster. Single-node deployments need ClickHouse Keeper enabled (a one-node Keeper is fine). +## Unmanaged legacy objects + +`analytics.web_vitals_hourly` and `analytics.web_vitals_hourly_mv` may remain +on existing clusters, but fresh installs do not create them. The materialized +view finalized percentiles and averages per insert block, then stored those +values in a `SummingMergeTree`; later merges therefore produced invalid +statistics. Nothing reads this aggregate, and `analytics.web_vitals_spans` +remains the source of truth. + +The verifier exempts exactly these two names while they await a separately +planned physical cleanup, and prints a warning for each one it finds. This is +not a wildcard for materialized views: every other live object still needs a +matching SQL definition. + +After explicit production approval, clean them up in this order so writes stop +before stored rows are removed: + +```sql +DROP TABLE IF EXISTS analytics.web_vitals_hourly_mv; +DROP TABLE IF EXISTS analytics.web_vitals_hourly; +``` + +Do not reverse the order: the materialized view targets the table. + ## Applying Apply base tables first, then materialized views (an MV requires both the table @@ -48,9 +73,11 @@ find analytics -name '*_mv.sql' | sort \ ## Changing an existing table -Never edit a shipped `.sql` in a way that assumes a fresh table — deployed -tables already exist. Instead: +`clickhouse:init` is a bootstrap command, not a migration runner. Never edit a +shipped `.sql` as if the table were new: `CREATE ... IF NOT EXISTS` will be a +no-op on deployed tables, while `ch:verify` will correctly report drift. -1. Add a forward-only, dated `ALTER ... IF NOT EXISTS` to `changes/`. -2. Apply it to every environment. -3. Update the table's `.sql` here so it still mirrors the live definition. +Prepare and review a forward-only migration separately, apply it to each +environment, and then update the reference SQL. Changes that reorder a sorting +key or move a replicated table to another Keeper path require an explicit +shadow-table or replica migration; they are not safe in-place DDL edits. diff --git a/packages/db/src/clickhouse/schema/analytics/web-vitals/web_vitals_hourly.sql b/packages/db/src/clickhouse/schema/analytics/web-vitals/web_vitals_hourly.sql deleted file mode 100644 index 705e0fb99..000000000 --- a/packages/db/src/clickhouse/schema/analytics/web-vitals/web_vitals_hourly.sql +++ /dev/null @@ -1,17 +0,0 @@ -CREATE TABLE IF NOT EXISTS analytics.web_vitals_hourly -( - `client_id` String CODEC(ZSTD(1)), - `path` String CODEC(ZSTD(1)), - `metric_name` LowCardinality(String) CODEC(ZSTD(1)), - `hour` DateTime CODEC(Delta(4), ZSTD(1)), - `sample_count` UInt64 CODEC(ZSTD(1)), - `p75` Float64 CODEC(ZSTD(1)), - `p50` Float64 CODEC(ZSTD(1)), - `avg_value` Float64 CODEC(ZSTD(1)), - `min_value` Float64 CODEC(ZSTD(1)), - `max_value` Float64 CODEC(ZSTD(1)) -) -ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{shard}/analytics_web_vitals_hourly', '{replica}') -PARTITION BY toYYYYMM(hour) -ORDER BY (client_id, metric_name, path, hour) -SETTINGS index_granularity = 8192 diff --git a/packages/db/src/clickhouse/schema/analytics/web-vitals/web_vitals_hourly_mv.sql b/packages/db/src/clickhouse/schema/analytics/web-vitals/web_vitals_hourly_mv.sql deleted file mode 100644 index af204800d..000000000 --- a/packages/db/src/clickhouse/schema/analytics/web-vitals/web_vitals_hourly_mv.sql +++ /dev/null @@ -1,16 +0,0 @@ -CREATE MATERIALIZED VIEW IF NOT EXISTS analytics.web_vitals_hourly_mv TO analytics.web_vitals_hourly -( - `client_id` String, - `path` String, - `metric_name` LowCardinality(String), - `hour` DateTime('UTC'), - `sample_count` UInt64, - `p75` Float64, - `p50` Float64, - `avg_value` Float64, - `min_value` Float64, - `max_value` Float64 -) -AS SELECT client_id, path, metric_name, toStartOfHour(timestamp) AS hour, count() AS sample_count, quantile(0.75)(metric_value) AS p75, quantile(0.5)(metric_value) AS p50, avg(metric_value) AS avg_value, min(metric_value) AS min_value, max(metric_value) AS max_value -FROM analytics.web_vitals_spans -GROUP BY client_id, path, metric_name, hour diff --git a/packages/db/src/clickhouse/schema/tables.generated.ts b/packages/db/src/clickhouse/schema/tables.generated.ts index 1fae9d702..6a7317bdc 100644 --- a/packages/db/src/clickhouse/schema/tables.generated.ts +++ b/packages/db/src/clickhouse/schema/tables.generated.ts @@ -333,32 +333,6 @@ export interface RevenueInsert { profile_id?: string; } -export interface WebVitalsHourlyRow { - client_id: string; - path: string; - metric_name: string; - hour: string; - sample_count: number; - p75: number; - p50: number; - avg_value: number; - min_value: number; - max_value: number; -} - -export interface WebVitalsHourlyInsert { - client_id: string; - path: string; - metric_name: string; - hour: number | string; - sample_count: number; - p75: number; - p50: number; - avg_value: number; - min_value: number; - max_value: number; -} - export interface WebVitalsSpansRow { client_id: string; anonymous_id: string; @@ -439,7 +413,6 @@ export interface ClickHouseTables { link_visits: LinkVisitsRow; outgoing_links: OutgoingLinksRow; revenue: RevenueRow; - web_vitals_hourly: WebVitalsHourlyRow; web_vitals_spans: WebVitalsSpansRow; uptime_monitor: UptimeMonitorRow; } @@ -454,7 +427,6 @@ export const TABLE_COLUMNS = { "analytics.link_visits": ["id", "link_id", "timestamp", "referrer", "user_agent", "ip_hash", "country", "region", "city", "browser_name", "device_type"], "analytics.outgoing_links": ["id", "client_id", "anonymous_id", "session_id", "href", "text", "properties", "timestamp"], "analytics.revenue": ["owner_id", "website_id", "transaction_id", "provider", "type", "status", "amount", "original_amount", "original_currency", "currency", "anonymous_id", "session_id", "customer_id", "product_id", "product_name", "metadata", "created", "synced_at", "profile_id"], - "analytics.web_vitals_hourly": ["client_id", "path", "metric_name", "hour", "sample_count", "p75", "p50", "avg_value", "min_value", "max_value"], "analytics.web_vitals_spans": ["client_id", "anonymous_id", "session_id", "timestamp", "path", "metric_name", "metric_value"], "uptime.uptime_monitor": ["site_id", "url", "timestamp", "status", "http_code", "ttfb_ms", "total_ms", "attempt", "retries", "failure_streak", "response_bytes", "content_hash", "redirect_count", "probe_region", "probe_ip", "ssl_expiry", "ssl_valid", "env", "check_type", "user_agent", "error", "json_data"], } as const satisfies Record; diff --git a/packages/db/src/clickhouse/verify-policy.test.ts b/packages/db/src/clickhouse/verify-policy.test.ts new file mode 100644 index 000000000..aaf4b8a7a --- /dev/null +++ b/packages/db/src/clickhouse/verify-policy.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, test } from "bun:test"; +import { + isUnmanagedClickHouseObject, + UNMANAGED_CLICKHOUSE_OBJECTS, +} from "./verify-policy"; + +describe("ClickHouse verification policy", () => { + test("acknowledges only the retired web-vitals aggregate objects", () => { + expect([...UNMANAGED_CLICKHOUSE_OBJECTS].sort()).toEqual([ + "analytics.web_vitals_hourly", + "analytics.web_vitals_hourly_mv", + ]); + }); + + test("does not generically exempt untracked materialized views", () => { + expect(isUnmanagedClickHouseObject("analytics.web_vitals_hourly_mv")).toBe( + true + ); + expect(isUnmanagedClickHouseObject("analytics.unknown_mv")).toBe(false); + }); +}); diff --git a/packages/db/src/clickhouse/verify-policy.ts b/packages/db/src/clickhouse/verify-policy.ts new file mode 100644 index 000000000..6fee5a08d --- /dev/null +++ b/packages/db/src/clickhouse/verify-policy.ts @@ -0,0 +1,8 @@ +export const UNMANAGED_CLICKHOUSE_OBJECTS: ReadonlySet = new Set([ + "analytics.web_vitals_hourly", + "analytics.web_vitals_hourly_mv", +]); + +export function isUnmanagedClickHouseObject(name: string): boolean { + return UNMANAGED_CLICKHOUSE_OBJECTS.has(name); +} diff --git a/packages/db/src/clickhouse/verify.ts b/packages/db/src/clickhouse/verify.ts index 33243a0cb..320071b05 100644 --- a/packages/db/src/clickhouse/verify.ts +++ b/packages/db/src/clickhouse/verify.ts @@ -1,9 +1,17 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { type ParsedTable, parseTable, readSql, sqlFiles } from "./schema-parse"; +import { + type ParsedTable, + parseTable, + readSql, + sqlFiles, +} from "./schema-parse"; +import { isUnmanagedClickHouseObject } from "./verify-policy"; const SCHEMA_DIR = join(dirname(fileURLToPath(import.meta.url)), "schema"); const DATABASES = ["analytics", "uptime"]; +const CREATE_DATABASE_PATTERN = + /CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)\./i; const NO_COLOR = Boolean(process.env.NO_COLOR); const c = { @@ -15,9 +23,7 @@ const c = { }; function dbNameOf(sql: string): string { - const m = sql.match( - /CREATE\s+(?:TABLE|MATERIALIZED\s+VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)\./i - ); + const m = sql.match(CREATE_DATABASE_PATTERN); return m ? m[1] : "analytics"; } @@ -38,12 +44,17 @@ async function fetchLive(): Promise> { url.searchParams.set("query", query); const res = await fetch(url, { headers }); if (!res.ok) { - throw new Error(`ClickHouse query failed: ${res.status} ${await res.text()}`); + throw new Error( + `ClickHouse query failed: ${res.status} ${await res.text()}` + ); } const text = await res.text(); const live = new Map(); for (const line of text.trim().split("\n").filter(Boolean)) { - const row = JSON.parse(line) as { database: string; create_table_query: string }; + const row = JSON.parse(line) as { + database: string; + create_table_query: string; + }; const parsed = parseTable(row.create_table_query); live.set(`${row.database}.${parsed.name}`, parsed); } @@ -57,7 +68,10 @@ function diffTable(repo: ParsedTable, live: ParsedTable): string[] { for (const [name, type] of repoCols) { if (!liveCols.has(name)) { - lines.push(c.green(` + ${name.padEnd(24)} ${type}`) + c.dim(" (in repo, missing on cluster)")); + lines.push( + c.green(` + ${name.padEnd(24)} ${type}`) + + c.dim(" (in repo, missing on cluster)") + ); } else if (liveCols.get(name) !== type) { lines.push( c.yellow(` ~ ${name.padEnd(24)} `) + @@ -69,17 +83,53 @@ function diffTable(repo: ParsedTable, live: ParsedTable): string[] { } for (const [name, type] of liveCols) { if (!repoCols.has(name)) { - lines.push(c.red(` - ${name.padEnd(24)} ${type}`) + c.dim(" (on cluster, missing in repo)")); + lines.push( + c.red(` - ${name.padEnd(24)} ${type}`) + + c.dim(" (on cluster, missing in repo)") + ); } } const repoNames = repo.columns.map((col) => col.name).join(","); const liveNames = live.columns.map((col) => col.name).join(","); - if (repoNames !== liveNames && repoCols.size === liveCols.size && - [...repoCols.keys()].every((k) => liveCols.has(k))) { + if ( + repoNames !== liveNames && + repoCols.size === liveCols.size && + [...repoCols.keys()].every((k) => liveCols.has(k)) + ) { lines.push(c.yellow(" ~ column order differs")); } + const repoIndexes = new Map( + repo.indexes.map((index) => [index.name, index.definition]) + ); + const liveIndexes = new Map( + live.indexes.map((index) => [index.name, index.definition]) + ); + for (const [name, definition] of repoIndexes) { + const liveDefinition = liveIndexes.get(name); + if (liveDefinition === undefined) { + lines.push( + c.green(` + INDEX ${name} ${definition}`) + + c.dim(" (in repo, missing on cluster)") + ); + } else if (liveDefinition !== definition) { + lines.push( + c.yellow(` ~ INDEX ${name}`) + + `\n ${c.red(`cluster: ${liveDefinition}`)}` + + `\n ${c.green(`repo: ${definition}`)}` + ); + } + } + for (const [name, definition] of liveIndexes) { + if (!repoIndexes.has(name)) { + lines.push( + c.red(` - INDEX ${name} ${definition}`) + + c.dim(" (on cluster, missing in repo)") + ); + } + } + for (const key of ["engine", "partitionBy", "orderBy", "settings"] as const) { if (repo[key] !== live[key]) { lines.push( @@ -96,6 +146,7 @@ const live = await fetchLive(); const repoFiles = sqlFiles(SCHEMA_DIR); const seenLive = new Set(); let drifted = 0; +let unmanaged = 0; for (const file of repoFiles) { const sql = readSql(file); @@ -104,7 +155,9 @@ for (const file of repoFiles) { const repo = parseTable(sql); const liveTable = live.get(key); if (!liveTable) { - console.info(`${c.red("✗")} ${c.bold(key)} ${c.red("— in repo, does not exist on cluster")}`); + console.info( + `${c.red("✗")} ${c.bold(key)} ${c.red("— in repo, does not exist on cluster")}` + ); drifted++; continue; } @@ -121,15 +174,29 @@ for (const file of repoFiles) { } for (const key of live.keys()) { - if (!seenLive.has(key) && !key.endsWith("_mv")) { - console.info(`${c.red("✗")} ${c.bold(key)} ${c.red("— on cluster, no .sql in repo")}`); - drifted++; + if (seenLive.has(key)) { + continue; + } + if (isUnmanagedClickHouseObject(key)) { + console.info( + `${c.yellow("⚠")} ${c.bold(key)} ${c.yellow("— unmanaged legacy object remains on cluster")}` + ); + unmanaged++; + continue; } + console.info( + `${c.red("✗")} ${c.bold(key)} ${c.red("— on cluster, no .sql in repo")}` + ); + drifted++; } console.info(""); if (drifted === 0) { - console.info(c.green(`✓ schema in sync — ${repoFiles.length} objects match the cluster`)); + console.info( + c.green( + `✓ schema in sync — ${repoFiles.length} managed objects match the cluster; ${unmanaged} unmanaged legacy object(s) acknowledged` + ) + ); process.exit(0); } console.info(c.red(`✗ ${drifted} object(s) drifted from the cluster`)); diff --git a/packages/db/src/drizzle/schema/analytics.test.ts b/packages/db/src/drizzle/schema/analytics.test.ts index 4cec03033..a5c797a7f 100644 --- a/packages/db/src/drizzle/schema/analytics.test.ts +++ b/packages/db/src/drizzle/schema/analytics.test.ts @@ -12,6 +12,15 @@ describe("analytics insights schema", () => { expect(column?.notNull).toBe(false); }); + test("preserves historical investigation provenance", () => { + const column = getTableConfig(analyticsInsights).columns.find( + (candidate) => candidate.name === "investigation_depth" + ); + + expect(column).toBeDefined(); + expect(column?.notNull).toBe(false); + }); + test("indexes the resolved history sort by organization and website", () => { const indexNames = getTableConfig(analyticsInsights).indexes.map( (index) => index.config.name diff --git a/packages/db/src/drizzle/schema/analytics.ts b/packages/db/src/drizzle/schema/analytics.ts index f6b0c10a2..b67d02e8d 100644 --- a/packages/db/src/drizzle/schema/analytics.ts +++ b/packages/db/src/drizzle/schema/analytics.ts @@ -224,6 +224,10 @@ export const analyticsInsights = pgTable( rootCause: text("root_cause"), remediationKind: text("remediation_kind").$type(), evidence: jsonb("evidence").$type(), + // Historical provenance; new investigations express depth through evidence. + investigationDepth: text("investigation_depth").$type< + "surface" | "investigated" | "deep" + >(), actions: jsonb().$type(), chainId: text("chain_id"), timezone: text().notNull().default("UTC"), diff --git a/packages/db/src/drizzle/schema/billing.test.ts b/packages/db/src/drizzle/schema/billing.test.ts new file mode 100644 index 000000000..458bba9ff --- /dev/null +++ b/packages/db/src/drizzle/schema/billing.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, test } from "bun:test"; +import { getTableConfig } from "drizzle-orm/pg-core"; +import { + autumnWebhookEvents, + autumnWebhookStatusValues, + usageAlertLog, +} from "./billing"; + +describe("Autumn webhook inbox schema", () => { + test("stores leases, retry scheduling, and retained dead letters", () => { + const config = getTableConfig(autumnWebhookEvents); + expect(config.columns.map((column) => column.name)).toEqual( + expect.arrayContaining([ + "lease_token", + "lease_expires_at", + "next_attempt_at", + "dead_lettered_at", + "alerted_at", + ]) + ); + expect(autumnWebhookStatusValues).toEqual([ + "pending", + "processing", + "deferred", + "completed", + "dead_letter", + ]); + expect(config.indexes.map((index) => index.config.name)).toEqual( + expect.arrayContaining([ + "autumn_webhook_events_status_next_attempt_idx", + "autumn_webhook_events_status_lease_idx", + "autumn_webhook_events_status_dead_letter_idx", + ]) + ); + }); +}); + +describe("usage alert log schema", () => { + test("scopes cooldown history to an organization", () => { + const config = getTableConfig(usageAlertLog); + expect(config.columns.map((column) => column.name)).toContain( + "organization_id" + ); + expect(config.indexes.map((index) => index.config.name)).toContain( + "usage_alert_log_org_user_feature_idx" + ); + expect(config.foreignKeys.map((key) => key.getName())).toContain( + "usage_alert_log_organization_id_fkey" + ); + }); +}); diff --git a/packages/db/src/drizzle/schema/billing.ts b/packages/db/src/drizzle/schema/billing.ts index a3ced03c3..db2472c34 100644 --- a/packages/db/src/drizzle/schema/billing.ts +++ b/packages/db/src/drizzle/schema/billing.ts @@ -2,6 +2,7 @@ import { boolean, foreignKey, index, + integer, jsonb, pgTable, text, @@ -11,11 +12,87 @@ import { import { organization, user } from "./auth"; import { websites } from "./websites"; +export const autumnWebhookStatusValues = [ + "pending", + "processing", + "deferred", + "completed", + "dead_letter", +] as const; +export type AutumnWebhookStatus = (typeof autumnWebhookStatusValues)[number]; + +/** + * Durable inbox for Autumn usage webhooks that cannot be routed yet. + * Payloads are normalized before insert and contain billing identifiers only. + */ +export const autumnWebhookEvents = pgTable( + "autumn_webhook_events", + { + id: text().primaryKey(), + type: text().notNull(), + payload: jsonb().$type>().notNull(), + status: text().$type().default("pending").notNull(), + attempts: integer().default(0).notNull(), + errorMessage: text("error_message"), + leaseToken: text("lease_token"), + leaseExpiresAt: timestamp("lease_expires_at", { + precision: 3, + withTimezone: true, + }), + nextAttemptAt: timestamp("next_attempt_at", { + precision: 3, + withTimezone: true, + }), + completedAt: timestamp("completed_at", { + precision: 3, + withTimezone: true, + }), + deadLetteredAt: timestamp("dead_lettered_at", { + precision: 3, + withTimezone: true, + }), + alertedAt: timestamp("alerted_at", { + precision: 3, + withTimezone: true, + }), + createdAt: timestamp("created_at", { precision: 3, withTimezone: true }) + .defaultNow() + .notNull(), + updatedAt: timestamp("updated_at", { precision: 3, withTimezone: true }) + .defaultNow() + .notNull() + .$onUpdate(() => new Date()), + }, + (table) => [ + index("autumn_webhook_events_status_updated_idx").on( + table.status, + table.updatedAt + ), + index("autumn_webhook_events_status_completed_idx").on( + table.status, + table.completedAt + ), + index("autumn_webhook_events_status_next_attempt_idx").on( + table.status, + table.nextAttemptAt + ), + index("autumn_webhook_events_status_lease_idx").on( + table.status, + table.leaseExpiresAt + ), + index("autumn_webhook_events_status_dead_letter_idx").on( + table.status, + table.deadLetteredAt + ), + ] +); + export const usageAlertLog = pgTable( "usage_alert_log", { id: text().primaryKey(), userId: text("user_id").notNull(), + organizationId: text("organization_id"), featureId: text("feature_id").notNull(), alertType: text("alert_type").notNull(), emailSentTo: text("email_sent_to").notNull(), @@ -25,12 +102,22 @@ export const usageAlertLog = pgTable( }, (table) => [ index("usage_alert_log_user_feature_idx").on(table.userId, table.featureId), + index("usage_alert_log_org_user_feature_idx").on( + table.organizationId, + table.userId, + table.featureId + ), index("usage_alert_log_created_at_idx").on(table.createdAt), foreignKey({ columns: [table.userId], foreignColumns: [user.id], name: "usage_alert_log_user_id_fkey", }).onDelete("cascade"), + foreignKey({ + columns: [table.organizationId], + foreignColumns: [organization.id], + name: "usage_alert_log_organization_id_fkey", + }).onDelete("set null"), ] ); diff --git a/packages/db/src/drizzle/schema/insights.test.ts b/packages/db/src/drizzle/schema/insights.test.ts index 62699bedf..4ccd65465 100644 --- a/packages/db/src/drizzle/schema/insights.test.ts +++ b/packages/db/src/drizzle/schema/insights.test.ts @@ -1,8 +1,8 @@ import { describe, expect, test } from "bun:test"; import { getTableConfig } from "drizzle-orm/pg-core"; import { + DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT, INSIGHT_RUN_ACTIVE_STATUSES, - INSIGHT_RUN_ACTIVE_UNIQUE_INDEX, insightGenerationConfigs, insightObservations, insightRunEffects, @@ -11,7 +11,7 @@ import { } from "./insights"; describe("insight generation config schema", () => { - test("stores only product settings and scheduling metadata", () => { + test("stores one minimal schedule per organization", () => { expect( getTableConfig(insightGenerationConfigs).columns.map( (column) => column.name @@ -21,24 +21,34 @@ describe("insight generation config schema", () => { "organization_id", "enabled", "frequency", + "model_tier", "timezone", "deliveries", "next_run_at", + "dispatch_due_at", "last_run_at", "created_at", "updated_at", ]); - const uniqueIndexes = getTableConfig(insightGenerationConfigs).indexes.filter( - (index) => index.config.unique - ); - expect(uniqueIndexes).toHaveLength(1); - expect(uniqueIndexes[0]?.config.name).toBe( - "insight_generation_configs_org_uidx" - ); - expect(uniqueIndexes[0]?.config.columns.map((column) => column.name)).toEqual( - ["organization_id"] - ); + const config = getTableConfig(insightGenerationConfigs); + const uniqueIndexes = config.indexes.filter((index) => index.config.unique); + expect( + uniqueIndexes.map((index) => ({ + columns: index.config.columns.map((column) => column.name), + name: index.config.name, + partial: Boolean(index.config.where), + })) + ).toEqual([ + { + columns: ["organization_id"], + name: "insight_generation_configs_org_uidx", + partial: false, + }, + ]); + expect(config.foreignKeys.map((key) => key.getName())).toEqual([ + "insight_generation_configs_organization_id_fkey", + ]); }); }); @@ -85,29 +95,38 @@ describe("insight observations schema", () => { }); describe("insight runs schema", () => { - test("enforces one active run per organization", () => { - const index = getTableConfig(insightRuns).indexes.find( - (candidate) => candidate.config.name === INSIGHT_RUN_ACTIVE_UNIQUE_INDEX + test("keeps one active run per organization across mixed-version deploys", () => { + const indexes = getTableConfig(insightRuns).indexes; + expect( + indexes.find( + (index) => index.config.name === "insight_runs_org_created_idx" + )?.config.columns.map((column) => column.name) + ).toEqual(["organization_id", "created_at"]); + const activeRunIndex = indexes.find( + (index) => index.config.name === "insight_runs_org_active_uidx" ); - - expect(index?.config.unique).toBe(true); - expect(index?.config.columns.map((column) => column.name)).toEqual([ - "organization_id", - ]); - expect(index?.config.where).toBeDefined(); + expect(activeRunIndex?.config.unique).toBe(true); + expect( + activeRunIndex?.config.columns.map((column) => column.name) + ).toEqual(["organization_id"]); + expect(activeRunIndex?.config.where).toBeDefined(); expect(INSIGHT_RUN_ACTIVE_STATUSES).toEqual(["queued", "running"]); }); test("stores prepared state and one durable effect per provider target", () => { - expect( - getTableConfig(insightRunItems).columns.map((column) => column.name) - ).toEqual( + const runItemColumns = getTableConfig(insightRunItems).columns; + expect(runItemColumns.map((column) => column.name)).toEqual( expect.arrayContaining([ "prepared_at", "prepared_status", "prepared_message", + "config_snapshot", ]) ); + expect( + runItemColumns.find((column) => column.name === "config_snapshot") + ?.default + ).toEqual(DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT); const effects = getTableConfig(insightRunEffects); expect(effects.columns.map((column) => column.name)).toEqual([ "id", diff --git a/packages/db/src/drizzle/schema/insights.ts b/packages/db/src/drizzle/schema/insights.ts index d47d10777..89649b1d7 100644 --- a/packages/db/src/drizzle/schema/insights.ts +++ b/packages/db/src/drizzle/schema/insights.ts @@ -18,7 +18,20 @@ import { import { organization, user } from "./auth"; import { websites } from "./websites"; +export const INSIGHT_GENERATION_DEFAULT_TOOLS = [ + "web_metrics", + "product_metrics", + "ops_context", +] as const; + export type InsightGenerationFrequency = "daily" | "weekly"; +export type InsightGenerationTool = + | "web_metrics" + | "product_metrics" + | "ops_context" + | "business_context"; +export type InsightGenerationDepth = "light" | "standard" | "deep"; +export type InsightGenerationModelTier = "fast" | "balanced" | "deep"; export type InsightGenerationReason = | "manual" | "scheduled" @@ -49,6 +62,34 @@ export interface InsightDelivery { type: "slack"; } +/** + * Deprecated worker payload retained for one mixed-version deploy window. + * The current engine owns its policy in code and does not read these knobs. + */ +export interface InsightGenerationConfigSnapshot { + allowedTools: InsightGenerationTool[]; + cooldownHours: number; + depth: InsightGenerationDepth; + lookbackDays: number; + maxInsightsPerWebsite: number; + maxSteps: number; + maxToolCalls: number; + modelTier: InsightGenerationModelTier; + timezone: string; +} + +export const DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT = { + allowedTools: [...INSIGHT_GENERATION_DEFAULT_TOOLS], + cooldownHours: 6, + depth: "standard", + lookbackDays: 7, + maxInsightsPerWebsite: 3, + maxSteps: 24, + maxToolCalls: 16, + modelTier: "balanced", + timezone: "UTC", +} satisfies InsightGenerationConfigSnapshot; + export const insightGenerationConfigs = pgTable( "insight_generation_configs", { @@ -59,6 +100,11 @@ export const insightGenerationConfigs = pgTable( .$type() .default("weekly") .notNull(), + // Kept while old clients still read this deprecated field. + legacyModelTier: text("model_tier") + .$type() + .default("balanced") + .notNull(), timezone: text().default("UTC").notNull(), deliveries: jsonb("deliveries") .$type() @@ -68,6 +114,10 @@ export const insightGenerationConfigs = pgTable( precision: 3, withTimezone: true, }), + dispatchDueAt: timestamp("dispatch_due_at", { + precision: 3, + withTimezone: true, + }), lastRunAt: timestamp("last_run_at", { precision: 3, withTimezone: true, @@ -154,6 +204,12 @@ export const insightRunItems = pgTable( queueJobId: text("queue_job_id"), status: text().$type().default("queued").notNull(), attempts: integer().default(0).notNull(), + // Keep the database default through the mixed-version rollout: old API + // instances do not send this newly added column. + configSnapshot: jsonb("config_snapshot") + .$type() + .default(DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT) + .notNull(), resultCount: integer("result_count").default(0).notNull(), preparedAt: timestamp("prepared_at", { precision: 3, diff --git a/packages/db/src/drizzle/schema/links.test.ts b/packages/db/src/drizzle/schema/links.test.ts new file mode 100644 index 000000000..538aa7e31 --- /dev/null +++ b/packages/db/src/drizzle/schema/links.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, test } from "bun:test"; +import { getTableConfig } from "drizzle-orm/pg-core"; +import { links } from "./links"; + +describe("links schema", () => { + test("indexes stable newest pagination within an organization", () => { + const index = getTableConfig(links).indexes.find( + (candidate) => candidate.config.name === "links_org_created_at_id_idx" + ); + + expect(index?.config.columns.map((column) => column.name)).toEqual([ + "organization_id", + "created_at", + "id", + ]); + expect( + index?.config.columns.map((column) => column.indexConfig.order) + ).toEqual(["asc", "desc", "desc"]); + }); +}); diff --git a/packages/db/src/drizzle/schema/links.ts b/packages/db/src/drizzle/schema/links.ts index b74484a43..606b3872a 100644 --- a/packages/db/src/drizzle/schema/links.ts +++ b/packages/db/src/drizzle/schema/links.ts @@ -80,6 +80,11 @@ export const links = pgTable( (table) => [ index("links_folder_id_idx").on(table.folderId), index("links_org_folder_idx").on(table.organizationId, table.folderId), + index("links_org_created_at_id_idx").on( + table.organizationId, + table.createdAt.desc(), + table.id.desc() + ), index("links_created_by_idx").on(table.createdBy), index("links_external_id_idx").on(table.externalId), index("links_org_source_type_idx").on( diff --git a/packages/db/src/drizzle/schema/relations.ts b/packages/db/src/drizzle/schema/relations.ts index e13e214ae..8edc24149 100644 --- a/packages/db/src/drizzle/schema/relations.ts +++ b/packages/db/src/drizzle/schema/relations.ts @@ -20,7 +20,12 @@ import { user, userPreferences, } from "./auth"; -import { alarmDestinations, alarms, usageAlertLog } from "./billing"; +import { + alarmDestinations, + alarms, + autumnWebhookEvents, + usageAlertLog, +} from "./billing"; import { feedback, feedbackRedemptions, insightUserFeedback } from "./feedback"; import { flags, flagsToTargetGroups, targetGroups } from "./flags"; import { slackChannelBindings, slackIntegrations } from "./integrations"; @@ -71,6 +76,7 @@ const schema = { revenueConfig, alarms, alarmDestinations, + autumnWebhookEvents, usageAlertLog, goals, annotations, diff --git a/packages/db/src/drizzle/schema/uptime.ts b/packages/db/src/drizzle/schema/uptime.ts index 4a30513f3..3f76065d9 100644 --- a/packages/db/src/drizzle/schema/uptime.ts +++ b/packages/db/src/drizzle/schema/uptime.ts @@ -9,6 +9,7 @@ import { timestamp, uniqueIndex, } from "drizzle-orm/pg-core"; +import type { UptimeGranularity } from "@databuddy/shared/uptime"; import { organization } from "./auth"; import { websites } from "./websites"; @@ -24,7 +25,7 @@ export const uptimeSchedules = pgTable( organizationId: text("organization_id").notNull(), url: text().notNull(), name: text(), - granularity: text().notNull(), + granularity: text().$type().notNull(), cron: text().notNull(), isPaused: boolean("is_paused").default(false).notNull(), isPublic: boolean("is_public").default(false).notNull(), diff --git a/packages/email/package.json b/packages/email/package.json index ba35d77dc..3f75df9bd 100644 --- a/packages/email/package.json +++ b/packages/email/package.json @@ -16,6 +16,7 @@ }, "main": "./src/emails/index.ts", "dependencies": { + "@databuddy/shared": "workspace:*", "@types/node": "^26.0.0", "react": "catalog:", "react-dom": "catalog:", diff --git a/packages/email/src/emails/auth-email-copy.test.tsx b/packages/email/src/emails/auth-email-copy.test.tsx index 22a0859c9..910f8bebb 100644 --- a/packages/email/src/emails/auth-email-copy.test.tsx +++ b/packages/email/src/emails/auth-email-copy.test.tsx @@ -1,6 +1,9 @@ import { describe, expect, test } from "bun:test"; import { render } from "react-email"; -import { AUTH_EMAIL_EXPIRY_LABELS } from "./auth-email-expiry"; +import { + AUTH_EMAIL_EXPIRY_LABELS, + AUTH_EMAIL_EXPIRY_SECONDS, +} from "./auth-email-expiry"; import { DeleteAccountEmail } from "./delete-account-email"; import { InvitationEmail } from "./invitation-email"; import { MagicLinkEmail } from "./magic-link-email"; @@ -11,6 +14,13 @@ const asText = (element: React.ReactElement) => render(element, { plainText: true }); describe("authentication email copy", () => { + test("derives customer-facing labels from the enforced token TTLs", () => { + expect(AUTH_EMAIL_EXPIRY_SECONDS.magicLink).toBe(15 * 60); + expect(AUTH_EMAIL_EXPIRY_LABELS.magicLink).toBe("15 minutes"); + expect(AUTH_EMAIL_EXPIRY_SECONDS.emailVerification).toBe(24 * 60 * 60); + expect(AUTH_EMAIL_EXPIRY_LABELS.emailVerification).toBe("24 hours"); + }); + test("magic-link and verification copy use the shared expiry labels", async () => { const magicLink = await asText( MagicLinkEmail({ url: "https://example.com/magic" }) diff --git a/packages/email/src/emails/auth-email-expiry.ts b/packages/email/src/emails/auth-email-expiry.ts index eb1e82699..af5439a2e 100644 --- a/packages/email/src/emails/auth-email-expiry.ts +++ b/packages/email/src/emails/auth-email-expiry.ts @@ -7,11 +7,26 @@ export const AUTH_EMAIL_EXPIRY_SECONDS = { passwordReset: 60 * 60, } as const; -export const AUTH_EMAIL_EXPIRY_LABELS = { - accountDeletion: "1 hour", - emailVerification: "24 hours", - invitation: "48 hours", - magicLink: "15 minutes", - oneTimeCode: "10 minutes", - passwordReset: "1 hour", -} as const; +function expiryLabel(totalSeconds: number): string { + const units = [ + [60 * 60, "hour"], + [60, "minute"], + [1, "second"], + ] as const; + for (const [unitSeconds, unit] of units) { + if (totalSeconds % unitSeconds === 0) { + const value = totalSeconds / unitSeconds; + return `${value} ${unit}${value === 1 ? "" : "s"}`; + } + } + return `${totalSeconds} seconds`; +} + +export const AUTH_EMAIL_EXPIRY_LABELS = Object.freeze( + Object.fromEntries( + Object.entries(AUTH_EMAIL_EXPIRY_SECONDS).map(([key, seconds]) => [ + key, + expiryLabel(seconds), + ]) + ) +) as Readonly>; diff --git a/packages/email/src/emails/usage-alert-email.tsx b/packages/email/src/emails/usage-alert-email.tsx index 1432d56b3..8d227839d 100644 --- a/packages/email/src/emails/usage-alert-email.tsx +++ b/packages/email/src/emails/usage-alert-email.tsx @@ -1,3 +1,4 @@ +import { DATABUNNY_USAGE } from "@databuddy/shared/billing"; import { Heading, Link, Section, Text } from "react-email"; import { emailBrand } from "./email-brand"; import { EmailButton } from "./email-button"; @@ -136,17 +137,16 @@ export const UsageAlertEmail = ({ }; UsageAlertEmail.PreviewProps = { - featureDescription: - "This allowance powers Databunny questions and AI analysis. More complex work uses more of it.", - featureName: "Databunny usage", + featureDescription: DATABUNNY_USAGE.description, + featureName: DATABUNNY_USAGE.name, limitAmount: 350, nextResetAt: Date.UTC(2026, 7, 1), organizationName: "Acme Inc", overageAllowed: false, - pausedActivity: "Databunny questions and AI analysis", + pausedActivity: DATABUNNY_USAGE.pausedActivity, remainingAmount: 70, usageAmount: 280, - usageUnit: "allowance units", + usageUnit: DATABUNNY_USAGE.unit, } satisfies UsageAlertEmailProps; export default UsageAlertEmail; diff --git a/packages/email/src/emails/usage-email-copy.test.tsx b/packages/email/src/emails/usage-email-copy.test.tsx index 80d842ebd..d5e1cd677 100644 --- a/packages/email/src/emails/usage-email-copy.test.tsx +++ b/packages/email/src/emails/usage-email-copy.test.tsx @@ -1,29 +1,43 @@ import { describe, expect, test } from "bun:test"; +import { DATABUNNY_USAGE } from "@databuddy/shared/billing"; import { render } from "react-email"; import { UsageAlertEmail } from "./usage-alert-email"; +import { + formatResetDate, + formatUsageNumber, + formatUsagePercentage, +} from "./usage-email-utils"; import { UsageLimitEmail } from "./usage-limit-email"; const FEATURE_COPY = { - featureDescription: - "This allowance powers Databunny questions and AI analysis. More complex work uses more of it.", - featureName: "Databunny usage", + featureDescription: DATABUNNY_USAGE.description, + featureName: DATABUNNY_USAGE.name, limitAmount: 350, nextResetAt: Date.UTC(2026, 7, 1), organizationName: "Acme", overageAllowed: false, - pausedActivity: "Databunny questions and AI analysis", + pausedActivity: DATABUNNY_USAGE.pausedActivity, remainingAmount: 62, usageAmount: 288, - usageUnit: "allowance units", + usageUnit: DATABUNNY_USAGE.unit, } as const; describe("billing usage email copy", () => { + test("rejects malformed usage values and invalid millisecond timestamps", () => { + expect(formatUsageNumber(Number.NaN)).toBe("—"); + expect(formatUsageNumber(Number.POSITIVE_INFINITY)).toBe("—"); + expect(formatUsagePercentage(Number.NaN, 100)).toBe("—"); + expect(formatResetDate(Number.MAX_VALUE)).toBeUndefined(); + expect(formatResetDate(Date.UTC(2026, 7, 1))).toContain("2026"); + expect(formatResetDate(1_782_864_000)).toContain("1970"); + }); + test("explains Databunny usage with real values and no owner greeting", async () => { const text = await render(UsageAlertEmail(FEATURE_COPY), { plainText: true, }); - expect(text).toContain("288 of 350 allowance units"); + expect(text).toContain("288 of 350 usage units"); expect(text).toContain("62 remain"); expect(text).toContain("powers Databunny questions and AI analysis"); expect(text).toContain("More complex work uses more of it"); @@ -46,7 +60,7 @@ describe("billing usage email copy", () => { expect(text).toContain( "Access to Databunny questions and AI analysis is currently paused" ); - expect(text).toContain("350 of 350 allowance units"); + expect(text).toContain("350 of 350 usage units"); expect(text).not.toContain("1.5x"); expect(text).not.toContain("10,000"); }); diff --git a/packages/email/src/emails/usage-email-utils.ts b/packages/email/src/emails/usage-email-utils.ts index 2697ab180..624cd022d 100644 --- a/packages/email/src/emails/usage-email-utils.ts +++ b/packages/email/src/emails/usage-email-utils.ts @@ -9,6 +9,9 @@ const resetDateFormatter = new Intl.DateTimeFormat("en-US", { }); export function formatUsageNumber(value: number): string { + if (!Number.isFinite(value)) { + return "—"; + } return usageNumberFormatter.format(value); } @@ -19,9 +22,16 @@ export function formatUsagePercentage(usage: number, limit: number): string { return `${Math.round((usage / limit) * 100)}%`; } -export function formatResetDate(timestamp?: number | null): string | undefined { - if (timestamp == null || !Number.isFinite(timestamp)) { +/** Formats Autumn's Unix timestamp, which is expressed in milliseconds. */ +export function formatResetDate( + timestampMs?: number | null +): string | undefined { + if (timestampMs == null || !Number.isFinite(timestampMs)) { + return; + } + const date = new Date(timestampMs); + if (Number.isNaN(date.getTime())) { return; } - return resetDateFormatter.format(new Date(timestamp)); + return resetDateFormatter.format(date); } diff --git a/packages/email/src/emails/usage-limit-email.tsx b/packages/email/src/emails/usage-limit-email.tsx index 064a50821..c60425677 100644 --- a/packages/email/src/emails/usage-limit-email.tsx +++ b/packages/email/src/emails/usage-limit-email.tsx @@ -1,3 +1,4 @@ +import { DATABUNNY_USAGE } from "@databuddy/shared/billing"; import { Heading, Link, Section, Text } from "react-email"; import { emailBrand } from "./email-brand"; import { EmailButton } from "./email-button"; @@ -145,19 +146,18 @@ export const UsageLimitEmail = ({ }; UsageLimitEmail.PreviewProps = { - featureDescription: - "This allowance powers Databunny questions and AI analysis. More complex work uses more of it.", - featureName: "Databunny usage", + featureDescription: DATABUNNY_USAGE.description, + featureName: DATABUNNY_USAGE.name, isAvailable: false, limitAmount: 350, limitType: "included", nextResetAt: Date.UTC(2026, 7, 1), organizationName: "Acme Inc", overageAllowed: false, - pausedActivity: "Databunny questions and AI analysis", + pausedActivity: DATABUNNY_USAGE.pausedActivity, remainingAmount: 0, usageAmount: 350, - usageUnit: "allowance units", + usageUnit: DATABUNNY_USAGE.unit, } satisfies UsageLimitEmailProps; export default UsageLimitEmail; diff --git a/packages/env/src/app.test.ts b/packages/env/src/app.test.ts index ca03718da..9be1eb999 100644 --- a/packages/env/src/app.test.ts +++ b/packages/env/src/app.test.ts @@ -88,16 +88,6 @@ describe("createConfig", () => { }); }); - it("reads the server-only OpenAI Ads Conversions API key", () => { - expect( - createConfig({ - OPENAI_ADS_CONVERSIONS_API_KEY: " capi_123 ", - }) - ).toMatchObject({ - integrations: { openAiAdsConversionsApiKey: "capi_123" }, - }); - }); - it("deduplicates API CORS origins from dashboard URLs", () => { expect( createConfig({ diff --git a/packages/env/src/app.ts b/packages/env/src/app.ts index 66424ed3a..2e656637b 100644 --- a/packages/env/src/app.ts +++ b/packages/env/src/app.ts @@ -57,7 +57,6 @@ export interface Config { from: string; }; integrations: { - openAiAdsConversionsApiKey?: string; openAiAdsPixelId?: string; }; urls: { @@ -144,10 +143,6 @@ export function createConfig(env: Env = process.env): Config { from: readEmail(env, EMAIL.from), }, integrations: { - openAiAdsConversionsApiKey: readOptional( - env, - "OPENAI_ADS_CONVERSIONS_API_KEY" - ), openAiAdsPixelId: readOptional(env, "NEXT_PUBLIC_OPENAI_ADS_PIXEL_ID"), }, urls: { diff --git a/packages/env/src/insights.ts b/packages/env/src/insights.ts index 46e489016..7ad80acaf 100644 --- a/packages/env/src/insights.ts +++ b/packages/env/src/insights.ts @@ -16,6 +16,8 @@ const insightsEnvSchema = z.object({ INSIGHTS_WORKER_ENABLED: optionalString, INSIGHTS_WORKER_CONCURRENCY: optionalString, INSIGHTS_DISPATCH_INTERVAL_MS: optionalString, + INSIGHTS_SCHEDULED_DISPATCH_ENABLED: optionalString, + INSIGHTS_SCHEDULED_ORGANIZATION_IDS: optionalString, INSIGHTS_MAINTENANCE_INTERVAL_MS: optionalString, INSIGHTS_STALE_ITEM_MS: optionalString, RAILWAY_ENVIRONMENT_NAME: optionalString, diff --git a/packages/evals/src/fixtures/insight-timelines.ts b/packages/evals/src/fixtures/insight-timelines.ts index 203b221fa..9d67d1993 100644 --- a/packages/evals/src/fixtures/insight-timelines.ts +++ b/packages/evals/src/fixtures/insight-timelines.ts @@ -99,10 +99,20 @@ interface VitalRow extends Record { samples: number; } +interface PaymentRow extends Record { + currency: string; + failed_payment_attempts: number; + observed_failure_event_types: number; + payment_failure_rate: number; + successful_payment_attempts: number; + top_payment_failure_reason: string; +} + interface MetricFrame { errors?: { current: ErrorRow; previous: ErrorRow }; failQueries?: string[]; hasData?: boolean; + payment?: { current: PaymentRow; previous: PaymentRow }; revenue?: { current: number; previous: number }; summary?: { current: SummaryRow; previous: SummaryRow }; vitals?: { current: VitalRow[]; previous: VitalRow[] }; @@ -232,7 +242,7 @@ function metricQuery(frame: MetricFrame, asOf: string): QueryFn { if (failed.has(request.type)) { return Promise.reject(new Error(`Synthetic ${request.type} failure`)); } - if (request.type === "events_by_date") { + if (request.type === "events_by_date" || request.type === "custom_events") { return Promise.resolve([]); } if (request.type === "summary_metrics") { @@ -242,9 +252,14 @@ function metricQuery(frame: MetricFrame, asOf: string): QueryFn { return Promise.resolve([periodValue(request, asOf, errors)]); } if (request.type === "revenue_overview") { + const payment = frame.payment + ? periodValue(request, asOf, frame.payment) + : {}; return Promise.resolve([ { + currency: frame.payment?.current.currency ?? "USD", total_revenue: periodValue(request, asOf, revenue), + ...payment, }, ]); } @@ -300,8 +315,74 @@ function definitionDeps(frame: DefinitionFrame, asOf: string): FunnelGoalDeps { } function evidenceForSignal( - signal: InvestigationSignal + signal: InvestigationSignal, + frame: MetricFrame ): InvestigationEvidence[] { + if ( + (signal.entity.type === "goal" || signal.entity.type === "funnel") && + signal.expectation?.confirmation + ) { + return [ + { + evidenceId: `fixture:${signal.entity.type}:verified`, + signalKey: signal.signalKey, + kind: "definition", + source: "product", + queryType: + signal.entity.type === "goal" ? "goals_summary" : "funnels_summary", + entity: signal.entity, + period: "current", + range: signal.period.current, + status: "ok", + rowCount: 1, + summary: `${signal.entity.label} was re-verified against its active definition.`, + remediation: signal.expectation, + }, + ]; + } + if (signal.metric.key === "payment_failure_rate" && frame.payment) { + return (["current", "previous"] as const).map((period) => { + const payment = frame.payment?.[period]; + if (!payment) { + throw new Error("Missing synthetic payment evidence"); + } + const reason = payment.top_payment_failure_reason.replace(/[_-]+/g, " "); + return { + evidenceId: `fixture:payment-failure:${period}`, + signalKey: signal.signalKey, + kind: "breakdown", + source: "business", + queryType: "revenue_overview", + period, + range: signal.period[period], + status: "ok", + rowCount: 1, + summary: `${period === "current" ? "Current" : "Previous"} tracked payment failure rate was ${payment.payment_failure_rate}%. Most common failure: ${reason}. ${payment.observed_failure_event_types} distinct Stripe failure event types were observed in this range.`, + metrics: [ + { + label: "Payment failure rate", + current: payment.payment_failure_rate, + format: "percent", + }, + { + label: "Failed payment attempts", + current: payment.failed_payment_attempts, + format: "number", + }, + { + label: "Successful payment attempts", + current: payment.successful_payment_attempts, + format: "number", + }, + { + label: "Observed failure event types", + current: payment.observed_failure_event_types, + format: "number", + }, + ], + }; + }); + } if (signal.metric.key === "revenue") { return (["current", "previous"] as const).map((period) => ({ evidenceId: `fixture:revenue:${period}`, @@ -313,7 +394,7 @@ function evidenceForSignal( range: signal.period[period], status: "ok", rowCount: 1, - summary: `${period === "current" ? "Current" : "Previous"} queried revenue was ${period === "current" ? signal.metric.current : signal.metric.previous}.`, + summary: `${period === "current" ? "Current" : "Previous"} queried revenue (${signal.currency ?? "unknown currency"}) was ${period === "current" ? signal.metric.current : signal.metric.previous}.`, metrics: [ { label: "Queried revenue", @@ -401,7 +482,7 @@ function createSources( new Error("Unexpected synthetic evidence read") ); } - return Promise.resolve(evidenceForSignal(params.signal)); + return Promise.resolve(evidenceForSignal(params.signal, metricFrame)); }), createServiceAuth: () => Promise.resolve(undefined), detectDefinitionSignals: (params, today, _deps, options) => @@ -487,7 +568,7 @@ const trafficLifecycle: InsightTimeline = { disposition: "needs_context", lifecycle: "detected", status: "completed", - title: "Visitors drop needs context", + title: "Visitors fell 60%", }, id: "detected", previous: { unique_visitors: 1000 }, @@ -513,7 +594,7 @@ const trafficLifecycle: InsightTimeline = { disposition: "needs_context", lifecycle: "worsened", status: "completed", - title: "Visitors drop needs context", + title: "Visitors fell 95%", }, id: "worsened", previous: { unique_visitors: 1000 }, @@ -545,7 +626,7 @@ const revenueDrop: InsightTimeline = { disposition: "needs_context", lifecycle: "detected", status: "completed", - title: "Revenue drop needs context", + title: "Revenue (USD) fell 60%", }, frame: { metrics: { revenue: { current: 400, previous: 1000 } } }, id: "drop", @@ -554,6 +635,46 @@ const revenueDrop: InsightTimeline = { ], }; +const paymentFailureRegression: InsightTimeline = { + id: "payment-failure-regression", + name: "Payment failure regression preserves safe cause and occurrence context", + stages: [ + stage({ + asOf: "2026-07-06", + expect: { + disposition: "needs_context", + lifecycle: "detected", + status: "completed", + title: "Payment failure rate (USD) rose to 20%", + }, + frame: { + metrics: { + payment: { + current: { + currency: "USD", + failed_payment_attempts: 10, + observed_failure_event_types: 2, + payment_failure_rate: 20, + successful_payment_attempts: 40, + top_payment_failure_reason: "insufficient_funds", + }, + previous: { + currency: "USD", + failed_payment_attempts: 2, + observed_failure_event_types: 2, + payment_failure_rate: 5, + successful_payment_attempts: 38, + top_payment_failure_reason: "generic_decline", + }, + }, + }, + }, + id: "regression", + timelineId: "payment-failure-regression", + }), + ], +}; + const errorSpike: InsightTimeline = { id: "error-spike", name: "Error spike localizes an unresolved fingerprint", @@ -616,7 +737,7 @@ const zeroGoal: InsightTimeline = { disposition: "needs_context", lifecycle: "detected", status: "completed", - title: "Signup needs context", + title: "Signup conversion stopped", }, frame: { definitions: { @@ -750,7 +871,7 @@ const incompleteValidSignal: InsightTimeline = { disposition: "needs_context", lifecycle: "detected", status: "completed", - title: "Visitors drop needs context", + title: "Visitors fell 70%", }, frame: { metrics: { @@ -875,6 +996,7 @@ const lowImpactErrorSpike: InsightTimeline = { export const insightTimelines: InsightTimeline[] = [ trafficLifecycle, revenueDrop, + paymentFailureRegression, errorSpike, vitalRegression, zeroGoal, diff --git a/packages/evals/src/insight-history.test.ts b/packages/evals/src/insight-history.test.ts index 8d146e87b..3a17fb048 100644 --- a/packages/evals/src/insight-history.test.ts +++ b/packages/evals/src/insight-history.test.ts @@ -48,23 +48,35 @@ describe("historical insight runner", () => { expect(result.aggregate).toMatchObject({ actions: 1, failures: 0, - insights: 8, - stages: 16, - timelines: 13, + insights: 9, + stages: 17, + timelines: 14, }); - expect( - result.timelines - .flatMap((timeline) => timeline.stages) - .find( - (stage) => - stage.artifact?.decision?.disposition === "action_ready" - ) - ?.artifact?.insight?.evidence?.some((item) => - item.description.includes( - "Independent revenue tracking recorded 12 transactions" - ) - ) - ).toBe(true); + const action = result.timelines + .flatMap((timeline) => timeline.stages) + .find( + (stage) => stage.artifact?.decision?.disposition === "action_ready" + )?.artifact?.insight; + expect(action?.impactSummary).toContain( + "Independent revenue tracking recorded 12 transactions" + ); + expect(action?.suggestion).toBe( + 'Restore the "purchase" event at the Purchase step in Checkout.' + ); + const payment = result.timelines.find( + (timeline) => timeline.id === "payment-failure-regression" + )?.stages[0]; + expect(payment?.lifecycle).toBe("detected"); + expect(payment?.artifact?.validationErrors).toEqual([]); + expect(payment?.artifact?.insight?.impactSummary).toContain( + "Most common failure: insufficient funds" + ); + expect(payment?.artifact?.insight?.impactSummary).toContain( + "2 distinct Stripe failure event types were observed" + ); + expect(payment?.artifact?.insight?.impactSummary).not.toContain( + "provider message" + ); }); it("keeps low-volume noise out of the investigation queue", async () => { @@ -92,12 +104,12 @@ describe("historical insight runner", () => { const report = formatInsightHistory(result); expect(result.passed).toBe(true); - expect(result.aggregate.maxVisibleWords).toBe(43); + expect(result.aggregate.maxVisibleWords).toBe(37); expect(result.aggregate.actions).toBe(0); expect(result.aggregate.contexts).toBe(1); - expect(report).toContain("Title: Visitors drop needs context"); + expect(report).toContain("Title: Visitors fell 60%"); expect(report).toContain( - "Next: Visitors fell from 1000 to 400. Was this expected?" + "Next: Was this traffic drop expected? If not, check source traffic" ); }); diff --git a/packages/evals/src/insight-history.ts b/packages/evals/src/insight-history.ts index cd78de253..bb292846d 100644 --- a/packages/evals/src/insight-history.ts +++ b/packages/evals/src/insight-history.ts @@ -397,6 +397,12 @@ async function runTimeline( ) : null; const retiredSignalKey = retiredSignalKeyForOutcome({ + coverage: { + definitions: + artifact.recoveryCoverage?.definitions ?? artifact.detectionComplete, + metrics: + artifact.recoveryCoverage?.metrics ?? artifact.detectionComplete, + }, disposition: artifact.decision?.disposition, hasInsight: artifact.insight !== null, signalKey, diff --git a/packages/evals/src/insight-production-shadow.test.ts b/packages/evals/src/insight-production-shadow.test.ts index 6f0bb04cc..d534a529d 100644 --- a/packages/evals/src/insight-production-shadow.test.ts +++ b/packages/evals/src/insight-production-shadow.test.ts @@ -1,5 +1,470 @@ import { describe, expect, test } from "bun:test"; -import { runCancellableAttempt } from "./insight-production-shadow"; +import type { + GeneratedInsight, + InvestigationSignal, +} from "@databuddy/shared/insights"; +import { + assertShadowEvidenceUsesClickHouse, + chronologicalOffsets, + configureIsolatedShadowRedis, + createSnapshotProductMetricsFetcher, + evaluateShadowGates, + HISTORICAL_LIMITATIONS, + observationsAt, + quarantineShadowPostgresEnvironment, + renderShadowInsight, + runCancellableAttempt, + summarizeValidationErrors, + withDefinitionSnapshot, +} from "./insight-production-shadow"; +import type { ShadowObservation } from "./insight-production-shadow"; +import type { + FunnelDef, + GoalDef, +} from "../../../apps/insights/src/funnel-detection"; +import type { LatestInsightObservation } from "../../../apps/insights/src/observations"; + +function signal(signalKey: string, current: number): InvestigationSignal { + return { + changePercent: -50, + detectedAt: "2026-01-08", + detection: { + method: "period_comparison", + reason: "Visitors fell against the prior complete period.", + }, + direction: "down", + entity: { id: "website", label: "Website", type: "website" }, + insightType: "traffic_drop", + kind: "change", + metric: { + current, + format: "number", + key: "visitors", + label: "Visitors", + previous: 100, + }, + period: { + current: { from: "2026-01-02", to: "2026-01-08" }, + previous: { from: "2025-12-26", to: "2026-01-01" }, + }, + priority: 7, + sentiment: "negative", + severity: "warning", + signalKey, + websiteId: "site-1", + }; +} + +function observation( + asOf: string, + current: number, + signalKey = "website:visitors" +): LatestInsightObservation { + return { + asOf: new Date(asOf), + decision: { disposition: "monitor" }, + evidence: [], + recheckAt: new Date("2026-02-01T00:00:00.000Z"), + signal: signal(signalKey, current), + }; +} + +describe("production shadow definition snapshots", () => { + test("removes the live definition loader", async () => { + const timestamp = new Date("2026-01-01T00:00:00.000Z"); + const funnels: FunnelDef[] = [ + { + createdAt: timestamp, + filters: null, + id: "snapshot-funnel", + name: "Checkout", + steps: [ + { name: "Start", target: "/", type: "PAGE_VIEW" }, + { name: "Buy", target: "purchase", type: "EVENT" }, + ], + updatedAt: timestamp, + }, + ]; + const goals: GoalDef[] = [ + { + createdAt: timestamp, + filters: null, + id: "snapshot-goal", + name: "Signup", + target: "signup", + type: "EVENT", + updatedAt: timestamp, + }, + ]; + const deps = withDefinitionSnapshot( + { + fetchDefinitionWindow: async () => { + throw new Error("live loader must not run"); + }, + funnelConversion: async () => ({ + completions: 0, + entrants: 0, + rate: 0, + steps: [], + }), + goalConversion: async () => ({ completions: 0, entrants: 0, rate: 0 }), + }, + funnels, + goals + ); + + expect(deps.fetchDefinitionWindow).toBeUndefined(); + expect(await deps.fetchFunnels?.()).toBe(funnels); + expect(await deps.fetchGoals?.()).toBe(goals); + }); + + test("builds private goal and funnel evidence from snapshot conversions", async () => { + const timestamp = new Date("2026-01-01T00:00:00.000Z"); + const fetch = createSnapshotProductMetricsFetcher( + { + funnelConversion: async () => ({ + completions: 20, + entrants: 100, + rate: 20, + steps: [ + { stepNumber: 1, users: 100 }, + { stepNumber: 2, users: 20 }, + ], + }), + goalConversion: async () => ({ + completions: 10, + entrants: 40, + rate: 25, + }), + }, + [ + { + createdAt: timestamp, + filters: null, + id: "snapshot-funnel", + name: "Checkout", + steps: [ + { name: "Start", target: "/", type: "PAGE_VIEW" }, + { name: "Buy", target: "purchase", type: "EVENT" }, + ], + updatedAt: timestamp, + }, + ], + [ + { + createdAt: timestamp, + filters: null, + id: "snapshot-goal", + name: "Signup", + target: "signup", + type: "EVENT", + updatedAt: timestamp, + }, + ] + ); + const appContext = { + chatId: "shadow:test", + currentDateTime: timestamp.toISOString(), + organizationId: "org_example", + timezone: "UTC", + userId: "system", + websiteId: "site_example", + }; + + await expect( + fetch( + appContext, + { from: "2025-12-01", to: "2025-12-07" }, + { id: "snapshot-goal", type: "goal" } + ) + ).resolves.toMatchObject({ + results: [ + { + goals: [ + { + id: "snapshot-goal", + overall_conversion_rate: 25, + total_users_completed: 10, + }, + ], + type: "goals_summary", + }, + ], + }); + await expect( + fetch( + appContext, + { from: "2025-12-01", to: "2025-12-07" }, + { id: "snapshot-funnel", type: "funnel" } + ) + ).resolves.toMatchObject({ + results: [ + { + funnels: [ + { + biggest_dropoff_rate: 80, + biggest_dropoff_step: 2, + id: "snapshot-funnel", + steps: [{ users: 100 }, { users: 20 }], + }, + ], + type: "funnels_summary", + }, + ], + }); + expect(appContext).not.toHaveProperty("serviceAuth"); + }); +}); + +describe("production shadow process isolation", () => { + test("allows only ClickHouse-backed operations evidence", () => { + expect(() => + assertShadowEvidenceUsesClickHouse({ + input: { + period: "current", + queries: [ + { type: "error_fingerprints" }, + { type: "uptime_summary" }, + ], + }, + name: "ops_context", + }) + ).not.toThrow(); + expect(() => + assertShadowEvidenceUsesClickHouse({ + input: { + period: "current", + queries: [{ type: "anomaly_summary" }], + }, + name: "ops_context", + }) + ).toThrow("blocks non-ClickHouse ops evidence: anomaly_summary"); + }); + + test("quarantines the process from the captured production database", () => { + const previous = { + databaseUrl: process.env.DATABASE_URL, + timeout: process.env.DB_CONNECTION_TIMEOUT_MS, + pgOptions: process.env.PGOPTIONS, + }; + try { + process.env.DATABASE_URL = + "postgresql://user:secret@production.example.test/databuddy"; + + quarantineShadowPostgresEnvironment(); + + const guardedUrl = new URL(process.env.DATABASE_URL ?? ""); + expect(guardedUrl.hostname).toBe("127.0.0.1"); + expect(guardedUrl.port).toBe("1"); + expect(process.env.DB_CONNECTION_TIMEOUT_MS).toBe("250"); + expect(process.env.PGOPTIONS).toBeUndefined(); + } finally { + for (const [key, value] of Object.entries({ + DATABASE_URL: previous.databaseUrl, + DB_CONNECTION_TIMEOUT_MS: previous.timeout, + PGOPTIONS: previous.pgOptions, + })) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + } + }); + + test("replaces production Redis and BullMQ URLs with loopback DB 15", () => { + const previous = { + bullmq: process.env.BULLMQ_REDIS_URL, + insights: process.env.INSIGHTS_BULLMQ_REDIS_URL, + redis: process.env.REDIS_URL, + }; + try { + process.env.REDIS_URL = "rediss://production.example.test/0"; + process.env.BULLMQ_REDIS_URL = "rediss://production.example.test/1"; + process.env.INSIGHTS_BULLMQ_REDIS_URL = + "rediss://production.example.test/2"; + + configureIsolatedShadowRedis(); + + expect(process.env.REDIS_URL).toBe("redis://127.0.0.1:6379/15"); + expect(process.env.BULLMQ_REDIS_URL).toBe( + "redis://127.0.0.1:6379/15" + ); + expect(process.env.INSIGHTS_BULLMQ_REDIS_URL).toBe( + "redis://127.0.0.1:6379/15" + ); + } finally { + for (const [key, value] of Object.entries({ + BULLMQ_REDIS_URL: previous.bullmq, + INSIGHTS_BULLMQ_REDIS_URL: previous.insights, + REDIS_URL: previous.redis, + })) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + } + }); +}); + +describe("production shadow historical replay", () => { + test("orders offsets from oldest to newest without mutating input", () => { + const offsets = [0, 60, 7, 30]; + + expect(chronologicalOffsets(offsets)).toEqual([60, 30, 7, 0]); + expect(offsets).toEqual([0, 60, 7, 30]); + }); + + test("loads backfills only after creation and overlays replay state", () => { + const persisted = [ + { + ...observation("2026-01-03T00:00:00.000Z", 30), + createdAt: new Date("2026-01-03T01:00:00.000Z"), + organizationId: "org-1", + signalKey: "website:visitors", + websiteId: "site-1", + }, + { + ...observation("2026-01-05T00:00:00.000Z", 40), + createdAt: new Date("2026-01-05T01:00:00.000Z"), + organizationId: "org-1", + signalKey: "website:visitors", + websiteId: "site-1", + }, + { + ...observation("2026-01-05T00:00:00.000Z", 50), + createdAt: new Date("2026-01-20T02:00:00.000Z"), + organizationId: "org-1", + signalKey: "website:visitors", + websiteId: "site-1", + }, + ] satisfies ShadowObservation[]; + const params = { + asOf: new Date("2026-01-10T00:00:00.000Z"), + organizationId: "org-1", + persisted, + signalKeys: ["website:visitors"], + websiteId: "site-1", + }; + + expect( + observationsAt({ ...params, simulated: new Map() }).get( + "website:visitors" + )?.signal.metric.current + ).toBe(40); + expect( + observationsAt({ + ...params, + asOf: new Date("2026-01-21T00:00:00.000Z"), + simulated: new Map(), + }).get("website:visitors")?.signal.metric.current + ).toBe(50); + expect( + observationsAt({ + ...params, + simulated: new Map([ + [ + "website:visitors", + observation("2026-01-08T00:00:00.000Z", 80), + ], + ]), + }).get("website:visitors")?.signal.metric.current + ).toBe(80); + }); + + test("discloses the historical states the replay cannot reconstruct", () => { + const disclosure = HISTORICAL_LIMITATIONS.join(" "); + + expect(disclosure).toContain("inactive, deleted, and pre-edit revisions"); + expect(disclosure).toContain("annotation edits"); + expect(disclosure).toContain("late-arriving"); + expect(disclosure).toContain("production state is never changed"); + }); +}); + +describe("production shadow pre-ship gates", () => { + const passingMetrics = { + detectionFailures: 0, + detectionIncomplete: 0, + evidenceFailed: 0, + evidenceTruncated: 0, + repeatAgreement: 1, + status: { completed: 4 }, + visibleWords: { max: 100 }, + }; + + test("passes clean output and ignores repeat agreement when repeat is off", () => { + expect(evaluateShadowGates(passingMetrics, true)).toEqual([]); + expect( + evaluateShadowGates( + { ...passingMetrics, repeatAgreement: null }, + false + ) + ).toEqual([]); + }); + + test("reports expected partial definition coverage without failing the release", () => { + expect( + evaluateShadowGates( + { ...passingMetrics, detectionIncomplete: 3 }, + true + ) + ).toEqual([]); + }); + + test("reports every failed PRD pre-ship gate", () => { + expect( + evaluateShadowGates( + { + detectionFailures: 3, + detectionIncomplete: 3, + evidenceFailed: 4, + evidenceTruncated: 5, + repeatAgreement: null, + status: { error: 1, invalid_output: 2 }, + visibleWords: { max: 101 }, + }, + true + ) + ).toEqual([ + { actual: 1, comparator: "eq", gate: "error_cases", required: 0 }, + { + actual: 2, + comparator: "eq", + gate: "invalid_output_cases", + required: 0, + }, + { + actual: 3, + comparator: "eq", + gate: "detection_failed_probes", + required: 0, + }, + { actual: 4, comparator: "eq", gate: "evidence_failed", required: 0 }, + { + actual: 5, + comparator: "eq", + gate: "evidence_truncated", + required: 0, + }, + { + actual: 101, + comparator: "lte", + gate: "visible_words_max", + required: 100, + }, + { + actual: null, + comparator: "eq", + gate: "repeat_agreement", + required: 1, + }, + ]); + }); +}); describe("production shadow attempt deadline", () => { test("aborts timed-out work with the same generic error", async () => { @@ -61,3 +526,71 @@ describe("production shadow attempt deadline", () => { expect(completedSignal?.aborted).toBe(false); }); }); + +describe("production shadow validation errors", () => { + test("projects a bounded sanitized invalid-output summary", () => { + const summary = summarizeValidationErrors( + [ + "First validation error for secret-token", + "Second validation error", + "Third validation error", + "Fourth validation error", + "Fifth validation error", + "This sixth error must be omitted", + ], + ["secret-token"] + ); + + expect(summary).toContain("[entity]"); + expect(summary).not.toContain("secret-token"); + expect(summary).not.toContain("sixth"); + expect(summary?.length).toBeLessThanOrEqual(300); + }); + + test("sanitizes whole tokens and counts exactly the rendered fields", () => { + const insight: GeneratedInsight = { + changePercent: 300, + confidence: 0.65, + description: "Failure remains for AI.", + evidence: [ + { + description: + "xAI:checkoutx and AI:checkout affected 21 sessions while Failure stayed isolated.", + type: "error", + }, + ], + impactSummary: "Thirty additional failures were measured.", + metrics: [ + { current: 40, format: "number", label: "Errors", previous: 10 }, + ], + priority: 9, + rootCause: "Failure originated in AI checkout handling.", + sentiment: "negative", + severity: "critical", + sources: ["ops"], + subjectKey: "error_count", + suggestion: "Inspect AI checkout handling immediately.", + title: "AI checkout failures increased sharply this week", + type: "error_spike", + }; + + const rendered = renderShadowInsight(insight, true, ["AI", "AI:checkout"]); + const visibleText = [ + rendered.title, + rendered.description, + rendered.impact, + rendered.rootCause, + ...rendered.evidence, + rendered.suggestion, + ] + .filter((value): value is string => Boolean(value)) + .join(" "); + + expect(rendered.title).toBe("Investigate [error]"); + expect(rendered.description).toBe("Failure remains for [entity]."); + expect(rendered.rootCause).toContain("Failure originated"); + expect(rendered.rootCause).not.toContain("F[entity]lure"); + expect(rendered.evidence[0]).toContain("xAI:checkoutx and [entity]"); + expect(rendered.visibleWords).toBe(visibleText.split(/\s+/).length); + }); +}); diff --git a/packages/evals/src/insight-production-shadow.ts b/packages/evals/src/insight-production-shadow.ts index 879ca8c58..5b15defb6 100644 --- a/packages/evals/src/insight-production-shadow.ts +++ b/packages/evals/src/insight-production-shadow.ts @@ -1,5 +1,10 @@ -import { resolve } from "node:path"; import { writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import type { + InsightEvidenceReadRequest, + ProductMetricsFetcher, +} from "@databuddy/ai/insights/evidence-reader"; +import type { GeneratedInsight } from "@databuddy/shared/insights"; import { Pool, type PoolClient } from "pg"; import type { InvestigationSources, @@ -7,9 +12,11 @@ import type { } from "../../../apps/insights/src/generation"; import type { FunnelDef, + FunnelGoalDeps, GoalDef, } from "../../../apps/insights/src/funnel-detection"; import type { InvestigationAnnotation } from "../../../apps/insights/src/investigation"; +import type { LatestInsightObservation } from "../../../apps/insights/src/observations"; const REQUIRED_CONFIRMATION = "--confirm-read-only-production"; const DEFAULT_OFFSETS = [60, 30, 7, 0]; @@ -18,6 +25,27 @@ const DEFAULT_CONCURRENCY = 2; const STATEMENT_TIMEOUT_MS = 60_000; const CASE_ATTEMPT_TIMEOUT_MS = 150_000; const WHITESPACE_PATTERN = /\s+/; +const LOCAL_SHADOW_REDIS_URL = "redis://127.0.0.1:6379/15"; +const LOCAL_SHADOW_POSTGRES_GUARD_URL = + "postgresql://shadow_guard:shadow_guard@127.0.0.1:1/shadow_guard?sslmode=disable"; +const SHADOW_DB_CONNECTION_TIMEOUT_MS = "250"; +const SECRET_TOKEN_CHARACTER_PATTERN = /^[\p{L}\p{N}_-]$/u; +const SECRET_TOKEN_BOUNDARY = "[\\p{L}\\p{N}_-]"; +const UNRESOLVED_IDENTIFIER_PATTERN = + /^Identifier [`']([A-Za-z_][A-Za-z0-9_.]*)[`'] cannot be resolved from table with name ([A-Za-z_][A-Za-z0-9_]*)\./; +const CLICKHOUSE_OPS_QUERIES = new Set([ + "errors_summary", + "errors_by_page", + "error_fingerprints", + "uptime_summary", +]); + +export const HISTORICAL_LIMITATIONS = [ + "Definitions are limited to records that are active and undeleted now; inactive, deleted, and pre-edit revisions cannot be reconstructed.", + "Configuration, website metadata, and annotation edits are not historized; the replay uses the current surviving records conservatively.", + "Persisted observations respect their creation timestamps, but warehouse queries read currently retained data for past windows, so late-arriving or deleted warehouse facts and what was known at the original run time cannot be reconstructed.", + "Persisted observations seed lifecycle state and replayed observations are appended in memory only; production state is never changed.", +] as const; interface CliOptions { concurrency: number; @@ -53,10 +81,175 @@ interface AnnotationRow { xValue: Date; } +export interface ShadowObservation extends LatestInsightObservation { + createdAt: Date; + organizationId: string; + signalKey: string; + websiteId: string; +} + +interface ShadowMetadata { + annotations: AnnotationRow[]; + funnels: FunnelRow[]; + goals: GoalRow[]; + observations: ShadowObservation[]; + sites: RankedWebsite[]; +} + +export function withDefinitionSnapshot( + base: FunnelGoalDeps, + funnels: FunnelDef[], + goals: GoalDef[] +): FunnelGoalDeps { + const snapshot = { + ...base, + fetchFunnels: async () => funnels, + fetchGoals: async () => goals, + }; + snapshot.fetchDefinitionWindow = undefined; + return snapshot; +} + +function percentage(part: number, total: number): number { + return total > 0 ? Math.round((part / total) * 10_000) / 100 : 0; +} + +export function createSnapshotProductMetricsFetcher( + deps: Pick, + funnels: FunnelDef[], + goals: GoalDef[] +): ProductMetricsFetcher { + const funnelsById = new Map(funnels.map((funnel) => [funnel.id, funnel])); + const goalsById = new Map(goals.map((goal) => [goal.id, goal])); + + return async (_appContext, range, target, abortSignal) => { + if (target.type === "goal") { + const goal = goalsById.get(target.id); + if (!goal) { + throw new Error( + `Goal ${target.id} is absent from the definition snapshot` + ); + } + const conversion = await deps.goalConversion(goal, range, abortSignal); + return { + results: [ + { + type: "goals_summary", + count: 1, + goals: [ + { + id: goal.id, + is_active: true, + name: goal.name, + type: goal.type, + target: goal.target, + definition_updated_at: goal.updatedAt.toISOString(), + overall_conversion_rate: conversion.rate, + total_users_entered: conversion.entrants, + total_users_completed: conversion.completions, + error_context_available: false, + error_rate: 0, + }, + ], + }, + ], + }; + } + + if (target.type === "funnel") { + const funnel = funnelsById.get(target.id); + if (!funnel) { + throw new Error( + `Funnel ${target.id} is absent from the definition snapshot` + ); + } + const conversion = await deps.funnelConversion( + funnel, + range, + abortSignal + ); + const usersByStep = new Map( + conversion.steps.map((step) => [step.stepNumber, step.users]) + ); + const steps = funnel.steps.map((step, index) => ({ + step_number: index + 1, + name: step.name, + target: step.target, + type: step.type, + users: usersByStep.get(index + 1) ?? 0, + })); + const dropoffs = steps.slice(1).map((step, index) => { + const previousUsers = steps[index]?.users ?? 0; + return { + rate: percentage(previousUsers - step.users, previousUsers), + step: step.step_number, + }; + }); + const biggestDropoff = dropoffs.reduce< + { rate: number; step: number } | undefined + >( + (current, item) => + !current || item.rate > current.rate ? item : current, + undefined + ); + + return { + results: [ + { + type: "funnels_summary", + count: 1, + funnels: [ + { + id: funnel.id, + is_active: true, + name: funnel.name, + definition_updated_at: funnel.updatedAt.toISOString(), + steps, + overall_conversion_rate: conversion.rate, + total_users_entered: conversion.entrants, + total_users_completed: conversion.completions, + biggest_dropoff_step: biggestDropoff?.step ?? 1, + biggest_dropoff_rate: biggestDropoff?.rate ?? 0, + error_context_available: false, + error_correlation_rate: 0, + }, + ], + }, + ], + }; + } + + throw new Error("Snapshot product metrics support goals and funnels only"); + }; +} + +export function assertShadowEvidenceUsesClickHouse( + request: InsightEvidenceReadRequest +): void { + if (request.name !== "ops_context") { + return; + } + const nonClickHouseQuery = request.input.queries.find( + (query) => !CLICKHOUSE_OPS_QUERIES.has(query.type) + ); + if (nonClickHouseQuery) { + throw new Error( + `Production shadow blocks non-ClickHouse ops evidence: ${nonClickHouseQuery.type}` + ); + } +} + interface ShadowCase { caseId: string; detectedSignalCount: number; detectionComplete: boolean; + detectionIssues: { + definitionFailureSummaries: string[]; + failedDefinitions: number; + failedMetricFamilies: number; + nonComparableDefinitions: number; + rotatedDefinitions: number; + }; disposition: string | null; durationMs: number; engineId: string; @@ -80,6 +273,7 @@ interface ShadowCase { }; offsetDays: number; repeatDifferences: string[]; + repeatDurationMs: number | null; repeatEqual: boolean | null; selectedSignal: null | { changePercent: number | null; @@ -95,27 +289,63 @@ interface ShadowCase { status: string; } +export interface ShadowGateMetrics { + detectionFailures: number; + detectionIncomplete: number; + evidenceFailed: number; + evidenceTruncated: number; + repeatAgreement: number | null; + status: Readonly>; + visibleWords: { max: number }; +} + +export interface ShadowGateFailure { + actual: number | null; + comparator: "eq" | "lte"; + gate: + | "detection_failed_probes" + | "error_cases" + | "evidence_failed" + | "evidence_truncated" + | "invalid_output_cases" + | "repeat_agreement" + | "visible_words_max"; + required: number; +} + interface ShadowReport { aggregate: { cases: number; cards: number; + detectionFailures: number; detectionIncomplete: number; + nonComparableDefinitions: number; + rotatedDefinitions: number; dispositions: Record; durationsMs: { p50: number; p95: number }; evidenceFailed: number; evidenceTruncated: number; metricFamilies: Record; repeatAgreement: number | null; + repeatDurationsMs: { p50: number; p95: number } | null; severity: Record; status: Record; visibleWords: { max: number; p50: number; p95: number }; }; cases: ShadowCase[]; + gateFailures: ShadowGateFailure[]; meta: { concurrency: number; engine: "current deterministic production path"; generatedAt: string; - history: "disabled"; + historicalLimitations: readonly string[]; + history: "persisted-seed/in-memory-replay/current-records"; + isolation: { + clickhouse: "read-only-user/verified"; + postgres: "explicit-read-only-transaction/process-quarantined"; + productEvidence: "definition-snapshot/direct-clickhouse"; + redis: "loopback-db-15"; + }; minEvents: number; offsets: number[]; productionWrites: false; @@ -186,6 +416,51 @@ function disableExternalEffects(): void { } } +function postgresUrl(databaseUrl: string): string { + let url: URL; + try { + url = new URL(databaseUrl); + } catch { + throw new Error("DATABASE_URL must be a valid PostgreSQL URL"); + } + if (url.protocol !== "postgres:" && url.protocol !== "postgresql:") { + throw new Error("DATABASE_URL must use the postgres protocol"); + } + return url.toString(); +} + +export function quarantineShadowPostgresEnvironment(): void { + process.env.DATABASE_URL = LOCAL_SHADOW_POSTGRES_GUARD_URL; + process.env.DB_CONNECTION_TIMEOUT_MS = SHADOW_DB_CONNECTION_TIMEOUT_MS; + delete process.env.PGOPTIONS; +} + +function createReadOnlyMetadataLoader(): ( + ranked: Array<{ events: number; id: string }> +) => Promise { + const productionUrl = process.env.DATABASE_URL; + if (!productionUrl) { + throw new Error("DATABASE_URL is required"); + } + const readOnlyProductionUrl = postgresUrl(productionUrl); + quarantineShadowPostgresEnvironment(); + + let used = false; + return (ranked) => { + if (used) { + throw new Error("Production shadow metadata may only be loaded once"); + } + used = true; + return loadMetadata(readOnlyProductionUrl, ranked); + }; +} + +export function configureIsolatedShadowRedis(): void { + process.env.REDIS_URL = LOCAL_SHADOW_REDIS_URL; + process.env.BULLMQ_REDIS_URL = LOCAL_SHADOW_REDIS_URL; + process.env.INSIGHTS_BULLMQ_REDIS_URL = LOCAL_SHADOW_REDIS_URL; +} + function configureReadOnlyClickHouse(): void { const readonlyUrl = process.env.CLICKHOUSE_READONLY_URL; if (!readonlyUrl) { @@ -226,31 +501,42 @@ function safeTimezone(value: unknown): string { } async function inReadOnlyTransaction( + databaseUrl: string, work: (client: PoolClient) => Promise ): Promise { - if (!process.env.DATABASE_URL) { - throw new Error("DATABASE_URL is required"); - } const pool = new Pool({ application_name: "databuddy_insights_shadow_readonly", - connectionString: process.env.DATABASE_URL, + connectionString: databaseUrl, + connectionTimeoutMillis: 10_000, max: 1, }); - const client = await pool.connect(); + let client: PoolClient | undefined; try { + client = await pool.connect(); + // Some managed Postgres proxies reject startup `options`. Make the private + // one-shot session read-only before beginning the only transaction instead. + await client.query("SET default_transaction_read_only = on"); + const defaultMode = await client.query<{ + default_transaction_read_only: string; + }>("SHOW default_transaction_read_only"); + if (defaultMode.rows[0]?.default_transaction_read_only !== "on") { + throw new Error("Postgres session is not read-only"); + } await client.query("BEGIN TRANSACTION READ ONLY"); await client.query(`SET LOCAL statement_timeout = ${STATEMENT_TIMEOUT_MS}`); await client.query("SET LOCAL lock_timeout = 1000"); - const mode = await client.query<{ transaction_read_only: string }>( - "SHOW transaction_read_only" - ); - if (mode.rows[0]?.transaction_read_only !== "on") { + const transactionMode = await client.query<{ + transaction_read_only: string; + }>("SHOW transaction_read_only"); + if (transactionMode.rows[0]?.transaction_read_only !== "on") { throw new Error("Postgres transaction is not read-only"); } return await work(client); } finally { - await client.query("ROLLBACK").catch(() => undefined); - client.release(); + if (client) { + await client.query("ROLLBACK").catch(() => undefined); + client.release(); + } await pool.end(); } } @@ -281,21 +567,20 @@ async function loadCohort( return rows.map((row) => ({ events: Number(row.events), id: row.id })); } -function loadMetadata(ranked: Array<{ events: number; id: string }>): Promise<{ - annotations: AnnotationRow[]; - funnels: FunnelRow[]; - goals: GoalRow[]; - sites: RankedWebsite[]; -}> { +function loadMetadata( + databaseUrl: string, + ranked: Array<{ events: number; id: string }> +): Promise { if (ranked.length === 0) { return Promise.resolve({ annotations: [], funnels: [], goals: [], + observations: [], sites: [], }); } - return inReadOnlyTransaction(async (client) => { + return inReadOnlyTransaction(databaseUrl, async (client) => { const ids = ranked.map((item) => item.id); const siteResult = await client.query<{ domain: string; @@ -307,6 +592,7 @@ function loadMetadata(ranked: Array<{ events: number; id: string }>): Promise<{ FROM websites w LEFT JOIN insight_generation_configs c ON c.organization_id = w.organization_id + AND to_jsonb(c) ->> 'website_id' IS NULL WHERE w.id = ANY($1::text[]) AND w."deletedAt" IS NULL`, [ids] @@ -358,6 +644,24 @@ function loadMetadata(ranked: Array<{ events: number; id: string }>): Promise<{ WHERE website_id = ANY($1::text[])`, [ids] ); + const observationResult = await client.query<{ + as_of: Date; + created_at: Date; + decision: LatestInsightObservation["decision"]; + evidence: LatestInsightObservation["evidence"]; + organization_id: string; + recheck_at: Date; + signal: LatestInsightObservation["signal"]; + signal_key: string; + website_id: string; + }>( + `SELECT organization_id, website_id, signal_key, as_of, + decision, evidence, signal, recheck_at, created_at + FROM insight_observations + WHERE website_id = ANY($1::text[]) + ORDER BY website_id, signal_key, as_of, created_at`, + [ids] + ); const rank = new Map(ranked.map((item, index) => [item.id, index])); const events = new Map(ranked.map((item) => [item.id, item.events])); const sites = siteResult.rows @@ -390,6 +694,17 @@ function loadMetadata(ranked: Array<{ events: number; id: string }>): Promise<{ updatedAt: row.updated_at, websiteId: row.website_id, })), + observations: observationResult.rows.map((row) => ({ + asOf: row.as_of, + createdAt: row.created_at, + decision: row.decision, + evidence: row.evidence, + organizationId: row.organization_id, + recheckAt: row.recheck_at, + signal: row.signal, + signalKey: row.signal_key, + websiteId: row.website_id, + })), annotations: annotationResult.rows.map((row) => ({ createdAt: row.created_at, deletedAt: row.deleted_at, @@ -402,15 +717,67 @@ function loadMetadata(ranked: Array<{ events: number; id: string }>): Promise<{ }); } -function dateAtOffset(offsetDays: number): string { - const now = new Date(); +function dateAtOffset(offsetDays: number, evaluationDate: Date): string { const date = new Date( - Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()) - + Date.UTC( + evaluationDate.getUTCFullYear(), + evaluationDate.getUTCMonth(), + evaluationDate.getUTCDate() + ) - offsetDays * 86_400_000 ); return date.toISOString().slice(0, 10); } +export function chronologicalOffsets(offsets: number[]): number[] { + return [...offsets].sort((left, right) => right - left); +} + +export function observationsAt(params: { + asOf: Date; + organizationId: string; + persisted: ShadowObservation[]; + signalKeys: string[]; + simulated: ReadonlyMap; + websiteId: string; +}): Map { + const requested = new Set(params.signalKeys); + const persisted = new Map(); + // A historical replay can use a backfill only after that observation existed. + for (const observation of params.persisted) { + if ( + observation.organizationId !== params.organizationId || + observation.websiteId !== params.websiteId || + !requested.has(observation.signalKey) || + observation.asOf > params.asOf || + observation.createdAt > params.asOf + ) { + continue; + } + const current = persisted.get(observation.signalKey); + if ( + !current || + observation.asOf > current.asOf || + (observation.asOf.getTime() === current.asOf.getTime() && + observation.createdAt > current.createdAt) + ) { + persisted.set(observation.signalKey, observation); + } + } + + const result = new Map(persisted); + for (const [signalKey, observation] of params.simulated) { + if (!requested.has(signalKey) || observation.asOf > params.asOf) { + continue; + } + const current = result.get(signalKey); + if (!current || observation.asOf >= current.asOf) { + result.set(signalKey, observation); + } + } + return result; +} + function definitionsAt( rows: T[], asOf: Date @@ -425,6 +792,7 @@ async function createSources(params: { asOf: Date; funnels: FunnelRow[]; goals: GoalRow[]; + loadObservations: InvestigationSources["loadObservations"]; site: RankedWebsite; attemptSignal: AbortSignal; }): Promise { @@ -453,31 +821,42 @@ async function createSources(params: { signal ? AbortSignal.any([params.attemptSignal, signal]) : params.attemptSignal; + const base = defaultFunnelGoalDeps(params.site.id, params.asOf); + const snapshotDeps = withDefinitionSnapshot( + { + ...base, + funnelConversion: (funnel, range, signal) => + base.funnelConversion(funnel, range, withAttemptSignal(signal)), + goalConversion: (goal, range, signal) => + base.goalConversion(goal, range, withAttemptSignal(signal)), + }, + siteFunnels, + siteGoals + ); + const fetchSnapshotProductMetrics = createSnapshotProductMetricsFetcher( + snapshotDeps, + siteFunnels, + siteGoals + ); return { createEvidenceReader: (readerParams) => { - const readEvidence = createInsightEvidenceReader(readerParams); - return Promise.resolve((request, appContext, signal) => - readEvidence(request, appContext, withAttemptSignal(signal)) - ); + const usesDefinitionSnapshot = + readerParams.signal.entity.type === "goal" || + readerParams.signal.entity.type === "funnel"; + const readEvidence = createInsightEvidenceReader({ + ...readerParams, + ...(usesDefinitionSnapshot + ? { fetchProductMetrics: fetchSnapshotProductMetrics } + : {}), + }); + return Promise.resolve((request, appContext, signal) => { + assertShadowEvidenceUsesClickHouse(request); + return readEvidence(request, appContext, withAttemptSignal(signal)); + }); }, createServiceAuth: () => Promise.resolve(undefined), - detectDefinitionSignals: (detectParams, today, _deps, options) => { - const base = defaultFunnelGoalDeps(params.site.id, params.asOf); - return detectFunnelGoalSignals( - detectParams, - today, - { - ...base, - fetchFunnels: async () => siteFunnels, - fetchGoals: async () => siteGoals, - funnelConversion: (funnel, range, signal) => - base.funnelConversion(funnel, range, withAttemptSignal(signal)), - goalConversion: (goal, range, signal) => - base.goalConversion(goal, range, withAttemptSignal(signal)), - }, - options - ); - }, + detectDefinitionSignals: (detectParams, today, _deps, options) => + detectFunnelGoalSignals(detectParams, today, snapshotDeps, options), detectMetricSignals: (detectParams, queryFn, today, signal, diagnostics) => detectSignals( detectParams, @@ -519,7 +898,7 @@ async function createSources(params: { timezone, withAttemptSignal(signal) ), - loadObservations: () => Promise.resolve(new Map()), + loadObservations: params.loadObservations, }; } @@ -584,12 +963,23 @@ function escapeRegExp(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } +function secretPattern(secret: string): RegExp { + const escaped = escapeRegExp(secret); + const prefix = SECRET_TOKEN_CHARACTER_PATTERN.test(secret[0] ?? "") + ? `(? item.length >= 2) .sort((a, b) => b.length - a.length)) { - output = output.replace(new RegExp(escapeRegExp(secret), "gi"), "[entity]"); + output = output.replace(secretPattern(secret), "[entity]"); } return output .replace(/https?:\/\/[^\s"'“”]+/gi, "[url]") @@ -600,6 +990,61 @@ function sanitizeText(value: string, secrets: string[]): string { .replace(/\/(?:[^\s.,;:!?()[\]{}]+\/)*[^\s.,;:!?()[\]{}]*/g, "[path]"); } +function sanitizeDefinitionFailure(value: string, secrets: string[]): string { + const identifier = value.match(UNRESOLVED_IDENTIFIER_PATTERN); + if (identifier) { + return `Unknown identifier ${identifier[1]} in ${identifier[2]}.`; + } + return sanitizeText(value, secrets).slice(0, 300); +} + +export function summarizeValidationErrors( + errors: string[] | undefined, + secrets: string[] = [] +): string | null { + if (!errors?.length) { + return null; + } + return sanitizeText(errors.slice(0, 5).join("; "), secrets).slice(0, 300); +} + +export function renderShadowInsight( + insight: GeneratedInsight, + isError: boolean, + secrets: string[] +): NonNullable { + const evidence = (insight.evidence ?? []).map((item) => + sanitizeText(item.description, secrets) + ); + const rendered = { + description: sanitizeText(insight.description, secrets), + evidence, + impact: insight.impactSummary + ? sanitizeText(insight.impactSummary, secrets) + : null, + rootCause: insight.rootCause + ? sanitizeText(insight.rootCause, secrets) + : null, + suggestion: isError + ? "No patch target is established yet. Reproduce [error] and trace its first application frame." + : sanitizeText(insight.suggestion, secrets), + title: isError + ? "Investigate [error]" + : sanitizeText(insight.title, secrets), + }; + return { + ...rendered, + visibleWords: wordCount([ + rendered.title, + rendered.description, + rendered.impact, + rendered.rootCause, + ...rendered.evidence, + rendered.suggestion, + ]), + }; +} + function safeQueryType(value: string): string { if (value.startsWith("detector:goal:")) { return "detector:goal"; @@ -641,6 +1086,7 @@ function semanticProjection(artifact: WebsiteInvestigationArtifact): unknown { insight: artifact.insight, signal: artifact.signal, status: artifact.status, + validationErrors: artifact.validationErrors, }; } @@ -661,21 +1107,38 @@ function projectCase(params: { durationMs: number; offsetDays: number; repeatDifferences: string[]; + repeatDurationMs: number | null; repeatEqual: boolean | null; secrets: string[]; }): ShadowCase { const { artifact } = params; + const recoveryCoverage = artifact.recoveryCoverage; + const activeDefinitionKeys = new Set( + recoveryCoverage?.activeDefinitionKeys ?? [] + ); + const eligibleDefinitionKeys = new Set( + recoveryCoverage?.eligibleDefinitionKeys ?? [] + ); const evidenceStatuses = countBy( artifact.evidence.map((item) => item.status) ); const insight = artifact.insight; const isError = artifact.signal?.entity.type === "error"; - const visibleEvidence = - insight?.evidence?.map((item) => item.description) ?? []; return { caseId: params.caseId, detectionComplete: artifact.detectionComplete, detectedSignalCount: artifact.detectedSignals.length, + detectionIssues: { + definitionFailureSummaries: [ + ...new Set(recoveryCoverage?.definitionFailureMessages ?? []), + ].map((message) => sanitizeDefinitionFailure(message, params.secrets)), + failedDefinitions: recoveryCoverage?.failedDefinitions ?? 0, + failedMetricFamilies: recoveryCoverage?.failedMetricFamilies ?? 0, + nonComparableDefinitions: [...activeDefinitionKeys].filter( + (key) => !eligibleDefinitionKeys.has(key) + ).length, + rotatedDefinitions: recoveryCoverage?.rotatedDefinitions ?? 0, + }, disposition: artifact.decision?.disposition ?? null, durationMs: params.durationMs, engineId: artifact.engineId, @@ -688,38 +1151,17 @@ function projectCase(params: { total: artifact.evidence.length, truncated: evidenceStatuses.truncated ?? 0, }, - errorType: null, - errorSummary: null, + errorType: artifact.validationErrors?.length ? "ValidationError" : null, + errorSummary: summarizeValidationErrors( + artifact.validationErrors, + params.secrets + ), insight: insight - ? { - description: sanitizeText(insight.description, params.secrets), - evidence: visibleEvidence.map((value) => - sanitizeText(value, params.secrets) - ), - impact: insight.impactSummary - ? sanitizeText(insight.impactSummary, params.secrets) - : null, - rootCause: insight.rootCause - ? sanitizeText(insight.rootCause, params.secrets) - : null, - suggestion: isError - ? "No patch target is established yet. Reproduce [error] and trace its first application frame." - : sanitizeText(insight.suggestion, params.secrets), - title: isError - ? "Investigate [error]" - : sanitizeText(insight.title, params.secrets), - visibleWords: wordCount([ - insight.title, - insight.description, - insight.impactSummary, - insight.rootCause, - ...visibleEvidence, - insight.suggestion, - ]), - } + ? renderShadowInsight(insight, isError, params.secrets) : null, offsetDays: params.offsetDays, repeatDifferences: params.repeatDifferences, + repeatDurationMs: params.repeatDurationMs, repeatEqual: params.repeatEqual, selectedSignal: artifact.signal ? { @@ -749,6 +1191,13 @@ function failedCase(params: { caseId: params.caseId, detectionComplete: false, detectedSignalCount: 0, + detectionIssues: { + definitionFailureSummaries: [], + failedDefinitions: 0, + failedMetricFamilies: 0, + nonComparableDefinitions: 0, + rotatedDefinitions: 0, + }, disposition: null, durationMs: params.durationMs, engineId: "deterministic/v1", @@ -764,6 +1213,7 @@ function failedCase(params: { insight: null, offsetDays: params.offsetDays, repeatDifferences: [], + repeatDurationMs: null, repeatEqual: null, selectedSignal: null, status: "error", @@ -794,10 +1244,28 @@ function aggregateCases(cases: ShadowCase[]): ShadowReport["aggregate"] { item.insight ? [item.insight.visibleWords] : [] ); const repeats = cases.filter((item) => item.repeatEqual !== null); + const repeatDurations = cases.flatMap((item) => + item.repeatDurationMs === null ? [] : [item.repeatDurationMs] + ); return { cases: cases.length, cards: words.length, + detectionFailures: cases.reduce( + (sum, item) => + sum + + item.detectionIssues.failedDefinitions + + item.detectionIssues.failedMetricFamilies, + 0 + ), detectionIncomplete: cases.filter((item) => !item.detectionComplete).length, + nonComparableDefinitions: cases.reduce( + (sum, item) => sum + item.detectionIssues.nonComparableDefinitions, + 0 + ), + rotatedDefinitions: cases.reduce( + (sum, item) => sum + item.detectionIssues.rotatedDefinitions, + 0 + ), dispositions: countBy(cases.map((item) => item.disposition)), durationsMs: { p50: percentile( @@ -821,6 +1289,13 @@ function aggregateCases(cases: ShadowCase[]): ShadowReport["aggregate"] { repeats.length === 0 ? null : repeats.filter((item) => item.repeatEqual).length / repeats.length, + repeatDurationsMs: + repeatDurations.length === 0 + ? null + : { + p50: percentile(repeatDurations, 0.5), + p95: percentile(repeatDurations, 0.95), + }, severity: countBy( cases.map((item) => item.selectedSignal?.severity ?? null) ), @@ -833,6 +1308,67 @@ function aggregateCases(cases: ShadowCase[]): ShadowReport["aggregate"] { }; } +export function evaluateShadowGates( + metrics: ShadowGateMetrics, + repeat: boolean +): ShadowGateFailure[] { + const checks: ShadowGateFailure[] = [ + { + actual: metrics.status.error ?? 0, + comparator: "eq", + gate: "error_cases", + required: 0, + }, + { + actual: metrics.status.invalid_output ?? 0, + comparator: "eq", + gate: "invalid_output_cases", + required: 0, + }, + { + actual: metrics.detectionFailures, + comparator: "eq", + gate: "detection_failed_probes", + required: 0, + }, + { + actual: metrics.evidenceFailed, + comparator: "eq", + gate: "evidence_failed", + required: 0, + }, + { + actual: metrics.evidenceTruncated, + comparator: "eq", + gate: "evidence_truncated", + required: 0, + }, + { + actual: metrics.visibleWords.max, + comparator: "lte", + gate: "visible_words_max", + required: 100, + }, + ]; + if (repeat) { + checks.push({ + actual: metrics.repeatAgreement, + comparator: "eq", + gate: "repeat_agreement", + required: 1, + }); + } + + return checks.filter((check) => { + if (check.actual === null) { + return true; + } + return check.comparator === "eq" + ? check.actual !== check.required + : check.actual > check.required; + }); +} + function assertOutsideRepository(output: string): string { const absolute = resolve(output); const repository = resolve(import.meta.dir, "../../.."); @@ -843,37 +1379,83 @@ function assertOutsideRepository(output: string): string { } async function closeShadowConnections(): Promise { - const [{ clickHouse }, { shutdownRedis }] = await Promise.all([ + const [ + { clickHouse }, + { shutdownPostgres }, + { shutdownRedis }, + { closeInsightsQueue }, + ] = await Promise.all([ import("@databuddy/db/clickhouse"), + import("@databuddy/db"), import("../../../packages/redis/redis"), + import("../../../packages/redis/insights-queue"), + ]); + await Promise.allSettled([ + clickHouse.close(), + shutdownPostgres(), + shutdownRedis(), + closeInsightsQueue(), ]); - await Promise.allSettled([clickHouse.close(), shutdownRedis()]); } export async function runProductionShadow( options: CliOptions ): Promise { disableExternalEffects(); + const loadProductionMetadata = createReadOnlyMetadataLoader(); + configureIsolatedShadowRedis(); configureReadOnlyClickHouse(); const restoreConsole = silenceLibraryConsole(); try { const ranked = await loadCohort(options.minEvents, options.limit); - const metadata = await loadMetadata(ranked); - const jobs = metadata.sites.flatMap((site, siteIndex) => - options.offsets.map((offsetDays) => ({ offsetDays, site, siteIndex })) - ); - const { investigateWebsiteWithSources } = await import( - "../../../apps/insights/src/generation" - ); - const cases = await mapConcurrent( - jobs, + const metadata = await loadProductionMetadata(ranked); + const [{ investigateWebsiteWithSources }, { nextRecheckAt }] = + await Promise.all([ + import("../../../apps/insights/src/generation"), + import("../../../apps/insights/src/observations"), + ]); + const offsets = chronologicalOffsets(options.offsets); + const evaluationDate = new Date(); + const siteCases = await mapConcurrent( + metadata.sites, options.concurrency, - async ({ offsetDays, site, siteIndex }) => { - const caseId = `site-${String(siteIndex + 1).padStart(2, "0")}@d-${offsetDays}`; - const asOfString = dateAtOffset(offsetDays); - const asOf = new Date(`${asOfString}T23:59:59.999Z`); - const startedAt = Date.now(); - try { + async (site, siteIndex) => { + const cases: ShadowCase[] = []; + const simulated = new Map(); + const persisted = metadata.observations.filter( + (observation) => + observation.organizationId === site.organizationId && + observation.websiteId === site.id + ); + const siteDefinitions = [ + ...metadata.funnels.filter((row) => row.websiteId === site.id), + ...metadata.goals.filter((row) => row.websiteId === site.id), + ]; + const siteSecrets = [ + site.id, + site.domain, + site.organizationId, + ...siteDefinitions.flatMap((definition) => [ + definition.id, + definition.name, + ]), + ]; + + for (const offsetDays of offsets) { + const caseId = `site-${String(siteIndex + 1).padStart(2, "0")}@d-${offsetDays}`; + const asOfString = dateAtOffset(offsetDays, evaluationDate); + const asOf = new Date(`${asOfString}T23:59:59.999Z`); + const preAppendSimulated = new Map(simulated); + const loadObservations: InvestigationSources["loadObservations"] = ( + params + ) => + Promise.resolve( + observationsAt({ + ...params, + persisted, + simulated: preAppendSimulated, + }) + ); const input = { asOf: asOfString, domain: site.domain, @@ -889,69 +1471,123 @@ export async function runProductionShadow( attemptSignal, funnels: metadata.funnels, goals: metadata.goals, + loadObservations, site, }); return investigateWebsiteWithSources(input, sources); }); - const artifact = await investigate(); - let repeatEqual: boolean | null = null; - let repeatDifferences: string[] = []; - if (options.repeat) { - const repeated = await investigate(); - repeatDifferences = semanticDifferences(artifact, repeated); - repeatEqual = repeatDifferences.length === 0; + + const firstStartedAt = Date.now(); + let artifact: WebsiteInvestigationArtifact; + try { + artifact = await investigate(); + } catch (error) { + cases.push( + failedCase({ + caseId, + durationMs: Date.now() - firstStartedAt, + error, + offsetDays, + secrets: siteSecrets, + }) + ); + continue; + } + const durationMs = Date.now() - firstStartedAt; + + try { + let repeatDifferences: string[] = []; + let repeatDurationMs: number | null = null; + let repeatEqual: boolean | null = null; + if (options.repeat) { + const repeatStartedAt = Date.now(); + try { + const repeated = await investigate(); + repeatDifferences = semanticDifferences(artifact, repeated); + repeatEqual = repeatDifferences.length === 0; + } catch (error) { + const type = + error instanceof Error + ? error.constructor.name + : typeof error; + repeatDifferences = [`repeat_failed:${type}`]; + repeatEqual = false; + } finally { + repeatDurationMs = Date.now() - repeatStartedAt; + } + } + + const { decision, signal } = artifact; + if (decision && signal) { + const observationAsOf = new Date(artifact.asOf); + simulated.set(signal.signalKey, { + asOf: observationAsOf, + decision, + evidence: artifact.evidence.filter( + (evidence) => evidence.signalKey === signal.signalKey + ), + recheckAt: nextRecheckAt( + observationAsOf, + decision.disposition, + signal + ), + signal, + }); + } + + cases.push( + projectCase({ + artifact, + caseId, + durationMs, + offsetDays, + repeatDifferences, + repeatDurationMs, + repeatEqual, + secrets: [ + ...siteSecrets, + artifact.signal?.entity.id ?? "", + artifact.signal?.entity.label ?? "", + artifact.signal?.expectation?.eventName ?? "", + artifact.signal?.expectation?.stepName ?? "", + ], + }) + ); + } catch (error) { + cases.push( + failedCase({ + caseId, + durationMs, + error, + offsetDays, + secrets: siteSecrets, + }) + ); } - const secrets = [ - site.id, - site.domain, - site.organizationId, - artifact.signal?.entity.id ?? "", - artifact.signal?.entity.label ?? "", - artifact.signal?.expectation?.eventName ?? "", - artifact.signal?.expectation?.stepName ?? "", - ]; - return projectCase({ - artifact, - caseId, - durationMs: Date.now() - startedAt, - offsetDays, - repeatDifferences, - repeatEqual, - secrets, - }); - } catch (error) { - const siteDefinitions = [ - ...metadata.funnels.filter((row) => row.websiteId === site.id), - ...metadata.goals.filter((row) => row.websiteId === site.id), - ]; - return failedCase({ - caseId, - durationMs: Date.now() - startedAt, - error, - offsetDays, - secrets: [ - site.id, - site.domain, - site.organizationId, - ...siteDefinitions.flatMap((definition) => [ - definition.id, - definition.name, - ]), - ], - }); } + return cases; } ); + const cases = siteCases.flat(); + const aggregate = aggregateCases(cases); return { - aggregate: aggregateCases(cases), + aggregate, cases, + gateFailures: evaluateShadowGates(aggregate, options.repeat), meta: { concurrency: options.concurrency, engine: "current deterministic production path", generatedAt: new Date().toISOString(), - history: "disabled", + historicalLimitations: HISTORICAL_LIMITATIONS, + history: "persisted-seed/in-memory-replay/current-records", + isolation: { + clickhouse: "read-only-user/verified", + postgres: "explicit-read-only-transaction/process-quarantined", + productEvidence: "definition-snapshot/direct-clickhouse", + redis: "loopback-db-15", + }, minEvents: options.minEvents, - offsets: options.offsets, + offsets, productionWrites: false, repeat: options.repeat, sites: metadata.sites.length, @@ -971,9 +1607,17 @@ if (import.meta.main) { await writeFile(output, `${JSON.stringify(result, null, 2)}\n`); } process.stdout.write( - `${JSON.stringify({ aggregate: result.aggregate, meta: result.meta }, null, 2)}\n` + `${JSON.stringify( + { + aggregate: result.aggregate, + gateFailures: result.gateFailures, + meta: result.meta, + }, + null, + 2 + )}\n` ); - if ((result.aggregate.status.error ?? 0) > 0) { + if (result.gateFailures.length > 0) { process.exitCode = 1; } } catch (error) { diff --git a/packages/evals/ui/index.html b/packages/evals/ui/index.html index b1b442c00..d26050e86 100644 --- a/packages/evals/ui/index.html +++ b/packages/evals/ui/index.html @@ -794,16 +794,16 @@

Latest model board

const id = escapeHtml(c.id); const cost = (c.metrics?.costUsd || 0) + (c.metrics?.judgeCostUsd || 0); return ` -
${id}
- ${escapeHtml(c.category || "case")} - ${c.passed ? "Pass" : "Fail"} - ${c.scores?.tool_routing ?? "--"} - ${c.scores?.quality ?? "--"} - ${((c.metrics?.latencyMs || 0) / 1000).toFixed(1)}s - ${c.metrics?.steps ?? "--"} - ${money(cost)} - -
${detail(c)}
`; +
${id}
+ ${escapeHtml(c.category || "case")} + ${c.passed ? "Pass" : "Fail"} + ${c.scores?.tool_routing ?? "--"} + ${c.scores?.quality ?? "--"} + ${((c.metrics?.latencyMs || 0) / 1000).toFixed(1)}s + ${c.metrics?.steps ?? "--"} + ${money(cost)} + +
${detail(c)}
`; } function detail(c) { @@ -815,7 +815,7 @@

Latest model board

.map((t) => `${escapeHtml(t)}`) .join("") || 'No tools called'; return `

Response

${escapeHtml(c.response || "No response captured.")}
-

Failures

    ${failures}

Tools

${tools}
`; +

Failures

    ${failures}

Tools

${tools}
`; } function toggle(id) { diff --git a/packages/notifications/src/__tests__/client.test.ts b/packages/notifications/src/__tests__/client.test.ts new file mode 100644 index 000000000..c686fac9b --- /dev/null +++ b/packages/notifications/src/__tests__/client.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from "bun:test"; +import { NotificationClient } from "../client"; +import type { NotificationChannel } from "../types"; + +describe("NotificationClient", () => { + test("snapshots caller-owned channels before awaiting providers", async () => { + const channels: NotificationChannel[] = ["slack", "email"]; + const client = new NotificationClient(); + const pending = client.send( + { title: "Status changed", message: "A monitor is down." }, + { channels } + ); + + channels.length = 0; + + const results = await pending; + expect(results.map((result) => result.channel)).toEqual([ + "slack", + "email", + ]); + expect(results.every((result) => !result.success)).toBe(true); + }); +}); diff --git a/packages/notifications/src/__tests__/providers/slack.test.ts b/packages/notifications/src/__tests__/providers/slack.test.ts index e17170b26..2c43c28f3 100644 --- a/packages/notifications/src/__tests__/providers/slack.test.ts +++ b/packages/notifications/src/__tests__/providers/slack.test.ts @@ -34,4 +34,19 @@ describe("buildSlackBlocks", () => { expect(blocks[0]?.text?.text.length).toBe(150); expect(blocks[1]?.text?.text.length).toBe(2900); }); + + test("keeps large payloads within Slack's 50-block limit", () => { + const blocks = buildSlackBlocks({ + title: "Anomaly detected", + message: "Traffic changed.", + metadata: Object.fromEntries( + Array.from({ length: 1000 }, (_, index) => [`field${index}`, index]) + ), + priority: "urgent", + }); + + expect(blocks).toHaveLength(50); + expect(blocks.at(-1)?.type).toBe("context"); + expect(blocks.at(-1)?.elements?.[0]?.text).toContain("URGENT"); + }); }); diff --git a/packages/notifications/src/client.ts b/packages/notifications/src/client.ts index 551d4c6df..d9ac0956a 100644 --- a/packages/notifications/src/client.ts +++ b/packages/notifications/src/client.ts @@ -28,7 +28,7 @@ export class NotificationClient { constructor(config: NotificationClientConfig = {}) { this.providers = new Map(); - this.defaultChannels = config.defaultChannels ?? []; + this.defaultChannels = [...(config.defaultChannels ?? [])]; const defaults = { timeout: config.defaultTimeout ?? 10_000, @@ -71,10 +71,11 @@ export class NotificationClient { payload: NotificationPayload, options?: NotificationOptions ): Promise { - const channels = - options?.channels && options.channels.length > 0 + const channels = [ + ...(options?.channels && options.channels.length > 0 ? options.channels - : this.defaultChannels; + : this.defaultChannels), + ]; if (channels.length === 0) { return []; diff --git a/packages/notifications/src/providers/slack.ts b/packages/notifications/src/providers/slack.ts index e8abb4114..e632e6d1b 100644 --- a/packages/notifications/src/providers/slack.ts +++ b/packages/notifications/src/providers/slack.ts @@ -9,6 +9,7 @@ const MAX_HEADER_LENGTH = 150; const MAX_MESSAGE_LENGTH = 2900; const MAX_FIELD_LENGTH = 1900; const MAX_FIELDS_PER_SECTION = 10; +const MAX_BLOCKS = 50; const FIRST_CHARACTER_PATTERN = /^./; function truncate(value: string, maxLength: number): string { @@ -53,10 +54,16 @@ export function buildSlackBlocks( }, }, ]; + const elevatedPriority = + payload.priority && payload.priority !== "normal" ? payload.priority : null; + const metadataSectionLimit = + MAX_BLOCKS - blocks.length - (elevatedPriority ? 1 : 0); + const metadataFieldLimit = metadataSectionLimit * MAX_FIELDS_PER_SECTION; const metadataFields = payload.metadata ? Object.entries(payload.metadata) .filter(([key]) => isUserFacingMetadata(key)) + .slice(0, metadataFieldLimit) .map(([key, value]) => ({ type: "mrkdwn" as const, text: truncate( @@ -76,19 +83,19 @@ export function buildSlackBlocks( }); } - if (payload.priority && payload.priority !== "normal") { + if (elevatedPriority) { blocks.push({ type: "context", elements: [ { type: "mrkdwn", - text: `Priority: *${payload.priority.toUpperCase()}*`, + text: `Priority: *${elevatedPriority.toUpperCase()}*`, }, ], }); } - return blocks; + return blocks.slice(0, MAX_BLOCKS); } export interface SlackProviderConfig { diff --git a/packages/redis/cache-invalidation.test.ts b/packages/redis/cache-invalidation.test.ts index 15c3025d8..e7dc1199b 100644 --- a/packages/redis/cache-invalidation.test.ts +++ b/packages/redis/cache-invalidation.test.ts @@ -117,8 +117,10 @@ mock.module("./redis", () => ({ })); const { + cacheNamespaces, cacheTags, getAgentContextSnapshotKey, + getCacheableKey, invalidateAgentContextSnapshot, invalidateAgentContextSnapshotsForOwner, invalidateAgentContextSnapshotsForWebsite, @@ -178,31 +180,51 @@ describe("agent context snapshot keys", () => { describe("website read cache invalidation", () => { it("invalidates known website cacheable keys and batch domain caches", async () => { const websiteId = "site-1"; - redisStore.set("cacheable:website_by_id:[site-1]", { + const keys = { + agentTelemetry: getCacheableKey( + cacheNamespaces.agentTelemetryWebsiteExists, + websiteId + ), + batch: getCacheableKey(cacheNamespaces.websiteDomainsBatch, [ + websiteId, + "site-2", + ]), + otherBatch: getCacheableKey(cacheNamespaces.websiteDomainsBatch, [ + "site-2", + ]), + website: getCacheableKey(cacheNamespaces.websiteById, websiteId), + websiteCache: getCacheableKey(cacheNamespaces.websiteCache, websiteId), + websiteDomain: getCacheableKey(cacheNamespaces.websiteDomain, websiteId), + websiteWithOwner: getCacheableKey( + cacheNamespaces.websiteWithOwner, + websiteId + ), + }; + redisStore.set(keys.website, { value: "{}", ttl: 100, }); - redisStore.set("cacheable:website_with_owner_v2:[site-1]", { + redisStore.set(keys.websiteWithOwner, { value: "{}", ttl: 100, }); - redisStore.set("cacheable:website-cache:[site-1]", { + redisStore.set(keys.websiteCache, { value: "{}", ttl: 100, }); - redisStore.set("cacheable:website-domain:[site-1]", { + redisStore.set(keys.websiteDomain, { value: "example.com", ttl: 100, }); - redisStore.set("cacheable:agent-telemetry:website-exists:[site-1]", { + redisStore.set(keys.agentTelemetry, { value: "true", ttl: 100, }); - redisStore.set("cacheable:website-domains-batch:[[site-1,site-2]]", { + redisStore.set(keys.batch, { value: "{}", ttl: 100, }); - redisStore.set("cacheable:website-domains-batch:[[site-2]]", { + redisStore.set(keys.otherBatch, { value: "{}", ttl: 100, }); @@ -210,21 +232,13 @@ describe("website read cache invalidation", () => { const result = await invalidateWebsiteReadCaches(websiteId); expect(result).toEqual({ attempted: 6, failed: 0 }); - expect(redisStore.has("cacheable:website_by_id:[site-1]")).toBe(false); - expect(redisStore.has("cacheable:website_with_owner_v2:[site-1]")).toBe( - false - ); - expect(redisStore.has("cacheable:website-cache:[site-1]")).toBe(false); - expect(redisStore.has("cacheable:website-domain:[site-1]")).toBe(false); - expect( - redisStore.has("cacheable:agent-telemetry:website-exists:[site-1]") - ).toBe(false); - expect( - redisStore.has("cacheable:website-domains-batch:[[site-1,site-2]]") - ).toBe(false); - expect(redisStore.has("cacheable:website-domains-batch:[[site-2]]")).toBe( - true - ); + expect(redisStore.has(keys.website)).toBe(false); + expect(redisStore.has(keys.websiteWithOwner)).toBe(false); + expect(redisStore.has(keys.websiteCache)).toBe(false); + expect(redisStore.has(keys.websiteDomain)).toBe(false); + expect(redisStore.has(keys.agentTelemetry)).toBe(false); + expect(redisStore.has(keys.batch)).toBe(false); + expect(redisStore.has(keys.otherBatch)).toBe(true); }); }); @@ -354,27 +368,54 @@ describe("flag read cache invalidation", () => { describe("organization membership cache invalidation", () => { it("invalidates role, owner, and billing caches for a member", async () => { - redisStore.set("cacheable:rpc:member_role:[user-1,org-1]", { + const keys = { + apiKeyOwner: getCacheableKey(cacheNamespaces.apiKeyOwnerId, "org-1"), + billingMember: getCacheableKey( + cacheNamespaces.billingOwner, + "user-1", + "org-1" + ), + billingOrganization: getCacheableKey( + cacheNamespaces.billingOwner, + "user-2", + "org-1" + ), + otherBillingOrganization: getCacheableKey( + cacheNamespaces.billingOwner, + "user-2", + "org-2" + ), + organizationOwner: getCacheableKey( + cacheNamespaces.organizationOwner, + "org-1" + ), + role: getCacheableKey( + cacheNamespaces.memberRole, + "user-1", + "org-1" + ), + }; + redisStore.set(keys.role, { value: "member", ttl: 100, }); - redisStore.set("cacheable:rpc:org_owner:[org-1]", { + redisStore.set(keys.organizationOwner, { value: "user-1", ttl: 100, }); - redisStore.set("cacheable:api_key_owner_id:[org-1]", { + redisStore.set(keys.apiKeyOwner, { value: "user-1", ttl: 100, }); - redisStore.set("cacheable:rpc:billing_owner:[user-1,org-1]", { + redisStore.set(keys.billingMember, { value: "{}", ttl: 100, }); - redisStore.set("cacheable:rpc:billing_owner:[user-2,org-1]", { + redisStore.set(keys.billingOrganization, { value: "{}", ttl: 100, }); - redisStore.set("cacheable:rpc:billing_owner:[user-2,org-2]", { + redisStore.set(keys.otherBillingOrganization, { value: "{}", ttl: 100, }); @@ -385,22 +426,12 @@ describe("organization membership cache invalidation", () => { }); expect(result).toEqual({ attempted: 5, failed: 0 }); - expect(redisStore.has("cacheable:rpc:member_role:[user-1,org-1]")).toBe( - false - ); - expect(redisStore.has("cacheable:rpc:org_owner:[org-1]")).toBe(false); - expect(redisStore.has("cacheable:api_key_owner_id:[org-1]")).toBe( - false - ); - expect(redisStore.has("cacheable:rpc:billing_owner:[user-1,org-1]")).toBe( - false - ); - expect(redisStore.has("cacheable:rpc:billing_owner:[user-2,org-1]")).toBe( - false - ); - expect(redisStore.has("cacheable:rpc:billing_owner:[user-2,org-2]")).toBe( - true - ); + expect(redisStore.has(keys.role)).toBe(false); + expect(redisStore.has(keys.organizationOwner)).toBe(false); + expect(redisStore.has(keys.apiKeyOwner)).toBe(false); + expect(redisStore.has(keys.billingMember)).toBe(false); + expect(redisStore.has(keys.billingOrganization)).toBe(false); + expect(redisStore.has(keys.otherBillingOrganization)).toBe(true); }); }); diff --git a/packages/redis/cache-invalidation.ts b/packages/redis/cache-invalidation.ts index 47dac91b8..69c77f43d 100644 --- a/packages/redis/cache-invalidation.ts +++ b/packages/redis/cache-invalidation.ts @@ -51,7 +51,7 @@ export function getCacheableTagIndexKey(prefix: string, tag: string): string { export const cacheNamespaces = { agentTelemetryWebsiteExists: "agent-telemetry:website-exists", apiKeyByHash: "api-key-by-hash", - apiKeyOwnerId: "api_key_owner_id", + apiKeyOwnerId: "api_key_owner_id_v2", billingOwner: "rpc:billing_owner", flag: "flag", flagsClient: "flags-client", @@ -69,7 +69,7 @@ export const cacheNamespaces = { websiteCache: "website-cache", websiteDomain: "website-domain", websiteDomainsBatch: "website-domains-batch", - websiteWithOwner: "website_with_owner_v2", + websiteWithOwner: "website_with_owner_v3", } as const; function cacheTag(scope: string, ...parts: string[]): string { diff --git a/packages/redis/insights-queue.ts b/packages/redis/insights-queue.ts index 5bc34981e..4e5222eba 100644 --- a/packages/redis/insights-queue.ts +++ b/packages/redis/insights-queue.ts @@ -2,7 +2,9 @@ import { Queue } from "bullmq"; import { getBullMQConnectionOptions } from "./bullmq"; export const INSIGHTS_QUEUE_ENV_PREFIX = "INSIGHTS"; -export const INSIGHTS_QUEUE_NAME = "insights-generation"; +// Version the queue whenever worker semantics stop being backward-compatible. +// This keeps rolling deploys from sending legacy jobs to the deterministic engine. +export const INSIGHTS_QUEUE_NAME = "insights-investigations-v1"; export const INSIGHTS_DISPATCH_JOB_NAME = "insights-dispatch"; export const INSIGHTS_GENERATE_WEBSITE_JOB_NAME = "insights-generate-website"; export const INSIGHTS_MAINTENANCE_JOB_NAME = "insights-maintenance"; diff --git a/packages/rpc/package.json b/packages/rpc/package.json index de71d0ee7..e679e510a 100644 --- a/packages/rpc/package.json +++ b/packages/rpc/package.json @@ -7,7 +7,7 @@ "types": "./src/index.ts", "scripts": { "check-types": "tsc --noEmit", - "test": "bun test src/routers src/services/insight-schedule.test.ts src/services/uptime-lifecycle.test.ts src/services/uptime-scheduler.test.ts src/utils/*.test.ts", + "test": "bun test src/routers src/lib/analytics-utils.integration.test.ts src/services/insight-schedule.test.ts src/services/uptime-lifecycle.test.ts src/services/uptime-scheduler.test.ts src/utils/*.test.ts", "test:integration": "bun test src/services/uptime-scheduler.integration.test.ts" }, "exports": { @@ -16,6 +16,7 @@ "./autumn": "./src/lib/autumn-client.ts", "./billing": "./src/utils/billing.ts", "./flags": "./src/utils/flags.ts", + "./funnel-steps": "./src/routers/funnel-steps.ts", "./insight-generation": "./src/routers/insight-generation.ts", "./insight-schedule": "./src/services/insight-schedule.ts", "./log-context": "./src/lib/rpc-log-context.ts", diff --git a/packages/rpc/src/lib/analytics-utils.integration.test.ts b/packages/rpc/src/lib/analytics-utils.integration.test.ts new file mode 100644 index 000000000..7c80978ef --- /dev/null +++ b/packages/rpc/src/lib/analytics-utils.integration.test.ts @@ -0,0 +1,328 @@ +import { beforeAll, describe, expect, it } from "bun:test"; +import { chCommand } from "@databuddy/db/clickhouse"; +import { randomUUIDv7 } from "bun"; +import { + getTotalWebsiteUsers, + processFunnelAnalytics, + processFunnelConversionCounts, + processGoalAnalytics, + queryLinkVisitorIds, +} from "./analytics-utils"; + +const describeIntegration = + process.env.CLICKHOUSE_INTEGRATION_TESTS === "true" ? describe : describe.skip; +const testPrefix = randomUUIDv7(); +const profileWebsiteId = `identity-profile-${testPrefix}`; +const sessionWebsiteId = `identity-session-${testPrefix}`; +const afterContextWebsiteId = `context-after-${testPrefix}`; +const crossSessionWebsiteId = `context-cross-session-${testPrefix}`; +const sameTimeWebsiteId = `context-same-time-${testPrefix}`; +const reassignedAnonymousWebsiteId = `identity-reassigned-${testPrefix}`; +const repeatedEntryWebsiteId = `funnel-repeated-entry-${testPrefix}`; +const startDate = "2026-01-01"; +const endDate = "2026-01-02 23:59:59"; + +function queryParams(websiteId: string) { + return { endDate, startDate, websiteId }; +} + +describeIntegration("goal and funnel visitor identity", () => { + beforeAll(async () => { + await chCommand( + `INSERT INTO analytics.events + (id, client_id, event_name, anonymous_id, time, session_id, url, path, ip, user_agent, properties, created_at, profile_id, utm_source) + VALUES + (toUUID({profileEventId:String}), {profileWebsiteId:String}, 'screen_view', 'profile-anon', toDateTime64('2026-01-01 12:00:00', 3), 'profile-session', 'https://example.test/start', '/start', '', '', '{}', toDateTime64('2026-01-01 12:00:00', 3), '', 'newsletter'), + (toUUID({sessionEventId:String}), {sessionWebsiteId:String}, 'screen_view', 'session-anon', toDateTime64('2026-01-01 12:00:00', 3), 'session-1', 'https://example.test/start', '/start', '', '', '{}', toDateTime64('2026-01-01 12:00:00', 3), '', 'newsletter')`, + { + profileEventId: randomUUIDv7(), + profileWebsiteId, + sessionEventId: randomUUIDv7(), + sessionWebsiteId, + } + ); + await chCommand( + `INSERT INTO analytics.custom_events + (owner_id, website_id, timestamp, event_name, properties, anonymous_id, session_id, profile_id) + VALUES + ({profileWebsiteId:String}, {profileWebsiteId:String}, toDateTime64('2026-01-01 11:59:00', 3), 'identify', '{}', 'profile-anon', NULL, 'profile-1'), + ({profileWebsiteId:String}, {profileWebsiteId:String}, toDateTime64('2026-01-01 12:01:00', 3), 'purchase', '{}', NULL, NULL, 'profile-1'), + ({sessionWebsiteId:String}, {sessionWebsiteId:String}, toDateTime64('2026-01-01 11:59:00', 3), 'purchase', '{}', NULL, 'session-1', ''), + ({sessionWebsiteId:String}, {sessionWebsiteId:String}, toDateTime64('2026-01-01 12:01:00', 3), 'purchase', '{}', NULL, 'session-1', '')`, + { profileWebsiteId, sessionWebsiteId } + ); + await chCommand( + `INSERT INTO analytics.events + (id, client_id, event_name, anonymous_id, time, session_id, url, path, ip, user_agent, properties, created_at, profile_id, utm_source) + VALUES + (toUUID({afterEventId:String}), {afterWebsiteId:String}, 'screen_view', 'after-anon', toDateTime64('2026-01-01 12:02:00', 3), 'after-session', 'https://example.test/start', '/start', '', '', '{}', toDateTime64('2026-01-01 12:02:00', 3), '', 'newsletter'), + (toUUID({crossEventId:String}), {crossWebsiteId:String}, 'screen_view', 'cross-anon', toDateTime64('2026-01-01 12:00:00', 3), 'cross-browser-session', 'https://example.test/start', '/start', '', '', '{}', toDateTime64('2026-01-01 12:00:00', 3), '', 'newsletter'), + (toUUID({sameEventId:String}), {sameWebsiteId:String}, 'screen_view', 'same-anon', toDateTime64('2026-01-01 12:00:00', 3), 'same-session', 'https://example.test/start', '/start', '', '', '{}', toDateTime64('2026-01-01 12:00:00', 3), '', 'newsletter')`, + { + afterEventId: randomUUIDv7(), + afterWebsiteId: afterContextWebsiteId, + crossEventId: randomUUIDv7(), + crossWebsiteId: crossSessionWebsiteId, + sameEventId: randomUUIDv7(), + sameWebsiteId: sameTimeWebsiteId, + } + ); + await chCommand( + `INSERT INTO analytics.custom_events + (owner_id, website_id, timestamp, event_name, properties, anonymous_id, session_id, profile_id) + VALUES + ({afterWebsiteId:String}, {afterWebsiteId:String}, toDateTime64('2026-01-01 12:01:00', 3), 'purchase', '{}', 'after-anon', 'after-session', ''), + ({crossWebsiteId:String}, {crossWebsiteId:String}, toDateTime64('2026-01-01 12:01:00', 3), 'purchase', '{}', 'cross-anon', 'cross-purchase-session', ''), + ({sameWebsiteId:String}, {sameWebsiteId:String}, toDateTime64('2026-01-01 12:00:00', 3), 'purchase', '{}', 'same-anon', 'same-session', '')`, + { + afterWebsiteId: afterContextWebsiteId, + crossWebsiteId: crossSessionWebsiteId, + sameWebsiteId: sameTimeWebsiteId, + } + ); + await chCommand( + `INSERT INTO analytics.events + (id, client_id, event_name, anonymous_id, time, session_id, url, path, ip, user_agent, properties, created_at, profile_id) + VALUES + (toUUID({firstViewId:String}), {websiteId:String}, 'screen_view', 'shared-browser', toDateTime64('2026-01-01 10:00:00', 3), 'first-session', 'https://example.test/start', '/start', '', '', '{}', toDateTime64('2026-01-01 10:00:00', 3), ''), + (toUUID({secondViewId:String}), {websiteId:String}, 'screen_view', 'shared-browser', toDateTime64('2026-01-02 10:00:00', 3), 'second-session', 'https://example.test/start', '/start', '', '', '{}', toDateTime64('2026-01-02 10:00:00', 3), '')`, + { + firstViewId: randomUUIDv7(), + secondViewId: randomUUIDv7(), + websiteId: reassignedAnonymousWebsiteId, + } + ); + await chCommand( + `INSERT INTO analytics.custom_events + (owner_id, website_id, timestamp, event_name, properties, anonymous_id, session_id, profile_id) + VALUES + ({websiteId:String}, {websiteId:String}, toDateTime64('2026-01-01 10:01:00', 3), 'identify', '{}', 'shared-browser', 'first-session', 'profile-one'), + ({websiteId:String}, {websiteId:String}, toDateTime64('2026-01-01 10:02:00', 3), 'purchase', '{}', 'shared-browser', 'first-session', 'profile-one'), + ({websiteId:String}, {websiteId:String}, toDateTime64('2026-01-02 10:01:00', 3), 'identify', '{}', 'shared-browser', 'second-session', 'profile-two'), + ({websiteId:String}, {websiteId:String}, toDateTime64('2026-01-02 10:02:00', 3), 'purchase', '{}', 'shared-browser', 'second-session', 'profile-two')`, + { websiteId: reassignedAnonymousWebsiteId } + ); + await chCommand( + `INSERT INTO analytics.events + (id, client_id, event_name, anonymous_id, time, session_id, url, path, ip, user_agent, properties, created_at, profile_id) + VALUES + (toUUID({oldEntryId:String}), {websiteId:String}, 'screen_view', 'repeat-browser', toDateTime64('2026-01-01 10:00:00', 3), 'repeat-session', 'https://example.test/start', '/start', '', '', '{}', toDateTime64('2026-01-01 10:00:00', 3), ''), + (toUUID({matchedEntryId:String}), {websiteId:String}, 'screen_view', 'repeat-browser', toDateTime64('2026-01-10 10:00:00', 3), 'repeat-session', 'https://example.test/start', '/start', '', '', '{}', toDateTime64('2026-01-10 10:00:00', 3), '')`, + { + matchedEntryId: randomUUIDv7(), + oldEntryId: randomUUIDv7(), + websiteId: repeatedEntryWebsiteId, + } + ); + await chCommand( + `INSERT INTO analytics.custom_events + (owner_id, website_id, timestamp, event_name, properties, anonymous_id, session_id, profile_id) + VALUES + ({websiteId:String}, {websiteId:String}, toDateTime64('2026-01-10 10:01:00', 3), 'purchase', '{}', 'repeat-browser', 'repeat-session', '')`, + { websiteId: repeatedEntryWebsiteId } + ); + }); + + for (const [identity, websiteId] of [ + ["profile-only", profileWebsiteId], + ["session-only", sessionWebsiteId], + ] as const) { + it(`counts a ${identity} custom event as a goal completion`, async () => { + const result = await processGoalAnalytics( + [ + { + name: "Purchase", + step_number: 1, + target: "purchase", + type: "EVENT", + }, + ], + [], + queryParams(websiteId), + 1 + ); + + expect(result.total_users_completed).toBe(1); + }); + + it(`joins a ${identity} custom event to its browser funnel`, async () => { + const result = await processFunnelAnalytics( + [ + { + name: "Start", + step_number: 1, + target: "/start", + type: "PAGE_VIEW", + }, + { + name: "Purchase", + step_number: 2, + target: "purchase", + type: "EVENT", + }, + ], + [], + queryParams(websiteId) + ); + + expect(result.total_users_entered).toBe(1); + expect(result.total_users_completed).toBe(1); + expect(result.duration_available).toBe(false); + expect(result.avg_completion_time).toBe(0); + expect(result.steps_analytics[1]?.avg_time_to_complete).toBe(0); + }); + } + + it("applies acquisition filters to the funnel cohort, not every later step", async () => { + const result = await processFunnelAnalytics( + [ + { + name: "Start", + step_number: 1, + target: "/start", + type: "PAGE_VIEW", + }, + { + name: "Purchase", + step_number: 2, + target: "purchase", + type: "EVENT", + }, + ], + [ + { field: "path", operator: "equals", value: "/start" }, + { field: "utm_source", operator: "equals", value: "newsletter" }, + ], + queryParams(sessionWebsiteId) + ); + + expect(result.total_users_completed).toBe(1); + }); + + it("resolves row-time identity in direct event denominators and link cohorts", async () => { + const [totalUsers, linkVisitors] = await Promise.all([ + getTotalWebsiteUsers( + profileWebsiteId, + startDate, + endDate + ), + queryLinkVisitorIds("missing-link", queryParams(profileWebsiteId)), + ]); + + expect(totalUsers).toBe(1); + expect(linkVisitors).toEqual(new Set()); + }); + + it("keeps the cheap detector counts in parity with deep funnel counts", async () => { + const steps = [ + { + name: "Start", + step_number: 1, + target: "/start", + type: "PAGE_VIEW" as const, + }, + { + name: "Purchase", + step_number: 2, + target: "purchase", + type: "EVENT" as const, + }, + ]; + const [deep, cheap] = await Promise.all([ + processFunnelAnalytics(steps, [], queryParams(sessionWebsiteId)), + processFunnelConversionCounts(steps, [], queryParams(sessionWebsiteId)), + ]); + + expect(cheap.entrants).toBe(deep.total_users_entered); + expect(cheap.completions).toBe(deep.total_users_completed); + expect(cheap.steps.map((step) => step.users)).toEqual( + deep.steps_analytics.map((step) => step.users) + ); + }); + + it("attributes a conversion to the entry that actually began its matched sequence", async () => { + const result = await processFunnelAnalytics( + [ + { + name: "Start", + step_number: 1, + target: "/start", + type: "PAGE_VIEW", + }, + { + name: "Purchase", + step_number: 2, + target: "purchase", + type: "EVENT", + }, + ], + [], + { + endDate: "2026-01-11 23:59:59", + startDate, + websiteId: repeatedEntryWebsiteId, + } + ); + + expect(result.total_users_entered).toBe(1); + expect(result.total_users_completed).toBe(1); + expect(result.time_series).toEqual([ + { + avg_time: 0, + conversion_rate: 100, + conversions: 1, + date: "2026-01-10", + dropoffs: 0, + users: 1, + }, + ]); + }); + + it("does not rewrite an earlier session when a browser is assigned to another profile", async () => { + const result = await processFunnelConversionCounts( + [ + { + name: "Start", + step_number: 1, + target: "/start", + type: "PAGE_VIEW", + }, + { + name: "Purchase", + step_number: 2, + target: "purchase", + type: "EVENT", + }, + ], + [], + queryParams(reassignedAnonymousWebsiteId) + ); + + expect(result.entrants).toBe(2); + expect(result.completions).toBe(2); + }); + + for (const [name, websiteId, expected] of [ + ["rejects browser context after the conversion", afterContextWebsiteId, 0], + ["does not leak context across sessions", crossSessionWebsiteId, 0], + ["accepts browser context at the exact same timestamp", sameTimeWebsiteId, 1], + ] as const) { + it(name, async () => { + const result = await processGoalAnalytics( + [ + { + name: "Purchase", + step_number: 1, + target: "purchase", + type: "EVENT", + }, + ], + [{ field: "utm_source", operator: "equals", value: "newsletter" }], + queryParams(websiteId), + 1 + ); + + expect(result.total_users_completed).toBe(expected); + }); + } +}); diff --git a/packages/rpc/src/lib/analytics-utils.ts b/packages/rpc/src/lib/analytics-utils.ts index 03af5d566..99f1347e9 100644 --- a/packages/rpc/src/lib/analytics-utils.ts +++ b/packages/rpc/src/lib/analytics-utils.ts @@ -2,6 +2,8 @@ import { chQuery } from "@databuddy/db/clickhouse"; import { goalFunnelFilterFieldSet } from "@databuddy/shared/analytics-filters"; import { parseReferrer } from "@databuddy/shared/utils/referrer"; +const DATE_ONLY_PATTERN = /^\d{4}-\d{2}-\d{2}$/; + export interface AnalyticsStep { name: string; step_number: number; @@ -20,6 +22,7 @@ export interface StepAnalytics { conversion_rate: number; dropoff_rate: number; dropoffs: number; + error_context_available: boolean; error_count: number; error_rate: number; step_name: string; @@ -43,7 +46,9 @@ export interface FunnelAnalytics { avg_completion_time_formatted: string; biggest_dropoff_rate: number; biggest_dropoff_step: number; + duration_available: boolean; error_insights: { + available: boolean; total_errors: number; sessions_with_errors: number; dropoffs_with_errors: number; @@ -94,6 +99,13 @@ interface FunnelAggRow { users: number; } +export interface FunnelConversionCounts { + completions: number; + entrants: number; + rate: number; + steps: { stepNumber: number; users: number }[]; +} + interface ReferrerRow { max_step: number; referrer: string; @@ -166,7 +178,9 @@ const OPS = new Set([ const buildFilterSQL = ( filters: Filter[], - params: ClickhouseQueryParams + params: ClickhouseQueryParams, + sourceAlias?: string, + paramPrefix = "f" ): string => { const parts: string[] = []; @@ -176,8 +190,15 @@ const buildFilterSQL = ( continue; } - const key = `f${i}`; - const sqlField = field === "path" ? normalizedPathExpression(field) : field; + const key = `${paramPrefix}${i}`; + const sourceField = field === "screen_resolution" ? "viewport_size" : field; + const qualifiedField = sourceAlias + ? `${sourceAlias}.${sourceField}` + : sourceField; + const sqlField = + field === "path" + ? normalizedPathExpression(qualifiedField) + : qualifiedField; const normalizeExactPathValue = field === "path" && (operator === "equals" || @@ -244,7 +265,7 @@ const buildFilterSQL = ( }; // Query building -const buildTimeRangeWhere = (timeColumn: "time" | "timestamp") => +const buildTimeRangeWhere = (timeColumn: string) => `${timeColumn} >= parseDateTimeBestEffort({startDate:String}) AND ${timeColumn} <= parseDateTimeBestEffort({endDate:String})`; @@ -253,6 +274,84 @@ const buildBaseWhere = ( ) => `client_id = {websiteId:String} AND ${buildTimeRangeWhere(timeColumn)}`; +const customEventRows = (projection: string): string => `SELECT ${projection} + FROM analytics.custom_events + WHERE owner_id = {websiteId:String} + AND ${buildTimeRangeWhere("timestamp")} + UNION ALL + SELECT ${projection} + FROM analytics.custom_events + WHERE website_id = {websiteId:String} + AND owner_id != {websiteId:String} + AND ${buildTimeRangeWhere("timestamp")}`; + +const visitorIdentityCtes = `visitor_identity_rows AS ( + SELECT + profile_id, + anonymous_id, + session_id, + time AS identity_time + FROM analytics.events + WHERE ${buildBaseWhere("time")} + UNION ALL + ${customEventRows(` + profile_id, + ifNull(anonymous_id, '') AS anonymous_id, + ifNull(session_id, '') AS session_id, + timestamp AS identity_time`)} +), +visitor_profiles_by_anonymous AS ( + SELECT + anonymous_id, + arraySort(groupArray((identity_time, profile_id))) AS profile_history + FROM visitor_identity_rows + WHERE anonymous_id != '' AND profile_id != '' + GROUP BY anonymous_id +), +visitor_identity_by_session AS ( + SELECT + session_id, + argMaxIf(profile_id, identity_time, profile_id != '') AS mapped_profile_id, + argMaxIf(anonymous_id, identity_time, anonymous_id != '') AS mapped_anonymous_id + FROM visitor_identity_rows + WHERE session_id != '' + GROUP BY session_id +)`; + +const identityJoins = (source: string): string => ` + LEFT JOIN visitor_profiles_by_anonymous direct_profile + ON ${source}.anonymous_id = direct_profile.anonymous_id + LEFT JOIN visitor_identity_by_session session_identity + ON ${source}.session_id = session_identity.session_id + LEFT JOIN visitor_profiles_by_anonymous session_profile + ON session_identity.mapped_anonymous_id = session_profile.anonymous_id`; + +const profileAtRowTime = ( + source: string, + profileSource: string, + identityTime = `${source}.identity_time` +): string => + `tupleElement( + arrayLast( + identity -> tupleElement(identity, 1) <= ${identityTime}, + ${profileSource}.profile_history + ), + 2 +)`; + +const canonicalVisitorExpression = ( + source: string, + identityTime?: string +): string => `coalesce( + nullIf(${source}.profile_id, ''), + nullIf(session_identity.mapped_profile_id, ''), + nullIf(${profileAtRowTime(source, "direct_profile", identityTime)}, ''), + nullIf(${profileAtRowTime(source, "session_profile", identityTime)}, ''), + nullIf(${source}.anonymous_id, ''), + nullIf(session_identity.mapped_anonymous_id, ''), + '' +)`; + const pathOnlyExpression = (field = "path") => `if(startsWith(${field}, 'http://') OR startsWith(${field}, 'https://'), path(${field}), ${field})`; @@ -281,248 +380,296 @@ const normalizeGoalPathTarget = (target: string): string => { return withoutTrailingSlash || "/"; }; -const buildStepQuery = ( - step: AnalyticsStep, - idx: number, - filterSQL: string, - params: ClickhouseQueryParams, - includeReferrer = false -): string => { - params[`n${idx}`] = step.name; - params[`t${idx}`] = step.target; - - const refCol = includeReferrer ? ", any(referrer) as ref" : ""; - const base = buildBaseWhere("time"); - - if (step.type === "PAGE_VIEW") { - params[`t${idx}`] = normalizeGoalPathTarget(step.target); - return `SELECT ${idx + 1} as step, {n${idx}:String} as name, anonymous_id as vid, MIN(time) as ts${refCol} - FROM analytics.events - WHERE ${base} AND event_name = 'screen_view' - AND ${normalizedPathExpression("path")} = {t${idx}:String}${filterSQL} - GROUP BY vid`; - } - - // EVENT: query both analytics.events (track) and analytics.custom_events (api/server) - const refJoin = includeReferrer - ? `LEFT JOIN ( - SELECT anonymous_id as vid, argMin(referrer, time) as vref - FROM analytics.events WHERE ${base} AND event_name = 'screen_view' AND referrer != '' - GROUP BY vid - ) r ON e.vid = r.vid` - : ""; - - return `SELECT ${idx + 1} as step, {n${idx}:String} as name, e.vid as vid, MIN(ts) as ts${includeReferrer ? ", COALESCE(r.vref, '') as ref" : ""} - FROM ( - SELECT anonymous_id as vid, time as ts FROM analytics.events - WHERE ${base} AND event_name = {t${idx}:String}${filterSQL} - UNION ALL - SELECT COALESCE(anonymous_id, session_id, '') as vid, timestamp as ts FROM analytics.custom_events - WHERE (owner_id = {websiteId:String} OR website_id = {websiteId:String}) - AND ${buildTimeRangeWhere("timestamp")} - AND event_name = {t${idx}:String} - AND coalesce(anonymous_id, session_id, '') != '' - ) e ${refJoin} - GROUP BY e.vid${includeReferrer ? ", r.vref" : ""}`; -}; - -interface ErrorRow { - error_count: number; - error_type: string; - message: string; - path: string; - vid: string; -} - -const buildStepSubquery = ( - step: AnalyticsStep, - paramKey: string, - params: ClickhouseQueryParams -): string => { - params[paramKey] = step.target; - if (step.type === "PAGE_VIEW") { - params[paramKey] = normalizeGoalPathTarget(step.target); - return `SELECT DISTINCT anonymous_id FROM analytics.events - WHERE client_id = {websiteId:String} - AND time >= parseDateTimeBestEffort({startDate:String}) - AND time <= parseDateTimeBestEffort({endDate:String}) - AND event_name = 'screen_view' - AND ${normalizedPathExpression("path")} = {${paramKey}:String}`; - } - return `(SELECT DISTINCT anonymous_id FROM analytics.events - WHERE client_id = {websiteId:String} - AND time >= parseDateTimeBestEffort({startDate:String}) - AND time <= parseDateTimeBestEffort({endDate:String}) - AND event_name = {${paramKey}:String} - UNION ALL - SELECT DISTINCT COALESCE(anonymous_id, session_id, '') FROM analytics.custom_events - WHERE (owner_id = {websiteId:String} OR website_id = {websiteId:String}) - AND timestamp >= parseDateTimeBestEffort({startDate:String}) - AND timestamp <= parseDateTimeBestEffort({endDate:String}) - AND event_name = {${paramKey}:String} - AND COALESCE(anonymous_id, session_id, '') != '')`; -}; - -const queryFunnelErrors = async ( +function buildIdentifiedEventStream( steps: AnalyticsStep[], - hasVisitors: boolean, + filters: Filter[], params: ClickhouseQueryParams, - abortSignal?: AbortSignal -): Promise<{ - errorsByPath: Map; - sessionsWithErrors: Set; - totalErrors: number; - dropoffsWithErrors: number; -}> => { - const empty = { - errorsByPath: new Map(), - sessionsWithErrors: new Set(), - totalErrors: 0, - dropoffsWithErrors: 0, - }; - - const firstStep = steps[0]; - if (!(hasVisitors && firstStep)) { - return empty; - } - - const firstStepSubquery = buildStepSubquery( - firstStep, - "firstStepTarget", - params + options: { includeReferrer?: boolean } = {} +): string { + const browserFilter = buildFilterSQL(filters, params, "row", "browserFilter"); + const directCustomFilters = filters.filter( + (filter) => filter.field === "event_name" || filter.field === "path" ); - - const lastStep = steps.at(-1); - const hasMultipleSteps = lastStep && steps.length > 1; - const lastStepSubquery = hasMultipleSteps - ? buildStepSubquery(lastStep, "lastStepTarget", params) - : null; - - const errorQuery = `SELECT - path, - anonymous_id as vid, - error_type, - any(message) as message, - count() as error_count - FROM analytics.error_spans - WHERE client_id = {websiteId:String} - AND timestamp >= parseDateTimeBestEffort({startDate:String}) - AND timestamp <= parseDateTimeBestEffort({endDate:String}) - AND anonymous_id IN (${firstStepSubquery}) - GROUP BY path, anonymous_id, error_type - ORDER BY error_count DESC - LIMIT 1000`; - - const dropoffQuery = lastStepSubquery - ? `SELECT count(DISTINCT anonymous_id) as count - FROM analytics.error_spans - WHERE client_id = {websiteId:String} - AND timestamp >= parseDateTimeBestEffort({startDate:String}) - AND timestamp <= parseDateTimeBestEffort({endDate:String}) - AND anonymous_id IN (${firstStepSubquery}) - AND anonymous_id NOT IN (${lastStepSubquery})` - : null; - - const [errorRows, dropoffResult] = await Promise.all([ - chQuery(errorQuery, params, { abort_signal: abortSignal }), - dropoffQuery - ? chQuery<{ count: number }>(dropoffQuery, params, { - abort_signal: abortSignal, - }) - : Promise.resolve([]), - ]); - - const errorsByPath = new Map(); - const sessionsWithErrors = new Set(); - let totalErrors = 0; - - for (const row of errorRows) { - const count = toFiniteNumber(row.error_count, 0); - const normalized: ErrorRow = { - path: normalizeGoalPathTarget(String(row.path ?? "")), - vid: String(row.vid ?? ""), - error_type: String(row.error_type ?? ""), - message: String(row.message ?? ""), - error_count: count, - }; - - sessionsWithErrors.add(normalized.vid); - totalErrors += count; - - const existing = errorsByPath.get(normalized.path); - if (existing) { - existing.push(normalized); - } else { - errorsByPath.set(normalized.path, [normalized]); + const contextFilters = filters.filter( + (filter) => filter.field !== "event_name" && filter.field !== "path" + ); + const directCustomFilter = buildFilterSQL( + directCustomFilters, + params, + "row", + "customFilter" + ); + const contextFilter = buildFilterSQL( + contextFilters, + params, + "identified", + "contextFilter" + ); + const stepCases = steps.map((step, index) => { + const stepNumber = index + 1; + const targetKey = `t${index}`; + params[targetKey] = + step.type === "PAGE_VIEW" + ? normalizeGoalPathTarget(step.target) + : step.target; + const baseMatch = + step.type === "PAGE_VIEW" + ? `row.source_kind = 1 + AND row.event_name = 'screen_view' + AND ${normalizedPathExpression("row.path")} = {${targetKey}:String}` + : `row.event_name = {${targetKey}:String}`; + let matches = baseMatch; + if (index === 0 && filters.length > 0) { + if (step.type === "PAGE_VIEW") { + matches = `${baseMatch}${browserFilter}`; + } else { + const customHasContext = contextFilters.length + ? `(row.source_kind = 1${browserFilter} + OR (row.source_kind = 2${directCustomFilter} + AND row.last_matching_context_ms > 0 + AND row.last_matching_context_ms <= toUnixTimestamp64Milli(row.identity_time) + AND row.last_matching_context_ms >= toUnixTimestamp64Milli(row.identity_time) - 86400000))` + : `(row.source_kind = 1${browserFilter} + OR (row.source_kind = 2${directCustomFilter}))`; + matches = `${baseMatch} AND ${customHasContext}`; + } } - } - - const dropoffsWithErrors = toFiniteNumber(dropoffResult[0]?.count, 0); - - return { errorsByPath, sessionsWithErrors, totalErrors, dropoffsWithErrors }; -}; + return `if(ifNull(${matches}, false), toUInt8(${stepNumber}), toUInt8(0))`; + }); + const visitor = canonicalVisitorExpression("source_row"); + return `analytics_rows AS ( + SELECT + toUInt8(1) AS source_kind, + profile_id, + anonymous_id, + session_id, + time AS identity_time, + event_name, + path, + referrer, + country, + city, + device_type, + browser_name, + os_name, + language, + utm_source, + utm_medium, + utm_campaign, + utm_term, + utm_content, + user_agent, + viewport_size + FROM analytics.events + WHERE ${buildBaseWhere("time")} + UNION ALL + ${customEventRows(` + toUInt8(2) AS source_kind, + profile_id, + ifNull(anonymous_id, '') AS anonymous_id, + ifNull(session_id, '') AS session_id, + timestamp AS identity_time, + event_name, + path, + CAST(NULL, 'Nullable(String)') AS referrer, + CAST(NULL, 'Nullable(String)') AS country, + CAST(NULL, 'Nullable(String)') AS city, + CAST(NULL, 'Nullable(String)') AS device_type, + CAST(NULL, 'Nullable(String)') AS browser_name, + CAST(NULL, 'Nullable(String)') AS os_name, + CAST(NULL, 'Nullable(String)') AS language, + CAST(NULL, 'Nullable(String)') AS utm_source, + CAST(NULL, 'Nullable(String)') AS utm_medium, + CAST(NULL, 'Nullable(String)') AS utm_campaign, + CAST(NULL, 'Nullable(String)') AS utm_term, + CAST(NULL, 'Nullable(String)') AS utm_content, + CAST(NULL, 'Nullable(String)') AS user_agent, + CAST(NULL, 'Nullable(String)') AS viewport_size`)} +), +identified_rows AS ( + SELECT + source_row.source_kind AS source_kind, + source_row.profile_id AS profile_id, + source_row.anonymous_id AS anonymous_id, + source_row.session_id AS session_id, + source_row.identity_time AS identity_time, + source_row.event_name AS event_name, + source_row.path AS path, + source_row.referrer AS referrer, + source_row.country AS country, + source_row.city AS city, + source_row.device_type AS device_type, + source_row.browser_name AS browser_name, + source_row.os_name AS os_name, + source_row.language AS language, + source_row.utm_source AS utm_source, + source_row.utm_medium AS utm_medium, + source_row.utm_campaign AS utm_campaign, + source_row.utm_term AS utm_term, + source_row.utm_content AS utm_content, + source_row.user_agent AS user_agent, + source_row.viewport_size AS viewport_size, + ${visitor} AS vid + FROM analytics_rows source_row${identityJoins("source_row")} + WHERE ${visitor} != '' +), +context_rows AS ( + SELECT + context.*, + if( + context.session_id != '', + context.last_matching_session_context_ms, + context.last_matching_visitor_context_ms + ) AS last_matching_context_ms, + ${ + options.includeReferrer + ? `argMaxIf( + ifNull(context.referrer, ''), + context.identity_time, + context.source_kind = 1 + AND context.event_name = 'screen_view' + AND ifNull(context.referrer, '') != '' + ) OVER ( + PARTITION BY context.vid + ORDER BY context.identity_time, context.source_kind + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + )` + : "''" + } AS last_browser_referrer + FROM ( + SELECT + identified.*, + ${ + contextFilters.length > 0 + ? `maxIf( + toUnixTimestamp64Milli(identified.identity_time), + identified.source_kind = 1${contextFilter} + ) OVER ( + PARTITION BY identified.vid + ORDER BY identified.identity_time, identified.source_kind + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + )` + : "toInt64(0)" + } AS last_matching_visitor_context_ms, + ${ + contextFilters.length > 0 + ? `maxIf( + toUnixTimestamp64Milli(identified.identity_time), + identified.source_kind = 1${contextFilter} + ) OVER ( + PARTITION BY identified.vid, identified.session_id + ORDER BY identified.identity_time, identified.source_kind + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + )` + : "toInt64(0)" + } AS last_matching_session_context_ms + FROM identified_rows identified + ) context +), +events AS ( + SELECT + arrayJoin(arrayFilter(value -> value > 0, [${stepCases.join(", ")}])) AS step, + row.vid AS vid, + row.identity_time AS ts, + if( + row.source_kind = 1, + ifNull(row.referrer, ''), + row.last_browser_referrer + ) AS ref + FROM context_rows row +)`; +} export const queryLinkVisitorIds = async ( linkId: string, params: ClickhouseQueryParams ): Promise> => { const refParams = { ...params, linkRefPattern: `%ref=${linkId}%` }; + const eventVisitor = canonicalVisitorExpression("event", "event.time"); const rows = await chQuery<{ vid: string }>( - `SELECT DISTINCT anonymous_id as vid - FROM analytics.events - WHERE client_id = {websiteId:String} - AND ${buildTimeRangeWhere("time")} - AND url LIKE {linkRefPattern:String}`, + `WITH ${visitorIdentityCtes} + SELECT DISTINCT ${eventVisitor} as vid + FROM analytics.events event${identityJoins("event")} + WHERE event.client_id = {websiteId:String} + AND ${buildTimeRangeWhere("event.time")} + AND event.url LIKE {linkRefPattern:String} + AND ${eventVisitor} != ''`, refParams ); return new Set(rows.map((r) => String(r.vid ?? ""))); }; -// Build chained step CTEs + visitor summary for ClickHouse-side funnel computation -// Aggregate per-step error insights from the error query results -const buildStepErrorInsights = ( - errorsByPath: Map, - target: string -): { - stepErrorCount: number; - usersWithErrors: number; - topErrors: StepErrorInsight[]; -} => { - const stepErrors = errorsByPath.get(normalizeGoalPathTarget(target)) ?? []; - const stepErrorCount = stepErrors.reduce( - (sum, e) => sum + toFiniteNumber(e.error_count, 0), - 0 - ); - const usersWithErrors = new Set(stepErrors.map((e) => e.vid)).size; - - const errorsByType = new Map< - string, - { message: string; count: number; type: string } - >(); - for (const e of stepErrors) { - const ec = toFiniteNumber(e.error_count, 0); - const existing = errorsByType.get(e.error_type); - if (existing) { - existing.count += ec; - } else { - errorsByType.set(e.error_type, { - message: e.message, - count: ec, - type: e.error_type, - }); - } +/** + * Cheap detector path: one event stream and one visitor aggregation. It deliberately + * excludes time series, duration, and error context; those belong to investigation. + */ +export const processFunnelConversionCounts = async ( + steps: AnalyticsStep[], + filters: Filter[], + params: ClickhouseQueryParams, + abortSignal?: AbortSignal +): Promise => { + if (steps.length === 0) { + return { completions: 0, entrants: 0, rate: 0, steps: [] }; } - const topErrors = [...errorsByType.values()] - .sort((a, b) => b.count - a.count) - .slice(0, 3) - .map((e) => ({ - message: e.message, - error_type: e.type, - count: toFiniteNumber(e.count, 0), - })); + const conditions = steps.map((_, index) => `step = ${index + 1}`).join(", "); + const query = `WITH ${visitorIdentityCtes}, +${buildIdentifiedEventStream(steps, filters, params)}, +visitor_progress AS ( + SELECT + vid, + windowFunnel(86400000)(toUInt64(toUnixTimestamp64Milli(ts)), ${conditions}) AS max_step + FROM events + GROUP BY vid +) +SELECT + toUInt8(step_number) AS step_num, + countIf(max_step >= step_number) AS users +FROM visitor_progress +ARRAY JOIN range(1, ${steps.length + 1}) AS step_number +GROUP BY step_number +ORDER BY step_number`; + const rows = await chQuery<{ step_num: number; users: number }>( + query, + params, + { + abort_signal: abortSignal, + } + ); + const usersByStep = new Map( + rows.map((row) => [ + toFiniteNumber(row.step_num, 0), + toFiniteNumber(row.users, 0), + ]) + ); + const stepCounts = steps.map((_, index) => ({ + stepNumber: index + 1, + users: usersByStep.get(index + 1) ?? 0, + })); + const entrants = stepCounts[0]?.users ?? 0; + const completions = stepCounts.at(-1)?.users ?? 0; + return { + completions, + entrants, + rate: pct(completions, entrants), + steps: stepCounts, + }; +}; - return { stepErrorCount, usersWithErrors, topErrors }; +export const processGoalConversionCount = async ( + step: AnalyticsStep, + filters: Filter[], + params: ClickhouseQueryParams, + abortSignal?: AbortSignal +): Promise => { + const query = `WITH ${visitorIdentityCtes}, +${buildIdentifiedEventStream([step], filters, params)} +SELECT uniqExact(vid) AS completions FROM events`; + const [row] = await chQuery<{ completions: number }>(query, params, { + abort_signal: abortSignal, + }); + return toFiniteNumber(row?.completions, 0); }; // Main funnel analytics — step matching, timing, and aggregation happen in ClickHouse @@ -533,49 +680,88 @@ export const processFunnelAnalytics = async ( visitorFilter?: Set, abortSignal?: AbortSignal ): Promise => { - const filterSQL = buildFilterSQL(filters, params); const totalSteps = steps.length; - const stepQueries = steps.map((s, i) => - buildStepQuery(s, i, filterSQL, params) - ); + if (totalSteps === 0) { + throw new Error("A funnel requires at least one step"); + } let visitorFilterClause = ""; if (visitorFilter && visitorFilter.size > 0) { params.visitorFilterIds = [...visitorFilter]; - visitorFilterClause = " AND vid IN {visitorFilterIds:Array(String)}"; + visitorFilterClause = " WHERE vid IN {visitorFilterIds:Array(String)}"; } - const stepConditions = steps.map((_, i) => `step = ${i + 1}`).join(", "); - - const fullQuery = `WITH events AS (${stepQueries.join("\nUNION ALL\n")}), -step_events AS (SELECT DISTINCT step, vid, ts FROM events${visitorFilterClause ? ` WHERE 1=1${visitorFilterClause}` : ""}), -visitor_funnel AS ( + const candidateStepConditions = steps + .map((_, index) => + index === 0 + ? "event.step = 1 AND toDate(event.ts) = candidate.entry_date" + : `event.step = ${index + 1}` + ) + .join(", "); + const fullQuery = `WITH ${visitorIdentityCtes}, +${buildIdentifiedEventStream(steps, filters, params)}, +filtered_events AS ( + SELECT DISTINCT step, vid, ts + FROM events${visitorFilterClause} +), +entry_dates AS ( + SELECT DISTINCT vid, toDate(ts) AS entry_date + FROM filtered_events + WHERE step = 1 +), +candidate_progress AS ( + SELECT + candidate.vid AS vid, + candidate.entry_date AS entry_date, + windowFunnel(86400000)( + toUInt64(toUnixTimestamp64Milli(event.ts)), + ${candidateStepConditions} + ) AS candidate_max_step + FROM entry_dates candidate + INNER JOIN filtered_events event ON event.vid = candidate.vid + WHERE event.ts >= toDateTime(candidate.entry_date) + AND event.ts < toDateTime(candidate.entry_date + INTERVAL 2 DAY) + GROUP BY candidate.vid, candidate.entry_date + HAVING candidate_max_step >= 1 +), +visitor_progress AS ( SELECT vid, - windowFunnel(86400)(toDateTime(ts), ${stepConditions}) as max_step, - min(ts) as first_ts - FROM step_events + argMax( + entry_date, + tuple(candidate_max_step, -toInt32(toRelativeDayNum(entry_date))) + ) AS entry_date, + max(candidate_max_step) AS max_step + FROM candidate_progress GROUP BY vid HAVING max_step >= 1 +), +expanded AS ( + SELECT + entry_date, + max_step, + arrayJoin(range(0, ${totalSteps + 2})) AS output_index + FROM visitor_progress ) -SELECT step_num, date, users, avg_time, conversions FROM ( - ${Array.from({ length: totalSteps }, (_, i) => { - const stepNum = i + 1; - return `SELECT toUInt8(${stepNum}) as step_num, '' as date, countIf(max_step >= ${stepNum}) as users, toFloat64(0) as avg_time, toUInt64(0) as conversions FROM visitor_funnel`; - }).join("\nUNION ALL\n")} - UNION ALL - SELECT toUInt8(${totalSteps + 1}), '', toUInt64(0), toFloat64(0), toUInt64(0) FROM visitor_funnel - UNION ALL - SELECT toUInt8(0), toString(toDate(first_ts)), count(), toFloat64(0), countIf(max_step >= ${totalSteps}) FROM visitor_funnel GROUP BY toDate(first_ts) -) -ORDER BY 1, 2`; +SELECT + toUInt8(output_index) AS step_num, + if(output_index = 0, toString(entry_date), '') AS date, + if( + output_index = 0, + count(), + if(output_index <= ${totalSteps}, countIf(max_step >= output_index), toUInt64(0)) + ) AS users, + toFloat64(0) AS avg_time, + if(output_index = 0, countIf(max_step >= ${totalSteps}), toUInt64(0)) AS conversions +FROM expanded +GROUP BY output_index, date +ORDER BY step_num, date`; const sentinelStep = totalSteps + 1; - const [aggRows, errorData] = await Promise.all([ - chQuery(fullQuery, params, { abort_signal: abortSignal }), - queryFunnelErrors(steps, true, params, abortSignal), - ]); + const aggRows = await chQuery(fullQuery, params, { + abort_signal: abortSignal, + }); const stepRows: FunnelAggRow[] = []; const tsRows: FunnelAggRow[] = []; @@ -596,10 +782,6 @@ ORDER BY 1, 2`; const totalUsers = toFiniteNumber(stepRows[0]?.users, 0); const completedUsers = toFiniteNumber(stepRows.at(-1)?.users, 0); - const totalDropoffs = totalUsers - completedUsers; - - const { errorsByPath, totalErrors, sessionsWithErrors, dropoffsWithErrors } = - errorData; const stepsAnalytics: StepAnalytics[] = steps.map((s, i) => { const stepNum = i + 1; @@ -611,21 +793,19 @@ ORDER BY 1, 2`; : users; const drops = i > 0 ? prev - users : 0; - const { stepErrorCount, usersWithErrors, topErrors } = - buildStepErrorInsights(errorsByPath, s.target); - return { step_number: stepNum, step_name: s.name, users, total_users: totalUsers, - conversion_rate: i > 0 ? pct(users, prev) : 100, + conversion_rate: i > 0 ? pct(users, prev) : users > 0 ? 100 : 0, dropoffs: drops, dropoff_rate: i > 0 ? pct(drops, prev) : 0, avg_time_to_complete: Math.round(toFiniteNumber(row?.avg_time, 0)), - error_count: stepErrorCount, - error_rate: pct(usersWithErrors, users), - top_errors: topErrors, + error_context_available: false, + error_count: 0, + error_rate: 0, + top_errors: [], }; }); @@ -659,13 +839,15 @@ ORDER BY 1, 2`; avg_completion_time_formatted: formatDuration(avgCompletionTime), biggest_dropoff_step: biggestDropoff?.step_number || 1, biggest_dropoff_rate: biggestDropoff?.dropoff_rate || 0, + duration_available: false, steps_analytics: stepsAnalytics, time_series: timeSeries.length > 0 ? timeSeries : undefined, error_insights: { - total_errors: totalErrors, - sessions_with_errors: sessionsWithErrors.size, - dropoffs_with_errors: dropoffsWithErrors, - error_correlation_rate: pct(dropoffsWithErrors, totalDropoffs), + available: false, + total_errors: 0, + sessions_with_errors: 0, + dropoffs_with_errors: 0, + error_correlation_rate: 0, }, }; }; @@ -677,26 +859,15 @@ export const processGoalAnalytics = async ( totalWebsiteUsers: number, abortSignal?: AbortSignal ): Promise => { - const filterSQL = buildFilterSQL(filters, params); const step = steps[0]; - - const sql = `WITH events AS (${buildStepQuery(step, 0, filterSQL, params)}) - SELECT DISTINCT step, vid, ts FROM events`; - const rows = await chQuery<{ step: number; vid: string; ts: number }>( - sql, + if (!step) { + throw new Error("A goal requires one step"); + } + const completions = await processGoalConversionCount( + step, + filters, params, - { abort_signal: abortSignal } - ); - - const goalVids = new Set(rows.map((r) => r.vid)); - const completions = goalVids.size; - - const { errorsByPath, sessionsWithErrors, totalErrors } = - await queryFunnelErrors(steps, goalVids.size > 0, params, abortSignal); - - const { stepErrorCount, usersWithErrors, topErrors } = buildStepErrorInsights( - errorsByPath, - step.target + abortSignal ); return { @@ -707,6 +878,7 @@ export const processGoalAnalytics = async ( avg_completion_time_formatted: "—", biggest_dropoff_step: 1, biggest_dropoff_rate: 0, + duration_available: false, steps_analytics: [ { step_number: 1, @@ -717,14 +889,16 @@ export const processGoalAnalytics = async ( dropoffs: 0, dropoff_rate: 0, avg_time_to_complete: 0, - error_count: stepErrorCount, - error_rate: pct(usersWithErrors, completions), - top_errors: topErrors, + error_context_available: false, + error_count: 0, + error_rate: 0, + top_errors: [], }, ], error_insights: { - total_errors: totalErrors, - sessions_with_errors: sessionsWithErrors.size, + available: false, + total_errors: 0, + sessions_with_errors: 0, dropoffs_with_errors: 0, error_correlation_rate: 0, }, @@ -737,20 +911,21 @@ export const processFunnelAnalyticsByReferrer = async ( filters: Filter[], params: ClickhouseQueryParams ): Promise<{ referrer_analytics: ReferrerAnalytics[] }> => { - const filterSQL = buildFilterSQL(filters, params); const totalSteps = steps.length; - const stepQueries = steps.map((s, i) => - buildStepQuery(s, i, filterSQL, params, true) - ); - - const stepConditions = steps.map((_, i) => `step = ${i + 1}`).join(", "); + if (totalSteps === 0) { + return { referrer_analytics: [] }; + } + const stepConditions = steps + .map((_, index) => `step = ${index + 1}`) + .join(", "); - const fullQuery = `WITH events AS (${stepQueries.join("\nUNION ALL\n")}), + const fullQuery = `WITH ${visitorIdentityCtes}, +${buildIdentifiedEventStream(steps, filters, params, { includeReferrer: true })}, step_events AS (SELECT DISTINCT step, vid, ts, ref FROM events) SELECT vid, - windowFunnel(86400)(toDateTime(ts), ${stepConditions}) as max_step, - any(ref) as referrer + windowFunnel(86400000)(toUInt64(toUnixTimestamp64Milli(ts)), ${stepConditions}) as max_step, + argMinIf(ref, ts, step = 1) as referrer FROM step_events GROUP BY vid HAVING max_step >= 1`; @@ -809,18 +984,22 @@ export const getTotalWebsiteUsers = async ( const params: ClickhouseQueryParams = { websiteId, startDate, - endDate: `${endDate} 23:59:59`, + endDate: DATE_ONLY_PATTERN.test(endDate) ? `${endDate} 23:59:59` : endDate, }; const denominatorFilters = filters.filter( (filter) => filter.field !== "event_name" ); - const filterSQL = buildFilterSQL(denominatorFilters, params); + const filterSQL = buildFilterSQL(denominatorFilters, params, "event"); + const eventVisitor = canonicalVisitorExpression("event", "event.time"); const [result] = await chQuery<{ count: number }>( - `SELECT COUNT(DISTINCT anonymous_id) as count FROM analytics.events - WHERE client_id = {websiteId:String} - AND time >= parseDateTimeBestEffort({startDate:String}) - AND time <= parseDateTimeBestEffort({endDate:String}) - AND event_name = 'screen_view'${filterSQL}`, + `WITH ${visitorIdentityCtes} + SELECT COUNT(DISTINCT ${eventVisitor}) as count + FROM analytics.events event${identityJoins("event")} + WHERE event.client_id = {websiteId:String} + AND event.time >= parseDateTimeBestEffort({startDate:String}) + AND event.time <= parseDateTimeBestEffort({endDate:String}) + AND event.event_name = 'screen_view' + AND ${eventVisitor} != ''${filterSQL}`, params, { abort_signal: abortSignal } ); diff --git a/packages/rpc/src/routers/funnel-steps.ts b/packages/rpc/src/routers/funnel-steps.ts new file mode 100644 index 000000000..bfc461449 --- /dev/null +++ b/packages/rpc/src/routers/funnel-steps.ts @@ -0,0 +1,36 @@ +import { z } from "zod"; +import { rpcError } from "../errors"; +import type { AnalyticsStep } from "../lib/analytics-utils"; + +export const funnelStepSchema = z.object({ + type: z.enum(["PAGE_VIEW", "EVENT", "CUSTOM"]), + target: z.string().min(1), + name: z.string().min(1), + conditions: z.record(z.string(), z.unknown()).optional(), +}); + +export type FunnelStep = z.infer; + +export function normalizeFunnelSteps(steps: unknown): FunnelStep[] { + const parsed = z.array(funnelStepSchema).safeParse(steps); + return parsed.success ? parsed.data : []; +} + +export function requireFunnelSteps(steps: unknown): FunnelStep[] { + const normalized = normalizeFunnelSteps(steps); + if (normalized.length < 2) { + throw rpcError.badRequest( + "Funnel must contain at least 2 valid steps and no malformed steps" + ); + } + return normalized; +} + +export function toAnalyticsSteps(steps: FunnelStep[]): AnalyticsStep[] { + return steps.map((step, index) => ({ + step_number: index + 1, + type: step.type === "PAGE_VIEW" ? "PAGE_VIEW" : "EVENT", + target: step.target, + name: step.name, + })); +} diff --git a/packages/rpc/src/routers/funnels.test.ts b/packages/rpc/src/routers/funnels.test.ts new file mode 100644 index 000000000..553db7742 --- /dev/null +++ b/packages/rpc/src/routers/funnels.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "bun:test"; + +const { normalizeFunnelSteps, requireFunnelSteps, toAnalyticsSteps } = + await import("./funnel-steps"); + +const persistedSteps = [ + null, + { name: "Landing", target: "/", type: "PAGE_VIEW", stale: true }, + { name: "Missing target", type: "EVENT" }, + "checkout", + { name: "Checkout", target: "checkout_started", type: "CUSTOM" }, +]; + +describe("persisted funnel step normalization", () => { + it("rejects the entire sequence when any persisted step is malformed", () => { + expect(normalizeFunnelSteps(persistedSteps)).toEqual([]); + + expect(() => requireFunnelSteps(persistedSteps)).toThrow( + "no malformed steps" + ); + }); + + it("preserves a complete valid sequence and maps custom events", () => { + const steps = [ + { name: "Landing", target: "/", type: "PAGE_VIEW" }, + { name: "Checkout", target: "checkout_started", type: "CUSTOM" }, + ]; + expect(normalizeFunnelSteps(steps)).toEqual(steps); + + expect(toAnalyticsSteps(requireFunnelSteps(steps))).toEqual([ + { + name: "Landing", + step_number: 1, + target: "/", + type: "PAGE_VIEW", + }, + { + name: "Checkout", + step_number: 2, + target: "checkout_started", + type: "EVENT", + }, + ]); + }); + + it("rejects persisted funnels with fewer than two valid steps", () => { + let error: unknown; + try { + requireFunnelSteps([ + { name: "Landing", target: "/", type: "PAGE_VIEW" }, + { name: "Broken", type: "EVENT" }, + ]); + } catch (caught) { + error = caught; + } + + expect(error).toMatchObject({ + code: "BAD_REQUEST", + message: + "Funnel must contain at least 2 valid steps and no malformed steps", + }); + }); +}); diff --git a/packages/rpc/src/routers/funnels.ts b/packages/rpc/src/routers/funnels.ts index 0d8b95429..3d53bd42d 100644 --- a/packages/rpc/src/routers/funnels.ts +++ b/packages/rpc/src/routers/funnels.ts @@ -10,7 +10,6 @@ import { randomUUIDv7 } from "bun"; import { z } from "zod"; import { rpcError } from "../errors"; import { - type AnalyticsStep, processFunnelAnalytics, processFunnelAnalyticsByReferrer, queryLinkVisitorIds, @@ -23,26 +22,23 @@ import { withWorkspace, } from "../procedures/with-workspace"; import { requireFeatureWithLimit } from "../types/billing"; +import { + funnelStepSchema, + requireFunnelSteps, + toAnalyticsSteps, +} from "./funnel-steps"; const cache = createDrizzleCache({ redis, namespace: "funnels" }); const CACHE_TTL = 300; const ANALYTICS_CACHE_TTL = 180; -const stepSchema = z.object({ - type: z.enum(["PAGE_VIEW", "EVENT", "CUSTOM"]), - target: z.string().min(1), - name: z.string().min(1), - conditions: z.record(z.string(), z.unknown()).optional(), -}); - const filterSchema = z.object({ field: z.string(), operator: z.enum(["equals", "contains", "not_equals", "in", "not_in"]), value: z.union([z.string(), z.array(z.string())]), }); -type Step = z.infer; type Filter = z.infer; const getDefaultDateRange = () => { @@ -83,14 +79,6 @@ const invalidateFunnelsCache = async (websiteId: string, funnelId?: string) => { await Promise.all(operations); }; -const toAnalyticsSteps = (steps: Step[]): AnalyticsStep[] => - steps.map((step, index) => ({ - step_number: index + 1, - type: step.type as "PAGE_VIEW" | "EVENT", - target: step.target, - name: step.name, - })); - const funnelListOutputSchema = z.object({ createdAt: z.coerce.date(), description: z.string().nullable(), @@ -135,6 +123,7 @@ const stepAnalyticsOutputSchema = z.object({ dropoffs: z.number(), dropoff_rate: z.number(), avg_time_to_complete: z.number(), + error_context_available: z.boolean(), error_count: z.number(), error_rate: z.number(), top_errors: z.array(stepErrorInsightOutputSchema), @@ -157,9 +146,11 @@ const funnelAnalyticsOutputSchema = z.object({ avg_completion_time_formatted: z.string(), biggest_dropoff_step: z.number(), biggest_dropoff_rate: z.number(), + duration_available: z.boolean(), steps_analytics: z.array(stepAnalyticsOutputSchema), time_series: z.array(timeSeriesPointSchema).optional(), error_insights: z.object({ + available: z.boolean(), total_errors: z.number(), sessions_with_errors: z.number(), dropoffs_with_errors: z.number(), @@ -288,7 +279,7 @@ export const funnelsRouter = { websiteId: z.string(), name: z.string().min(1).max(100), description: z.string().optional(), - steps: z.array(stepSchema).min(2).max(10), + steps: z.array(funnelStepSchema).min(2).max(10), filters: z.array(filterSchema).optional(), ignoreHistoricData: z.boolean().optional(), }) @@ -351,7 +342,7 @@ export const funnelsRouter = { id: z.string(), name: z.string().min(1).max(100).optional(), description: z.string().optional(), - steps: z.array(stepSchema).min(2).max(10).optional(), + steps: z.array(funnelStepSchema).min(2).max(10).optional(), filters: z.array(filterSchema).optional(), ignoreHistoricData: z.boolean().optional(), isActive: z.boolean().optional(), @@ -478,10 +469,7 @@ export const funnelsRouter = { throw rpcError.notFound("funnel", input.funnelId); } - const steps = funnel.steps as Step[]; - if (!steps?.length) { - throw rpcError.badRequest("Funnel has no steps"); - } + const steps = requireFunnelSteps(funnel.steps); const effectiveStartDate = getEffectiveStartDate( startDate, @@ -550,10 +538,7 @@ export const funnelsRouter = { throw rpcError.notFound("funnel", input.funnelId); } - const steps = funnel.steps as Step[]; - if (!steps?.length) { - throw rpcError.badRequest("Funnel has no steps"); - } + const steps = requireFunnelSteps(funnel.steps); const effectiveStartDate = getEffectiveStartDate( startDate, @@ -623,10 +608,7 @@ export const funnelsRouter = { throw rpcError.notFound("funnel", input.funnelId); } - const steps = funnel.steps as Step[]; - if (!steps?.length) { - throw rpcError.badRequest("Funnel has no steps"); - } + const steps = requireFunnelSteps(funnel.steps); const effectiveStartDate = getEffectiveStartDate( startDate, @@ -651,8 +633,10 @@ export const funnelsRouter = { avg_completion_time_formatted: "—", biggest_dropoff_step: 1, biggest_dropoff_rate: 0, + duration_available: false, steps_analytics: [], error_insights: { + available: false, total_errors: 0, sessions_with_errors: 0, dropoffs_with_errors: 0, diff --git a/packages/rpc/src/routers/goals.ts b/packages/rpc/src/routers/goals.ts index d9e03f365..844d65293 100644 --- a/packages/rpc/src/routers/goals.ts +++ b/packages/rpc/src/routers/goals.ts @@ -85,6 +85,7 @@ const stepAnalyticsOutputSchema = z.object({ dropoffs: z.number(), dropoff_rate: z.number(), avg_time_to_complete: z.number(), + error_context_available: z.boolean(), error_count: z.number(), error_rate: z.number(), top_errors: z.array(stepErrorInsightOutputSchema), @@ -107,9 +108,11 @@ const goalAnalyticsOutputSchema = z.object({ avg_completion_time_formatted: z.string(), biggest_dropoff_step: z.number(), biggest_dropoff_rate: z.number(), + duration_available: z.boolean(), steps_analytics: z.array(stepAnalyticsOutputSchema), time_series: z.array(timeSeriesPointSchema).optional(), error_insights: z.object({ + available: z.boolean(), total_errors: z.number(), sessions_with_errors: z.number(), dropoffs_with_errors: z.number(), diff --git a/packages/rpc/src/routers/insight-generation.test.ts b/packages/rpc/src/routers/insight-generation.test.ts index 5dae329de..2137ace5c 100644 --- a/packages/rpc/src/routers/insight-generation.test.ts +++ b/packages/rpc/src/routers/insight-generation.test.ts @@ -5,7 +5,35 @@ process.env.BULLMQ_REDIS_URL ??= process.env.REDIS_URL; process.env.BETTER_AUTH_SECRET ??= "test-auth-secret"; process.env.BETTER_AUTH_URL ??= "http://localhost:3001"; -const { assertSingleActiveSlackBinding } = await import("./insight-generation"); +const { + applyInsightGenerationConfigPatch, + assertSingleActiveSlackBinding, + insightGenerationRouter, +} = await import("./insight-generation"); + +const compatibilityConfig = { + allowedTools: ["web_metrics", "product_metrics", "ops_context"] as const, + cooldownHours: 6, + createdAt: new Date("2026-01-01T00:00:00.000Z"), + cron: null, + deliveries: [], + depth: "standard" as const, + enabled: false, + frequency: "daily" as const, + id: "config-1", + lastRunAt: null, + lookbackDays: 7, + maxInsightsPerWebsite: 3, + maxSteps: 24, + maxToolCalls: 16, + modelTier: "balanced" as const, + nextRunAt: null, + organizationId: "org-1", + source: "organization" as const, + timezone: "UTC", + updatedAt: new Date("2026-01-01T00:00:00.000Z"), + websiteId: null, +}; describe("assertSingleActiveSlackBinding", () => { it("accepts exactly one binding", () => { @@ -21,3 +49,99 @@ describe("assertSingleActiveSlackBinding", () => { ); }); }); + +describe("insight generation compatibility contract", () => { + it("keeps run history and deprecated config metadata available", () => { + expect(insightGenerationRouter.listRuns["~orpc"].route.path).toBe( + "/insights/generation/listRuns" + ); + expect( + Object.keys(insightGenerationRouter.getConfig["~orpc"].outputSchema.shape) + ).toEqual( + expect.arrayContaining([ + "allowedTools", + "cooldownHours", + "cron", + "depth", + "lookbackDays", + "maxInsightsPerWebsite", + "maxSteps", + "maxToolCalls", + "modelTier", + "websiteId", + ]) + ); + expect( + insightGenerationRouter.getConfig["~orpc"].outputSchema.shape.source + .options + ).toEqual(["default", "organization"]); + }); + + it("accepts and ignores origin/main retired settings", () => { + const legacyPayload = { + allowedTools: ["web_metrics", "business_context"], + cooldownHours: 48, + cron: "0 3 * * 1", + depth: "deep", + enabled: true, + frequency: "weekly", + lookbackDays: 30, + maxInsightsPerWebsite: 2, + maxSteps: 50, + maxToolCalls: 50, + modelTier: "deep", + organizationId: "org-1", + timezone: "Europe/Berlin", + } as const; + const upsertSchema = + insightGenerationRouter.upsertConfig["~orpc"].inputSchema; + const result = applyInsightGenerationConfigPatch( + compatibilityConfig, + upsertSchema.parse(legacyPayload) + ); + + expect(result).toEqual({ + ...compatibilityConfig, + enabled: true, + frequency: "weekly", + timezone: "Europe/Berlin", + }); + expect(() => + upsertSchema.parse({ ...legacyPayload, maxSteps: 65 }) + ).toThrow(); + expect( + insightGenerationRouter.triggerRun["~orpc"].inputSchema.parse({ + ...legacyPayload, + websiteIds: ["site-1"], + }) + ).toMatchObject(legacyPayload); + }); + + it("ignores retired hourly, custom, and cron scheduling", () => { + const custom = applyInsightGenerationConfigPatch(compatibilityConfig, { + cron: "*/15 * * * *", + frequency: "custom", + }); + const hourly = applyInsightGenerationConfigPatch(compatibilityConfig, { + frequency: "hourly", + }); + + expect(custom.frequency).toBe("daily"); + expect(custom.cron).toBeNull(); + expect(hourly.frequency).toBe("daily"); + }); + + it("applies supported schedule settings", () => { + expect( + applyInsightGenerationConfigPatch(compatibilityConfig, { + enabled: true, + frequency: "weekly", + timezone: "America/New_York", + }) + ).toMatchObject({ + enabled: true, + frequency: "weekly", + timezone: "America/New_York", + }); + }); +}); diff --git a/packages/rpc/src/routers/insight-generation.ts b/packages/rpc/src/routers/insight-generation.ts index eacd98840..fd4ce2811 100644 --- a/packages/rpc/src/routers/insight-generation.ts +++ b/packages/rpc/src/routers/insight-generation.ts @@ -6,9 +6,11 @@ import { inArray, isNull, isUniqueViolationFor, + sql, withTransaction, } from "@databuddy/db"; import { + DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT, INSIGHT_RUN_ACTIVE_STATUSES, INSIGHT_RUN_ACTIVE_UNIQUE_INDEX, insightGenerationConfigs, @@ -17,6 +19,7 @@ import { slackChannelBindings, slackIntegrations, type InsightGenerationConfig, + type InsightGenerationConfigSnapshot, websites, } from "@databuddy/db/schema"; import { @@ -40,6 +43,15 @@ import { const queueStatusSchema = z.enum(["queued", "skipped", "disabled"]); const frequencySchema = z.enum(["daily", "weekly"]); +const legacyFrequencySchema = z.enum(["hourly", "daily", "weekly", "custom"]); +const generationToolSchema = z.enum([ + "web_metrics", + "product_metrics", + "ops_context", + "business_context", +]); +const depthSchema = z.enum(["light", "standard", "deep"]); +const modelTierSchema = z.enum(["fast", "balanced", "deep"]); const queueReasonSchema = z.enum(["manual", "scheduled"]); const reasonSchema = z.enum(["manual", "scheduled", "cooldown_refresh"]); const deliverySchema = z.object({ @@ -57,8 +69,17 @@ type ConfigExecutor = | Parameters[0]>[0]; const configPatchSchema = z.object({ + allowedTools: z.array(generationToolSchema).min(1).max(4).optional(), + cooldownHours: z.number().int().min(1).max(168).optional(), + cron: z.string().trim().min(1).max(120).nullable().optional(), + depth: depthSchema.optional(), enabled: z.boolean().optional(), - frequency: frequencySchema.optional(), + frequency: legacyFrequencySchema.optional(), + lookbackDays: z.number().int().min(1).max(90).optional(), + maxInsightsPerWebsite: z.number().int().min(1).max(10).optional(), + maxSteps: z.number().int().min(1).max(64).optional(), + maxToolCalls: z.number().int().min(1).max(64).optional(), + modelTier: modelTierSchema.optional(), timezone: z .string() .trim() @@ -67,26 +88,36 @@ const configPatchSchema = z.object({ .refine(isValidTimezone, "Invalid IANA timezone") .optional(), }); -const runPatchSchema = configPatchSchema.pick({ - timezone: true, -}); const organizationScopeSchema = z.object({ organizationId: z.string().nullish(), - websiteId: z.never().optional(), + websiteId: z.string().nullish(), }); const configOutputSchema = z.object({ + allowedTools: z.array(generationToolSchema).describe("Deprecated"), + cooldownHours: z.number().describe("Deprecated"), createdAt: z.union([z.date(), z.string()]).nullable(), + cron: z.string().nullable().describe("Deprecated"), deliveries: z.array(deliverySchema), + depth: depthSchema.describe("Deprecated"), enabled: z.boolean(), frequency: frequencySchema, id: z.string().nullable(), lastRunAt: z.union([z.date(), z.string()]).nullable(), + lookbackDays: z.number().describe("Deprecated"), + maxInsightsPerWebsite: z.number().describe("Deprecated"), + maxSteps: z.number().describe("Deprecated"), + maxToolCalls: z.number().describe("Deprecated"), + modelTier: modelTierSchema.describe("Deprecated"), nextRunAt: z.union([z.date(), z.string()]).nullable(), organizationId: z.string(), source: z.enum(["default", "organization"]), timezone: z.string(), updatedAt: z.union([z.date(), z.string()]).nullable(), + websiteId: z + .string() + .nullable() + .describe("Deprecated read scope; settings inherit from the organization"), }); const runOutputSchema = z.object({ @@ -116,6 +147,7 @@ const runOutputSchema = z.object({ const runItemOutputSchema = z.object({ attempts: z.number(), + configSnapshot: z.unknown(), createdAt: z.union([z.date(), z.string()]), errorMessage: z.string().nullable(), finishedAt: z.union([z.date(), z.string()]).nullable(), @@ -139,14 +171,25 @@ const DEFAULT_CONFIG: Omit< | "source" | "updatedAt" > = { + allowedTools: [...DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.allowedTools], + cooldownHours: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.cooldownHours, + cron: null, deliveries: [], + depth: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.depth, enabled: false, frequency: "weekly", + lookbackDays: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.lookbackDays, + maxInsightsPerWebsite: + DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.maxInsightsPerWebsite, + maxSteps: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.maxSteps, + maxToolCalls: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.maxToolCalls, + modelTier: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.modelTier, timezone: "UTC", + websiteId: null, }; export interface QueueInsightGenerationRunInput - extends z.infer { + extends z.infer { organizationId: string; reason?: z.infer; requestedByUserId?: string | null; @@ -183,17 +226,28 @@ function rowToConfig( } return { + allowedTools: [...DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.allowedTools], + cooldownHours: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.cooldownHours, createdAt: row.createdAt, + cron: null, deliveries: row.deliveries, + depth: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.depth, enabled: row.enabled, frequency: normalizeInsightScheduleFrequency(row.frequency), id: row.id, lastRunAt: row.lastRunAt, + lookbackDays: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.lookbackDays, + maxInsightsPerWebsite: + DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.maxInsightsPerWebsite, + maxSteps: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.maxSteps, + maxToolCalls: DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.maxToolCalls, + modelTier: row.legacyModelTier, nextRunAt: row.enabled ? row.nextRunAt : null, organizationId: row.organizationId, source, timezone: normalizeInsightTimezone(row.timezone), updatedAt: row.updatedAt, + websiteId: null, }; } @@ -212,24 +266,62 @@ function defaultConfig( }; } -function applyPatch( +function compatibilitySnapshot( + timezone: string +): InsightGenerationConfigSnapshot { + return { + ...DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT, + allowedTools: [...DEFAULT_INSIGHT_GENERATION_CONFIG_SNAPSHOT.allowedTools], + timezone, + }; +} + +export function applyInsightGenerationConfigPatch( config: z.infer, patch: z.infer ): z.infer { const parsed = configPatchSchema.parse(patch); + const frequency = + parsed.frequency === "daily" || parsed.frequency === "weekly" + ? parsed.frequency + : config.frequency; return { ...config, enabled: parsed.enabled ?? config.enabled, - frequency: parsed.frequency ?? config.frequency, + frequency, timezone: parsed.timezone ?? config.timezone, }; } async function resolveOrganization( context: Context, - input: { organizationId?: string | null }, + input: { + organizationId?: string | null; + websiteId?: string | null; + }, permission: "read" | "update" ): Promise { + if (input.websiteId) { + const workspace = await withWorkspace(context, { + websiteId: input.websiteId, + resource: "website", + permissions: [permission === "read" ? "view_analytics" : "update"], + }); + if ( + input.organizationId && + input.organizationId !== workspace.website.organizationId + ) { + throw rpcError.badRequest("Website does not belong to organization"); + } + await withWorkspace(context, { + organizationId: workspace.website.organizationId, + resource: "organization", + permissions: [permission], + allowCrossOrg: true, + }); + return workspace.website.organizationId; + } + const organizationId = input.organizationId?.trim() || context.organizationId; if (!organizationId) { throw rpcError.badRequest("Organization ID is required"); @@ -242,6 +334,19 @@ async function resolveOrganization( return organizationId; } +async function resolveOrganizationForMutation( + context: Context, + input: z.infer +): Promise { + const organizationId = await resolveOrganization(context, input, "update"); + if (input.websiteId) { + throw rpcError.badRequest( + "Website-specific insight settings are retired. Remove websiteId to update the organization settings." + ); + } + return organizationId; +} + async function findConfig( organizationId: string, executor: ConfigExecutor = db @@ -250,7 +355,6 @@ async function findConfig( .select() .from(insightGenerationConfigs) .where(eq(insightGenerationConfigs.organizationId, organizationId)) - .orderBy(insightGenerationConfigs.createdAt) .limit(1); return rows[0] ?? null; } @@ -300,8 +404,10 @@ function runConfigMutation( } const values = { deliveries: next.deliveries, + dispatchDueAt: scheduleChanged ? null : (row?.dispatchDueAt ?? null), enabled: next.enabled, frequency: next.frequency, + legacyModelTier: next.modelTier, nextRunAt, timezone: next.timezone, }; @@ -375,9 +481,10 @@ async function listTargetWebsites( } async function findActiveInsightRun( - organizationId: string + organizationId: string, + executor: ConfigExecutor = db ): Promise<{ id: string; totalItems: number } | null> { - const [active] = await db + const [active] = await executor .select({ id: insightRuns.id, totalItems: insightRuns.totalItems }) .from(insightRuns) .where( @@ -412,13 +519,21 @@ async function insertInsightRunOrFindActive( let conflict: unknown; for (let attempt = 0; attempt < 2; attempt += 1) { try { - await withTransaction(async (tx) => { + return await withTransaction(async (tx) => { + await tx.execute( + sql`SELECT pg_advisory_xact_lock(hashtextextended(${`insight-run:${organizationId}`}, 0))` + ); + const active = await findActiveInsightRun(organizationId, tx); + if (active) { + return active; + } + await tx.insert(insightRuns).values(run); if (items.length > 0) { await tx.insert(insightRunItems).values(items); } + return null; }); - return null; } catch (error) { if (!isUniqueViolationFor(error, INSIGHT_RUN_ACTIVE_UNIQUE_INDEX)) { throw error; @@ -440,8 +555,7 @@ export async function queueInsightGenerationRun( throw rpcError.badRequest("Select at least one website"); } const baseConfig = await getConfig(input.organizationId); - const runPatch = runPatchSchema.parse(input); - const runConfig = applyPatch(baseConfig, runPatch); + const runConfig = applyInsightGenerationConfigPatch(baseConfig, input); const reason = input.reason ?? "manual"; const active = await findActiveInsightRun(input.organizationId); @@ -468,8 +582,10 @@ export async function queueInsightGenerationRun( }); const requestedByUserId = input.requestedByUserId ?? null; const now = new Date(); + const configSnapshot = compatibilitySnapshot(runConfig.timezone); const runItems = queueItems.map((item) => ({ + configSnapshot, id: item.itemId, runId, organizationId: input.organizationId, @@ -519,8 +635,8 @@ export async function queueInsightGenerationRun( { error, organizationId: input.organizationId, runId }, "Failed to queue insight generation" ); - await Promise.all([ - db + await withTransaction(async (tx) => { + await tx .update(insightRuns) .set({ errorMessage: QUEUE_INSIGHT_GENERATION_ERROR, @@ -528,16 +644,16 @@ export async function queueInsightGenerationRun( finishedAt: new Date(), status: "failed", }) - .where(eq(insightRuns.id, runId)), - db + .where(eq(insightRuns.id, runId)); + await tx .update(insightRunItems) .set({ errorMessage: QUEUE_INSIGHT_GENERATION_ERROR, finishedAt: new Date(), status: "failed", }) - .where(eq(insightRunItems.runId, runId)), - ]); + .where(eq(insightRunItems.runId, runId)); + }); throw rpcError.internal("Failed to queue insight generation"); } @@ -560,7 +676,10 @@ export const insightGenerationRouter = { .output(configOutputSchema) .handler(async ({ context, input }) => { const organizationId = await resolveOrganization(context, input, "read"); - return getConfig(organizationId); + const config = await getConfig(organizationId); + return input.websiteId + ? { ...config, websiteId: input.websiteId } + : config; }), upsertConfig: protectedProcedure @@ -573,13 +692,12 @@ export const insightGenerationRouter = { .input(organizationScopeSchema.extend(configPatchSchema.shape)) .output(configOutputSchema) .handler(async ({ context, input }) => { - const organizationId = await resolveOrganization( + const organizationId = await resolveOrganizationForMutation( context, - input, - "update" + input ); return mutateConfig(organizationId, (current) => - applyPatch(current, input) + applyInsightGenerationConfigPatch(current, input) ); }), @@ -593,15 +711,14 @@ export const insightGenerationRouter = { .input( organizationScopeSchema.extend({ channelId: z.string().min(1).max(120), - frequency: frequencySchema.optional(), + frequency: legacyFrequencySchema.optional(), }) ) .output(configOutputSchema) .handler(async ({ context, input }) => { - const organizationId = await resolveOrganization( + const organizationId = await resolveOrganizationForMutation( context, - input, - "update" + input ); const bindings = await db .select({ id: slackChannelBindings.id }) @@ -630,7 +747,7 @@ export const insightGenerationRouter = { `Cannot route to more than ${MAX_SLACK_DELIVERIES} Slack channels` ); } - const base = applyPatch( + const base = applyInsightGenerationConfigPatch( current, input.frequency ? { enabled: true, frequency: input.frequency } @@ -660,10 +777,9 @@ export const insightGenerationRouter = { ) .output(configOutputSchema) .handler(async ({ context, input }) => { - const organizationId = await resolveOrganization( + const organizationId = await resolveOrganizationForMutation( context, - input, - "update" + input ); return mutateConfig(organizationId, (current) => ({ ...current, @@ -690,7 +806,7 @@ export const insightGenerationRouter = { organizationId: z.string().nullish(), websiteIds: z.array(z.string().min(1)).min(1).max(100).optional(), }) - .extend(runPatchSchema.shape) + .extend(configPatchSchema.shape) ) .output( z.object({ @@ -707,10 +823,9 @@ export const insightGenerationRouter = { "update" ); return queueInsightGenerationRun({ + ...input, organizationId, requestedByUserId: context.user?.id ?? null, - timezone: input.timezone, - websiteIds: input.websiteIds, }); }), @@ -745,4 +860,29 @@ export const insightGenerationRouter = { return { items, run }; }), + + listRuns: protectedProcedure + .route({ + method: "POST", + path: "/insights/generation/listRuns", + summary: "List insight generation runs", + description: "Deprecated compatibility endpoint for run history.", + tags: ["Insights"], + }) + .input( + organizationScopeSchema.extend({ + limit: z.number().int().min(1).max(100).default(20), + }) + ) + .output(z.object({ runs: z.array(runOutputSchema) })) + .handler(async ({ context, input }) => { + const organizationId = await resolveOrganization(context, input, "read"); + const runs = await db + .select() + .from(insightRuns) + .where(eq(insightRuns.organizationId, organizationId)) + .orderBy(desc(insightRuns.createdAt)) + .limit(input.limit); + return { runs }; + }), }; diff --git a/packages/rpc/src/routers/insights-compatibility.test.ts b/packages/rpc/src/routers/insights-compatibility.test.ts new file mode 100644 index 000000000..1fea4d224 --- /dev/null +++ b/packages/rpc/src/routers/insights-compatibility.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "bun:test"; + +process.env.REDIS_URL ??= "redis://localhost:6379/1"; +process.env.BULLMQ_REDIS_URL ??= process.env.REDIS_URL; +process.env.BETTER_AUTH_SECRET ??= "test-auth-secret-for-insights-contract"; +process.env.BETTER_AUTH_URL ??= "http://localhost:3001"; + +const { buildInsightLink, insightsRouter } = await import("./insights"); + +describe("insights compatibility contract", () => { + it("keeps deprecated feed and dismissal paths routable", () => { + expect(insightsRouter.feed["~orpc"].route.path).toBe("/insights/feed"); + expect(insightsRouter.setDismissed["~orpc"].route.path).toBe( + "/insights/setDismissed" + ); + expect(insightsRouter.clearDismissed["~orpc"].route.path).toBe( + "/insights/clearDismissed" + ); + }); + + it("keeps historical investigation provenance in the output schema", () => { + const history = insightsRouter.history["~orpc"].outputSchema.shape.insights; + expect(Object.keys(history.element.shape)).toEqual( + expect.arrayContaining(["chainId", "investigationDepth"]) + ); + }); + + it("links revenue findings to the surface that can resolve them", () => { + expect(buildInsightLink("site-1", "quality_shift", "revenue:usd")).toBe( + "/websites/site-1/revenue" + ); + expect( + buildInsightLink( + "site-1", + "quality_shift", + "payment_failure_rate:eur" + ) + ).toBe("/websites/site-1/revenue"); + }); +}); diff --git a/packages/rpc/src/routers/insights-votes.integration.test.ts b/packages/rpc/src/routers/insights-votes.integration.test.ts new file mode 100644 index 000000000..9459ee5db --- /dev/null +++ b/packages/rpc/src/routers/insights-votes.integration.test.ts @@ -0,0 +1,472 @@ +import { afterAll, afterEach, describe, expect, it } from "bun:test"; +import type { Context } from "../orpc"; + +const localDatabaseUrl = + "postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy_test"; +const localRedisUrl = "redis://localhost:6379/1"; + +if (process.env.CI !== "true") { + process.env.DATABASE_URL = localDatabaseUrl; + process.env.REDIS_URL = localRedisUrl; + process.env.BULLMQ_REDIS_URL = localRedisUrl; +} +process.env.BETTER_AUTH_SECRET ??= "test-auth-secret-for-integration"; +process.env.BETTER_AUTH_URL ??= "http://localhost:3001"; +process.env.NODE_ENV = "test"; + +const { and, db, eq, inArray, shutdownPostgres, sql } = await import( + "@databuddy/db" +); +const { + analyticsInsights, + insightGenerationConfigs, + insightUserFeedback, + member, + organization, + user, + websites, +} = await import("@databuddy/db/schema"); +const { auth } = await import("@databuddy/auth"); +const { shutdownRedis } = await import("@databuddy/redis/redis"); +const { insightGenerationRouter } = await import("./insight-generation"); +const { insightsRouter } = await import("./insights"); + +const handler = { + getById: insightsRouter.getById["~orpc"].handler, + getVotes: insightsRouter.getVotes["~orpc"].handler, + setVote: insightsRouter.setVote["~orpc"].handler, +}; +const generationHandler = { + getConfig: insightGenerationRouter.getConfig["~orpc"].handler, + upsertConfig: insightGenerationRouter.upsertConfig["~orpc"].handler, +}; +const runIntegrationTests = + process.env.CI === "true" || + process.env.INSIGHTS_INTEGRATION_TESTS === "true"; +if (runIntegrationTests) { + await db.execute(sql`select 1`); +} +const iit = runIntegrationTests ? it : it.skip; +const organizationIds: string[] = []; +const userIds: string[] = []; + +afterEach(async () => { + if (organizationIds.length > 0) { + await db + .delete(organization) + .where(inArray(organization.id, organizationIds.splice(0))); + } + if (userIds.length > 0) { + await db.delete(user).where(inArray(user.id, userIds.splice(0))); + } +}); + +afterAll(async () => { + await Promise.all([shutdownPostgres(), shutdownRedis()]); +}); + +function nextId(prefix: string): string { + return `${prefix}-${crypto.randomUUID()}`; +} + +async function insertUser() { + const id = nextId("user"); + const now = new Date(); + const [row] = await db + .insert(user) + .values({ + id, + name: "Insight reviewer", + email: `${id}@example.com`, + emailVerified: true, + createdAt: now, + updatedAt: now, + }) + .returning(); + userIds.push(id); + return row; +} + +async function insertOrganization() { + const id = nextId("org"); + const [row] = await db + .insert(organization) + .values({ + id, + name: `Organization ${id}`, + slug: id, + createdAt: new Date(), + }) + .returning(); + organizationIds.push(id); + return row; +} + +async function addMember( + userId: string, + organizationId: string, + role: "member" | "owner" = "member" +) { + await db.insert(member).values({ + id: nextId("member"), + userId, + organizationId, + role, + createdAt: new Date(), + }); +} + +async function insertWebsite(organizationId: string) { + const id = nextId("website"); + const [row] = await db + .insert(websites) + .values({ + id, + organizationId, + domain: `${id}.example.com`, + name: "Checkout", + }) + .returning(); + return row; +} + +async function insertInsight(input: { + id: string; + organizationId: string; + websiteId: string; +}) { + await db.insert(analyticsInsights).values({ + id: input.id, + organizationId: input.organizationId, + websiteId: input.websiteId, + runId: `run-${input.id}`, + title: "Checkout completion fell", + description: "Fewer visitors completed checkout.", + suggestion: "Inspect the checkout flow.", + severity: "warning", + sentiment: "negative", + type: "funnel_regression", + priority: 8, + subjectKey: "funnel:checkout", + sources: ["product"], + confidence: 0.9, + currentPeriodFrom: "2026-07-01", + currentPeriodTo: "2026-07-07", + previousPeriodFrom: "2026-06-24", + previousPeriodTo: "2026-06-30", + }); +} + +function context(userId: string, email: string, organizationId: string): Context { + const now = new Date(); + return { + db, + auth, + user: { + id: userId, + name: "Insight reviewer", + email, + emailVerified: true, + image: null, + firstName: null, + lastName: null, + status: "ACTIVE", + createdAt: now, + updatedAt: now, + deletedAt: null, + role: "USER", + twoFactorEnabled: false, + }, + session: { + id: nextId("session"), + expiresAt: new Date(now.getTime() + 86_400_000), + token: nextId("token"), + createdAt: now, + updatedAt: now, + ipAddress: "127.0.0.1", + userAgent: "test", + userId, + activeOrganizationId: organizationId, + }, + apiKey: undefined, + getBilling: async () => undefined, + organizationId, + headers: new Headers(), + anonymousId: null, + sessionId: null, + }; +} + +async function expectCode(promise: Promise, code: string) { + await expect(promise).rejects.toMatchObject({ code }); +} + +describe("insight feedback tenancy", () => { + iit("reads and writes a deep-linked finding in its stored organization", async () => { + const storedUser = await insertUser(); + const activeOrg = await insertOrganization(); + const insightOrg = await insertOrganization(); + await addMember(storedUser.id, activeOrg.id); + await addMember(storedUser.id, insightOrg.id); + const website = await insertWebsite(insightOrg.id); + const insightId = nextId("insight-org-b"); + await insertInsight({ + id: insightId, + organizationId: insightOrg.id, + websiteId: website.id, + }); + + await db.insert(insightUserFeedback).values({ + id: nextId("legacy-wrong-org-vote"), + userId: storedUser.id, + organizationId: activeOrg.id, + insightId, + vote: "up", + }); + + const activeContext = context( + storedUser.id, + storedUser.email, + activeOrg.id + ); + expect( + ( + await handler.getById({ + context: activeContext, + input: { insightId }, + }) + ).insight?.id + ).toBe(insightId); + expect( + await handler.getVotes({ + context: activeContext, + input: { insightIds: [insightId] }, + }) + ).toEqual({ votes: {} }); + + await handler.setVote({ + context: activeContext, + input: { insightId, vote: "down" }, + }); + + const rows = await db + .select({ + organizationId: insightUserFeedback.organizationId, + vote: insightUserFeedback.vote, + }) + .from(insightUserFeedback) + .where( + and( + eq(insightUserFeedback.userId, storedUser.id), + eq(insightUserFeedback.insightId, insightId) + ) + ); + + expect(rows).toEqual([ + { organizationId: insightOrg.id, vote: "down" }, + ]); + expect( + await handler.getVotes({ + context: activeContext, + input: { insightIds: [insightId] }, + }) + ).toEqual({ votes: { [insightId]: "down" } }); + + await handler.setVote({ + context: activeContext, + input: { insightId, vote: null }, + }); + expect( + await db + .select({ id: insightUserFeedback.id }) + .from(insightUserFeedback) + .where(eq(insightUserFeedback.insightId, insightId)) + ).toEqual([]); + }); + + iit("does not expose or mutate votes for an inaccessible organization", async () => { + const storedUser = await insertUser(); + const activeOrg = await insertOrganization(); + const inaccessibleOrg = await insertOrganization(); + await addMember(storedUser.id, activeOrg.id); + const website = await insertWebsite(inaccessibleOrg.id); + const insightId = nextId("insight-inaccessible"); + await insertInsight({ + id: insightId, + organizationId: inaccessibleOrg.id, + websiteId: website.id, + }); + const activeContext = context( + storedUser.id, + storedUser.email, + activeOrg.id + ); + + expect( + await handler.getVotes({ + context: activeContext, + input: { insightIds: [insightId] }, + }) + ).toEqual({ votes: {} }); + await expectCode( + handler.setVote({ + context: activeContext, + input: { insightId, vote: "up" }, + }), + "FORBIDDEN" + ); + }); +}); + +describe("insight generation compatibility authorization", () => { + iit("requires organization update permission for a legacy website-scoped write", async () => { + const storedUser = await insertUser(); + const storedOrganization = await insertOrganization(); + await addMember(storedUser.id, storedOrganization.id); + const website = await insertWebsite(storedOrganization.id); + const activeContext = context( + storedUser.id, + storedUser.email, + storedOrganization.id + ); + + expect( + await generationHandler.getConfig({ + context: activeContext, + input: { websiteId: website.id }, + }) + ).toMatchObject({ + organizationId: storedOrganization.id, + source: "default", + websiteId: website.id, + }); + await expectCode( + generationHandler.upsertConfig({ + context: activeContext, + input: { enabled: true, websiteId: website.id }, + }), + "FORBIDDEN" + ); + expect( + await db + .select({ id: insightGenerationConfigs.id }) + .from(insightGenerationConfigs) + .where( + eq( + insightGenerationConfigs.organizationId, + storedOrganization.id + ) + ) + ).toEqual([]); + }); + + iit("rejects website-scoped mutations after authorizing an organization owner", async () => { + const storedUser = await insertUser(); + const storedOrganization = await insertOrganization(); + await addMember(storedUser.id, storedOrganization.id, "owner"); + const website = await insertWebsite(storedOrganization.id); + + await expect( + generationHandler.upsertConfig({ + context: context( + storedUser.id, + storedUser.email, + storedOrganization.id + ), + input: { enabled: true, websiteId: website.id }, + }) + ).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: expect.stringContaining( + "Website-specific insight settings are retired" + ), + }); + }); + + iit("accepts and ignores retired config values from old clients", async () => { + const storedUser = await insertUser(); + const storedOrganization = await insertOrganization(); + await addMember(storedUser.id, storedOrganization.id, "owner"); + const website = await insertWebsite(storedOrganization.id); + const activeContext = context( + storedUser.id, + storedUser.email, + storedOrganization.id + ); + const legacyPayload = { + allowedTools: ["web_metrics", "business_context"] as const, + cooldownHours: 48, + cron: "0 3 * * 1", + depth: "deep" as const, + enabled: true, + frequency: "custom" as const, + lookbackDays: 30, + maxInsightsPerWebsite: 2, + maxSteps: 50, + maxToolCalls: 50, + modelTier: "deep" as const, + organizationId: storedOrganization.id, + timezone: "Europe/Berlin", + }; + + expect( + await generationHandler.upsertConfig({ + context: activeContext, + input: legacyPayload, + }) + ).toMatchObject({ + enabled: true, + frequency: "weekly", + timezone: "Europe/Berlin", + }); + expect( + await generationHandler.getConfig({ + context: activeContext, + input: { websiteId: website.id }, + }) + ).toMatchObject({ + source: "organization", + websiteId: website.id, + }); + expect( + await generationHandler.upsertConfig({ + context: activeContext, + input: { + ...legacyPayload, + frequency: "hourly", + maxSteps: 25, + timezone: "America/New_York", + }, + }) + ).toMatchObject({ + frequency: "weekly", + maxSteps: 24, + timezone: "America/New_York", + }); + + const [stored] = await db + .select() + .from(insightGenerationConfigs) + .where(eq(insightGenerationConfigs.organizationId, storedOrganization.id)); + expect(stored).toMatchObject({ + deliveries: [], + enabled: true, + frequency: "weekly", + legacyModelTier: "balanced", + organizationId: storedOrganization.id, + timezone: "America/New_York", + }); + for (const retiredField of [ + "allowedTools", + "cooldownHours", + "cron", + "depth", + "lookbackDays", + "maxInsightsPerWebsite", + "maxSteps", + "maxToolCalls", + ]) { + expect(stored).not.toHaveProperty(retiredField); + } + }); +}); diff --git a/packages/rpc/src/routers/insights.ts b/packages/rpc/src/routers/insights.ts index 2e499063a..c444ccfae 100644 --- a/packages/rpc/src/routers/insights.ts +++ b/packages/rpc/src/routers/insights.ts @@ -27,13 +27,14 @@ import { ORPCError } from "@orpc/server"; import { randomUUIDv7 } from "bun"; import { z } from "zod"; import { rpcError } from "../errors"; -import { sessionProcedure } from "../orpc"; +import { type Context, sessionProcedure } from "../orpc"; import { withWorkspace } from "../procedures/with-workspace"; const voteSchema = z.enum(["up", "down"]); const rangeSchema = z.enum(["7d", "30d", "90d"]); const insightStatusSchema = z.enum(["open", "resolved"]); const insightResolvedReasonSchema = z.enum(["recovered", "stale"]); +const investigationDepthSchema = z.enum(["surface", "investigated", "deep"]); const NARRATIVE_RATE_LIMIT = 30; const NARRATIVE_RATE_WINDOW_SECS = 3600; @@ -48,6 +49,25 @@ function isAccessDenied(error: unknown): boolean { ); } +function canReadInsightOrganization( + context: Context, + organizationId: string +): Promise { + return withWorkspace(context, { + organizationId, + resource: "organization", + permissions: ["read"], + allowCrossOrg: true, + }) + .then(() => true) + .catch((error) => { + if (isAccessDenied(error)) { + return false; + } + throw error; + }); +} + async function enforceRateLimit( key: string, limit: number, @@ -67,6 +87,7 @@ const historyInsightEvidenceSchema = insightEvidenceSchema.extend({ const historyInsightSchema = z.object({ actions: z.array(storedInsightActionSchema).nullable().optional(), + chainId: z.string().nullable().optional(), changePercent: z.number().optional(), confidence: z.number(), createdAt: z.string(), @@ -76,6 +97,7 @@ const historyInsightSchema = z.object({ evidence: z.array(historyInsightEvidenceSchema).nullable().optional(), id: z.string(), impactSummary: z.string().optional(), + investigationDepth: investigationDepthSchema.nullable().optional(), link: z.string(), metrics: z.array(insightMetricSchema), previousPeriodFrom: z.string().nullable(), @@ -100,8 +122,23 @@ const historyInsightSchema = z.object({ websiteName: z.string().nullable(), }); +const legacyWebsiteInsightSchema = historyInsightSchema.omit({ + createdAt: true, + currentPeriodFrom: true, + currentPeriodTo: true, + previousPeriodFrom: true, + previousPeriodTo: true, + remediationKind: true, + resolvedAt: true, + resolvedReason: true, + runId: true, + status: true, + timezone: true, +}); + const insightSelection = { actions: analyticsInsights.actions, + chainId: analyticsInsights.chainId, changePercent: analyticsInsights.changePercent, confidence: analyticsInsights.confidence, createdAt: analyticsInsights.createdAt, @@ -111,6 +148,7 @@ const insightSelection = { evidence: analyticsInsights.evidence, id: analyticsInsights.id, impactSummary: analyticsInsights.impactSummary, + investigationDepth: analyticsInsights.investigationDepth, metrics: analyticsInsights.metrics, organizationId: analyticsInsights.organizationId, previousPeriodFrom: analyticsInsights.previousPeriodFrom, @@ -144,8 +182,20 @@ function selectInsights() { type InsightRow = Awaited>[number]; -function buildInsightLink(websiteId: string, type: string): string { +export function buildInsightLink( + websiteId: string, + type: string, + subjectKey: string +): string { const base = `/websites/${websiteId}`; + if ( + subjectKey === "revenue" || + subjectKey.startsWith("revenue:") || + subjectKey === "payment_failure_rate" || + subjectKey.startsWith("payment_failure_rate:") + ) { + return `${base}/revenue`; + } if ( [ "error_spike", @@ -180,6 +230,7 @@ function serializeInsight( ): z.infer { return { actions: row.actions ?? null, + chainId: row.chainId ?? null, changePercent: row.changePercent ?? undefined, confidence: row.confidence, createdAt: row.createdAt.toISOString(), @@ -189,7 +240,8 @@ function serializeInsight( evidence: row.evidence ?? null, id: row.id, impactSummary: row.impactSummary ?? undefined, - link: buildInsightLink(row.websiteId, row.type), + investigationDepth: row.investigationDepth ?? null, + link: buildInsightLink(row.websiteId, row.type, row.subjectKey), metrics: row.metrics ?? [], previousPeriodFrom: row.previousPeriodFrom, previousPeriodTo: row.previousPeriodTo, @@ -257,6 +309,71 @@ const loadNarrativeCached = cacheable( ); export const insightsRouter = { + feed: sessionProcedure + .route({ + method: "POST", + path: "/insights/feed", + tags: ["Insights"], + summary: "Get current insight feed", + description: + "Deprecated read-only compatibility endpoint. Reads persisted findings without starting an analysis.", + }) + .input( + z.object({ + organizationId: z.string().min(1), + timezone: z.string().min(1).max(80).default("UTC"), + }) + ) + .output( + z.object({ + generation: z + .object({ + queuedItems: z.number().optional(), + runId: z.string().optional(), + status: z.enum(["queued", "skipped", "disabled", "unavailable"]), + }) + .optional(), + insights: z.array(legacyWebsiteInsightSchema), + source: z.enum(["ai", "fallback"]), + success: z.literal(true), + }) + ) + .handler(async ({ context, input }) => { + await withWorkspace(context, { + organizationId: input.organizationId, + resource: "organization", + permissions: ["read"], + }); + await enforceRateLimit( + `insights:feed:${input.organizationId}:${context.user.id}`, + INSIGHT_READ_RATE_LIMIT, + INSIGHT_READ_RATE_WINDOW_SECS + ); + + const rows = await selectInsights() + .where( + and( + eq(analyticsInsights.organizationId, input.organizationId), + eq(analyticsInsights.status, "open"), + isNull(websites.deletedAt) + ) + ) + .orderBy( + desc(analyticsInsights.priority), + desc(analyticsInsights.createdAt) + ) + .limit(10); + const insights = rows.map((row) => + legacyWebsiteInsightSchema.parse(serializeInsight(row)) + ); + + return { + insights, + source: insights.length > 0 ? ("ai" as const) : ("fallback" as const), + success: true as const, + }; + }), + history: sessionProcedure .route({ method: "POST", @@ -350,19 +467,10 @@ export const insightsRouter = { return { success: true as const, insight: null }; } - const canRead = await withWorkspace(context, { - organizationId: row.organizationId, - resource: "organization", - permissions: ["read"], - allowCrossOrg: true, - }) - .then(() => true) - .catch((error) => { - if (isAccessDenied(error)) { - return false; - } - throw error; - }); + const canRead = await canReadInsightOrganization( + context, + row.organizationId + ); if (!canRead) { return { success: true as const, insight: null }; @@ -423,19 +531,10 @@ export const insightsRouter = { return { success: true as const, insights: [] }; } - const canRead = await withWorkspace(context, { - organizationId: current.organizationId, - resource: "organization", - permissions: ["read"], - allowCrossOrg: true, - }) - .then(() => true) - .catch((error) => { - if (isAccessDenied(error)) { - return false; - } - throw error; - }); + const canRead = await canReadInsightOrganization( + context, + current.organizationId + ); if (!canRead) { return { success: true as const, insights: [] }; @@ -572,7 +671,7 @@ export const insightsRouter = { tags: ["Insights"], summary: "Get insight feedback votes", description: - "Returns thumbs up/down votes for the given insight ids for the current user in the active organization.", + "Returns thumbs up/down votes for findings the current user can access, resolved against each finding's organization.", }) .input( z.object({ @@ -585,24 +684,65 @@ export const insightsRouter = { }) ) .handler(async ({ context, input }) => { - if (!context.organizationId) { - throw rpcError.badRequest("Organization context is required"); - } if (input.insightIds.length === 0) { return { votes: {} }; } + const insightRows = await context.db + .select({ + id: analyticsInsights.id, + organizationId: analyticsInsights.organizationId, + }) + .from(analyticsInsights) + .innerJoin(websites, eq(analyticsInsights.websiteId, websites.id)) + .where( + and( + inArray(analyticsInsights.id, input.insightIds), + isNull(websites.deletedAt) + ) + ); + const organizationIds = [ + ...new Set(insightRows.map((row) => row.organizationId)), + ]; + const access = await Promise.all( + organizationIds.map(async (organizationId) => ({ + organizationId, + canRead: await canReadInsightOrganization(context, organizationId), + })) + ); + const accessibleOrganizations = new Set( + access + .filter((result) => result.canRead) + .map((result) => result.organizationId) + ); + const accessibleInsightIds = insightRows + .filter((row) => accessibleOrganizations.has(row.organizationId)) + .map((row) => row.id); + + if (accessibleInsightIds.length === 0) { + return { votes: {} }; + } + const rows = await context.db .select({ insightId: insightUserFeedback.insightId, vote: insightUserFeedback.vote, }) .from(insightUserFeedback) + .innerJoin( + analyticsInsights, + and( + eq(insightUserFeedback.insightId, analyticsInsights.id), + eq( + insightUserFeedback.organizationId, + analyticsInsights.organizationId + ) + ) + ) .where( and( eq(insightUserFeedback.userId, context.user.id), - eq(insightUserFeedback.organizationId, context.organizationId), - inArray(insightUserFeedback.insightId, input.insightIds), + inArray(analyticsInsights.id, accessibleInsightIds), inArray(insightUserFeedback.vote, ["up", "down"]) ) ); @@ -623,7 +763,7 @@ export const insightsRouter = { tags: ["Insights"], summary: "Set or clear insight vote", description: - "Sets thumbs up/down for an insight, or clears the vote when vote is null.", + "Sets thumbs up/down for a finding in its stored organization, or clears the vote when vote is null.", }) .input( z.object({ @@ -633,46 +773,189 @@ export const insightsRouter = { ) .output(z.object({ success: z.literal(true) })) .handler(async ({ context, input }) => { - if (!context.organizationId) { - throw rpcError.badRequest("Organization context is required"); + const [insight] = await context.db + .select({ organizationId: analyticsInsights.organizationId }) + .from(analyticsInsights) + .innerJoin(websites, eq(analyticsInsights.websiteId, websites.id)) + .where( + and( + eq(analyticsInsights.id, input.insightId), + isNull(websites.deletedAt) + ) + ) + .limit(1); + + if (!insight) { + throw rpcError.notFound("insight", input.insightId); } - if (input.vote === null) { - await context.db + await withWorkspace(context, { + organizationId: insight.organizationId, + resource: "organization", + permissions: ["read"], + allowCrossOrg: true, + }); + + await context.db.transaction(async (tx) => { + if (input.vote === null) { + await tx + .delete(insightUserFeedback) + .where( + and( + eq(insightUserFeedback.userId, context.user.id), + eq(insightUserFeedback.insightId, input.insightId) + ) + ); + return; + } + + await tx .delete(insightUserFeedback) .where( and( eq(insightUserFeedback.userId, context.user.id), - eq(insightUserFeedback.organizationId, context.organizationId), - eq(insightUserFeedback.insightId, input.insightId) + eq(insightUserFeedback.insightId, input.insightId), + ne(insightUserFeedback.organizationId, insight.organizationId) ) ); - return { success: true as const }; - } - const now = new Date(); - await context.db - .insert(insightUserFeedback) - .values({ - id: randomUUIDv7(), - userId: context.user.id, - organizationId: context.organizationId, - insightId: input.insightId, - vote: input.vote, - createdAt: now, - updatedAt: now, - }) - .onConflictDoUpdate({ - target: [ - insightUserFeedback.userId, - insightUserFeedback.organizationId, - insightUserFeedback.insightId, - ], - set: { + const now = new Date(); + await tx + .insert(insightUserFeedback) + .values({ + id: randomUUIDv7(), + userId: context.user.id, + organizationId: insight.organizationId, + insightId: input.insightId, vote: input.vote, + createdAt: now, updatedAt: now, - }, - }); + }) + .onConflictDoUpdate({ + target: [ + insightUserFeedback.userId, + insightUserFeedback.organizationId, + insightUserFeedback.insightId, + ], + set: { + vote: input.vote, + updatedAt: now, + }, + }); + }); + + return { success: true as const }; + }), + + setDismissed: sessionProcedure + .route({ + method: "POST", + path: "/insights/setDismissed", + tags: ["Insights"], + summary: "Dismiss or restore an insight", + description: + "Deprecated compatibility endpoint. Persists dismissal for older clients; the current dashboard keeps view state locally.", + }) + .input( + z.object({ + dismissed: z.boolean(), + insightId: z.string().min(1).max(256), + }) + ) + .output(z.object({ success: z.literal(true) })) + .handler(async ({ context, input }) => { + const [insight] = await context.db + .select({ organizationId: analyticsInsights.organizationId }) + .from(analyticsInsights) + .innerJoin(websites, eq(analyticsInsights.websiteId, websites.id)) + .where( + and( + eq(analyticsInsights.id, input.insightId), + isNull(websites.deletedAt) + ) + ) + .limit(1); + + if (!insight) { + throw rpcError.notFound("insight", input.insightId); + } + + await withWorkspace(context, { + organizationId: insight.organizationId, + resource: "organization", + permissions: ["read"], + allowCrossOrg: true, + }); + + await context.db.transaction(async (tx) => { + await tx + .delete(insightUserFeedback) + .where( + and( + eq(insightUserFeedback.userId, context.user.id), + eq(insightUserFeedback.insightId, input.insightId), + eq(insightUserFeedback.vote, "dismissed") + ) + ); + + if (!input.dismissed) { + return; + } + + const now = new Date(); + await tx + .insert(insightUserFeedback) + .values({ + id: randomUUIDv7(), + userId: context.user.id, + organizationId: insight.organizationId, + insightId: input.insightId, + vote: "dismissed", + createdAt: now, + updatedAt: now, + }) + .onConflictDoUpdate({ + target: [ + insightUserFeedback.userId, + insightUserFeedback.organizationId, + insightUserFeedback.insightId, + ], + set: { vote: "dismissed", updatedAt: now }, + }); + }); + + return { success: true as const }; + }), + + clearDismissed: sessionProcedure + .route({ + method: "POST", + path: "/insights/clearDismissed", + tags: ["Insights"], + summary: "Clear dismissed insights", + description: + "Deprecated compatibility endpoint. Clears persisted dismissals for the active organization.", + }) + .input(z.object({})) + .output(z.object({ success: z.literal(true) })) + .handler(async ({ context }) => { + if (!context.organizationId) { + throw rpcError.badRequest("Organization context is required"); + } + await withWorkspace(context, { + organizationId: context.organizationId, + resource: "organization", + permissions: ["read"], + }); + await context.db + .delete(insightUserFeedback) + .where( + and( + eq(insightUserFeedback.userId, context.user.id), + eq(insightUserFeedback.organizationId, context.organizationId), + eq(insightUserFeedback.vote, "dismissed") + ) + ); return { success: true as const }; }), diff --git a/packages/rpc/src/routers/link-folders.ts b/packages/rpc/src/routers/link-folders.ts index ee4c785f4..d3e43b963 100644 --- a/packages/rpc/src/routers/link-folders.ts +++ b/packages/rpc/src/routers/link-folders.ts @@ -2,8 +2,10 @@ import { and, asc, eq, + getTableColumns, isNull, isUniqueViolationFor, + sql, withTransaction, } from "@databuddy/db"; import { linkFolders, links } from "@databuddy/db/schema"; @@ -17,6 +19,7 @@ import { createLinkFolderSchema, deleteLinkFolderSchema, linkFolderOutputSchema, + linkFolderWithUsageOutputSchema, listLinkFoldersSchema, slugifyFolderName, updateLinkFolderSchema, @@ -81,7 +84,7 @@ export const linkFoldersRouter = { spec: (s) => ({ ...s, "x-required-scopes": ["read:links"] as const }), }) .input(listLinkFoldersSchema) - .output(z.array(linkFolderOutputSchema)) + .output(z.array(linkFolderWithUsageOutputSchema)) .handler(async ({ context, input }) => { const organizationId = requireOrganizationId( input.organizationId ?? context.organizationId @@ -90,7 +93,16 @@ export const linkFoldersRouter = { await requireLinkAccess(context, organizationId, "read"); return context.db - .select() + .select({ + ...getTableColumns(linkFolders), + linkCount: sql`( + select count(*)::int + from ${links} + where ${links.organizationId} = ${organizationId} + and ${links.folderId} = ${linkFolders.id} + and ${links.deletedAt} is null + )`.mapWith(Number), + }) .from(linkFolders) .where( and( diff --git a/packages/rpc/src/routers/links.schemas.ts b/packages/rpc/src/routers/links.schemas.ts index 0dcf7b6ab..7debcc5b2 100644 --- a/packages/rpc/src/routers/links.schemas.ts +++ b/packages/rpc/src/routers/links.schemas.ts @@ -46,13 +46,21 @@ export const linkTypeFilterSchema = z .default("all"); export const listLinksPageSchema = listLinksSchema.unwrap().extend({ + includeTotal: z.boolean().default(false), limit: z.number().int().min(1).max(100).default(50), offset: z.number().int().min(0).default(0), - search: z.string().max(255).optional(), + search: z.string().trim().min(1).max(255).optional(), sort: linkSortSchema, type: linkTypeFilterSchema, }); +export const linksSummarySchema = z + .object({ + organizationId: z.string().optional(), + search: z.string().trim().min(1).max(255).optional(), + }) + .default({}); + export const listLinkFoldersSchema = z .object({ organizationId: z.string().optional(), @@ -224,6 +232,12 @@ export const linkOutputSchema = linkSelectSchema.pick({ export const listLinksPageOutputSchema = z.object({ items: z.array(linkOutputSchema), hasMore: z.boolean(), + total: z.number().int().nonnegative().optional(), +}); + +export const linksSummaryOutputSchema = z.object({ + total: z.number().int().nonnegative(), + unfiledTotal: z.number().int().nonnegative(), }); export const linkFolderOutputSchema = linkFolderSelectSchema.pick({ @@ -237,6 +251,10 @@ export const linkFolderOutputSchema = linkFolderSelectSchema.pick({ updatedAt: true, }); +export const linkFolderWithUsageOutputSchema = linkFolderOutputSchema.extend({ + linkCount: z.number().int().nonnegative(), +}); + export function slugifyFolderName(name: string): string { const slug = name .trim() diff --git a/packages/rpc/src/routers/links.test.ts b/packages/rpc/src/routers/links.test.ts index 96c8822b8..8e06bbd6a 100644 --- a/packages/rpc/src/routers/links.test.ts +++ b/packages/rpc/src/routers/links.test.ts @@ -4,6 +4,8 @@ import { deleteLinkSchema, getLinkSchema, linkOutputSchema, + linksSummaryOutputSchema, + linksSummarySchema, listLinksPageSchema, listLinksSchema, updateLinkSchema, @@ -210,6 +212,7 @@ describe("listLinksPageSchema validation", () => { expect(result.success).toBe(true); if (result.success) { + expect(result.data.includeTotal).toBe(false); expect(result.data.limit).toBe(50); expect(result.data.offset).toBe(0); expect(result.data.sort).toBe("newest"); @@ -222,6 +225,7 @@ describe("listLinksPageSchema validation", () => { listLinksPageSchema.safeParse({ organizationId: "org-123", folderId: null, + includeTotal: true, search: "campaign", sort: "name-asc", type: "deep", @@ -244,6 +248,34 @@ describe("listLinksPageSchema validation", () => { }); }); +describe("linksSummarySchema validation", () => { + it("accepts active-organization fallback and optional search", () => { + expect(linksSummarySchema.parse({})).toEqual({}); + expect( + linksSummarySchema.parse({ + organizationId: "org-123", + search: "campaign", + }) + ).toEqual({ organizationId: "org-123", search: "campaign" }); + expect(linksSummarySchema.safeParse({ search: " " }).success).toBe( + false + ); + expect(linksSummarySchema.safeParse({ search: "x".repeat(256) }).success).toBe( + false + ); + }); + + it("validates exact aggregate output", () => { + expect( + linksSummaryOutputSchema.parse({ total: 1200, unfiledTotal: 35 }) + ).toEqual({ total: 1200, unfiledTotal: 35 }); + expect( + linksSummaryOutputSchema.safeParse({ total: -1, unfiledTotal: 0 }) + .success + ).toBe(false); + }); +}); + describe("id input schemas", () => { it("accept valid ids and reject missing ids", () => { expect(getLinkSchema.safeParse({ id: "link-123" }).success).toBe(true); diff --git a/packages/rpc/src/routers/links.ts b/packages/rpc/src/routers/links.ts index 8cadab940..de957b8ee 100644 --- a/packages/rpc/src/routers/links.ts +++ b/packages/rpc/src/routers/links.ts @@ -1,6 +1,7 @@ import { and, asc, + count, desc, eq, ilike, @@ -8,6 +9,7 @@ import { isNull, isUniqueViolationFor, or, + sql, } from "@databuddy/db"; import { linkFolders, links } from "@databuddy/db/schema"; import { @@ -29,6 +31,8 @@ import { deleteLinkSchema, getLinkSchema, linkOutputSchema, + linksSummaryOutputSchema, + linksSummarySchema, listLinksPageOutputSchema, listLinksPageSchema, listLinksSchema, @@ -167,6 +171,7 @@ function requireLinkAccess( } const LINKS_LIST_MAX = 1000; +const ILIKE_PATTERN_CHARACTER_REGEX = /[\\%_]/g; function buildLinkListConditions( input: { @@ -179,7 +184,10 @@ function buildLinkListConditions( }, organizationId: string ) { - const conditions = [eq(links.organizationId, organizationId)]; + const conditions = [ + eq(links.organizationId, organizationId), + isNull(links.deletedAt), + ]; if (input.externalId) { conditions.push(eq(links.externalId, input.externalId)); } @@ -206,6 +214,25 @@ function buildLinkListConditions( return conditions; } +function buildLinkSearchCondition(search: string | undefined) { + const trimmed = search?.trim(); + if (!trimmed) { + return; + } + + const term = `%${trimmed.replace(ILIKE_PATTERN_CHARACTER_REGEX, "\\$&")}%`; + return or( + ilike(links.name, term), + ilike(links.slug, term), + ilike(links.targetUrl, term), + ilike(links.externalId, term), + ilike(links.sourceType, term), + ilike(links.sourceId, term), + ilike(links.sourceOwnerId, term), + ilike(links.targetDomain, term) + ); +} + const linkSortOrder = { newest: [desc(links.createdAt), desc(links.id)], oldest: [asc(links.createdAt), asc(links.id)], @@ -264,7 +291,7 @@ export const linksRouter = { tags: ["Links"], summary: "List links (paginated)", description: - "Returns a page of links for the organization with server-side search, sort, type filter, and offset pagination. Requires read:links scope.", + "Returns a page of links for the organization with server-side search, sort, type filter, and offset pagination. Set includeTotal only when an exact filtered count is needed. Requires read:links scope.", spec: (s) => ({ ...s, "x-required-scopes": ["read:links"] as const }), }) .input(listLinksPageSchema) @@ -284,37 +311,83 @@ export const linksRouter = { conditions.push(isNotNull(links.deepLinkApp)); } - const search = input.search?.trim(); - if (search) { - const term = `%${search}%`; - const matches = or( - ilike(links.name, term), - ilike(links.slug, term), - ilike(links.targetUrl, term), - ilike(links.externalId, term), - ilike(links.sourceType, term), - ilike(links.sourceId, term), - ilike(links.sourceOwnerId, term), - ilike(links.targetDomain, term) - ); - if (matches) { - conditions.push(matches); - } + const matches = buildLinkSearchCondition(input.search); + if (matches) { + conditions.push(matches); } - const rows = await context.db + const where = and(...conditions); + const pageQuery = context.db .select() .from(links) - .where(and(...conditions)) + .where(where) .orderBy(...linkSortOrder[input.sort]) .limit(input.limit + 1) .offset(input.offset); + let rows: LinkRow[]; + let total: number | undefined; + + if (input.includeTotal) { + const [page, [summary]] = await Promise.all([ + pageQuery, + context.db.select({ total: count() }).from(links).where(where), + ]); + rows = page; + total = summary?.total ?? 0; + } else { + rows = await pageQuery; + } const hasMore = rows.length > input.limit; return { items: hasMore ? rows.slice(0, input.limit) : rows, hasMore, + ...(total === undefined ? {} : { total }), + }; + }), + + summary: protectedProcedure + .route({ + method: "POST", + path: "/links/summary", + tags: ["Links"], + summary: "Summarize links", + description: + "Returns exact total and unfiled counts. When search is set, both counts cover only matching links. Requires read:links scope.", + spec: (s) => ({ ...s, "x-required-scopes": ["read:links"] as const }), + }) + .input(linksSummarySchema) + .output(linksSummaryOutputSchema) + .handler(async ({ context, input }) => { + const organizationId = requireOrganizationId( + input.organizationId ?? context.organizationId + ); + + await requireLinkAccess(context, organizationId, "read"); + + const matches = buildLinkSearchCondition(input.search); + const unfiledMatches = matches + ? and(isNull(links.folderId), matches) + : isNull(links.folderId); + const [summary] = await context.db + .select({ + total: matches + ? sql`count(*) filter (where ${matches})`.mapWith(Number) + : count(), + unfiledTotal: + sql`count(*) filter (where ${unfiledMatches})`.mapWith( + Number + ), + }) + .from(links) + .where( + and(eq(links.organizationId, organizationId), isNull(links.deletedAt)) + ); + + return { + total: summary?.total ?? 0, + unfiledTotal: summary?.unfiledTotal ?? 0, }; }), diff --git a/packages/rpc/src/routers/revenue.schemas.ts b/packages/rpc/src/routers/revenue.schemas.ts new file mode 100644 index 000000000..ee0861e79 --- /dev/null +++ b/packages/rpc/src/routers/revenue.schemas.ts @@ -0,0 +1,15 @@ +import { isCurrencyCode } from "@databuddy/shared/currency"; +import { z } from "zod"; + +export const revenueCurrencySchema = z + .string() + .trim() + .toUpperCase() + .refine(isCurrencyCode, "Enter a valid ISO 4217 currency code"); + +export const revenueUpsertInputSchema = z.object({ + websiteId: z.string().optional(), + stripeWebhookSecret: z.string().optional(), + paddleWebhookSecret: z.string().optional(), + currency: revenueCurrencySchema.optional(), +}); diff --git a/packages/rpc/src/routers/revenue.test.ts b/packages/rpc/src/routers/revenue.test.ts new file mode 100644 index 000000000..132264e4c --- /dev/null +++ b/packages/rpc/src/routers/revenue.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, test } from "bun:test"; +import { revenueUpsertInputSchema } from "./revenue.schemas"; + +describe("revenue upsert input", () => { + test("normalizes valid ISO currency codes", () => { + expect(revenueUpsertInputSchema.parse({ currency: " eur " })).toEqual({ + currency: "EUR", + }); + }); + + test.each(["US", "ZZZ", "dollars", ""])( + "rejects invalid currency %p", + (currency) => { + expect(revenueUpsertInputSchema.safeParse({ currency }).success).toBe(false); + } + ); +}); diff --git a/packages/rpc/src/routers/revenue.ts b/packages/rpc/src/routers/revenue.ts index 6d99d34f0..fc5a82d51 100644 --- a/packages/rpc/src/routers/revenue.ts +++ b/packages/rpc/src/routers/revenue.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { rpcError } from "../errors"; import { protectedProcedure, sessionProcedure } from "../orpc"; import { withWorkspace } from "../procedures/with-workspace"; +import { revenueUpsertInputSchema } from "./revenue.schemas"; function generateHash(): string { const bytes = crypto.getRandomValues(new Uint8Array(24)); @@ -71,14 +72,7 @@ export const revenueRouter = { summary: "Upsert revenue config", tags: ["Revenue"], }) - .input( - z.object({ - websiteId: z.string().optional(), - stripeWebhookSecret: z.string().optional(), - paddleWebhookSecret: z.string().optional(), - currency: z.string().length(3).optional(), - }) - ) + .input(revenueUpsertInputSchema) .output(revenueOutputSchema) .handler(async ({ context, input }) => { const workspace = input.websiteId diff --git a/packages/rpc/src/routers/status-page-health.test.ts b/packages/rpc/src/routers/status-page-health.test.ts index 307ebb884..b6ff4fca1 100644 --- a/packages/rpc/src/routers/status-page-health.test.ts +++ b/packages/rpc/src/routers/status-page-health.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "bun:test"; +import { parseUptimeGranularity } from "@databuddy/shared/uptime"; import { deriveMonitorFreshness, deriveMonitorStatus, @@ -16,6 +17,20 @@ describe("status page health", () => { expect( deriveMonitorFreshness("2026-07-11 11:50:00", "minute", now) ).toBe("stale"); + expect( + deriveMonitorFreshness( + "2026-07-11 11:59:00", + parseUptimeGranularity("toString"), + now + ) + ).toBe("unknown"); + expect( + deriveMonitorFreshness( + "2026-07-11 11:59:00", + parseUptimeGranularity("__proto__"), + now + ) + ).toBe("unknown"); }); it("uses the monitor cadence when deciding freshness", () => { diff --git a/packages/rpc/src/routers/status-page-health.ts b/packages/rpc/src/routers/status-page-health.ts index 7ee0d18d6..14732e2ce 100644 --- a/packages/rpc/src/routers/status-page-health.ts +++ b/packages/rpc/src/routers/status-page-health.ts @@ -1,8 +1,10 @@ +import type { UptimeGranularity } from "@databuddy/shared/uptime"; + export type MonitorStatus = "up" | "down" | "degraded" | "unknown"; export type MonitorFreshness = "fresh" | "stale" | "unknown"; export type OverallStatus = "operational" | "degraded" | "outage" | "unknown"; -const GRANULARITY_MS: Record = { +const GRANULARITY_MS = { minute: 60_000, five_minutes: 5 * 60_000, ten_minutes: 10 * 60_000, @@ -11,7 +13,7 @@ const GRANULARITY_MS: Record = { six_hours: 6 * 60 * 60_000, twelve_hours: 12 * 60 * 60_000, day: 24 * 60 * 60_000, -}; +} satisfies Record; function parseCheckTimestamp(value: string): number { const normalized = value.includes("T") @@ -30,10 +32,10 @@ export function normalizeCheckTimestamp(value: string | null): string | null { export function deriveMonitorFreshness( lastCheckedAt: string | null, - granularity: string, + granularity: UptimeGranularity | null, now = Date.now() ): MonitorFreshness { - if (!lastCheckedAt) { + if (!(lastCheckedAt && granularity)) { return "unknown"; } diff --git a/packages/rpc/src/routers/status-page.ts b/packages/rpc/src/routers/status-page.ts index e9e762d09..5dbc16272 100644 --- a/packages/rpc/src/routers/status-page.ts +++ b/packages/rpc/src/routers/status-page.ts @@ -17,6 +17,7 @@ import { } from "@databuddy/redis"; import { randomUUIDv7 } from "bun"; import { z } from "zod"; +import { parseUptimeGranularity } from "@databuddy/shared/uptime"; import { rpcError } from "../errors"; import { setTrackProperties } from "../middleware/track-mutation"; import { protectedProcedure, publicProcedure, trackedProcedure } from "../orpc"; @@ -418,7 +419,7 @@ async function _fetchStatusPageData( const freshness: MonitorFreshness = deriveMonitorFreshness( lastCheckedAt, - schedule.granularity + parseUptimeGranularity(schedule.granularity) ); const currentStatus: MonitorStatus = deriveMonitorStatus({ lastStatus: latestCheck?.last_status ?? null, diff --git a/packages/rpc/src/routers/uptime.ts b/packages/rpc/src/routers/uptime.ts index 3ce2f04f3..b3586dee9 100644 --- a/packages/rpc/src/routers/uptime.ts +++ b/packages/rpc/src/routers/uptime.ts @@ -5,6 +5,10 @@ import { uptimeSchedules, } from "@databuddy/db/schema"; import { invalidateStatusPageCache } from "@databuddy/redis"; +import { + uptimeGranularitySchema, + type UptimeGranularity, +} from "@databuddy/shared/uptime"; import { randomUUIDv7 } from "bun"; import { z } from "zod"; import { rpcError } from "../errors"; @@ -27,21 +31,8 @@ import { hasUptimeSchedule, } from "../services/uptime-scheduler"; -const granularityEnum = z.enum([ - "minute", - "five_minutes", - "ten_minutes", - "thirty_minutes", - "hour", - "six_hours", - "twelve_hours", - "day", -]); - -function parseStoredGranularity( - value: string -): z.infer { - const parsed = granularityEnum.safeParse(value); +function parseStoredGranularity(value: string): UptimeGranularity { + const parsed = uptimeGranularitySchema.safeParse(value); if (!parsed.success) { throw rpcError.internal("Invalid monitor granularity"); } @@ -78,7 +69,7 @@ const getScheduleOutputSchema = z organizationId: z.string(), url: z.string(), name: z.string().nullable(), - granularity: z.string(), + granularity: uptimeGranularitySchema, cron: z.string(), isPaused: z.boolean(), timeout: z.number().nullable().optional(), @@ -224,7 +215,7 @@ export const uptimeRouter = { name: z.string().optional(), organizationId: z.string().optional(), websiteId: z.string().optional(), - granularity: granularityEnum, + granularity: uptimeGranularitySchema, timeout: z.number().int().min(1000).max(120_000).optional(), cacheBust: z.boolean().optional(), jsonParsingConfig: z @@ -304,7 +295,7 @@ export const uptimeRouter = { z.object({ scheduleId: z.string(), name: z.string().nullish(), - granularity: granularityEnum.optional(), + granularity: uptimeGranularitySchema.optional(), timeout: z.number().int().min(1000).max(120_000).nullish(), cacheBust: z.boolean().optional(), jsonParsingConfig: z diff --git a/packages/services/src/identity.ts b/packages/services/src/identity.ts index c8490433a..acd815c1b 100644 --- a/packages/services/src/identity.ts +++ b/packages/services/src/identity.ts @@ -332,39 +332,110 @@ export interface TraitDistributionRow { value: string; } -const TRAIT_DISTRIBUTION_LIMIT = 200; -const TRAIT_VALUES_PER_KEY = 20; +interface TraitDistributionQueryRow + extends TraitDistributionRow, + Record { + distinct_value_count: number; + total_trait_keys: number; + values_per_key: number; +} + +const TRAIT_DISTRIBUTION_ROW_LIMIT = 200; +const MAX_TRAIT_VALUES_PER_KEY = 20; export async function getTraitDistribution(websiteId: string): Promise<{ + hasMoreKeys: boolean; + hasMoreValues: boolean; identifiedProfiles: number; + returnedTraitKeys: number; + totalTraitKeys: number; traits: TraitDistributionRow[]; + valuesPerKey: number; }> { const [totals] = await db .select({ identifiedProfiles: sql`count(*)::int` }) .from(profiles) .where(eq(profiles.websiteId, websiteId)); - const result = await db.execute(sql` - with ranked as ( + const result = await db.execute(sql` + with trait_values as ( select t.key as key, t.value as value, - count(*)::int as profiles, - row_number() over (partition by t.key order by count(*) desc) as value_rank + count(*)::int as profiles from ${profiles} p, jsonb_each_text(p.traits) as t(key, value) where p.website_id = ${websiteId} and jsonb_typeof(p.traits) = 'object' group by t.key, t.value + ), + ranked as ( + select + key, + value, + profiles, + (row_number() over ( + partition by key + order by profiles desc, value asc + ))::int as value_rank, + (count(*) over (partition by key))::int as distinct_value_count + from trait_values + ), + key_coverage as ( + select key, sum(profiles)::bigint as profile_coverage + from trait_values + group by key + ), + selected_keys as ( + select key, total_trait_keys + from ( + select + key, + profile_coverage, + (count(*) over ())::int as total_trait_keys + from key_coverage + ) keys + order by profile_coverage desc, key asc + limit ${TRAIT_DISTRIBUTION_ROW_LIMIT} + ), + budget as ( + select greatest( + 1, + least( + ${MAX_TRAIT_VALUES_PER_KEY}, + floor(${TRAIT_DISTRIBUTION_ROW_LIMIT}::numeric / greatest(count(*), 1))::int + ) + )::int as values_per_key + from selected_keys ) - select key, value, profiles + select + ranked.key, + ranked.value, + ranked.profiles, + ranked.distinct_value_count, + selected_keys.total_trait_keys, + budget.values_per_key from ranked - where value_rank <= ${TRAIT_VALUES_PER_KEY} - order by key asc, profiles desc - limit ${TRAIT_DISTRIBUTION_LIMIT} + inner join selected_keys on selected_keys.key = ranked.key + cross join budget + where ranked.value_rank <= budget.values_per_key + order by ranked.key asc, ranked.profiles desc, ranked.value asc `); + const rows = result.rows; + const valuesPerKey = rows[0]?.values_per_key ?? 0; + const totalTraitKeys = rows[0]?.total_trait_keys ?? 0; + const returnedTraitKeys = new Set(rows.map((row) => row.key)).size; return { + hasMoreKeys: returnedTraitKeys < totalTraitKeys, + hasMoreValues: rows.some((row) => row.distinct_value_count > valuesPerKey), identifiedProfiles: totals?.identifiedProfiles ?? 0, - traits: result.rows as unknown as TraitDistributionRow[], + returnedTraitKeys, + totalTraitKeys, + traits: rows.map((row) => ({ + key: row.key, + profiles: row.profiles, + value: row.value, + })), + valuesPerKey, }; } diff --git a/packages/shared/package.json b/packages/shared/package.json index fa5431d88..a51e553fc 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -20,9 +20,13 @@ "./custom-events": "./src/custom-events.ts", "./analytics-filters": "./src/analytics-filters.ts", "./agent-credits": "./src/agent-credits.ts", + "./billing": "./src/billing.ts", + "./currency": "./src/currency.ts", "./tracking-blocks": "./src/tracking-blocks.ts", + "./uptime": "./src/uptime.ts", "./http-error-response": "./src/http-error-response.ts", "./insights": "./src/insights.ts", + "./stripe-webhooks": "./src/stripe-webhooks.ts", "./evlog-fields": "./src/evlog-fields.ts", "./evlog-redaction": "./src/evlog-redaction.ts", "./evlog-superlog": "./src/evlog-superlog.ts" diff --git a/packages/shared/src/billing.ts b/packages/shared/src/billing.ts new file mode 100644 index 000000000..6f559f8e5 --- /dev/null +++ b/packages/shared/src/billing.ts @@ -0,0 +1,13 @@ +export const DATABUNNY_USAGE = { + description: + "This allowance powers Databunny questions and AI analysis. More complex work uses more of it.", + name: "Databunny usage", + pausedActivity: "Databunny questions and AI analysis", + unit: "usage units", + upgradeMessage: "Add more Databunny usage or upgrade your plan", +} as const; + +export const LEGACY_SCALE_PLAN = { + id: "scale", + name: "Enterprise", +} as const; diff --git a/packages/shared/src/currency.test.ts b/packages/shared/src/currency.test.ts new file mode 100644 index 000000000..ab4d0d832 --- /dev/null +++ b/packages/shared/src/currency.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, test } from "bun:test"; +import { isCurrencyCode, normalizeCurrencyCode } from "./currency"; + +describe("currency codes", () => { + test("normalizes supported ISO 4217 codes", () => { + expect(normalizeCurrencyCode(" eur ")).toBe("EUR"); + expect(isCurrencyCode("JPY")).toBe(true); + }); + + test("rejects missing, malformed, and unsupported codes", () => { + expect(normalizeCurrencyCode(undefined)).toBeNull(); + expect(normalizeCurrencyCode("US")).toBeNull(); + expect(normalizeCurrencyCode("ZZZ")).toBeNull(); + expect(isCurrencyCode("ZZZ")).toBe(false); + }); +}); diff --git a/packages/shared/src/currency.ts b/packages/shared/src/currency.ts new file mode 100644 index 000000000..f4f9b85e0 --- /dev/null +++ b/packages/shared/src/currency.ts @@ -0,0 +1,14 @@ +const ISO_CURRENCY_CODES = new Set(Intl.supportedValuesOf("currency")); + +export function normalizeCurrencyCode(value: unknown): string | null { + if (typeof value !== "string") { + return null; + } + + const currency = value.trim().toUpperCase(); + return ISO_CURRENCY_CODES.has(currency) ? currency : null; +} + +export function isCurrencyCode(value: string): boolean { + return normalizeCurrencyCode(value) !== null; +} diff --git a/packages/shared/src/insights.ts b/packages/shared/src/insights.ts index 563cb9c53..b2c9a918a 100644 --- a/packages/shared/src/insights.ts +++ b/packages/shared/src/insights.ts @@ -178,6 +178,7 @@ export const generatedInsightSchema = z .strict(); const investigationKeySchema = z.string().trim().min(1).max(160); +const investigationEntityIdSchema = z.string().trim().min(1).max(256); const investigationEntitySchema = z .object({ @@ -194,7 +195,7 @@ const investigationEntitySchema = z "campaign", "uptime_monitor", ]), - id: investigationKeySchema, + id: investigationEntityIdSchema, label: z.string().trim().min(1).max(120), }) .strict(); @@ -236,6 +237,7 @@ export const investigationSignalSchema = z }) .strict(), changePercent: z.number().nullable(), + currency: z.string().trim().length(3).optional(), direction: z.enum(["up", "down", "flat"]), severity: insightSeveritySchema, sentiment: insightSentimentSchema, diff --git a/packages/shared/src/stripe-webhooks.test.ts b/packages/shared/src/stripe-webhooks.test.ts new file mode 100644 index 000000000..4c28e285a --- /dev/null +++ b/packages/shared/src/stripe-webhooks.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, test } from "bun:test"; +import { + STRIPE_FAILURE_WEBHOOK_EVENTS, + STRIPE_WEBHOOK_EVENTS, +} from "./stripe-webhooks"; + +describe("STRIPE_WEBHOOK_EVENTS", () => { + test("keeps one unique canonical list with modern invoice allocations", () => { + const names = [ + ...STRIPE_WEBHOOK_EVENTS.required, + ...STRIPE_WEBHOOK_EVENTS.optional, + ].map(({ event }) => event); + + expect(new Set(names).size).toBe(names.length); + expect(STRIPE_WEBHOOK_EVENTS.required.map(({ event }) => event)).toContain( + "invoice_payment.paid" + ); + expect(STRIPE_WEBHOOK_EVENTS.required.map(({ event }) => event)).toContain( + "payment_intent.payment_failed" + ); + expect(STRIPE_WEBHOOK_EVENTS.required.map(({ event }) => event)).toContain( + "invoice.payment_failed" + ); + expect(STRIPE_WEBHOOK_EVENTS.optional).toHaveLength(0); + expect( + STRIPE_FAILURE_WEBHOOK_EVENTS.map(({ event }) => event) + ).toEqual([ + "payment_intent.payment_failed", + "invoice.payment_failed", + ]); + }); +}); diff --git a/packages/shared/src/stripe-webhooks.ts b/packages/shared/src/stripe-webhooks.ts new file mode 100644 index 000000000..4983f20f3 --- /dev/null +++ b/packages/shared/src/stripe-webhooks.ts @@ -0,0 +1,38 @@ +export const STRIPE_FAILURE_WEBHOOK_EVENTS = [ + { + event: "payment_intent.payment_failed", + purpose: "Tracks failed payment attempts", + }, + { + event: "invoice.payment_failed", + purpose: "Tracks failed invoice attempts and retries", + }, +] as const; + +export const STRIPE_WEBHOOK_EVENTS = { + required: [ + { + event: "payment_intent.succeeded", + purpose: "Records successful one-time payments and payment context", + }, + { + event: "invoice.paid", + purpose: "Records invoice and subscription context", + }, + { + event: "invoice_payment.paid", + purpose: "Records exact invoice allocations on modern Stripe versions", + }, + STRIPE_FAILURE_WEBHOOK_EVENTS[0], + { + event: "payment_intent.canceled", + purpose: "Tracks canceled payment attempts", + }, + STRIPE_FAILURE_WEBHOOK_EVENTS[1], + { + event: "charge.refunded", + purpose: "Records each refund", + }, + ], + optional: [], +} as const; diff --git a/packages/shared/src/types/features.ts b/packages/shared/src/types/features.ts index 5b30f43da..4f6228c8e 100644 --- a/packages/shared/src/types/features.ts +++ b/packages/shared/src/types/features.ts @@ -1,8 +1,10 @@ +import { DATABUNNY_USAGE, LEGACY_SCALE_PLAN } from "../billing"; + export const PLAN_IDS = { FREE: "free", HOBBY: "hobby", PRO: "pro", - SCALE: "scale", + SCALE: LEGACY_SCALE_PLAN.id, } as const; export type PlanId = (typeof PLAN_IDS)[keyof typeof PLAN_IDS]; @@ -142,11 +144,10 @@ export const FEATURE_METADATA: Record = upgradeMessage: "Upgrade to track more events", }, [FEATURE_IDS.AGENT_CREDITS]: { - name: "Databunny Usage", - description: - "AI processing allowance used by Databunny questions and analysis. Deeper work uses more of the allowance.", - upgradeMessage: "Add more Databunny usage or upgrade your plan", - unit: "usage units", + name: DATABUNNY_USAGE.name, + description: DATABUNNY_USAGE.description, + upgradeMessage: DATABUNNY_USAGE.upgradeMessage, + unit: DATABUNNY_USAGE.unit, }, [GATED_FEATURES.FUNNELS]: { name: "Funnels", diff --git a/packages/shared/src/uptime.ts b/packages/shared/src/uptime.ts new file mode 100644 index 000000000..2dcb273a9 --- /dev/null +++ b/packages/shared/src/uptime.ts @@ -0,0 +1,21 @@ +import { z } from "zod"; + +export const uptimeGranularitySchema = z.enum([ + "minute", + "five_minutes", + "ten_minutes", + "thirty_minutes", + "hour", + "six_hours", + "twelve_hours", + "day", +]); + +export type UptimeGranularity = z.infer; + +export function parseUptimeGranularity( + value: unknown +): UptimeGranularity | null { + const parsed = uptimeGranularitySchema.safeParse(value); + return parsed.success ? parsed.data : null; +} diff --git a/packages/tracker/src/core/utils.ts b/packages/tracker/src/core/utils.ts index fe8509520..db4936e29 100644 --- a/packages/tracker/src/core/utils.ts +++ b/packages/tracker/src/core/utils.ts @@ -75,6 +75,9 @@ export function sanitizePageUrl(value: string): string { } try { const url = new URL(value); + if (!(url.protocol === "http:" || url.protocol === "https:")) { + return ""; + } return `${url.origin}${url.pathname}`; } catch { return ""; diff --git a/packages/tracker/tests/unit/utils.test.ts b/packages/tracker/tests/unit/utils.test.ts index 71b7d22cd..05940b1f1 100644 --- a/packages/tracker/tests/unit/utils.test.ts +++ b/packages/tracker/tests/unit/utils.test.ts @@ -123,6 +123,15 @@ describe("privacy-safe page context", () => { ).toBe("https://referrer.example/path"); }); + test("only accepts HTTP page and referrer URLs", () => { + expect(sanitizePageUrl("http://example.com/path?secret=value")).toBe( + "http://example.com/path" + ); + expect(sanitizePageUrl("javascript:alert(1)")).toBe(""); + expect(sanitizePageUrl("data:text/plain,private")).toBe(""); + expect(sanitizePageUrl("file:///tmp/private.txt")).toBe(""); + }); + test("honors Global Privacy Control", () => { Object.defineProperty(globalThis, "window", { configurable: true, diff --git a/turbo.json b/turbo.json index 6d1d07cf3..2f1897acd 100644 --- a/turbo.json +++ b/turbo.json @@ -58,6 +58,10 @@ "TCC_API_KEY", "UPTIME_ROUTER_INTEGRATION" ], + "globalPassThroughEnv": [ + "INSIGHTS_SCHEDULED_DISPATCH_ENABLED", + "INSIGHTS_SCHEDULED_ORGANIZATION_IDS" + ], "tasks": { "build": { "dependsOn": ["^build"], @@ -82,7 +86,8 @@ "persistent": true }, "test": { - "dependsOn": ["^build"] + "dependsOn": ["^build"], + "env": ["CLICKHOUSE_INTEGRATION_TESTS"] }, "check-types": { "dependsOn": ["^build"]