From ece04b3661cadde973b440f619030a1d727ee9fe Mon Sep 17 00:00:00 2001 From: shrugs Date: Thu, 11 Jun 2026 10:38:04 -0500 Subject: [PATCH 01/10] checkpoint: monorepo enscli+ensskills, fix pnpm warnings --- .gitignore | 3 +++ package.json | 6 +++++- pnpm-lock.yaml | 9 +++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 898696d363..13ffe429e6 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,6 @@ apps/ensrainbow/test-* apps/fallback-ensapi/dist + +# Agent skills from npm packages (managed by skills-npm) +**/skills/npm-* diff --git a/package.json b/package.json index 3bfddad501..25a9cf666c 100644 --- a/package.json +++ b/package.json @@ -33,16 +33,20 @@ "generate": "pnpm run -w \"/^generate:.*/\"", "generate:openapi": "pnpm -r generate:openapi", "generate:gqlschema": "pnpm -r generate:gqlschema", - "generate:skills": "pnpm -F ensskills generate" + "generate:skills": "pnpm -F ensskills generate", + "prepare": "skills-npm --cwd ." }, "devDependencies": { "@biomejs/biome": "^2.3.1", "@changesets/changelog-github": "^0.5.2", "@changesets/cli": "^2.29.8", "@typescript/native-preview": "7.0.0-dev.20260128.1", + "enscli": "workspace:*", + "ensskills": "workspace:*", "jsdom": "^27.0.1", "prettier": "catalog:", "prettier-plugin-astro": "catalog:", + "skills-npm": "^1", "tsup": "catalog:", "typescript": "catalog:", "vitest": "catalog:" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1472d8e993..25d1c6cd2a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -177,6 +177,12 @@ importers: '@typescript/native-preview': specifier: 7.0.0-dev.20260128.1 version: 7.0.0-dev.20260128.1 + enscli: + specifier: workspace:* + version: link:packages/enscli + ensskills: + specifier: workspace:* + version: link:packages/ensskills jsdom: specifier: ^27.0.1 version: 27.0.1(postcss@8.5.12) @@ -186,6 +192,9 @@ importers: prettier-plugin-astro: specifier: 'catalog:' version: 0.14.1 + skills-npm: + specifier: ^1 + version: 1.1.1 tsup: specifier: 'catalog:' version: 8.5.0(jiti@2.6.1)(postcss@8.5.12)(tsx@4.22.3)(typescript@5.9.3)(yaml@2.8.3) From f2e6cb937f5579dfb949ce8a6e697bd62cd714e0 Mon Sep 17 00:00:00 2001 From: shrugs Date: Thu, 11 Jun 2026 10:47:05 -0500 Subject: [PATCH 02/10] minimize pnpm, commit shared agents folder --- .agents/skills/fix-audit/SKILL.md | 108 ++++++++++++++++++++++++++++++ package.json | 43 ++++++------ pnpm-lock.yaml | 47 ++++--------- 3 files changed, 140 insertions(+), 58 deletions(-) create mode 100644 .agents/skills/fix-audit/SKILL.md diff --git a/.agents/skills/fix-audit/SKILL.md b/.agents/skills/fix-audit/SKILL.md new file mode 100644 index 0000000000..0fd78a47cc --- /dev/null +++ b/.agents/skills/fix-audit/SKILL.md @@ -0,0 +1,108 @@ +--- +name: fix-audit +description: > + Fix security vulnerabilities found by pnpm audit using a git worktree off of `main`. + Also use to maintain `pnpm.overrides` hygiene: prune overrides that are no longer + necessary and scope overrides so they don't rewrite peerDependency ranges. +--- + +# Fix Audit + +Fix security vulnerabilities reported by `pnpm audit` / `pnpm audit:osv`, keeping +`pnpm.overrides` minimal and scoped. + +## Steps + +1. Create a git worktree off of `main` (branch `fix/deps`). +2. Run `pnpm audit` and `pnpm audit:osv` to identify vulnerabilities. +3. For each vulnerability, trace who pulls it in: `pnpm why -r `. + - If a direct dependency: bump to the fixed version using a caret range (e.g. `^1.2.3`). + - If transitive: try bumping the parent direct dependency first. + - If the parent can't be bumped: add a `pnpm.overrides` entry (see format below). +4. Run `pnpm install` and read the output — new "unmet peer" or "deprecated" warnings + mean an override is too broad (see Peer-range rewriting below). +5. Re-run the audits to verify; repeat until clean. +6. Prune obsolete overrides (see below) while you're here. +7. Validate from the repo root: `pnpm lint`, `pnpm typecheck`, `pnpm test`. +8. Propose a PR using lite /pr-notes. + +## Override format + +Use a ranged selector key that names the vulnerable range, and a caret target. This +self-documents what the override guards and makes obsolescence checkable later: + +```json +{ + "pnpm": { + "overrides": { + "vulnerable-package@<1.2.3": "^1.2.3" + } + } +} +``` + +## Peer-range rewriting: prefer scoped overrides + +pnpm applies range-selector overrides to **peerDependency ranges too**. A global +override like `"vite@>=5.0.0 <=6.4.1": "^6.4.2"` rewrites every package's +`vite` peer range that intersects the selector (e.g. `^6 || ^7 || ^8` becomes +`^6.4.2`), producing false "unmet peer" warnings and duplicate installs across +the workspace. + +If only one dependency subtree pulls the vulnerable version, scope the override +to that parent instead: + +```json +{ + "pnpm": { + "overrides": { + "ponder>vite": "^6.4.2", + "vite-node>vite": "^6.4.2" + } + } +} +``` + +Find the subtree(s) with `pnpm why -r ` before choosing global vs scoped. + +## Pruning obsolete overrides + +An override is obsolete when natural resolution no longer lands in the vulnerable +range (dependents widened their ranges, or a fixed version is now latest-in-range). +To check all overrides in one pass: + +1. Back up `package.json` and `pnpm-lock.yaml`. +2. Set `pnpm.overrides` to `{}` and run `pnpm install --lockfile-only`. +3. For each override selector `pkg@`, check whether any resolved version of + `pkg` in the regenerated lockfile satisfies the vulnerable range + (`semver.satisfies` against versions grepped from the lockfile's `packages:` section). + - No match → override is unnecessary; remove it. + - Match → still load-bearing; keep it. +4. Restore the backups, apply the pruning, and run `pnpm install`. + +Expect the final lockfile diff to contain **no resolved-version changes** for pruned +overrides — pruning only removes override metadata. If versions change, the override +was still doing work; re-check. + +## Unfixable warnings + +- Deprecated transitive deps with no fixed upstream release (check + `npm view @ deprecated` and whether the parent has a newer version): + acknowledge them in `pnpm.allowedDeprecatedVersions`, pinned to the exact current + version so a future, different deprecated version still warns. +- Genuinely unmet peers that work in practice (e.g. an unmaintained package peering + on an old major): acknowledge in `pnpm.peerDependencyRules.allowedVersions`, scoped + as `"parent>peer": "^N"`. +- Verify suppressions under a *fresh* resolution (warnings only print when resolution + runs): back up the lockfile, delete it, `pnpm install --lockfile-only`, read the + output, restore the lockfile. + +## Rules + +- ALWAYS use caret ranges (`^x.y.z`) for override targets, never bare `>=`, to prevent + accidental major version bumps. +- Prefer bumping direct dependencies over adding overrides; prefer scoped overrides + over global ones. +- Group related changes logically. +- If a fix requires a major version bump, warn me before proceeding. +- If a vulnerability cannot be fixed (no patched version available), report it at the end. diff --git a/package.json b/package.json index 25a9cf666c..e11010555d 100644 --- a/package.json +++ b/package.json @@ -58,40 +58,35 @@ "overrides": { "@adraffy/ens-normalize": "catalog:", "esbuild@<=0.24.2": ">=0.25.0", - "lodash@>=4.0.0 <=4.17.23": "^4.18.0", - "lodash-es@>=4.0.0 <=4.17.23": "^4.18.0", - "markdown-it@>=14.0.0 <14.1.1": ">=14.1.1", "tar@<=7.5.10": "^7.5.11", "ajv@<8.18.0": ">=8.18.0", "minimatch@<10.2.3": ">=10.2.3", - "rollup@>=4.0.0 <4.59.0": ">=4.59.0", - "svgo@>=3.0.0 <3.3.3": "^3.3.3", "ponder>@hono/node-server@<1.19.13": "catalog:", - "devalue@<5.8.1": "^5.8.1", - "undici@>=7.0.0 <7.24.0": "^7.24.0", - "undici@>=6.0.0 <6.24.0": "^6.24.0", "yauzl@<3.2.1": "^3.2.1", "fast-xml-parser@>=5.0.0 <5.7.0": ">=5.7.0", "kysely@>=0.26.0 <0.28.17": "0.28.17", - "h3@<1.15.9": ">=1.15.9", "yaml@>=2.0.0 <2.8.3": ">=2.8.3", - "picomatch@<2.3.2": "^2.3.2", - "picomatch@>=4.0.0 <4.0.4": ">=4.0.4", - "smol-toml@<1.6.1": ">=1.6.1", - "brace-expansion@>=4.0.0 <5.0.6": ">=5.0.6", - "defu@<=6.1.4": "^6.1.5", - "vite@>=5.0.0 <=6.4.1": "^6.4.2", - "axios@<1.15.2": "^1.15.2", - "follow-redirects@<1.16.0": "^1.16.0", + "ponder>vite": "^6.4.2", + "vite-node>vite": "^6.4.2", "dompurify@<3.4.0": "^3.4.0", - "protobufjs@>=7.0.0 <7.5.8": "7.5.8", "protobufjs@>=8.0.0 <8.3.0": "8.3.0", - "postcss@>=8.0.0 <8.5.12": "^8.5.12", - "uuid@>=10.0.0 <11.1.1": "^11.1.1", - "fast-uri@<3.1.2": "^3.1.2", - "fast-xml-builder@<1.1.7": "^1.1.7", - "ws@<8.20.1": "^8.20.1", - "shell-quote@<1.8.4": "^1.8.4" + "postcss@>=8.0.0 <8.5.12": "^8.5.12" + }, + "peerDependencyRules": { + "allowedVersions": { + "graphiql-explorer>graphql": "^16", + "graphiql-explorer>react": "^19", + "graphiql-explorer>react-dom": "^19", + "abitype>zod": "^4", + "@scalar/astro>astro": "^6" + } + }, + "allowedDeprecatedVersions": { + "@esbuild-kit/core-utils": "3.3.2", + "@esbuild-kit/esm-loader": "2.6.5", + "glob": "10.5.0", + "source-map": "0.8.0-beta.0", + "whatwg-encoding": "3.1.1" }, "ignoredBuiltDependencies": [ "bun" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 25d1c6cd2a..c3c64ada02 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -112,40 +112,19 @@ catalogs: overrides: '@adraffy/ens-normalize': 1.11.1 esbuild@<=0.24.2: '>=0.25.0' - lodash@>=4.0.0 <=4.17.23: ^4.18.0 - lodash-es@>=4.0.0 <=4.17.23: ^4.18.0 - markdown-it@>=14.0.0 <14.1.1: '>=14.1.1' tar@<=7.5.10: ^7.5.11 ajv@<8.18.0: '>=8.18.0' minimatch@<10.2.3: '>=10.2.3' - rollup@>=4.0.0 <4.59.0: '>=4.59.0' - svgo@>=3.0.0 <3.3.3: ^3.3.3 ponder>@hono/node-server@<1.19.13: ^1.19.13 - devalue@<5.8.1: ^5.8.1 - undici@>=7.0.0 <7.24.0: ^7.24.0 - undici@>=6.0.0 <6.24.0: ^6.24.0 yauzl@<3.2.1: ^3.2.1 fast-xml-parser@>=5.0.0 <5.7.0: '>=5.7.0' kysely@>=0.26.0 <0.28.17: 0.28.17 - h3@<1.15.9: '>=1.15.9' yaml@>=2.0.0 <2.8.3: '>=2.8.3' - picomatch@<2.3.2: ^2.3.2 - picomatch@>=4.0.0 <4.0.4: '>=4.0.4' - smol-toml@<1.6.1: '>=1.6.1' - brace-expansion@>=4.0.0 <5.0.6: '>=5.0.6' - defu@<=6.1.4: ^6.1.5 - vite@>=5.0.0 <=6.4.1: ^6.4.2 - axios@<1.15.2: ^1.15.2 - follow-redirects@<1.16.0: ^1.16.0 + ponder>vite: ^6.4.2 + vite-node>vite: ^6.4.2 dompurify@<3.4.0: ^3.4.0 - protobufjs@>=7.0.0 <7.5.8: 7.5.8 protobufjs@>=8.0.0 <8.3.0: 8.3.0 postcss@>=8.0.0 <8.5.12: ^8.5.12 - uuid@>=10.0.0 <11.1.1: ^11.1.1 - fast-uri@<3.1.2: ^3.1.2 - fast-xml-builder@<1.1.7: ^1.1.7 - ws@<8.20.1: ^8.20.1 - shell-quote@<1.8.4: ^1.8.4 patchedDependencies: '@changesets/assemble-release-plan@6.0.9': @@ -4380,7 +4359,7 @@ packages: resolution: {integrity: sha512-5EdhGZtnu3V88ces7s53hhfK5KSASnJZv8Lulpc04cWO3REESroJXg73DFsOmgbU2BhwV0E20bu2IDZb3VKW4Q==} engines: {node: '>=14.0.0'} peerDependencies: - rollup: '>=4.59.0' + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 peerDependenciesMeta: rollup: optional: true @@ -5060,7 +5039,7 @@ packages: '@tailwindcss/vite@4.3.0': resolution: {integrity: sha512-t6J3OrB5Fc0ExuhohouH0fWUGMYL6PTLhW+E7zIk/pdbnJARZDCwjBznFnkh5ynRnIRSI4YjtTH0t6USjJISrw==} peerDependencies: - vite: ^6.4.2 + vite: ^5.2.0 || ^6 || ^7 || ^8 '@tanstack/query-core@5.90.5': resolution: {integrity: sha512-wLamYp7FaDq6ZnNehypKI5fNvxHPfTYylE0m/ZpuuzJfJqhR5Pxg9gvGBHZx4n7J+V5Rg5mZxHHTlv25Zt5u+w==} @@ -5399,13 +5378,13 @@ packages: resolution: {integrity: sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA==} engines: {node: ^14.18.0 || >=16.0.0} peerDependencies: - vite: ^6.4.2 + vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 '@vitejs/plugin-react@5.2.0': resolution: {integrity: sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw==} engines: {node: ^20.19.0 || >=22.12.0} peerDependencies: - vite: ^6.4.2 + vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 '@vitest/expect@4.1.8': resolution: {integrity: sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==} @@ -5414,7 +5393,7 @@ packages: resolution: {integrity: sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==} peerDependencies: msw: ^2.4.9 - vite: ^6.4.2 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: msw: optional: true @@ -5521,7 +5500,7 @@ packages: resolution: {integrity: sha512-SDobKBbPIOe0cVL7QxMzGkuUGHvWTdihi9zOrrWaWUgFKe15cwEcwfWmgrcNzjT6kHnNmWuTajPHoIzUjYNYYQ==} peerDependencies: async-validator: ^4 - axios: ^1.15.2 + axios: ^1 change-case: ^5 drauu: ^0.4 focus-trap: ^7 @@ -7029,7 +7008,7 @@ packages: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} peerDependencies: - picomatch: '>=4.0.4' + picomatch: ^3 || ^4 peerDependenciesMeta: picomatch: optional: true @@ -7630,7 +7609,7 @@ packages: isows@1.0.7: resolution: {integrity: sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==} peerDependencies: - ws: ^8.20.1 + ws: '*' jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} @@ -10127,7 +10106,7 @@ packages: vite-tsconfig-paths@4.3.1: resolution: {integrity: sha512-cfgJwcGOsIxXOLU/nELPny2/LUD/lcf1IbfyeKTv2bsupVbTH/xpFtdQlBmIP1GEK2CjjLxYhFfB+QODFAx5aw==} peerDependencies: - vite: ^6.4.2 + vite: '*' peerDependenciesMeta: vite: optional: true @@ -10215,7 +10194,7 @@ packages: vitefu@1.1.3: resolution: {integrity: sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg==} peerDependencies: - vite: ^6.4.2 + vite: ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: vite: optional: true @@ -10236,7 +10215,7 @@ packages: '@vitest/ui': 4.1.8 happy-dom: '*' jsdom: '*' - vite: ^6.4.2 + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 peerDependenciesMeta: '@edge-runtime/vm': optional: true From 8c579f8cbc621593327f2c421a3987d4bf605934 Mon Sep 17 00:00:00 2001 From: shrugs Date: Thu, 11 Jun 2026 11:32:42 -0500 Subject: [PATCH 03/10] feat: agent overhaul --- .agents/skills/edit-ponder-sync/SKILL.md | 206 +++++++++++++ .../skills/ensindexer-perf-testing/SKILL.md | 285 ++++++++++++++++++ .agents/skills/fix-audit/SKILL.md | 2 +- ...omnigraph-skill-acceleration-and-status.md | 5 + .gitignore | 2 +- .mcp.json | 8 + AGENTS.md | 89 ++++-- apps/ensadmin/AGENTS.md | 32 ++ apps/ensadmin/CLAUDE.md | 1 + apps/ensapi/AGENTS.md | 48 +++ apps/ensapi/CLAUDE.md | 1 + apps/ensindexer/AGENTS.md | 57 ++++ apps/ensindexer/CLAUDE.md | 1 + apps/ensindexer/src/lib/graphnode-helpers.ts | 4 + apps/ensrainbow/AGENTS.md | 42 +++ apps/ensrainbow/CLAUDE.md | 1 + apps/fallback-ensapi/AGENTS.md | 31 ++ apps/fallback-ensapi/CLAUDE.md | 1 + context7.json | 12 +- docs/ensnode.io/AGENTS.md | 42 +++ docs/ensnode.io/CLAUDE.md | 1 + package.json | 2 +- packages/datasources/AGENTS.md | 26 ++ packages/datasources/CLAUDE.md | 1 + .../ensskills/skills/ens-protocol/SKILL.md | 16 +- .../references/ensv1-and-ensv2.md | 4 + packages/ensskills/skills/omnigraph/SKILL.md | 8 + packages/integration-test-env/AGENTS.md | 33 ++ packages/integration-test-env/CLAUDE.md | 1 + scripts/link-local-skills.mjs | 39 +++ skills-npm.config.ts | 7 + 31 files changed, 965 insertions(+), 43 deletions(-) create mode 100644 .agents/skills/edit-ponder-sync/SKILL.md create mode 100644 .agents/skills/ensindexer-perf-testing/SKILL.md create mode 100644 .changeset/omnigraph-skill-acceleration-and-status.md create mode 100644 .mcp.json create mode 100644 apps/ensadmin/AGENTS.md create mode 120000 apps/ensadmin/CLAUDE.md create mode 100644 apps/ensapi/AGENTS.md create mode 120000 apps/ensapi/CLAUDE.md create mode 100644 apps/ensindexer/AGENTS.md create mode 120000 apps/ensindexer/CLAUDE.md create mode 100644 apps/ensrainbow/AGENTS.md create mode 120000 apps/ensrainbow/CLAUDE.md create mode 100644 apps/fallback-ensapi/AGENTS.md create mode 120000 apps/fallback-ensapi/CLAUDE.md create mode 100644 docs/ensnode.io/AGENTS.md create mode 120000 docs/ensnode.io/CLAUDE.md create mode 100644 packages/datasources/AGENTS.md create mode 120000 packages/datasources/CLAUDE.md create mode 100644 packages/integration-test-env/AGENTS.md create mode 120000 packages/integration-test-env/CLAUDE.md create mode 100644 scripts/link-local-skills.mjs create mode 100644 skills-npm.config.ts diff --git a/.agents/skills/edit-ponder-sync/SKILL.md b/.agents/skills/edit-ponder-sync/SKILL.md new file mode 100644 index 0000000000..7646f3f913 --- /dev/null +++ b/.agents/skills/edit-ponder-sync/SKILL.md @@ -0,0 +1,206 @@ +--- +name: edit-ponder-sync +description: > + Inspect and surgically edit Ponder's RPC sync cache (the `ponder_sync` Postgres schema): + understand what's cached, diagnose stale/contaminated cache (e.g. a restarted ens-test-env + devnet reusing chain id 31337), and clip/purge a block range so Ponder re-fetches it cleanly + without re-syncing the whole chain. Use when a handler throws RecordNotFoundError on an entity + that should exist, when cached block hashes diverge from the live chain, or when splicing one + chain's sync data into another (upstream chain → a fork of it). +--- + +# Editing Ponder Sync (`ponder_sync`) + +`ponder_sync` is Ponder's **RPC cache** — blocks, logs, transactions, receipts, traces, and the +per-filter "what ranges have I synced" bookkeeping. It is separate from the **entity** schema +(`ENSINDEXER_SCHEMA_NAME`, e.g. `ensindexer_temp_dev`) which holds the indexed app data (`domains`, +`registries`, …) plus Ponder's `_ponder_checkpoint` / `_ponder_meta`. + +Editing `ponder_sync` is the right tool when the cache is **wrong or contaminated** and you want +Ponder to re-fetch a range from the RPC. It is reversible (anything deleted is re-fetchable), but +you must keep the `intervals` table consistent with the row tables or Ponder will trust a hole. + +Local Postgres in this repo: `postgresql://postgres@localhost:5432/ponder`. Always scope edits by `chain_id`. + +## Schema cheat-sheet + +``` +ponder_sync.blocks -- one row per CACHED block. KEY COLUMN: number (NOT block_number). + -- columns: number, hash, parent_hash, timestamp, chain_id, … +ponder_sync.logs -- block_number, log_index, address, topic0..3, block_hash, data, … +ponder_sync.transactions -- block_number, hash, from, to, … +ponder_sync.transaction_receipts -- block_number, transaction_hash, status, … (often empty: Ponder + -- only fetches receipts when a handler/config needs them) +ponder_sync.traces -- block_number, trace_index, … (often empty) +ponder_sync.rpc_request_results -- block_number (nullable), request_hash, result -- cached eth_call etc. +ponder_sync.intervals -- THE SOURCE OF TRUTH for "synced". fragment_id (PK), chain_id, + -- blocks (nummultirange). +ponder_sync.factories / -- factory-pattern (CREATE2 child) address discovery. Children +ponder_sync.factory_addresses -- discovered above a fork are valid current-chain data. +``` + +Gotchas that will bite you: + +- **`address`, `hash`, `topic*` are stored as TEXT** (`0x…` lowercase strings), **not bytea**. + `WHERE address = '\x0000…'` silently matches nothing and returns count 0. Use + `WHERE address = '0x00000000000c2e074ec69a0dfb2997ba6c7d2e1e'`. +- **`blocks` is sparse.** Ponder only caches blocks that contain a matching log (plus interval + boundaries / finalized blocks). Most heights have no row — a missing block in `blocks` is normal, + not a gap. Don't infer sync coverage from `blocks`; infer it from `intervals`. +- **`blocks` uses `number`; every other table uses `block_number`.** +- A row existing in `logs`/`blocks` does **not** mean Ponder will use it. Ponder decides what to + fetch/replay from `intervals`. If an interval says a range is synced, Ponder trusts the cache and + will **not** re-fetch — even if the cached blocks are stale. + +## How `intervals` works + +One row per **filter fragment** — a specific (chain, address, topic0, …) combination Ponder syncs. +`fragment_id` looks like: + +``` +log_______ +``` + +`blocks` is a Postgres `nummultirange` of the synced ranges, e.g. +`{[3702728,10770756],[10887910,10888672]}`. A **gap** between segments = unsynced → Ponder +re-fetches it. A continuous range = fully trusted cache. + +**Bounds must be inclusive `[a,b]`.** `nummultirange` is over `numeric`, which (unlike +`int8range`) does **not** canonicalize `[)` → `[]`. If you intersect with a default-bound range you +get a half-open `[a,b)` that Ponder mis-reads. Always force the inclusive upper bound: + +```sql +-- WRONG: leaves an exclusive upper bound [3702728,10770757) +blocks * nummultirange(numrange(NULL, 10770757)) +-- RIGHT: yields inclusive [3702728,10770756] +blocks * nummultirange(numrange(NULL, 10770756, '(]')) +``` + +Sanity check after any interval edit — this must return 0: + +```sql +SELECT count(*) FROM ponder_sync.intervals +WHERE chain_id = AND blocks::text LIKE '%)%'; +``` + +## Recipe: purge a chain wholesale (devnet restart) + +The ens-test-env devnet (anvil, chain id `31337`) is a fresh chain on every restart — but the chain +id stays the same, so `ponder_sync` keeps trusting cache from the previous run and Ponder serves +stale blocks instead of re-fetching. There is no clean prefix worth keeping (the chain is tiny and +free to re-sync), so purge everything for the chain: + +```sql +BEGIN; +DELETE FROM ponder_sync.intervals WHERE chain_id = 31337; +DELETE FROM ponder_sync.logs WHERE chain_id = 31337; +DELETE FROM ponder_sync.transaction_receipts WHERE chain_id = 31337; +DELETE FROM ponder_sync.transactions WHERE chain_id = 31337; +DELETE FROM ponder_sync.traces WHERE chain_id = 31337; +DELETE FROM ponder_sync.rpc_request_results WHERE chain_id = 31337; +DELETE FROM ponder_sync.blocks WHERE chain_id = 31337; +COMMIT; +``` + +(Deterministic deploys may reproduce identical early blocks across runs — don't rely on it.) + +## Recipe: clip the cache to block N and re-fetch the tail + +Use when cache above some block N is stale/contaminated and you want Ponder to re-fetch `(N, head]` +from the RPC while keeping the (expensive) deep history below N. Pick N = the **last block whose +cached hash matches the live chain** (see diagnosis below). + +```sql +BEGIN; +-- 1. clip every synced interval to [.., N] inclusive +UPDATE ponder_sync.intervals +SET blocks = blocks * nummultirange(numrange(NULL, , '(]')) +WHERE chain_id = ; + +-- 2. delete cached rows above N so they re-fetch clean (note: blocks uses `number`) +DELETE FROM ponder_sync.logs WHERE chain_id= AND block_number > ; +DELETE FROM ponder_sync.transaction_receipts WHERE chain_id= AND block_number > ; +DELETE FROM ponder_sync.transactions WHERE chain_id= AND block_number > ; +DELETE FROM ponder_sync.traces WHERE chain_id= AND block_number > ; +DELETE FROM ponder_sync.rpc_request_results WHERE chain_id= AND block_number > ; +DELETE FROM ponder_sync.blocks WHERE chain_id= AND number > ; +COMMIT; +``` + +Then restart the indexer. If the chain has **no** `_ponder_checkpoint` row and no entity rows, Ponder +indexes it from scratch over the (now-correct) cache + re-fetched tail. If it has a checkpoint, it +resumes from there. + +Always do interval clip + row deletes in **one transaction**, and keep them consistent: an interval +that claims a range is synced while the rows are gone makes Ponder skip real events; rows present +under a clipped interval are harmless (they get overwritten on re-fetch). + +## Diagnosing a stale / contaminated cache + +Symptom: a handler throws `RecordNotFoundError: No existing record found in table 'domains'` on a +`db.update` (e.g. `ENSv1Registry:Transfer`) for an entity whose creating event (`NewOwner`) is +earlier on-chain. The creating event was never indexed because the cache around it is stale. + +The usual culprit in this repo is a **restarted ens-test-env devnet** (chain id `31337`): the new +run is a different chain under the same chain id, so every cached block from the old run diverges — +purge the chain wholesale (recipe above). The same pattern appears with re-created forked testnets: +re-forking gives the chain a **new fork block**, so cached blocks from the old incarnation diverge +from the live chain, and clearing the cache only _above_ your resume point leaves the old-fork +window _below_ it intact. The diagnosis below finds the clip point for that partial-divergence case. + +Steps: + +1. **Identify the failing node and its creating tx.** From the error, get `node`, `tx hash`, + `block`. Pull the receipt via the chain's RPC (`RPC_URL_` in `apps/ensindexer/.env.local`) + to see the real event ordering (`NewOwner` creates, `Transfer`/`NewResolver` mutate). +2. **Compare cached block hashes to the live chain** at the failing block and the creating block: + + ```sql + SELECT number, hash FROM ponder_sync.blocks WHERE chain_id= AND number IN (,); + ``` + + vs `eth_getBlockByNumber` on the live RPC. A mismatch at the creating block but a match at the + failing block = the cache below your resume point is from a different fork. + +3. **Find the true fork block** (last block where live testnet == real upstream chain) by binary + search over `eth_getBlockByNumber` hashes (testnet RPC vs a real upstream RPC, e.g. a public + node for the upstream chain). Test activity lives strictly **above** this block. (For the + ens-test-env devnet there is no upstream — skip the search and purge the chain wholesale.) +4. **Find where the cache goes stale** (last cached block whose hash matches the real upstream + chain). Sample cached block numbers ascending and compare hashes; binary-search the boundary. + That boundary is the **old** fork — your clip point N. +5. Clip + purge with the recipe above, using N = last clean block. + +Helper to compare a live RPC hash to the cache: + +```bash +RPC=$(grep RPC_URL_ apps/ensindexer/.env.local | cut -d= -f2-) +rpchash(){ curl -s -X POST "$1" -H 'content-type: application/json' \ + --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$(printf 0x%x $2)\",false]}" \ + | grep -o '"hash":"0x[0-9a-f]*"' | head -1 | sed 's/"hash":"//;s/"//'; } +cachehash(){ psql "postgresql://postgres@localhost:5432/ponder" -tA \ + -c "select hash from ponder_sync.blocks where chain_id= and number=$1;"; } +``` + +## Splicing one chain's sync into a forked chain + +A forked testnet _is_ the upstream chain below its fork block, so you can avoid re-syncing millions +of blocks through the fork's (often rate-limited) RPC by reusing real-upstream sync data: + +- For blocks `≤ forkBlock`: forked testnet ≡ upstream chain. Cache built against the upstream chain + (with `chain_id` rewritten to the testnet's id) is valid. +- For blocks `> forkBlock`: only the testnet has the data; must come from the testnet RPC. + +The clip-and-refetch recipe handles the common case: keep valid cache below the (true, current) +fork, let Ponder fetch the small tail above it from the fork's RPC. Only hand-splice upstream rows if even +that tail is too large for the testnet RPC. When splicing, rewrite `chain_id` on every table **and** +`intervals`, and keep `intervals` bounds inclusive. + +## Don't + +- Don't propose a Drizzle/Ponder schema migration — `ponder_sync` is Ponder-owned; edit it directly + in SQL, never via app migrations. +- Don't delete rows without clipping the matching `intervals` (Ponder will trust the now-empty range + and skip events). +- Don't trust the `blocks` table for coverage (it's sparse); trust `intervals`. +- Don't use `\x…` bytea literals against the TEXT `address`/`hash`/`topic*` columns. diff --git a/.agents/skills/ensindexer-perf-testing/SKILL.md b/.agents/skills/ensindexer-perf-testing/SKILL.md new file mode 100644 index 0000000000..a7869a7d75 --- /dev/null +++ b/.agents/skills/ensindexer-perf-testing/SKILL.md @@ -0,0 +1,285 @@ +--- +name: ensindexer-perf-testing +description: > + Run perf tests on the ENSIndexer (Ponder app under apps/ensindexer). Establishes a clean, + reproducible baseline by disabling ENSRainbow healing on the hot path + (DISABLE_ENSRAINBOW_HEAL), isolating the schema, wiping prometheus between runs, and + snapshotting per-handler metrics at realtime. Use when the user + asks to compare branches/configs, build a perf baseline, or measure a plugin combination. +--- + +# ENSIndexer perf testing + +Captures wall-clock + per-handler timings for an ENSIndexer run. Always works in the same shape: +configure → disable heal → wipe → start → wait → snapshot → restore. + +## Required state before starting + +- Postgres at `postgresql://postgres@localhost:5432/ponder` is up. +- ENSRainbow data at `apps/ensrainbow/data/data-subgraph_0/` is hydrated. (The `data-ens-test-env_0/` dir is for ETE; it lacks subgraph labels.) +- Docker daemon is running (for prometheus/grafana sidecars). +- Working tree is clean (perf numbers must come from the branch as committed — no stray edits). + +If any of these aren't true, fix that first — don't paper over it. + +## Step 1 — start prometheus + grafana + +```bash +cd packages/ensindexer-perf-testing && pnpm start +# prometheus :9090, grafana :3001 (anonymous admin) +``` + +Confirm scrape target is reachable: + +```bash +curl -s http://localhost:9090/api/v1/targets?state=active | jq -r '.data.activeTargets[].health' +# should be 'up' once indexer starts +``` + +## Step 2 — start ENSRainbow (subgraph label set) + +ENSIndexer performs an explicit health + ready check against ENSRainbow at startup, so the +service must be running even when healing is disabled. Use the `subgraph` data dir for mainnet +runs: + +```bash +cd apps/ensrainbow && \ + DATA_DIR=$PWD/data/data-subgraph_0 PORT=3223 LOG_LEVEL=warn \ + pnpm tsx src/cli.ts serve > /tmp/perf-test/ensrainbow.log 2>&1 & +# wait ~5s, then verify +curl -s http://localhost:3223/health +``` + +For ens-test-env runs, swap `DATA_DIR=…/data-ens-test-env_0` and set +`LABEL_SET_ID=ens-test-env` in `.env.local`. + +## Step 3 — disable ENSRainbow healing + +Set the undocumented env var `DISABLE_ENSRAINBOW_HEAL=true` (add it to `.env.local` in Step 4, or +export it before launching). `labelByLabelHash` (`src/lib/graphnode-helpers.ts`) early-returns +`null` — every label is treated as unhealable — giving an apples-to-apples baseline without +ENSRainbow round-trips. No code edit needed. + +This changes indexing output: never set it outside perf runs, and always remove it from +`.env.local` at end of run. + +## Step 4 — configure `.env.local` + +Pick a unique schema per run so concurrent/sequential runs don't pollute each other: + +```env +NAMESPACE=mainnet +PLUGINS=ensv2,protocol-acceleration # or whatever combo +ENSINDEXER_SCHEMA_NAME=ensindexer_perf_