From 93ee8f76a103e4fc7fe31ba94720b13d46c40699 Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Mon, 5 Oct 2026 15:39:42 -0700 Subject: [PATCH 01/11] chore: add the @distilled.cloud/iceberg dependency The Prisma example pins `typescript` so @prisma/orm-postgres keeps resolving against the workspace TypeScript after the lockfile refresh. --- examples/cloudflare-neon-prisma/package.json | 3 +- packages/alchemy/package.json | 1 + pnpm-lock.yaml | 205 +++---------------- 3 files changed, 35 insertions(+), 174 deletions(-) diff --git a/examples/cloudflare-neon-prisma/package.json b/examples/cloudflare-neon-prisma/package.json index 167334f4d23..35f52cee82c 100644 --- a/examples/cloudflare-neon-prisma/package.json +++ b/examples/cloudflare-neon-prisma/package.json @@ -27,6 +27,7 @@ "effect": "catalog:" }, "devDependencies": { - "prisma": "catalog:" + "prisma": "catalog:", + "typescript": "catalog:" } } diff --git a/packages/alchemy/package.json b/packages/alchemy/package.json index bbfa1e4315f..fabb83be96b 100644 --- a/packages/alchemy/package.json +++ b/packages/alchemy/package.json @@ -416,6 +416,7 @@ "@distilled.cloud/fly-io": "workspace:*", "@distilled.cloud/gcp": "workspace:*", "@distilled.cloud/hetzner": "workspace:*", + "@distilled.cloud/iceberg": "workspace:*", "@distilled.cloud/infisical": "workspace:*", "@distilled.cloud/neon": "workspace:*", "@distilled.cloud/planetscale": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b04c08d114f..2ef0375ea9b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2074,7 +2074,7 @@ importers: version: 4.0.0(effect@4.0.0)(redis@6.2.1) '@prisma/orm-postgres': specifier: 'catalog:' - version: 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) + version: 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) alchemy: specifier: workspace:* version: link:../../packages/alchemy @@ -2087,7 +2087,10 @@ importers: devDependencies: prisma: specifier: 'catalog:' - version: 8.0.0-rc.15(6be7c66ac81756dccb064fd430eeb14d) + version: 8.0.0-rc.15(80bc071e98d758a11a24c1ff712bfbf2) + typescript: + specifier: 'catalog:' + version: 7.0.2 examples/cloudflare-neon-prisma-psl: dependencies: @@ -7011,6 +7014,9 @@ importers: '@distilled.cloud/hetzner': specifier: workspace:* version: link:../../submodules/distilled/packages/hetzner + '@distilled.cloud/iceberg': + specifier: workspace:* + version: link:../../submodules/distilled/packages/iceberg '@distilled.cloud/infisical': specifier: workspace:* version: link:../../submodules/distilled/packages/infisical @@ -24913,29 +24919,6 @@ snapshots: ansi-styles: 6.2.3 is-fullwidth-code-point: 5.1.0 - '@alchemy.run/cloudflare-runtime@2.0.0-beta.74(@distilled.cloud/cloudflare@1.0.0-rc.6(effect@4.0.0))(@effect/platform-bun@4.0.0(effect@4.0.0))(@effect/platform-node@4.0.0(effect@4.0.0)(redis@6.2.1))(@types/node@26.2.0)(effect@4.0.0)(rolldown@1.2.5)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))': - dependencies: - '@alchemy.run/node-utils': 2.0.0-beta.74 - '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20261002.1) - '@distilled.cloud/cloudflare': 1.0.0-rc.6(effect@4.0.0) - '@puppeteer/browsers': 3.2.1(yauzl@3.4.0) - capnp-es: 0.0.14(typescript@5.9.3) - effect: 4.0.0 - magic-string: 0.30.21 - sharp: 0.35.4(@types/node@26.2.0) - unenv: 2.0.0-rc.24 - workerd: 1.20261002.1 - yauzl: 3.4.0 - optionalDependencies: - '@effect/platform-bun': 4.0.0(effect@4.0.0) - '@effect/platform-node': 4.0.0(effect@4.0.0)(redis@6.2.1) - rolldown: 1.2.5 - vite: 8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0) - transitivePeerDependencies: - - '@types/node' - - proxy-agent - - typescript - '@alchemy.run/cloudflare-runtime@2.0.0-beta.74(@distilled.cloud/cloudflare@1.0.0-rc.6(effect@4.0.0))(@effect/platform-bun@4.0.0(effect@4.0.0))(@effect/platform-node@4.0.0(effect@4.0.0)(redis@6.2.1))(@types/node@26.2.0)(effect@4.0.0)(rolldown@1.2.5)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))': dependencies: '@alchemy.run/node-utils': 2.0.0-beta.74 @@ -30955,11 +30938,11 @@ snapshots: transitivePeerDependencies: - magicast - '@prisma/composer-cli@0.20.0(7d0073e65db52600bd3bf5c101578546)': + '@prisma/composer-cli@0.20.0(0d1b88465c5d4bc00bc45c7482ecc773)': dependencies: '@prisma/cli-engine': 0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4) - '@prisma/composer': 0.20.0(721d9b52bd64b4bc42305cb806399661) - alchemy: 2.0.0-beta.74(c239be5cc6766aba2fa3996f297d9953) + '@prisma/composer': 0.20.0(8074b3e1ef63f5f2277ebc6151c99765) + alchemy: 2.0.0-beta.74(89aebfa01b75756e4425a71763d372be) c12: 3.3.4(magicast@0.5.4) cross-spawn: 7.0.6 effect: 4.0.0 @@ -30989,11 +30972,11 @@ snapshots: - vitest - ws - '@prisma/composer-cli@0.20.0(e2698dcc0ed2f35317fd8fd30485dcd6)': + '@prisma/composer-cli@0.20.0(7d0073e65db52600bd3bf5c101578546)': dependencies: '@prisma/cli-engine': 0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4) - '@prisma/composer': 0.20.0(1f607ed179bcb6f10dc8bf9a330cb5eb) - alchemy: 2.0.0-beta.74(0d42197b89ac596214ac8b427948c4c4) + '@prisma/composer': 0.20.0(721d9b52bd64b4bc42305cb806399661) + alchemy: 2.0.0-beta.74(c239be5cc6766aba2fa3996f297d9953) c12: 3.3.4(magicast@0.5.4) cross-spawn: 7.0.6 effect: 4.0.0 @@ -31023,11 +31006,11 @@ snapshots: - vitest - ws - '@prisma/composer-cli@0.20.0(fc2e5377daac4a3cc4248d90abdcf078)': + '@prisma/composer-cli@0.20.0(e2698dcc0ed2f35317fd8fd30485dcd6)': dependencies: '@prisma/cli-engine': 0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4) - '@prisma/composer': 0.20.0(3182b8fd3b2b1f0001d9587f1f13fcd4) - alchemy: 2.0.0-beta.74(697bd10098ddbc4dd1ca83f9a2da9c19) + '@prisma/composer': 0.20.0(1f607ed179bcb6f10dc8bf9a330cb5eb) + alchemy: 2.0.0-beta.74(0d42197b89ac596214ac8b427948c4c4) c12: 3.3.4(magicast@0.5.4) cross-spawn: 7.0.6 effect: 4.0.0 @@ -31095,14 +31078,14 @@ snapshots: - vitest - ws - '@prisma/composer@0.20.0(3182b8fd3b2b1f0001d9587f1f13fcd4)': + '@prisma/composer@0.20.0(721d9b52bd64b4bc42305cb806399661)': dependencies: '@effect/sql-d1': 4.0.0-rc.112(effect@4.0.0) '@effect/sql-sqlite-do': 4.0.0-rc.112(effect@4.0.0) '@effect/vitest': 4.0.0-rc.112(effect@4.0.0)(vitest@5.0.3(@types/node@26.2.0)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))) '@prisma/management-api-sdk': 1.69.0 '@standard-schema/spec': 1.1.0 - alchemy: 2.0.0-beta.74(697bd10098ddbc4dd1ca83f9a2da9c19) + alchemy: 2.0.0-beta.74(c239be5cc6766aba2fa3996f297d9953) arktype: 2.2.3 c12: 3.3.4(magicast@0.5.4) cross-spawn: 7.0.6 @@ -31133,14 +31116,14 @@ snapshots: - vitest - ws - '@prisma/composer@0.20.0(721d9b52bd64b4bc42305cb806399661)': + '@prisma/composer@0.20.0(8074b3e1ef63f5f2277ebc6151c99765)': dependencies: '@effect/sql-d1': 4.0.0-rc.112(effect@4.0.0) '@effect/sql-sqlite-do': 4.0.0-rc.112(effect@4.0.0) '@effect/vitest': 4.0.0-rc.112(effect@4.0.0)(vitest@5.0.3(@types/node@26.2.0)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))) '@prisma/management-api-sdk': 1.69.0 '@standard-schema/spec': 1.1.0 - alchemy: 2.0.0-beta.74(c239be5cc6766aba2fa3996f297d9953) + alchemy: 2.0.0-beta.74(89aebfa01b75756e4425a71763d372be) arktype: 2.2.3 c12: 3.3.4(magicast@0.5.4) cross-spawn: 7.0.6 @@ -31195,28 +31178,6 @@ snapshots: '@prisma/debug@7.2.0': {} - '@prisma/dev@0.20.0(typescript@5.9.3)': - dependencies: - '@electric-sql/pglite': 0.3.15 - '@electric-sql/pglite-socket': 0.0.20(@electric-sql/pglite@0.3.15) - '@electric-sql/pglite-tools': 0.2.20(@electric-sql/pglite@0.3.15) - '@hono/node-server': 1.19.9(hono@4.11.4) - '@mrleebo/prisma-ast': 0.13.1 - '@prisma/get-platform': 7.2.0 - '@prisma/query-plan-executor': 7.2.0 - foreground-child: 3.3.1 - get-port-please: 3.2.0 - hono: 4.11.4 - http-status-codes: 2.3.0 - pathe: 2.0.3 - proper-lockfile: 4.1.2 - remeda: 2.33.4 - std-env: 3.10.0 - valibot: 1.2.0(typescript@5.9.3) - zeptomatch: 2.1.0 - transitivePeerDependencies: - - typescript - '@prisma/dev@0.20.0(typescript@7.0.2)': dependencies: '@electric-sql/pglite': 0.3.15 @@ -31247,23 +31208,6 @@ snapshots: dependencies: openapi-fetch: 0.14.0 - '@prisma/orm-family-sql@8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))': - dependencies: - '@prisma/orm-framework': 8.0.0-rc.11(typescript@5.9.3) - '@prisma/orm-toolchain': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) - '@standard-schema/spec': 1.1.0 - arktype: 2.2.3 - pathe: 2.0.3 - pluralize: 8.0.0 - ts-toolbelt: 9.6.0 - optionalDependencies: - typescript: 5.9.3 - transitivePeerDependencies: - - '@prisma/cli-engine' - - magicast - - typanion - - vite - '@prisma/orm-family-sql@8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))': dependencies: '@prisma/orm-framework': 8.0.0-rc.11(typescript@7.0.2) @@ -31281,15 +31225,6 @@ snapshots: - typanion - vite - '@prisma/orm-framework@8.0.0-rc.11(typescript@5.9.3)': - dependencies: - '@standard-schema/spec': 1.1.0 - arktype: 2.2.3 - pathe: 2.0.3 - uniku: 0.5.0 - optionalDependencies: - typescript: 5.9.3 - '@prisma/orm-framework@8.0.0-rc.11(typescript@7.0.2)': dependencies: '@standard-schema/spec': 1.1.0 @@ -31299,25 +31234,6 @@ snapshots: optionalDependencies: typescript: 7.0.2 - '@prisma/orm-postgres@8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))': - dependencies: - '@prisma/orm-family-sql': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) - '@prisma/orm-framework': 8.0.0-rc.11(typescript@5.9.3) - '@prisma/orm-target-postgres': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) - '@prisma/orm-toolchain': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) - '@types/node': 26.1.2 - '@types/pg': 8.20.4 - pathe: 2.0.3 - pg: 8.22.0 - optionalDependencies: - typescript: 5.9.3 - transitivePeerDependencies: - - '@prisma/cli-engine' - - magicast - - pg-native - - typanion - - vite - '@prisma/orm-postgres@8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))': dependencies: '@prisma/orm-family-sql': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) @@ -31337,27 +31253,6 @@ snapshots: - typanion - vite - '@prisma/orm-target-postgres@8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))': - dependencies: - '@prisma/orm-family-sql': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) - '@prisma/orm-framework': 8.0.0-rc.11(typescript@5.9.3) - '@prisma/orm-toolchain': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) - '@standard-schema/spec': 1.1.0 - '@types/node': 26.1.2 - '@types/pg': 8.20.4 - arktype: 2.2.3 - pathe: 2.0.3 - pg: 8.22.0 - pg-cursor: 2.22.0(pg@8.22.0) - optionalDependencies: - typescript: 5.9.3 - transitivePeerDependencies: - - '@prisma/cli-engine' - - magicast - - pg-native - - typanion - - vite - '@prisma/orm-target-postgres@8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))': dependencies: '@prisma/orm-family-sql': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) @@ -31379,34 +31274,6 @@ snapshots: - typanion - vite - '@prisma/orm-toolchain@8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))': - dependencies: - '@prisma/cli-engine': 0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4) - '@prisma/orm-framework': 8.0.0-rc.11(typescript@5.9.3) - '@vercel/detect-agent': 1.2.5 - arktype: 2.2.3 - c12: 3.3.4(magicast@0.5.4) - ci-info: 4.4.0 - clipanion: 4.0.0-rc.4(typanion@3.14.0) - closest-match: 1.3.3 - colorette: 2.0.20 - esbuild: 0.28.2 - jsonc-parser: 3.3.1 - package-manager-detector: 1.8.0 - pathe: 2.0.3 - prettier: 3.9.6 - string-width: 8.2.2 - strip-ansi: 7.2.0 - vscode-languageserver: 10.1.0 - vscode-languageserver-textdocument: 1.0.12 - wrap-ansi: 10.0.0 - optionalDependencies: - typescript: 5.9.3 - vite: 8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0) - transitivePeerDependencies: - - magicast - - typanion - '@prisma/orm-toolchain@8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))': dependencies: '@prisma/cli-engine': 0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4) @@ -36185,9 +36052,9 @@ snapshots: - utf-8-validate - vitest - alchemy@2.0.0-beta.74(697bd10098ddbc4dd1ca83f9a2da9c19): + alchemy@2.0.0-beta.74(89aebfa01b75756e4425a71763d372be): dependencies: - '@alchemy.run/cloudflare-runtime': 2.0.0-beta.74(@distilled.cloud/cloudflare@1.0.0-rc.6(effect@4.0.0))(@effect/platform-bun@4.0.0(effect@4.0.0))(@effect/platform-node@4.0.0(effect@4.0.0)(redis@6.2.1))(@types/node@26.2.0)(effect@4.0.0)(rolldown@1.2.5)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) + '@alchemy.run/cloudflare-runtime': 2.0.0-beta.74(@distilled.cloud/cloudflare@1.0.0-rc.6(effect@4.0.0))(@effect/platform-bun@4.0.0(effect@4.0.0))(@effect/platform-node@4.0.0(effect@4.0.0)(redis@6.2.1))(@types/node@26.2.0)(effect@4.0.0)(rolldown@1.2.5)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) '@alchemy.run/floci': 2.0.0-beta.74(effect@4.0.0) '@alchemy.run/node-utils': 2.0.0-beta.74 '@aws-sdk/credential-providers': 3.1136.0 @@ -36206,7 +36073,7 @@ snapshots: '@libsql/client': 0.17.4 '@octokit/rest': 22.0.1 '@octokit/webhooks': 14.2.0 - '@prisma/dev': 0.20.0(typescript@5.9.3) + '@prisma/dev': 0.20.0(typescript@7.0.2) '@smithy/node-config-provider': 4.6.0 '@smithy/shared-ini-file-loader': 4.7.2 '@smithy/types': 4.18.0 @@ -36233,7 +36100,7 @@ snapshots: '@effect/sql-pg': 4.0.0(effect@4.0.0) '@vercel/nft': 1.10.2(rollup@4.62.4)(supports-color@10.2.2) drizzle-kit: 1.0.0-rc.5-ab785fc - drizzle-orm: 1.0.0-rc.5-ab785fc(@cloudflare/workers-types@5.20261002.1)(@effect/sql-d1@4.0.0(effect@4.0.0))(@effect/sql-mysql2@4.0.0(@types/node@26.2.0)(effect@4.0.0))(@effect/sql-pg@4.0.0(effect@4.0.0))(@effect/sql-sqlite-do@4.0.0(effect@4.0.0))(@electric-sql/pglite@0.3.15)(@libsql/client@0.17.4)(@neondatabase/serverless@1.1.0)(@types/pg@8.20.4)(arktype@2.2.3)(bun-types@1.3.14)(effect@4.0.0)(mysql2@3.24.2(@types/node@26.2.0))(pg@8.22.0)(valibot@1.2.0(typescript@5.9.3))(zod@4.5.4) + drizzle-orm: 1.0.0-rc.5-ab785fc(@cloudflare/workers-types@5.20261002.1)(@effect/sql-d1@4.0.0(effect@4.0.0))(@effect/sql-mysql2@4.0.0(@types/node@26.2.0)(effect@4.0.0))(@effect/sql-pg@4.0.0(effect@4.0.0))(@effect/sql-sqlite-do@4.0.0(effect@4.0.0))(@electric-sql/pglite@0.3.15)(@libsql/client@0.17.4)(@neondatabase/serverless@1.1.0)(@types/pg@8.20.4)(arktype@2.2.3)(bun-types@1.3.14)(effect@4.0.0)(mysql2@3.24.2(@types/node@26.2.0))(pg@8.22.0)(valibot@1.2.0(typescript@7.0.2))(zod@4.5.4) mongodb: 6.21.0(@aws-sdk/credential-providers@3.1136.0)(socks@2.8.9) mysql2: 3.24.2(@types/node@26.2.0) pg: 8.22.0 @@ -36951,10 +36818,6 @@ snapshots: caniuse-lite@1.0.30001809: {} - capnp-es@0.0.14(typescript@5.9.3): - optionalDependencies: - typescript: 5.9.3 - capnp-es@0.0.14(typescript@7.0.2): optionalDependencies: typescript: 7.0.2 @@ -37521,7 +37384,7 @@ snapshots: zod: 4.5.4 optional: true - drizzle-orm@1.0.0-rc.5-ab785fc(@cloudflare/workers-types@5.20261002.1)(@effect/sql-d1@4.0.0(effect@4.0.0))(@effect/sql-mysql2@4.0.0(@types/node@26.2.0)(effect@4.0.0))(@effect/sql-pg@4.0.0(effect@4.0.0))(@effect/sql-sqlite-do@4.0.0(effect@4.0.0))(@electric-sql/pglite@0.3.15)(@libsql/client@0.17.4)(@neondatabase/serverless@1.1.0)(@types/pg@8.20.4)(arktype@2.2.3)(bun-types@1.3.14)(effect@4.0.0)(mysql2@3.24.2(@types/node@26.2.0))(pg@8.22.0)(valibot@1.2.0(typescript@5.9.3))(zod@4.5.4): + drizzle-orm@1.0.0-rc.5-ab785fc(@cloudflare/workers-types@5.20261002.1)(@effect/sql-d1@4.0.0(effect@4.0.0))(@effect/sql-mysql2@4.0.0(@types/node@26.2.0)(effect@4.0.0))(@effect/sql-pg@4.0.0(effect@4.0.0))(@effect/sql-sqlite-do@4.0.0(effect@4.0.0))(@electric-sql/pglite@0.3.15)(@libsql/client@0.17.4)(@neondatabase/serverless@1.1.0)(@types/pg@8.20.4)(arktype@2.2.3)(bun-types@1.3.14)(effect@4.0.0)(mysql2@3.24.2(@types/node@26.2.0))(pg@8.22.0)(valibot@1.2.0(typescript@7.0.2))(zod@4.5.4): optionalDependencies: '@cloudflare/workers-types': 5.20261002.1 '@effect/sql-d1': 4.0.0(effect@4.0.0) @@ -37537,7 +37400,7 @@ snapshots: effect: 4.0.0 mysql2: 3.24.2(@types/node@26.2.0) pg: 8.22.0 - valibot: 1.2.0(typescript@5.9.3) + valibot: 1.2.0(typescript@7.0.2) zod: 4.5.4 optional: true @@ -42088,14 +41951,14 @@ snapshots: dependencies: parse-ms: 4.0.0 - prisma@8.0.0-rc.15(6be7c66ac81756dccb064fd430eeb14d): + prisma@8.0.0-rc.15(7a4d27bd99bbffe1e250da2a0de1478d): dependencies: '@manypkg/tools': 2.1.2 '@prisma/cli-engine': 0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4) - '@prisma/composer-cli': 0.20.0(fc2e5377daac4a3cc4248d90abdcf078) + '@prisma/composer-cli': 0.20.0(7d0073e65db52600bd3bf5c101578546) '@prisma/compute-sdk': 0.42.0(@prisma/management-api-sdk@1.69.0)(rollup@4.62.4)(supports-color@10.2.2) '@prisma/management-api-sdk': 1.69.0 - '@prisma/orm-toolchain': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@5.9.3)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) + '@prisma/orm-toolchain': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) '@vercel/detect-agent': 1.2.5 better-result: 2.10.0 cross-spawn: 7.0.6 @@ -42134,11 +41997,11 @@ snapshots: - vitest - ws - prisma@8.0.0-rc.15(7a4d27bd99bbffe1e250da2a0de1478d): + prisma@8.0.0-rc.15(80bc071e98d758a11a24c1ff712bfbf2): dependencies: '@manypkg/tools': 2.1.2 '@prisma/cli-engine': 0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4) - '@prisma/composer-cli': 0.20.0(7d0073e65db52600bd3bf5c101578546) + '@prisma/composer-cli': 0.20.0(0d1b88465c5d4bc00bc45c7482ecc773) '@prisma/compute-sdk': 0.42.0(@prisma/management-api-sdk@1.69.0)(rollup@4.62.4)(supports-color@10.2.2) '@prisma/management-api-sdk': 1.69.0 '@prisma/orm-toolchain': 8.0.0-rc.11(@prisma/cli-engine@0.4.0(@prisma/management-api-sdk@1.69.0)(magicast@0.5.4))(magicast@0.5.4)(typanion@3.14.0)(typescript@7.0.2)(vite@8.2.1(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) @@ -44380,10 +44243,6 @@ snapshots: uuid@14.0.1: {} - valibot@1.2.0(typescript@5.9.3): - optionalDependencies: - typescript: 5.9.3 - valibot@1.2.0(typescript@7.0.2): optionalDependencies: typescript: 7.0.2 From a8fbbf43568f7cbefd3615b51f4506778306f53b Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Mon, 5 Oct 2026 16:11:29 -0700 Subject: [PATCH 02/11] feat(cloudflare)!: K2 streams and Basin (Pipelines, Catalog, Tables) - Cloudflare.K2: Stream, Subscription, WriteStream (Binding/Http/Local), StreamSink, ReadSubscription, consumeStreamRecords + StreamEventSourcePolling - Cloudflare.Basin: Pipelines + Catalog re-exports, Basin.Table over Iceberg REST - Pipelines: typed WriteStream, Http/Local producers, StreamSink, Effect Schema and list/struct stream schemas, basin_catalog sinks taking the catalog, plan-time SQL validation BREAKING: a Pipelines Stream declared without `http` no longer exposes a public ingest endpoint; set `http: { enabled: true, authentication: false }` to keep it public. --- .../Cloudflare/ApiToken/PermissionGroups.ts | 24 + .../alchemy/src/Cloudflare/Basin/Table.ts | 933 ++++++++++++++++++ .../alchemy/src/Cloudflare/Basin/index.ts | 24 + .../src/Cloudflare/Basin/internal/schema.ts | 559 +++++++++++ packages/alchemy/src/Cloudflare/K2/K2Codec.ts | 153 +++ packages/alchemy/src/Cloudflare/K2/K2Http.ts | 71 ++ packages/alchemy/src/Cloudflare/K2/K2Types.ts | 88 ++ .../src/Cloudflare/K2/ReadSubscription.ts | 85 ++ .../src/Cloudflare/K2/ReadSubscriptionHttp.ts | 99 ++ .../Cloudflare/K2/ReadSubscriptionLocal.ts | 49 + packages/alchemy/src/Cloudflare/K2/Stream.ts | 368 +++++++ .../src/Cloudflare/K2/StreamEventSource.ts | 247 +++++ .../alchemy/src/Cloudflare/K2/StreamPoller.ts | 134 +++ .../alchemy/src/Cloudflare/K2/StreamSink.ts | 71 ++ .../src/Cloudflare/K2/StreamSinkBatch.ts | 86 ++ .../src/Cloudflare/K2/StreamSinkBinding.ts | 32 + .../src/Cloudflare/K2/StreamSinkHttp.ts | 23 + .../src/Cloudflare/K2/StreamSinkLocal.ts | 32 + .../alchemy/src/Cloudflare/K2/Subscription.ts | 229 +++++ .../alchemy/src/Cloudflare/K2/WriteStream.ts | 103 ++ .../src/Cloudflare/K2/WriteStreamBinding.ts | 131 +++ .../src/Cloudflare/K2/WriteStreamClient.ts | 26 + .../src/Cloudflare/K2/WriteStreamHttp.ts | 62 ++ .../src/Cloudflare/K2/WriteStreamLocal.ts | 46 + packages/alchemy/src/Cloudflare/K2/index.ts | 15 + .../src/Cloudflare/Pipelines/Pipeline.ts | 87 +- .../alchemy/src/Cloudflare/Pipelines/Sink.ts | 321 ++++-- .../src/Cloudflare/Pipelines/Stream.ts | 395 ++++++-- .../src/Cloudflare/Pipelines/StreamCodec.ts | 101 ++ .../src/Cloudflare/Pipelines/StreamHttp.ts | 133 +++ .../src/Cloudflare/Pipelines/StreamSchema.ts | 347 +++++++ .../src/Cloudflare/Pipelines/StreamSink.ts | 72 ++ .../Cloudflare/Pipelines/StreamSinkBatch.ts | 72 ++ .../Cloudflare/Pipelines/StreamSinkBinding.ts | 21 + .../Cloudflare/Pipelines/StreamSinkHttp.ts | 21 + .../Cloudflare/Pipelines/StreamSinkLocal.ts | 26 + .../src/Cloudflare/Pipelines/WriteStream.ts | 112 ++- .../Pipelines/WriteStreamBinding.ts | 41 +- .../Cloudflare/Pipelines/WriteStreamHttp.ts | 36 + .../Cloudflare/Pipelines/WriteStreamLocal.ts | 40 + .../alchemy/src/Cloudflare/Pipelines/index.ts | 7 + packages/alchemy/src/Cloudflare/Providers.ts | 8 + .../alchemy/src/Cloudflare/R2/DataCatalog.ts | 15 +- .../src/Cloudflare/Workers/InferEnv.ts | 14 +- .../src/Cloudflare/Workers/RuntimeBindings.ts | 7 + .../Cloudflare/Workers/WorkerAsyncBindings.ts | 7 + .../src/Cloudflare/Workers/WorkerBinding.ts | 8 + packages/alchemy/src/Cloudflare/index.ts | 2 + .../test/Cloudflare/Basin/Pipeline.test.ts | 99 ++ .../test/Cloudflare/Basin/Sink.test.ts | 163 +++ .../test/Cloudflare/Basin/Stream.test.ts | 123 +++ .../test/Cloudflare/Basin/Table.test.ts | 337 +++++++ .../test/Cloudflare/Basin/WriteStream.test.ts | 160 +++ .../Basin/fixtures/pipelines-http-worker.ts | 43 + .../Basin/fixtures/pipelines-shared.ts | 44 + .../Basin/fixtures/pipelines-stream.ts | 29 + .../Basin/fixtures/pipelines-worker.ts | 62 ++ .../Cloudflare/K2/ReadSubscription.test.ts | 246 +++++ .../alchemy/test/Cloudflare/K2/Stream.test.ts | 108 ++ .../test/Cloudflare/K2/StreamSink.test.ts | 183 ++++ .../test/Cloudflare/K2/Subscription.test.ts | 103 ++ .../test/Cloudflare/K2/WriteStream.test.ts | 193 ++++ .../Cloudflare/K2/fixtures/k2-async-stack.ts | 13 + .../Cloudflare/K2/fixtures/k2-async-worker.ts | 28 + .../K2/fixtures/k2-binding-worker.ts | 33 + .../Cloudflare/K2/fixtures/k2-http-worker.ts | 32 + .../test/Cloudflare/K2/fixtures/k2-shared.ts | 64 ++ .../test/Cloudflare/K2/k2-test-utils.ts | 82 ++ .../Pipelines/LegacyPipeline.test.ts | 8 +- .../Cloudflare/Pipelines/Pipeline.test.ts | 8 +- .../Cloudflare/Pipelines/Pipelines.test.ts | 18 +- .../test/Cloudflare/Pipelines/Sink.test.ts | 7 +- 72 files changed, 7523 insertions(+), 246 deletions(-) create mode 100644 packages/alchemy/src/Cloudflare/Basin/Table.ts create mode 100644 packages/alchemy/src/Cloudflare/Basin/index.ts create mode 100644 packages/alchemy/src/Cloudflare/Basin/internal/schema.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/K2Codec.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/K2Http.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/K2Types.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/ReadSubscription.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/ReadSubscriptionHttp.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/ReadSubscriptionLocal.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/Stream.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/StreamPoller.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/StreamSink.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/StreamSinkBatch.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/StreamSinkBinding.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/StreamSinkHttp.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/StreamSinkLocal.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/Subscription.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/WriteStream.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/WriteStreamBinding.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/WriteStreamClient.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/WriteStreamHttp.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/WriteStreamLocal.ts create mode 100644 packages/alchemy/src/Cloudflare/K2/index.ts create mode 100644 packages/alchemy/src/Cloudflare/Pipelines/StreamCodec.ts create mode 100644 packages/alchemy/src/Cloudflare/Pipelines/StreamHttp.ts create mode 100644 packages/alchemy/src/Cloudflare/Pipelines/StreamSchema.ts create mode 100644 packages/alchemy/src/Cloudflare/Pipelines/StreamSink.ts create mode 100644 packages/alchemy/src/Cloudflare/Pipelines/StreamSinkBatch.ts create mode 100644 packages/alchemy/src/Cloudflare/Pipelines/StreamSinkBinding.ts create mode 100644 packages/alchemy/src/Cloudflare/Pipelines/StreamSinkHttp.ts create mode 100644 packages/alchemy/src/Cloudflare/Pipelines/StreamSinkLocal.ts create mode 100644 packages/alchemy/src/Cloudflare/Pipelines/WriteStreamHttp.ts create mode 100644 packages/alchemy/src/Cloudflare/Pipelines/WriteStreamLocal.ts create mode 100644 packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts create mode 100644 packages/alchemy/test/Cloudflare/Basin/Sink.test.ts create mode 100644 packages/alchemy/test/Cloudflare/Basin/Stream.test.ts create mode 100644 packages/alchemy/test/Cloudflare/Basin/Table.test.ts create mode 100644 packages/alchemy/test/Cloudflare/Basin/WriteStream.test.ts create mode 100644 packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-http-worker.ts create mode 100644 packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-shared.ts create mode 100644 packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-stream.ts create mode 100644 packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-worker.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/Stream.test.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/Subscription.test.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/fixtures/k2-async-stack.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/fixtures/k2-async-worker.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/fixtures/k2-binding-worker.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/fixtures/k2-http-worker.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/fixtures/k2-shared.ts create mode 100644 packages/alchemy/test/Cloudflare/K2/k2-test-utils.ts diff --git a/packages/alchemy/src/Cloudflare/ApiToken/PermissionGroups.ts b/packages/alchemy/src/Cloudflare/ApiToken/PermissionGroups.ts index d1cbf0547fe..1b04c7c6c1d 100644 --- a/packages/alchemy/src/Cloudflare/ApiToken/PermissionGroups.ts +++ b/packages/alchemy/src/Cloudflare/ApiToken/PermissionGroups.ts @@ -1078,6 +1078,30 @@ export const PERMISSION_GROUPS = [ description: "Grants write access to Intel API", scopes: ["com.cloudflare.api.account"], }, + { + id: "9159aeca78b949818ec1556aa42ffd2e", + name: "K2 Config Read", + description: "Grants read access to K2 streams", + scopes: ["com.cloudflare.api.account"], + }, + { + id: "0a50f569ebae4364a205dfc3137dbc5f", + name: "K2 Config Write", + description: "Grants write access to K2 streams", + scopes: ["com.cloudflare.api.account"], + }, + { + id: "3a6cae593f454607a4db52e67a45b19f", + name: "K2 Consume", + description: "Grants access to manage subscriptions and consume records from K2 streams", + scopes: ["com.cloudflare.api.account"], + }, + { + id: "47b72b140c694f90bfcd21b7da1e8a06", + name: "K2 Produce", + description: "Grants access to produce records to K2 streams", + scopes: ["com.cloudflare.api.account"], + }, { id: "4657621393f94f83b8ef94adba382e48", name: "L4 DDoS Managed Ruleset Read", diff --git a/packages/alchemy/src/Cloudflare/Basin/Table.ts b/packages/alchemy/src/Cloudflare/Basin/Table.ts new file mode 100644 index 00000000000..6e5b7310b5b --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Basin/Table.ts @@ -0,0 +1,933 @@ +import * as basin from "@distilled.cloud/cloudflare/basin_catalog"; +import * as Iceberg from "@distilled.cloud/iceberg"; +import * as Data from "effect/Data"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Predicate from "effect/Predicate"; +import type * as Redacted from "effect/Redacted"; +import * as S from "effect/Schema"; +import { Unowned } from "../../AdoptPolicy.ts"; +import { deepEqual, isResolved } from "../../Diff.ts"; +import { createPhysicalName } from "../../PhysicalName.ts"; +import * as Provider from "../../Provider.ts"; +import { Resource } from "../../Resource.ts"; +import { createInternalTags, hasAlchemyTags } from "../../Tags.ts"; +import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; +import type { Providers } from "../Providers.ts"; +import type { DataCatalog } from "../R2/DataCatalog.ts"; +import { + assignFieldIds, + BasinTableSchemaUnsupported, + destructiveChanges, + fieldIdsByPath, + fromIcebergFields, + maxFieldId, + normalizeColumns, + parsePartitionTerm, + partitionFieldName, + samePartitionTerms, + toTableSchema, + type PartitionTerm, +} from "./internal/schema.ts"; + +export { BasinTableSchemaUnsupported } from "./internal/schema.ts"; + +const TypeId = "Cloudflare.Basin.Table" as const; +type TypeId = typeof TypeId; + +/** + * An Iceberg primitive type. Decimals are written `decimal(P, S)` and + * fixed-length binaries `fixed[N]`. + */ +export type TablePrimitiveType = + | "boolean" + | "int" + | "long" + | "float" + | "double" + | "date" + | "time" + | "timestamp" + | "timestamptz" + | "timestamp_ns" + | "timestamptz_ns" + | "string" + | "uuid" + | "binary" + | `decimal(${number}, ${number})` + | `decimal(${number},${number})` + | `fixed[${number}]`; + +/** A nested Iceberg struct column type. */ +export interface TableStructType { + type: "struct"; + /** Columns of the struct. */ + fields: TableColumn[]; +} + +/** An Iceberg list column type. */ +export interface TableListType { + type: "list"; + /** Type of each element. */ + element: TableColumnType; + /** + * Whether elements are non-null. + * @default false + */ + elementRequired?: boolean; +} + +/** An Iceberg map column type. */ +export interface TableMapType { + type: "map"; + /** Type of the keys. */ + key: TableColumnType; + /** Type of the values. */ + value: TableColumnType; + /** + * Whether values are non-null. + * @default false + */ + valueRequired?: boolean; +} + +/** Any Iceberg column type. */ +export type TableColumnType = TablePrimitiveType | TableStructType | TableListType | TableMapType; + +/** One column of a table schema. Field ids are assigned by Alchemy. */ +export interface TableColumn { + /** Column name. */ + name: string; + /** Column type. */ + type: TableColumnType; + /** + * Whether the column is non-null. + * @default false + */ + required?: boolean; + /** Column documentation stored in the Iceberg schema. */ + doc?: string; +} + +/** + * A plain Iceberg schema descriptor: the table's top-level columns. + */ +export interface TableSchema { + /** Top-level columns. */ + fields: TableColumn[]; +} + +/** + * A table schema: an Effect Schema `Struct` or `Class`, or a plain + * {@link TableSchema} descriptor (for types Effect Schema cannot express, + * like `float`, `decimal(P, S)`, or `date`). + */ +export type TableSchemaInput = S.Schema | TableSchema; + +/** + * What happens to the Iceberg table when the resource is removed from the + * stack or the stack is destroyed. + * + * - `"retain"` — leave the table and its data in the catalog. + * - `"drop"` — drop the table from the catalog without purging its files. + * - `"purge"` — drop the table and ask the catalog to purge its data and + * metadata files. + */ +export type TableDeletePolicy = "retain" | "drop" | "purge"; + +/** Target output file size, in MB, of compaction. */ +export type TableCompactionTargetSizeMb = "64" | "128" | "256" | "512"; + +/** Per-table maintenance managed by the Basin catalog. */ +export interface TableMaintenance { + /** + * Compaction rewrites small data files into larger ones. Omit to leave + * the table's compaction settings unmanaged. + */ + compaction?: { + /** + * Whether compaction runs. + * @default true + */ + enabled?: boolean; + /** Target output file size, in MB. */ + targetSizeMb?: TableCompactionTargetSizeMb; + }; + /** + * Snapshot expiration removes old snapshots. Omit to leave the table's + * snapshot expiration settings unmanaged. + */ + snapshotExpiration?: { + /** + * Whether snapshot expiration runs. + * @default true + */ + enabled?: boolean; + /** + * Expire snapshots older than this, e.g. `"7 days"` or + * `Duration.hours(36)`. Sent to Cloudflare in whole days, hours, or + * minutes. + */ + olderThan?: Duration.Input; + /** Always keep at least this many snapshots. */ + retainLast?: number; + }; +} + +export interface TableProps { + /** + * The Basin (R2 Data) catalog that owns the table. Pass the catalog + * resource; the table is created in its warehouse. Changing the catalog + * fails the plan. + */ + catalog: DataCatalog; + /** + * Iceberg namespace of the table. Created if it does not exist; never + * deleted (other tables may share it). Changing the namespace fails the + * plan. + */ + namespace: string; + /** + * Table name. Changing it fails the plan (Iceberg renames would orphan + * readers and writers). + * @default ${app}_${id}_${stage}_${suffix} (lowercase, underscores) + */ + name?: string; + /** + * Table schema. Additive changes (new optional columns, relaxing + * `required`, `int`→`long`, `float`→`double`, wider decimals, docs, + * column order) evolve the table in place. Destructive changes (dropping + * a column, adding a required column, narrowing or changing a type) fail + * the plan — Alchemy never drops data to replace a table. + */ + schema: TableSchemaInput; + /** + * Partition spec as Iceberg transform terms: a bare column (`"region"`, + * identity), `"year(at)"`, `"month(at)"`, `"day(at)"`, `"hour(at)"`, + * `"bucket[16](id)"`, `"truncate[4](name)"`. Immutable: changing it fails + * the plan. + */ + partitionBy?: string[]; + /** + * Iceberg table properties. Synced in place; keys removed from this map + * are removed from the table. Properties set by the catalog or other + * writers are left alone. + */ + properties?: Record; + /** + * Per-table maintenance (compaction and snapshot expiration). Only the + * settings you specify are enforced. Enabling a job on a table requires + * the same job to be enabled on the catalog (`compaction` / + * `snapshotExpiration` with `state: "enabled"` on the DataCatalog). + */ + maintenance?: TableMaintenance; + /** + * What happens to the table when it is removed from the stack. + * @default "retain" + */ + delete?: TableDeletePolicy; + /** + * Bearer token for the Iceberg REST catalog. Defaults to the deploying + * Cloudflare credentials (API token or OAuth access token), which need + * R2 Data Catalog access. + */ + token?: Redacted.Redacted; +} + +export interface TableAttributes { + /** Iceberg namespace of the table. */ + namespace: string; + /** Table name. */ + tableName: string; + /** `namespace.tableName`, as engines like Spark and DuckDB address it. */ + identifier: string; + /** Iceberg table UUID. Stable across in-place schema evolution. */ + tableUuid: string; + /** Base location of the table's files in R2. */ + location: string | undefined; + /** Location of the current metadata file. */ + metadataLocation: string | undefined; + /** Iceberg REST catalog URI of the warehouse. */ + catalogUri: string; + /** Warehouse name (`{accountId}_{bucketName}`). */ + warehouse: string; + /** The Cloudflare account that owns the catalog. */ + accountId: string; + /** R2 bucket that backs the catalog. */ + bucketName: string; + /** Iceberg table format version. */ + formatVersion: number; + /** Id of the table's current schema. */ + currentSchemaId: number | undefined; +} + +export type Table = Resource; + +/** + * A plan-time rejection: the change cannot be applied in place and Alchemy + * will not replace (drop) an Iceberg table to apply it. + */ +export class BasinTableChangeRejected extends Data.TaggedError( + "Cloudflare.Basin.TableChangeRejected", +)<{ + message: string; + table: string; + reasons: string[]; +}> {} + +/** + * The deploying Cloudflare credentials cannot authenticate against the + * Iceberg REST catalog (global API keys are not bearer tokens). + */ +export class BasinTableCredentialsUnavailable extends Data.TaggedError( + "Cloudflare.Basin.TableCredentialsUnavailable", +)<{ + message: string; +}> {} + +const TableResource = Resource(TypeId); + +const toProps =

(props: P): Effect.Effect

=> + Effect.try({ + try: () => { + const p = props as { schema?: unknown } | undefined; + return p?.schema !== undefined && S.isSchema(p.schema) + ? ({ ...p, schema: toTableSchema(p.schema as S.Schema) } as P) + : props; + }, + catch: (error) => error, + }).pipe(Effect.orDie); + +/** + * An Apache Iceberg table in a Cloudflare Basin (R2 Data) catalog. + * + * Declare the table's schema with Effect Schema; Alchemy derives the Iceberg + * schema (`String`→`string`, `Boolean`→`boolean`, `Int`→`long` (`int` when + * bounded to 32 bits), `Number`→`double`, `Date`→`timestamptz`, + * `Uint8Array`→`binary`, arrays→`list`, `Struct`/`Class`→`struct`, + * `Record`→`map`, optional/nullable→not required). Additive schema changes + * evolve the table in place, keeping its UUID and field ids; destructive + * changes fail the plan. By default the table is retained when removed + * from the stack. + * + * Writing to a table from a Pipelines sink and querying it with Basin SQL + * are not part of this resource yet. + * ### Creating a table + * **Example:** Table from an Effect Schema + * ```typescript + * const Order = Schema.Struct({ + * id: Schema.String, + * amount: Schema.Number, + * at: Schema.Date, + * note: Schema.optional(Schema.String), + * }); + * + * const bucket = yield* Cloudflare.R2.Bucket("Analytics"); + * const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", { + * bucketName: bucket.bucketName, + * }); + * const orders = yield* Cloudflare.Basin.Table("Orders", { + * catalog, + * namespace: "sales", + * schema: Order, + * partitionBy: ["day(at)"], + * }); + * // orders.identifier → "sales." + * ``` + * + * **Example:** Plain Iceberg column descriptors + * ```typescript + * const prices = yield* Cloudflare.Basin.Table("Prices", { + * catalog, + * namespace: "sales", + * name: "prices", + * schema: { + * fields: [ + * { name: "sku", type: "string", required: true }, + * { name: "price", type: "decimal(10, 2)" }, + * { name: "day", type: "date" }, + * ], + * }, + * }); + * ``` + * + * ### Evolving the schema + * **Example:** Add an optional column in place + * ```typescript + * const Order = Schema.Struct({ + * id: Schema.String, + * amount: Schema.Number, + * at: Schema.Date, + * note: Schema.optional(Schema.String), + * coupon: Schema.optional(Schema.String), // new: evolves the table + * }); + * ``` + * + * ### Properties and maintenance + * **Example:** Table properties, compaction, and snapshot expiration + * ```typescript + * // Table maintenance needs the jobs enabled on the catalog first. + * const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", { + * bucketName: bucket.bucketName, + * compaction: { state: "enabled" }, + * snapshotExpiration: { state: "enabled" }, + * token: maintenanceToken, + * }); + * const orders = yield* Cloudflare.Basin.Table("Orders", { + * catalog, + * namespace: "sales", + * schema: Order, + * properties: { "write.parquet.compression-codec": "zstd" }, + * maintenance: { + * compaction: { targetSizeMb: "256" }, + * snapshotExpiration: { olderThan: "7 days", retainLast: 10 }, + * }, + * }); + * ``` + * + * ### Deleting + * **Example:** Purge the table's data on destroy + * ```typescript + * const scratch = yield* Cloudflare.Basin.Table("Scratch", { + * catalog, + * namespace: "tmp", + * schema: Order, + * delete: "purge", + * }); + * ``` + * + * @see https://developers.cloudflare.com/r2/data-catalog/ + * + * @resource + * @product Basin + * @category Storage & Databases + */ +export const Table: typeof TableResource = Object.assign( + (...args: [id: string, props: unknown] | [methods: object]) => { + if (typeof args[0] !== "string") { + return (TableResource as (methods: object) => unknown)(args[0]); + } + const [id, props] = args as [string, unknown]; + const resolved = Effect.isEffect(props) + ? Effect.flatMap(props as Effect.Effect, toProps) + : Effect.suspend(() => toProps(props)); + return TableResource(id, resolved as Effect.Effect); + }, + TableResource, +) as typeof TableResource; + +/** + * Returns true if the given value is a Basin Table resource. + */ +export const isTable = (value: unknown): value is Table => + Predicate.hasProperty(value, "Type") && value.Type === TypeId; + +interface CatalogRef { + catalogUri: string; + warehouse: string; + accountId: string; + bucketName: string; +} + +const CATALOG_HOST = "https://catalog.cloudflarestorage.com"; + +/** + * The catalog prop is resolved to the DataCatalog's attributes before a + * lifecycle operation runs. During plan it may still be unresolved. + */ +const catalogRefOf = (catalog: unknown): CatalogRef | undefined => { + if (catalog === null || typeof catalog !== "object") return undefined; + const c = catalog as Record; + const str = (v: unknown) => (typeof v === "string" && v.length > 0 ? v : undefined); + const fromUri = str(c.catalogUri)?.match(/\/([^/]+)\/([^/]+)\/?$/); + const accountId = str(c.accountId) ?? fromUri?.[1]; + const bucketName = str(c.bucketName) ?? str(c.bucket) ?? fromUri?.[2]; + if (!accountId || !bucketName) return undefined; + return { + accountId, + bucketName, + catalogUri: str(c.catalogUri) ?? `${CATALOG_HOST}/${accountId}/${bucketName}`, + warehouse: str(c.warehouse) ?? str(c.name) ?? `${accountId}_${bucketName}`, + }; +}; + +const refFromOutput = (output: TableAttributes): CatalogRef => ({ + catalogUri: output.catalogUri, + warehouse: output.warehouse, + accountId: output.accountId, + bucketName: output.bucketName, +}); + +const bearerToken = (override: Redacted.Redacted | undefined) => + Effect.gen(function* () { + if (override !== undefined) return override; + const creds = yield* yield* CloudflareEnvironment; + if (creds.type === "apiToken") return creds.apiToken; + if (creds.type === "oauth") return creds.accessToken; + return yield* Effect.fail( + new BasinTableCredentialsUnavailable({ + message: + "Cloudflare.Basin.Table: the Iceberg catalog needs a bearer token; global API key credentials cannot be used. Log in with an API token or OAuth, or pass `token`.", + }), + ); + }); + +/** Scope Iceberg operations to one warehouse of the Basin catalog. */ +const withCatalog = + (ref: CatalogRef, token: Redacted.Redacted) => + (effect: Effect.Effect) => + effect.pipe( + Effect.provide( + Layer.mergeAll( + Iceberg.IcebergProtocol, + Iceberg.fromCatalogConfig({ + uri: ref.catalogUri, + warehouse: ref.warehouse, + token, + }), + ), + ), + ); + +const tableNameFor = (id: string, name: string | undefined) => + name !== undefined + ? Effect.succeed(name) + : createPhysicalName({ id, lowercase: true, delimiter: "_" }).pipe( + Effect.map((generated) => generated.replaceAll(/[^a-z0-9_]/g, "_")), + ); + +const schemaOf = (schema: TableSchemaInput) => + Effect.try({ + try: () => toTableSchema(schema), + catch: (error) => + error instanceof BasinTableSchemaUnsupported + ? error + : new BasinTableSchemaUnsupported({ path: "", message: String(error) }), + }); + +const partitionTermsOf = (partitionBy: readonly string[] | undefined) => + Effect.try({ + try: () => (partitionBy ?? []).map(parsePartitionTerm), + catch: (error) => + error instanceof BasinTableSchemaUnsupported + ? error + : new BasinTableSchemaUnsupported({ path: "", message: String(error) }), + }); + +const rejected = (table: string, reasons: string[]) => + new BasinTableChangeRejected({ + table, + reasons, + message: `Cloudflare.Basin.Table '${table}': this change cannot be applied in place, and Alchemy never drops an Iceberg table to replace it:\n${reasons.map((r) => ` - ${r}`).join("\n")}`, + }); + +/** HEAD the table; a missing table (or namespace) answers 404 with no body. */ +const tableExists = (namespace: string, table: string) => + Iceberg.tableExists({ namespace, table }).pipe( + Effect.as(true), + Effect.catchTag("NotFound", () => Effect.succeed(false)), + ); + +const loadTable = (namespace: string, table: string) => + Iceberg.loadTable({ namespace, table }).pipe( + Effect.catchTag("NoSuchTableException", () => Effect.succeed(undefined)), + ); + +const requirement = (type: string, fields: Record): Iceberg.TableRequirement => ({ + type, + ...fields, +}); + +const currentSchemaOf = (metadata: Iceberg.TableMetadata) => + metadata.schemas?.find((s) => s.schema_id === metadata.current_schema_id) ?? + metadata.schemas?.[0]; + +const observedPartitionTerms = ( + metadata: Iceberg.TableMetadata, + fields: readonly Iceberg.StructField[], +): PartitionTerm[] => { + const spec = + metadata.partition_specs?.find((s) => s.spec_id === metadata.default_spec_id) ?? + metadata.partition_specs?.[0]; + const pathById = new Map([...fieldIdsByPath(fields)].map(([path, fid]) => [fid, path])); + return (spec?.fields ?? []).map((f) => ({ + column: pathById.get(f.source_id) ?? `#${f.source_id}`, + transform: f.transform.replaceAll(/\s+/g, "").toLowerCase(), + })); +}; + +interface Observed { + metadata: Iceberg.TableMetadata; + metadataLocation: string | undefined; +} + +/** + * Converge an existing table's schema and properties onto the desired state + * in one optimistic commit. Observes first; skips the commit on a no-op. + */ +const syncTable = (input: { + namespace: string; + table: string; + schema: TableSchema; + partition: PartitionTerm[]; + properties: Record; + ownedPropertyKeys: string[]; +}) => + Effect.gen(function* () { + const loaded = yield* Iceberg.loadTable({ namespace: input.namespace, table: input.table }); + const metadata = loaded.metadata; + const current = currentSchemaOf(metadata); + const observedFields = current?.fields ?? []; + const identifier = `${input.namespace}.${input.table}`; + + const reasons = destructiveChanges(fromIcebergFields(observedFields), input.schema.fields); + const observedPartition = observedPartitionTerms(metadata, observedFields); + if (!samePartitionTerms(observedPartition, input.partition)) { + reasons.push( + `changes the partition spec from [${observedPartition.map(describeTerm).join(", ")}] to [${input.partition.map(describeTerm).join(", ")}] (partition specs are immutable)`, + ); + } + if (reasons.length > 0) return yield* Effect.fail(rejected(identifier, reasons)); + + const requirements: Iceberg.TableRequirement[] = [ + requirement("assert-table-uuid", { uuid: metadata.table_uuid }), + ]; + const updates: Iceberg.TableUpdateInput[] = []; + + if ( + !deepEqual( + normalizeColumns(fromIcebergFields(observedFields)), + normalizeColumns(input.schema.fields), + ) + ) { + let next = Math.max(metadata.last_column_id ?? 0, maxFieldId(observedFields)) + 1; + const fields = assignFieldIds(input.schema.fields, observedFields, () => next++); + requirements.push( + requirement("assert-current-schema-id", { + "current-schema-id": metadata.current_schema_id, + }), + ); + updates.push( + { action: "add-schema", schema: { type: "struct", fields } }, + { action: "set-current-schema", schema_id: -1 }, + ); + } + + const observedProps = metadata.properties ?? {}; + const changed = Object.fromEntries( + Object.entries(input.properties).filter(([key, value]) => observedProps[key] !== value), + ); + if (Object.keys(changed).length > 0) { + updates.push({ action: "set-properties", updates: changed }); + } + const removals = input.ownedPropertyKeys.filter( + (key) => !(key in input.properties) && observedProps[key] !== undefined, + ); + if (removals.length > 0) { + updates.push({ action: "remove-properties", removals }); + } + + if (updates.length === 0) { + return { + metadata, + metadataLocation: loaded.metadata_location ?? undefined, + } satisfies Observed; + } + const committed = yield* Iceberg.updateTable({ + namespace: input.namespace, + table: input.table, + requirements, + updates, + }); + return { + metadata: committed.metadata, + metadataLocation: committed.metadata_location, + } satisfies Observed; + }); + +const describeTerm = (t: PartitionTerm) => + t.transform === "identity" ? t.column : `${t.transform}(${t.column})`; + +// --------------------------------------------------------------------------- +// Maintenance +// --------------------------------------------------------------------------- + +const AGE_UNITS = [ + ["d", 86_400_000], + ["h", 3_600_000], + ["m", 60_000], +] as const; + +/** Whole days, hours, or minutes (rounded up) — the API's `7d` / `36h` form. */ +const formatAge = (millis: number): string => { + for (const [unit, size] of AGE_UNITS) { + if (millis % size === 0) return `${millis / size}${unit}`; + } + return `${Math.ceil(millis / 60_000)}m`; +}; + +const parseAge = (age: string | undefined): number | undefined => { + const m = age?.trim().match(/^(\d+)\s*([dhms])$/); + if (!m) return undefined; + const size = { d: 86_400_000, h: 3_600_000, m: 60_000, s: 1_000 }[m[2] as "d" | "h" | "m" | "s"]; + return Number(m[1]) * size; +}; + +const syncMaintenance = ( + ref: CatalogRef, + namespace: string, + tableName: string, + maintenance: TableMaintenance, +) => + Effect.gen(function* () { + const key = { + accountId: ref.accountId, + bucketName: ref.bucketName, + namespace, + tableName, + }; + const { maintenanceConfig: observed } = yield* basin.getNamespacesTablesMaintenanceConfig(key); + const request: basin.UpdateNamespacesTablesMaintenanceConfigRequest = { ...key }; + let dirty = false; + + if (maintenance.compaction !== undefined) { + const desired = { + state: maintenance.compaction.enabled === false ? "disabled" : "enabled", + targetSizeMb: maintenance.compaction.targetSizeMb, + }; + const o = observed.compaction ?? undefined; + if ( + o?.state !== desired.state || + (desired.targetSizeMb !== undefined && o?.targetSizeMb !== desired.targetSizeMb) + ) { + request.compaction = desired; + dirty = true; + } + } + + if (maintenance.snapshotExpiration !== undefined) { + const se = maintenance.snapshotExpiration; + const olderThan = + se.olderThan === undefined + ? undefined + : yield* Effect.try({ + try: () => Duration.toMillis(se.olderThan!), + catch: () => + new BasinTableSchemaUnsupported({ + path: "maintenance.snapshotExpiration.olderThan", + message: `Cloudflare.Basin.Table: invalid duration ${String(se.olderThan)}`, + }), + }); + const desired = { + state: se.enabled === false ? "disabled" : "enabled", + maxSnapshotAge: olderThan === undefined ? undefined : formatAge(olderThan), + minSnapshotsToKeep: se.retainLast, + }; + const o = observed.snapshotExpiration ?? undefined; + if ( + o?.state !== desired.state || + (olderThan !== undefined && parseAge(o?.maxSnapshotAge) !== olderThan) || + (desired.minSnapshotsToKeep !== undefined && + o?.minSnapshotsToKeep !== desired.minSnapshotsToKeep) + ) { + request.snapshotExpiration = desired; + dirty = true; + } + } + + if (dirty) yield* basin.updateNamespacesTablesMaintenanceConfig(request); + }); + +const toAttributes = ( + ref: CatalogRef, + namespace: string, + tableName: string, + { metadata, metadataLocation }: Observed, +): TableAttributes => ({ + namespace, + tableName, + identifier: `${namespace}.${tableName}`, + tableUuid: metadata.table_uuid, + location: metadata.location, + metadataLocation, + catalogUri: ref.catalogUri, + warehouse: ref.warehouse, + accountId: ref.accountId, + bucketName: ref.bucketName, + formatVersion: metadata.format_version, + currentSchemaId: metadata.current_schema_id, +}); + +export const TableProvider = () => + Provider.succeed(Table, { + stables: [ + "tableUuid", + "namespace", + "tableName", + "identifier", + "location", + "catalogUri", + "warehouse", + "accountId", + "bucketName", + ], + + diff: Effect.fn(function* ({ olds, news, output }) { + const table = output?.identifier ?? ""; + const reasons: string[] = []; + if (output !== undefined) { + if (typeof news.namespace === "string" && news.namespace !== output.namespace) { + reasons.push(`changes the namespace from '${output.namespace}' to '${news.namespace}'`); + } + if (typeof news.name === "string" && news.name !== output.tableName) { + reasons.push(`renames the table from '${output.tableName}' to '${news.name}'`); + } + const ref = isResolved(news.catalog) ? catalogRefOf(news.catalog) : undefined; + if (ref !== undefined && ref.warehouse !== output.warehouse) { + reasons.push( + `moves the table from warehouse '${output.warehouse}' to '${ref.warehouse}'`, + ); + } + } + if (olds !== undefined) { + if (olds.schema !== undefined && isResolved(news.schema)) { + const previous = yield* schemaOf(olds.schema); + const next = yield* schemaOf(news.schema); + reasons.push(...destructiveChanges(previous.fields, next.fields)); + } + if (isResolved(news.partitionBy)) { + const previous = yield* partitionTermsOf(olds.partitionBy); + const next = yield* partitionTermsOf(news.partitionBy); + if (!samePartitionTerms(previous, next)) { + reasons.push( + `changes the partition spec from [${previous.map(describeTerm).join(", ")}] to [${next.map(describeTerm).join(", ")}] (partition specs are immutable)`, + ); + } + } + } + if (reasons.length > 0) return yield* Effect.fail(rejected(table, reasons)); + return undefined; + }), + + read: Effect.fn(function* ({ id, olds, output }) { + const ref = output ? refFromOutput(output) : catalogRefOf(olds?.catalog); + const namespace = output?.namespace ?? olds?.namespace; + if (ref === undefined || typeof namespace !== "string") return undefined; + const tableName = output?.tableName ?? (yield* tableNameFor(id, olds?.name)); + const token = yield* bearerToken(olds?.token); + const loaded = yield* Effect.gen(function* () { + if (!(yield* tableExists(namespace, tableName))) return undefined; + return yield* loadTable(namespace, tableName); + }).pipe(withCatalog(ref, token)); + if (loaded === undefined) return undefined; + const attrs = toAttributes(ref, namespace, tableName, { + metadata: loaded.metadata, + metadataLocation: loaded.metadata_location ?? undefined, + }); + // Tables carry Alchemy's ownership tags as table properties. + if (output !== undefined) return attrs; + const owned = yield* hasAlchemyTags(id, loaded.metadata.properties ?? {}); + return owned ? attrs : Unowned(attrs); + }), + + reconcile: Effect.fn(function* ({ id, news, olds, output }) { + const ref = catalogRefOf(news.catalog) ?? (output ? refFromOutput(output) : undefined); + const namespace = news.namespace; + const tableName = yield* tableNameFor(id, news.name ?? output?.tableName); + const identifier = `${namespace}.${tableName}`; + if (ref === undefined) { + return yield* Effect.fail( + rejected(identifier, ["has no resolved catalog (pass a Cloudflare.R2.DataCatalog)"]), + ); + } + if (output !== undefined && output.identifier !== identifier) { + return yield* Effect.fail( + rejected(output.identifier, [`changes the table identifier to '${identifier}'`]), + ); + } + + const token = yield* bearerToken(news.token); + const schema = yield* schemaOf(news.schema); + const partition = yield* partitionTermsOf(news.partitionBy); + const properties = { ...news.properties, ...(yield* createInternalTags(id)) }; + + const observed = yield* Effect.gen(function* () { + // Ensure the namespace — creation is idempotent under a race. + yield* Iceberg.createNamespace({ namespace: [namespace] }).pipe( + Effect.catchTag("AlreadyExistsException", () => Effect.void), + ); + + // Ensure the table. A concurrent create (or a create whose state + // write was lost) surfaces as AlreadyExists and falls through to sync. + if ((yield* loadTable(namespace, tableName)) === undefined) { + let next = 1; + const fields = assignFieldIds(schema.fields, undefined, () => next++); + const ids = fieldIdsByPath(fields); + const missing = partition.filter((t) => !ids.has(t.column)); + if (missing.length > 0) { + return yield* Effect.fail( + rejected( + identifier, + missing.map((t) => `partitions by unknown column '${t.column}'`), + ), + ); + } + yield* Iceberg.createTable({ + namespace, + name: tableName, + schema: { type: "struct", fields }, + partition_spec: + partition.length > 0 + ? { + fields: partition.map((t) => ({ + source_id: ids.get(t.column)!, + transform: t.transform, + name: partitionFieldName(t), + })), + } + : undefined, + properties, + }).pipe(Effect.catchTag("AlreadyExistsException", () => Effect.void)); + } + + // Sync schema + properties against observed metadata; a concurrent + // commit invalidates the requirements — reload and retry. + return yield* syncTable({ + namespace, + table: tableName, + schema, + partition, + properties, + ownedPropertyKeys: Object.keys(olds?.properties ?? {}), + }).pipe( + Effect.retry({ + while: (e) => e._tag === "CommitFailedException", + times: 3, + }), + ); + }).pipe(withCatalog(ref, token)); + + if (news.maintenance !== undefined) { + yield* syncMaintenance(ref, namespace, tableName, news.maintenance); + } + + return toAttributes(ref, namespace, tableName, observed); + }), + + delete: Effect.fn(function* ({ olds, output }) { + const policy = olds?.delete ?? "retain"; + if (policy === "retain") return; + const token = yield* bearerToken(olds?.token); + // Basin answers drop on a missing table with an ambiguous 403, so + // observe first: HEAD is a clean 404 when the table is gone. + yield* Effect.gen(function* () { + if (!(yield* tableExists(output.namespace, output.tableName))) return; + yield* Iceberg.dropTable({ + namespace: output.namespace, + table: output.tableName, + purgeRequested: policy === "purge", + }).pipe(Effect.catchTag("NoSuchTableException", () => Effect.void)); + }).pipe(withCatalog(refFromOutput(output), token)); + }), + }); diff --git a/packages/alchemy/src/Cloudflare/Basin/index.ts b/packages/alchemy/src/Cloudflare/Basin/index.ts new file mode 100644 index 00000000000..98ab3b75cf1 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Basin/index.ts @@ -0,0 +1,24 @@ +export * from "./Table.ts"; + +// Basin = Pipelines + Catalog + SQL. These are the same values as +// `Cloudflare.Pipelines.*` / `Cloudflare.R2.DataCatalog` (persisted resource +// types are unchanged), re-exported under the product's name. +export * from "../Pipelines/Pipeline.ts"; +export * from "../Pipelines/Sink.ts"; +export * from "../Pipelines/Stream.ts"; +export * from "../Pipelines/StreamSchema.ts"; +export * from "../Pipelines/StreamSink.ts"; +export * from "../Pipelines/StreamSinkBinding.ts"; +export * from "../Pipelines/StreamSinkHttp.ts"; +export * from "../Pipelines/StreamSinkLocal.ts"; +export * from "../Pipelines/WriteStream.ts"; +export * from "../Pipelines/WriteStreamBinding.ts"; +export * from "../Pipelines/WriteStreamHttp.ts"; +export * from "../Pipelines/WriteStreamLocal.ts"; +export { + DataCatalog as Catalog, + DataCatalogProvider as CatalogProvider, + isDataCatalog as isCatalog, + type DataCatalogAttributes as CatalogAttributes, + type DataCatalogProps as CatalogProps, +} from "../R2/DataCatalog.ts"; diff --git a/packages/alchemy/src/Cloudflare/Basin/internal/schema.ts b/packages/alchemy/src/Cloudflare/Basin/internal/schema.ts new file mode 100644 index 00000000000..643c44e2c60 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Basin/internal/schema.ts @@ -0,0 +1,559 @@ +import type * as Iceberg from "@distilled.cloud/iceberg"; +import * as Data from "effect/Data"; +import * as S from "effect/Schema"; +import * as AST from "effect/SchemaAST"; +import type { + TableColumn, + TableColumnType, + TablePrimitiveType, + TableSchema, + TableSchemaInput, +} from "../Table.ts"; + +/** + * An Effect Schema (or column) that has no Iceberg representation. + */ +export class BasinTableSchemaUnsupported extends Data.TaggedError( + "Cloudflare.Basin.TableSchemaUnsupported", +)<{ + message: string; + path: string; +}> {} + +const unsupported = (path: string, what: string) => + new BasinTableSchemaUnsupported({ + path, + message: `Cloudflare.Basin.Table: column '${path || ""}' is ${what}, which has no Iceberg type`, + }); + +// --------------------------------------------------------------------------- +// Effect Schema → column descriptor +// --------------------------------------------------------------------------- + +const INT32_MIN = -2147483648; +const INT32_MAX = 2147483647; + +interface CheckRep { + id: string; + payload: unknown; +} + +const checkReps = (ast: AST.AST): CheckRep[] => { + const out: CheckRep[] = []; + const visit = (check: unknown) => { + const c = check as { + annotations?: { representation?: CheckRep }; + checks?: readonly unknown[]; + }; + if (c.annotations?.representation?.id) out.push(c.annotations.representation); + c.checks?.forEach(visit); + }; + ((ast as { checks?: readonly unknown[] }).checks ?? []).forEach(visit); + return out; +}; + +const hasCheck = (reps: CheckRep[], id: string) => reps.some((r) => r.id === `effect/schema/${id}`); + +const numberType = (ast: AST.AST): TablePrimitiveType => { + const reps = checkReps(ast); + if (!hasCheck(reps, "isInt")) return "double"; + const fitsInt32 = reps.some((r) => { + if (r.id !== "effect/schema/isBetween") return false; + const p = r.payload as { minimum?: number; maximum?: number } | null; + return ( + typeof p?.minimum === "number" && + typeof p?.maximum === "number" && + p.minimum >= INT32_MIN && + p.maximum <= INT32_MAX + ); + }); + return fitsInt32 ? "int" : "long"; +}; + +const representationId = (ast: AST.AST): string | undefined => + (ast.annotations as { representation?: { id?: string } } | undefined)?.representation?.id; + +const descriptionOf = (ast: AST.AST): string | undefined => { + const own = (ast.annotations as { description?: unknown } | undefined)?.description; + if (typeof own === "string") return own; + if (AST.isUnion(ast)) { + for (const member of ast.types) { + const d = (member.annotations as { description?: unknown } | undefined)?.description; + if (typeof d === "string") return d; + } + } + return undefined; +}; + +const NUMERIC_RANK: Record = { int: 0, long: 1, double: 2 }; + +const columnTypeOf = (ast: AST.AST, path: string): { type: TableColumnType; nullable: boolean } => { + switch (ast._tag) { + case "Suspend": + return columnTypeOf(ast.thunk(), path); + case "Union": { + const members = ast.types.filter( + (t) => !(AST.isNull(t) || AST.isUndefined(t) || AST.isVoid(t)), + ); + const nullable = members.length !== ast.types.length; + if (members.length === 0) throw unsupported(path, "always null"); + const resolved = members.map((m) => columnTypeOf(m, path)); + const anyNullable = nullable || resolved.some((r) => r.nullable); + if (resolved.length === 1) return { type: resolved[0]!.type, nullable: anyNullable }; + const types = resolved.map((r) => r.type); + if (types.every((t) => typeof t === "string" && t in NUMERIC_RANK)) { + const widest = types.reduce((a, b) => + NUMERIC_RANK[a as string]! >= NUMERIC_RANK[b as string]! ? a : b, + ); + return { type: widest, nullable: anyNullable }; + } + if (types.every((t) => typeof t === "string" && t === types[0])) { + return { type: types[0]!, nullable: anyNullable }; + } + throw unsupported(path, "a union of different types"); + } + case "Literal": { + const literal = ast.literal; + if (typeof literal === "string") return { type: "string", nullable: false }; + if (typeof literal === "boolean") return { type: "boolean", nullable: false }; + if (typeof literal === "bigint") return { type: "long", nullable: false }; + return { type: Number.isInteger(literal) ? "long" : "double", nullable: false }; + } + case "Enum": { + const values = ast.enums.map(([, value]) => value); + if (values.every((v) => typeof v === "string")) return { type: "string", nullable: false }; + if (values.every((v) => typeof v === "number")) return { type: "long", nullable: false }; + throw unsupported(path, "an enum mixing strings and numbers"); + } + case "TemplateLiteral": + return { type: "string", nullable: false }; + case "String": + return { + type: hasCheck(checkReps(ast), "isUUID") ? "uuid" : "string", + nullable: false, + }; + case "Number": + return { type: numberType(ast), nullable: false }; + case "Boolean": + return { type: "boolean", nullable: false }; + case "BigInt": + return { type: "long", nullable: false }; + case "Declaration": { + const rep = representationId(ast); + if (rep === "effect/schema/Date" || rep === "effect/schema/DateTimeUtc") { + return { type: "timestamptz", nullable: false }; + } + if (rep === "effect/schema/Uint8Array") return { type: "binary", nullable: false }; + if (rep === "effect/schema/ReadonlyMap" && ast.typeParameters.length === 2) { + return { + type: mapTypeOf(ast.typeParameters[0]!, ast.typeParameters[1]!, path), + nullable: false, + }; + } + if (rep === "effect/schema/ReadonlySet" && ast.typeParameters.length === 1) { + return { type: listTypeOf(ast.typeParameters[0]!, path), nullable: false }; + } + // Schema.Class / TaggedClass: a declaration over the struct of its fields. + const [inner] = ast.typeParameters; + if (ast.typeParameters.length === 1 && inner !== undefined && AST.isObjects(inner)) { + return columnTypeOf(inner, path); + } + throw unsupported(path, `a declaration (${rep ?? "unknown"})`); + } + case "Objects": { + if (ast.propertySignatures.length === 0 && ast.indexSignatures.length === 1) { + const index = ast.indexSignatures[0]!; + return { type: mapTypeOf(index.parameter, index.type, path), nullable: false }; + } + if (ast.indexSignatures.length > 0) { + throw unsupported(path, "a struct with an index signature"); + } + return { type: { type: "struct", fields: structFieldsOf(ast, path) }, nullable: false }; + } + case "Arrays": { + if (ast.elements.length === 0 && ast.rest.length === 1) { + return { type: listTypeOf(ast.rest[0]!, path), nullable: false }; + } + throw unsupported(path, "a tuple"); + } + default: + throw unsupported(path, `of schema type '${ast._tag}'`); + } +}; + +const listTypeOf = (element: AST.AST, path: string): TableColumnType => { + const resolved = columnTypeOf(element, `${path}.element`); + return { type: "list", element: resolved.type, elementRequired: !resolved.nullable }; +}; + +const mapTypeOf = (key: AST.AST, value: AST.AST, path: string): TableColumnType => { + const k = columnTypeOf(key, `${path}.key`); + if (k.nullable) throw unsupported(`${path}.key`, "a nullable map key"); + const v = columnTypeOf(value, `${path}.value`); + return { type: "map", key: k.type, value: v.type, valueRequired: !v.nullable }; +}; + +const structFieldsOf = (ast: AST.Objects, path: string): TableColumn[] => + ast.propertySignatures.map((property) => { + if (typeof property.name !== "string") { + throw unsupported(path, "keyed by a symbol"); + } + const columnPath = path ? `${path}.${property.name}` : property.name; + const resolved = columnTypeOf(property.type, columnPath); + const optional = AST.isOptional(property.type); + const keyDoc = (property.type.context?.annotations as { description?: unknown } | undefined) + ?.description; + const doc = typeof keyDoc === "string" ? keyDoc : descriptionOf(property.type); + return { + name: property.name, + type: resolved.type, + required: !(optional || resolved.nullable), + ...(doc !== undefined ? { doc } : {}), + }; + }); + +/** + * Convert an Effect Schema struct (or `Schema.Class`) into the plain column + * descriptor persisted in Table props. Plain descriptors pass through. + */ +export const toTableSchema = (schema: TableSchemaInput): TableSchema => { + if (!S.isSchema(schema)) return schema; + const root = columnTypeOf(schema.ast, "").type; + if (typeof root === "string" || root.type !== "struct") { + throw unsupported("", "not a struct (a table schema must be a Struct or Class)"); + } + return { fields: root.fields }; +}; + +// --------------------------------------------------------------------------- +// Normalization + comparison +// --------------------------------------------------------------------------- + +/** Canonical primitive spelling: lowercase, no whitespace. */ +const normalizePrimitive = (t: string): string => t.replaceAll(/\s+/g, "").toLowerCase(); + +/** Fill defaults so two descriptors compare structurally. */ +export const normalizeColumnType = (t: TableColumnType): TableColumnType => { + if (typeof t === "string") return normalizePrimitive(t) as TablePrimitiveType; + switch (t.type) { + case "struct": + return { type: "struct", fields: normalizeColumns(t.fields) }; + case "list": + return { + type: "list", + element: normalizeColumnType(t.element), + elementRequired: t.elementRequired ?? false, + }; + case "map": + return { + type: "map", + key: normalizeColumnType(t.key), + value: normalizeColumnType(t.value), + valueRequired: t.valueRequired ?? false, + }; + } +}; + +export const normalizeColumns = (fields: readonly TableColumn[]): TableColumn[] => + fields.map((f) => ({ + name: f.name, + type: normalizeColumnType(f.type), + required: f.required ?? false, + ...(f.doc ? { doc: f.doc } : {}), + })); + +/** Iceberg wire struct fields → column descriptors. */ +export const fromIcebergFields = (fields: readonly Iceberg.StructField[]): TableColumn[] => + fields.map((f) => ({ + name: f.name, + type: fromIcebergType(f.type), + required: f.required, + ...(f.doc ? { doc: f.doc } : {}), + })); + +const fromIcebergType = (t: Iceberg.Type): TableColumnType => { + if (typeof t === "string") return normalizePrimitive(t) as TablePrimitiveType; + switch (t.type) { + case "struct": + return { type: "struct", fields: fromIcebergFields((t as Iceberg.StructType).fields) }; + case "list": { + const l = t as Iceberg.ListType; + return { + type: "list", + element: fromIcebergType(l.element), + elementRequired: l.element_required, + }; + } + case "map": { + const m = t as Iceberg.MapType; + return { + type: "map", + key: fromIcebergType(m.key), + value: fromIcebergType(m.value), + valueRequired: m.value_required, + }; + } + default: + return normalizePrimitive(t.type) as TablePrimitiveType; + } +}; + +const decimalOf = (t: string) => /^decimal\((\d+),(\d+)\)$/.exec(t); + +/** Iceberg's legal in-place type promotions. */ +const isPromotion = (from: string, to: string): boolean => { + if (from === to) return true; + if (from === "int" && to === "long") return true; + if (from === "float" && to === "double") return true; + const a = decimalOf(from); + const b = decimalOf(to); + return !!a && !!b && a[2] === b[2] && Number(b[1]) >= Number(a[1]); +}; + +const describe = (t: TableColumnType): string => + typeof t === "string" ? t : t.type === "struct" ? "struct" : t.type; + +/** + * Compare an observed (or previously declared) schema with the desired one + * and list every change Iceberg cannot apply without rewriting or losing + * data: dropped columns, new required columns, required-ness tightened, and + * type changes other than Iceberg's promotions (int→long, float→double, + * decimal precision widening). + */ +export const destructiveChanges = ( + observed: readonly TableColumn[], + desired: readonly TableColumn[], +): string[] => { + const reasons: string[] = []; + compareFields("", normalizeColumns(observed), normalizeColumns(desired), reasons); + return reasons; +}; + +const compareFields = ( + prefix: string, + observed: TableColumn[], + desired: TableColumn[], + reasons: string[], +) => { + for (const o of observed) { + const path = `${prefix}${o.name}`; + const d = desired.find((f) => f.name === o.name); + if (!d) { + reasons.push(`drops column '${path}'`); + continue; + } + if (!o.required && d.required) reasons.push(`makes optional column '${path}' required`); + compareTypes(path, o.type, d.type, reasons); + } + for (const d of desired) { + if (d.required && !observed.some((f) => f.name === d.name)) { + reasons.push( + `adds required column '${prefix}${d.name}' (new columns must be optional so existing rows stay valid)`, + ); + } + } +}; + +const compareTypes = (path: string, o: TableColumnType, d: TableColumnType, reasons: string[]) => { + if (typeof o === "string" || typeof d === "string") { + if (typeof o === "string" && typeof d === "string" && isPromotion(o, d)) return; + reasons.push(`changes the type of '${path}' from ${describe(o)} to ${describe(d)}`); + return; + } + if (o.type !== d.type) { + reasons.push(`changes the type of '${path}' from ${describe(o)} to ${describe(d)}`); + return; + } + if (o.type === "struct" && d.type === "struct") { + compareFields(`${path}.`, o.fields, d.fields, reasons); + } else if (o.type === "list" && d.type === "list") { + if (!o.elementRequired && d.elementRequired) { + reasons.push(`makes the elements of '${path}' required`); + } + compareTypes(`${path}.element`, o.element, d.element, reasons); + } else if (o.type === "map" && d.type === "map") { + if (JSON.stringify(o.key) !== JSON.stringify(d.key)) { + reasons.push(`changes the key type of map '${path}'`); + } + if (!o.valueRequired && d.valueRequired) { + reasons.push(`makes the values of '${path}' required`); + } + compareTypes(`${path}.value`, o.value, d.value, reasons); + } +}; + +// --------------------------------------------------------------------------- +// Field ids +// --------------------------------------------------------------------------- + +/** + * Build the Iceberg wire schema for `desired`, reusing the field ids of + * columns that already exist in `observed` (matched by name at the same + * nesting) and assigning fresh ids from `nextId` to new columns. Ids at one + * struct level are assigned before descending, like Iceberg's own clients. + */ +export const assignFieldIds = ( + desired: readonly TableColumn[], + observed: readonly Iceberg.StructField[] | undefined, + nextId: () => number, +): Iceberg.StructField[] => { + const normalized = normalizeColumns(desired); + const matches = normalized.map((f) => observed?.find((o) => o.name === f.name)); + const ids = matches.map((m) => m?.id ?? nextId()); + return normalized.map((f, i) => ({ + id: ids[i]!, + name: f.name, + required: f.required ?? false, + type: buildType(f.type, matches[i]?.type, nextId), + ...(f.doc ? { doc: f.doc } : {}), + })); +}; + +const buildType = ( + t: TableColumnType, + observed: Iceberg.Type | undefined, + nextId: () => number, +): Iceberg.Type => { + if (typeof t === "string") return t; + const obs = typeof observed === "object" ? observed : undefined; + switch (t.type) { + case "struct": + return { + type: "struct", + fields: assignFieldIds( + t.fields, + obs?.type === "struct" ? (obs as Iceberg.StructType).fields : undefined, + nextId, + ), + }; + case "list": { + const ol = obs?.type === "list" ? (obs as Iceberg.ListType) : undefined; + const elementId = ol?.element_id ?? nextId(); + return { + type: "list", + element_id: elementId, + element: buildType(t.element, ol?.element, nextId), + element_required: t.elementRequired ?? false, + }; + } + case "map": { + const om = obs?.type === "map" ? (obs as Iceberg.MapType) : undefined; + const keyId = om?.key_id ?? nextId(); + const valueId = om?.value_id ?? nextId(); + return { + type: "map", + key_id: keyId, + key: buildType(t.key, om?.key, nextId), + value_id: valueId, + value: buildType(t.value, om?.value, nextId), + value_required: t.valueRequired ?? false, + }; + } + } +}; + +/** Highest field id anywhere in a wire schema. */ +export const maxFieldId = (fields: readonly Iceberg.StructField[]): number => { + let max = 0; + const visitType = (t: Iceberg.Type) => { + if (typeof t === "string") return; + if (t.type === "struct") visitFields((t as Iceberg.StructType).fields); + else if (t.type === "list") { + const l = t as Iceberg.ListType; + max = Math.max(max, l.element_id); + visitType(l.element); + } else if (t.type === "map") { + const m = t as Iceberg.MapType; + max = Math.max(max, m.key_id, m.value_id); + visitType(m.key); + visitType(m.value); + } + }; + const visitFields = (fs: readonly Iceberg.StructField[]) => { + for (const f of fs) { + max = Math.max(max, f.id); + visitType(f.type); + } + }; + visitFields(fields); + return max; +}; + +/** Dotted column path → field id, for partition source resolution. */ +export const fieldIdsByPath = (fields: readonly Iceberg.StructField[]): Map => { + const out = new Map(); + const visit = (prefix: string, fs: readonly Iceberg.StructField[]) => { + for (const f of fs) { + const path = `${prefix}${f.name}`; + out.set(path, f.id); + if (typeof f.type === "object" && f.type.type === "struct") { + visit(`${path}.`, (f.type as Iceberg.StructType).fields); + } + } + }; + visit("", fields); + return out; +}; + +// --------------------------------------------------------------------------- +// Partition specs +// --------------------------------------------------------------------------- + +export interface PartitionTerm { + /** Dotted source column path. */ + column: string; + /** Iceberg transform, e.g. `identity`, `day`, `bucket[16]`. */ + transform: string; +} + +const UNARY: Record = { + identity: "identity", + year: "year", + years: "year", + month: "month", + months: "month", + day: "day", + days: "day", + date: "day", + hour: "hour", + hours: "hour", + void: "void", +}; + +const COLUMN = "([A-Za-z_][\\w]*(?:\\.[A-Za-z_][\\w]*)*)"; +const UNARY_RE = new RegExp(`^(\\w+)\\(${COLUMN}\\)$`); +const BRACKET_RE = new RegExp(`^(bucket|truncate)\\[(\\d+)\\]\\(${COLUMN}\\)$`); +const ARG_RE = new RegExp(`^(bucket|truncate)\\((\\d+),${COLUMN}\\)$`); +const BARE_RE = new RegExp(`^${COLUMN}$`); + +/** + * Parse a `partitionBy` term: `col`, `identity(col)`, `year|month|day|hour(col)`, + * `bucket[N](col)` / `bucket(N, col)`, `truncate[N](col)` / `truncate(N, col)`. + */ +export const parsePartitionTerm = (term: string): PartitionTerm => { + const t = term.replaceAll(/\s+/g, ""); + let m = BRACKET_RE.exec(t) ?? ARG_RE.exec(t); + if (m) return { column: m[3]!, transform: `${m[1]}[${m[2]}]` }; + m = UNARY_RE.exec(t); + if (m && UNARY[m[1]!.toLowerCase()]) { + return { column: m[2]!, transform: UNARY[m[1]!.toLowerCase()]! }; + } + m = BARE_RE.exec(t); + if (m) return { column: m[1]!, transform: "identity" }; + throw new BasinTableSchemaUnsupported({ + path: term, + message: `Cloudflare.Basin.Table: cannot parse partition term '${term}' (expected e.g. 'day(at)', 'bucket[16](id)', 'region')`, + }); +}; + +export const partitionFieldName = ({ column, transform }: PartitionTerm): string => { + const base = column.replaceAll(".", "_"); + if (transform === "identity") return base; + if (transform.startsWith("bucket")) return `${base}_bucket`; + if (transform.startsWith("truncate")) return `${base}_trunc`; + return `${base}_${transform}`; +}; + +export const samePartitionTerms = (a: readonly PartitionTerm[], b: readonly PartitionTerm[]) => + a.length === b.length && + a.every((t, i) => t.column === b[i]!.column && t.transform === b[i]!.transform); diff --git a/packages/alchemy/src/Cloudflare/K2/K2Codec.ts b/packages/alchemy/src/Cloudflare/K2/K2Codec.ts new file mode 100644 index 00000000000..c12d193df90 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/K2Codec.ts @@ -0,0 +1,153 @@ +import type * as k2 from "@distilled.cloud/cloudflare/k2"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import { + type ConsumedRecord, + type DecodedRecord, + type EncodedRecord, + K2SchemaError, + type Record as K2Record, + type RecordSchema, +} from "./K2Types.ts"; + +/** + * Record encoding shared by the K2 producer and consumer clients. Internal — + * not exported from the K2 barrel. + */ + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +/** Header K2 producers set on schema-encoded (JSON) records. */ +export const JSON_HEADERS = { "content-type": "application/json" } as const; + +/** Normalize record content to bytes; strings are UTF-8 encoded. */ +export const toBytes = (content: Uint8Array | ArrayBuffer | string): Uint8Array => + typeof content === "string" + ? encoder.encode(content) + : content instanceof Uint8Array + ? content + : new Uint8Array(content); + +/** Standard (padded) base64, chunked so large payloads don't overflow `apply`. */ +export const toBase64 = (bytes: Uint8Array): string => { + let binary = ""; + const chunk = 0x8000; + for (let i = 0; i < bytes.length; i += chunk) { + binary += String.fromCharCode(...bytes.subarray(i, i + chunk)); + } + return btoa(binary); +}; + +export const fromBase64 = (base64: string): Uint8Array => { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +}; + +/** Encode untyped records: content to bytes, headers passed through. */ +export const encodeRecords = (records: ReadonlyArray): ReadonlyArray => + records.map((record) => ({ + content: toBytes(record.content), + ...(record.headers && Object.keys(record.headers).length > 0 + ? { headers: record.headers } + : {}), + })); + +/** Encode typed values to JSON records with the binding's schema. */ +export const encodeWithSchema = + (schema: RecordSchema) => + (values: ReadonlyArray): Effect.Effect, K2SchemaError> => + Effect.forEach(values, (value) => + Schema.encodeEffect(schema)(value).pipe( + Effect.mapError( + (cause) => + new K2SchemaError({ + message: `Could not encode K2 record: ${cause.message}`, + cause, + }), + ), + Effect.flatMap((encoded) => + Effect.try({ + try: (): EncodedRecord => ({ + content: encoder.encode(JSON.stringify(encoded)), + headers: { ...JSON_HEADERS }, + }), + catch: (cause) => + new K2SchemaError({ message: "Could not serialize K2 record as JSON", cause }), + }), + ), + ), + ); + +/** The `produce` request body shape for encoded records. */ +export const toProduceRecords = (records: ReadonlyArray): Array => + records.map((record) => ({ + content: toBase64(record.content), + ...(record.headers ? { headers: record.headers } : {}), + })); + +/** Decode the `consume` response records into bytes + headers + timestamp. */ +export const fromConsumedRecords = ( + records: ReadonlyArray, +): ReadonlyArray => + records.map((record) => ({ + content: fromBase64(record.content), + headers: Object.fromEntries( + Object.entries(record.headers ?? {}).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ), + timestamp: new Date(record.timestampMs), + })); + +/** Decode JSON record content with a schema into `record.value`. */ +export const decodeWithSchema = + (schema: RecordSchema) => + ( + records: ReadonlyArray, + ): Effect.Effect>, K2SchemaError> => + Effect.forEach(records, (record) => + Effect.try({ + try: () => JSON.parse(decoder.decode(record.content)) as unknown, + catch: (cause) => + new K2SchemaError({ message: "K2 record content is not valid JSON", cause }), + }).pipe( + Effect.flatMap((json) => + Schema.decodeUnknownEffect(schema)(json).pipe( + Effect.mapError( + (cause) => + new K2SchemaError({ + message: `Could not decode K2 record: ${cause.message}`, + cause, + }), + ), + ), + ), + Effect.map((value): DecodedRecord => ({ ...record, value })), + ), + ); + +/** + * Approximate size K2 counts against the 1 MB record limit: content plus + * header names and values. + */ +export const recordSize = (record: EncodedRecord): number => { + let size = record.content.byteLength; + for (const [key, value] of Object.entries(record.headers ?? {})) { + size += encoder.encode(key).byteLength + encoder.encode(value).byteLength; + } + return size; +}; + +/** + * Approximate wire size of a record inside a `produce` request: base64 + * content (4/3 of the bytes) plus headers and JSON framing. + */ +export const wireSize = (record: EncodedRecord): number => + Math.ceil(record.content.byteLength / 3) * 4 + + (recordSize(record) - record.content.byteLength) + + 64; diff --git a/packages/alchemy/src/Cloudflare/K2/K2Http.ts b/packages/alchemy/src/Cloudflare/K2/K2Http.ts new file mode 100644 index 00000000000..83fe809f2d8 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/K2Http.ts @@ -0,0 +1,71 @@ +import * as Effect from "effect/Effect"; +import type * as HttpClient from "effect/http/HttpClient"; +import type { RuntimeContext } from "../../RuntimeContext.ts"; +import { Self } from "../../Self.ts"; +import { AccountApiToken } from "../ApiToken/AccountApiToken.ts"; +import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; +import type { Credentials } from "../Credentials.ts"; +import { authorizeWith } from "../HttpClientUtils.ts"; + +/** + * Shared scaffolding for the HTTP-backed K2 services. Internal — not + * exported from the K2 barrel. + */ + +/** + * Injectable auth for the K2 HTTP clients. The scoped-token variant + * (`*Http`) and the current-credentials variant (`*Local`) share the same + * request path and differ only in how credentials reach the SDK operation. + * K2 data-plane operations are addressed by stream id, so no account id is + * needed. + */ +export interface K2Auth { + authorize: ( + eff: Effect.Effect, + ) => Effect.Effect; +} + +type K2PermissionGroup = "K2 Produce" | "K2 Consume"; + +/** + * Mint (or extend) the host's scoped {@link AccountApiToken} with the given + * K2 permission group, bind it into the host at deploy time, and return + * the {@link K2Auth} that authorizes requests with it. + */ +export const makeK2HttpAuth = Effect.gen(function* () { + const Token = yield* AccountApiToken; + const self = yield* Self; + const env = yield* CloudflareEnvironment; + + return Effect.fn(function* (resource: { LogicalId: string }, permissionGroup: K2PermissionGroup) { + const { accountId } = yield* env; + const token = yield* Token(`${self.LogicalId}Token`); + if (!globalThis.__ALCHEMY_RUNTIME__) { + yield* token.bind`${resource.LogicalId}`({ + policies: [ + { + effect: "allow", + permissionGroups: [permissionGroup], + resources: { + [`com.cloudflare.api.account.${accountId}`]: "*", + }, + }, + ], + }); + } + const value = yield* token.value; + return { authorize: authorizeWith({ value }) } satisfies K2Auth; + }); +}); + +/** + * Capture the ambient credentials (the stack's providers layer during + * stack-eval) so a client can run with the current credentials instead of a + * scoped token — for Actions and other deploy-time Effects. + */ +export const makeK2LocalAuth = Effect.gen(function* () { + const context = yield* Effect.context(); + return { + authorize: (eff) => eff.pipe(Effect.provideContext(context)), + } satisfies K2Auth; +}); diff --git a/packages/alchemy/src/Cloudflare/K2/K2Types.ts b/packages/alchemy/src/Cloudflare/K2/K2Types.ts new file mode 100644 index 00000000000..e9b37a21946 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/K2Types.ts @@ -0,0 +1,88 @@ +import type * as k2 from "@distilled.cloud/cloudflare/k2"; +import * as Data from "effect/Data"; +import type * as Schema from "effect/Schema"; + +/** + * A record to append to a K2 stream. + * + * `content` is the payload: raw bytes, or a string encoded as UTF-8. + * `headers` are up to 32 string headers describing the content. A record + * (content and headers combined) can be at most 1 MB. + */ +export interface Record { + content: Uint8Array | ArrayBuffer | string; + headers?: { [key: string]: string }; +} + +/** + * A record in the shape every K2 producer path sends: bytes plus headers. + * Built by a client's `encode` and accepted by its `sendEncoded`. + */ +export interface EncodedRecord { + content: Uint8Array; + headers?: { [key: string]: string }; +} + +/** + * A record read from a K2 subscription. `content` is the decoded payload, + * `timestamp` is when K2 received the record. + */ +export interface ConsumedRecord { + content: Uint8Array; + headers: { [key: string]: string }; + timestamp: Date; +} + +/** + * A consumed record whose JSON content was decoded with a schema + * (`consumeStreamRecords(stream, { schema }, handler)`). + */ +export interface DecodedRecord extends ConsumedRecord { + value: A; +} + +/** + * A batch of records leased to one worker id for five minutes. Ack it once + * the records are processed, nack it to have them redelivered, or extend it + * while processing takes longer than the lease. + */ +export interface Batch { + batchId: string; + workerId: string; + /** When the lease expires. */ + leasedUntil: Date; + records: ReadonlyArray; +} + +/** Where a new subscription starts reading. */ +export type StartAt = "earliest" | "latest"; + +/** + * A JSON codec for record payloads. `send` encodes values to JSON with it; + * `consumeStreamRecords` decodes record content with it. + */ +export type RecordSchema = Schema.Codec; + +/** + * Raised when a value cannot be encoded with (or a record decoded with) the + * binding's schema. + */ +export class K2SchemaError extends Data.TaggedError("K2SchemaError")<{ + message: string; + cause?: unknown; +}> {} + +/** Errors a K2 producer (`WriteStream`, `StreamSink`) can fail with. */ +export type ProduceError = k2.ProduceError | K2SchemaError; + +/** Errors {@link ReadSubscriptionClient.consume} can fail with. */ +export type ConsumeError = k2.ConsumeError; + +/** Errors {@link ReadSubscriptionClient.ack} can fail with. */ +export type AckError = k2.AckBatchError; + +/** Errors {@link ReadSubscriptionClient.nack} can fail with. */ +export type NackError = k2.NackBatchError; + +/** Errors {@link ReadSubscriptionClient.extend} can fail with. */ +export type ExtendError = k2.ExtendLeaseError; diff --git a/packages/alchemy/src/Cloudflare/K2/ReadSubscription.ts b/packages/alchemy/src/Cloudflare/K2/ReadSubscription.ts new file mode 100644 index 00000000000..061a2d2170f --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/ReadSubscription.ts @@ -0,0 +1,85 @@ +import type * as Effect from "effect/Effect"; +import type * as Option from "effect/Option"; +import * as Binding from "../../Binding.ts"; +import type { RuntimeContext } from "../../RuntimeContext.ts"; +import type { AckError, Batch, ConsumeError, ExtendError, NackError } from "./K2Types.ts"; +import type { Subscription } from "./Subscription.ts"; + +/** Options for {@link ReadSubscriptionClient.consume}. */ +export interface ConsumeOptions { + /** + * The consumer requesting the lease, 1 to 256 characters. A worker that + * already holds a lease gets the same batch back. Use a distinct id per + * concurrent consumer. + */ + workerId: string; + /** + * At most this many records, from 1 to 10000. + * @default 100 + */ + maxRecords?: number; +} + +/** + * Binding service that turns a K2 {@link Subscription} into a pull client: + * lease a batch of records, then acknowledge it (advance the subscription), + * nack it (redeliver), or extend its five-minute lease. + * + * The native `k2` Worker binding can only produce, so this service ships + * HTTP implementations only: {@link ReadSubscriptionHttp} (scoped + * `K2 Consume` token) and {@link ReadSubscriptionLocal} (current + * credentials). For a long-running consumer loop, see + * `consumeStreamRecords`. + * ### Pulling Records + * **Example:** Lease, process, and acknowledge a batch + * ```typescript + * const inbox = yield* Cloudflare.K2.ReadSubscription(Analytics); + * + * const batch = yield* inbox.consume({ workerId: "worker-1", maxRecords: 100 }); + * if (Option.isSome(batch)) { + * yield* Effect.forEach(batch.value.records, (record) => + * Effect.log(new TextDecoder().decode(record.content)), + * ); + * yield* inbox.ack(batch.value); + * } + * ``` + * + * ### Redelivery + * **Example:** Release a batch for redelivery + * ```typescript + * yield* process(batch.records).pipe( + * Effect.andThen(inbox.ack(batch)), + * Effect.catch(() => inbox.nack(batch)), + * ); + * ``` + * + * @binding + * @product K2 + * @category Storage & Databases + */ +export interface ReadSubscription extends Binding.Service< + ReadSubscription, + "Cloudflare.K2.ReadSubscription", + (subscription: Subscription) => Effect.Effect +> {} + +export const ReadSubscription = Binding.Service("Cloudflare.K2.ReadSubscription"); + +export interface ReadSubscriptionClient { + /** + * Lease the next batch of records for five minutes. `None` when there are + * no records to read. + */ + consume( + options: ConsumeOptions, + ): Effect.Effect, ConsumeError, RuntimeContext>; + /** Mark a batch processed and release its lease. Acknowledging twice succeeds. */ + ack(batch: Batch): Effect.Effect; + /** Release a batch for redelivery (under a new batch id). */ + nack(batch: Batch): Effect.Effect; + /** + * Extend a held lease to five minutes from now. Fails with `K2LeaseLost` + * when the worker no longer holds it. + */ + extend(batch: Batch): Effect.Effect; +} diff --git a/packages/alchemy/src/Cloudflare/K2/ReadSubscriptionHttp.ts b/packages/alchemy/src/Cloudflare/K2/ReadSubscriptionHttp.ts new file mode 100644 index 00000000000..cae806d0158 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/ReadSubscriptionHttp.ts @@ -0,0 +1,99 @@ +import * as k2 from "@distilled.cloud/cloudflare/k2"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { fromConsumedRecords } from "./K2Codec.ts"; +import { type K2Auth, makeK2HttpAuth } from "./K2Http.ts"; +import type { Batch } from "./K2Types.ts"; +import { ReadSubscription, type ReadSubscriptionClient } from "./ReadSubscription.ts"; +import type { Subscription } from "./Subscription.ts"; + +/** Default `maxRecords` per `consume`. */ +export const DEFAULT_MAX_RECORDS = 100; + +/** + * HTTP-backed implementation of the {@link ReadSubscription} service. Mints + * a scoped `K2 Consume` {@link AccountApiToken}, binds it into the host, and + * leases batches through the stream's data-plane API + * (`https://.k2.cloudflarestorage.com/subscriptions/...`). + * ### Providing the Layer + * **Example:** Pull from a container + * ```typescript + * Effect.gen(function* () { + * const inbox = yield* Cloudflare.K2.ReadSubscription(Analytics); + * // ... + * }).pipe(Effect.provide(Cloudflare.K2.ReadSubscriptionHttp)); + * ``` + * + * @layer + * @provides Cloudflare.K2.ReadSubscription + * @product K2 + * @category Storage & Databases + */ +export const ReadSubscriptionHttp = Layer.effect( + ReadSubscription, + Effect.gen(function* () { + const auth = yield* makeK2HttpAuth; + return Effect.fn(function* (subscription: Subscription) { + const subscriptionAuth = yield* auth(subscription, "K2 Consume"); + return makeReadSubscriptionClient(subscriptionAuth, { + streamId: yield* subscription.streamId, + subscriptionId: yield* subscription.subscriptionId, + }); + }); + }), +); + +/** + * Build the pull client over the K2 subscription HTTP API. Shared by + * {@link ReadSubscriptionHttp} and `ReadSubscriptionLocal`. + */ +export const makeReadSubscriptionClient = ( + auth: K2Auth, + ids: { + streamId: Effect.Effect; + subscriptionId: Effect.Effect; + }, +): ReadSubscriptionClient => { + const target = Effect.all({ streamId: ids.streamId, subscriptionId: ids.subscriptionId }); + const onBatch = (batch: Batch) => + target.pipe(Effect.map((t) => ({ ...t, batchId: batch.batchId, workerId: batch.workerId }))); + return { + consume: ({ workerId, maxRecords }) => + target.pipe( + Effect.flatMap((t) => + auth.authorize( + k2.consume({ ...t, workerId, maxRecords: maxRecords ?? DEFAULT_MAX_RECORDS }), + ), + ), + Effect.map((response) => + response.batchId + ? Option.some({ + batchId: response.batchId, + workerId, + leasedUntil: new Date(response.leasedUntilMs ?? 0), + records: fromConsumedRecords(response.records), + }) + : Option.none(), + ), + ), + ack: (batch) => + onBatch(batch).pipe( + Effect.flatMap((request) => auth.authorize(k2.ackBatch(request))), + Effect.asVoid, + ), + nack: (batch) => + onBatch(batch).pipe( + Effect.flatMap((request) => auth.authorize(k2.nackBatch(request))), + Effect.asVoid, + ), + extend: (batch) => + onBatch(batch).pipe( + Effect.flatMap((request) => auth.authorize(k2.extendLease(request))), + Effect.map((response): Batch => ({ + ...batch, + leasedUntil: new Date(response.leasedUntilMs), + })), + ), + }; +}; diff --git a/packages/alchemy/src/Cloudflare/K2/ReadSubscriptionLocal.ts b/packages/alchemy/src/Cloudflare/K2/ReadSubscriptionLocal.ts new file mode 100644 index 00000000000..bf26493dc64 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/ReadSubscriptionLocal.ts @@ -0,0 +1,49 @@ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { makeK2LocalAuth } from "./K2Http.ts"; +import { ReadSubscription } from "./ReadSubscription.ts"; +import { makeReadSubscriptionClient } from "./ReadSubscriptionHttp.ts"; +import type { Subscription } from "./Subscription.ts"; + +/** + * Local implementation of the {@link ReadSubscription} service — leases + * batches over the K2 subscription HTTP API with the **current + * credentials** instead of a scoped token. Provide it on an Action (or any + * deploy-time Effect), or on a polling event source run in-process. + * + * The subscription's ids are resolved at apply time, so the subscription + * can be created in the same deploy. + * ### Providing the Layer + * **Example:** Drain a subscription from an Action + * ```typescript + * const Drain = Alchemy.Action( + * "Drain", + * Effect.gen(function* () { + * const inbox = yield* Cloudflare.K2.ReadSubscription(Analytics); + * return Effect.fn(function* () { + * const batch = yield* inbox.consume({ workerId: "drain" }); + * if (Option.isSome(batch)) yield* inbox.ack(batch.value); + * }); + * }).pipe(Effect.provide(Cloudflare.K2.ReadSubscriptionLocal)), + * ); + * ``` + * + * @layer + * @provides Cloudflare.K2.ReadSubscription + * @product K2 + * @category Storage & Databases + */ +export const ReadSubscriptionLocal = Layer.effect( + ReadSubscription, + Effect.gen(function* () { + const auth = yield* makeK2LocalAuth; + return Effect.fn(function* (subscription: Subscription) { + // Deferred accessors — resolved at apply time. No `host.bind`: the + // local variant registers no binding. + return makeReadSubscriptionClient(auth, { + streamId: yield* subscription.streamId, + subscriptionId: yield* subscription.subscriptionId, + }); + }); + }), +); diff --git a/packages/alchemy/src/Cloudflare/K2/Stream.ts b/packages/alchemy/src/Cloudflare/K2/Stream.ts new file mode 100644 index 00000000000..2e6b54650af --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/Stream.ts @@ -0,0 +1,368 @@ +import * as k2 from "@distilled.cloud/cloudflare/k2"; +import type * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Predicate from "effect/Predicate"; +import * as EffectStream from "effect/Stream"; +import { Unowned } from "../../AdoptPolicy.ts"; +import { isResolved } from "../../Diff.ts"; +import { createPhysicalName } from "../../PhysicalName.ts"; +import * as Provider from "../../Provider.ts"; +import { Resource } from "../../Resource.ts"; +import { toWireSeconds } from "../../Util/Duration.ts"; +import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; +import type { Providers } from "../Providers.ts"; + +const StreamTypeId = "Cloudflare.K2.Stream" as const; +type StreamTypeId = typeof StreamTypeId; + +/** K2's default retention: seven days. */ +const DEFAULT_RETENTION_SECONDS = 604_800; + +/** + * HTTP input configuration of a K2 stream. + */ +export interface StreamHttpInput { + /** + * Require an API token with the `K2 Produce` permission on `/produce`. + * When `false` the endpoint is public. + * @default true + */ + authentication?: boolean; + /** + * Origins allowed to produce from a browser: `http(s)://host[:port]` + * without a path, or `*` alone. Up to five. + */ + cors?: string[]; +} + +export interface StreamProps { + /** + * Name of the stream: 1 to 128 letters, numbers, or underscores, unique in + * the account (not case-sensitive). If omitted, a unique name is generated + * from the app, stage, and logical ID. Changing the name replaces the + * stream. + * @default ${app}_${id}_${stage}_${suffix} + */ + name?: string; + /** + * How long records are retained, from one hour to 30 days. Rounded to whole + * seconds. Mutable in place. + * @default "7 days" + */ + retention?: Duration.Input; + /** + * The stream's HTTP input (`POST https://.k2.cloudflarestorage.com/produce`). + * `false` disables it, `true` enables it with token authentication, and an + * object enables it with the given settings. Mutable in place. + * @default false + */ + http?: boolean | StreamHttpInput; + /** + * Whether Workers can produce to the stream through a `k2` binding + * (`Cloudflare.K2.WriteStream` with `WriteStreamBinding`). Mutable in place. + * At least one of `http` and `workerBinding` must be enabled. + * @default true + */ + workerBinding?: boolean; +} + +export interface StreamAttributes { + /** The stream ID: 32 lowercase hexadecimal characters. */ + streamId: string; + /** The stream name (unique in the account). */ + streamName: string; + /** The stream's HTTP endpoint, `https://.k2.cloudflarestorage.com`. */ + endpoint: string; + /** How long records are retained, in seconds. */ + retentionSeconds: number; + /** Whether the HTTP input accepts records. */ + httpEnabled: boolean; + /** Whether the HTTP input requires a `K2 Produce` API token. */ + httpAuthentication: boolean; + /** Origins allowed to produce from a browser. */ + corsOrigins: string[] | undefined; + /** Whether Workers can produce through a `k2` binding. */ + workerBindingEnabled: boolean; + /** Account that owns the stream. */ + accountId: string; + /** When the stream was created. */ + createdAt: string; + /** When the stream was last modified. */ + modifiedAt: string; +} + +export type Stream = Resource; + +/** + * A Cloudflare K2 stream — a durable, ordered log of records. Producers + * append records over HTTP or from Workers through a `k2` binding; + * consumers read them through a {@link Subscription}, each with its own + * position in the stream. + * + * The name is fixed at creation (changing it replaces the stream); + * retention and the HTTP / Worker-binding inputs are mutable in place. + * ### Creating a Stream + * **Example:** Stream with default settings + * ```typescript + * // Worker-binding input enabled, HTTP input disabled, 7-day retention. + * const orders = yield* Cloudflare.K2.Stream("Orders"); + * ``` + * + * **Example:** Custom retention + * ```typescript + * const orders = yield* Cloudflare.K2.Stream("Orders", { + * retention: "30 days", + * }); + * ``` + * + * ### HTTP Input + * **Example:** Authenticated HTTP input + * ```typescript + * // POST records to orders.endpoint + "/produce" with a K2 Produce token. + * const orders = yield* Cloudflare.K2.Stream("Orders", { + * http: true, + * }); + * ``` + * + * **Example:** Public HTTP input for browsers + * ```typescript + * const clicks = yield* Cloudflare.K2.Stream("Clicks", { + * http: { authentication: false, cors: ["https://app.example.com"] }, + * workerBinding: false, + * }); + * ``` + * + * ### Producing from a Worker + * **Example:** Send records through the Worker binding + * ```typescript + * export default Cloudflare.Worker( + * "Api", + * { main: import.meta.url }, + * Effect.gen(function* () { + * const orders = yield* Cloudflare.K2.WriteStream(Orders); + * return { + * fetch: Effect.gen(function* () { + * yield* orders.send([{ content: JSON.stringify({ id: 1 }) }]); + * return HttpServerResponse.empty({ status: 202 }); + * }), + * }; + * }).pipe(Effect.provide(Cloudflare.K2.WriteStreamBinding)), + * ); + * ``` + * + * ### Consuming + * **Example:** A subscription reading from the start of the stream + * ```typescript + * const analytics = yield* Cloudflare.K2.Subscription("Analytics", { + * streamId: orders.streamId, + * startAt: "earliest", + * }); + * ``` + * + * @see https://developers.cloudflare.com/k2/ + * + * @resource + * @product K2 + * @category Storage & Databases + */ +export const Stream = Resource(StreamTypeId); + +/** Returns true if the given value is a K2 Stream resource. */ +export const isStream = (value: unknown): value is Stream => + Predicate.hasProperty(value, "Type") && value.Type === StreamTypeId; + +export const StreamProvider = () => + Provider.succeed(Stream, { + stables: ["streamId", "streamName", "endpoint", "accountId", "createdAt"], + + diff: Effect.fn(function* ({ id, olds, news = {}, output }) { + const { accountId } = yield* yield* CloudflareEnvironment; + if (!isResolved(news)) return undefined; + if ((output?.accountId ?? accountId) !== accountId) { + return { action: "replace" } as const; + } + // A generated name is engine-owned: the deployed name stays + // authoritative. Only a different explicit name forces a replace. + const oldName = output?.streamName ?? (yield* streamName(id, olds?.name)); + const newName = news.name ?? oldName; + if (newName.toLowerCase() !== oldName.toLowerCase()) { + return { action: "replace" } as const; + } + return undefined; + }), + + read: Effect.fn(function* ({ id, output, olds }) { + const { accountId } = yield* yield* CloudflareEnvironment; + const acct = output?.accountId ?? accountId; + if (output?.streamId) { + const observed = yield* getStream(acct, output.streamId); + if (observed) return toAttributes(observed, acct); + } + // Cold read: names are unique per account. A generated name embeds the + // instance id (proof of ownership); an explicit name does not, so gate + // takeover behind the adopt policy. + const name = yield* streamName(id, olds?.name); + const match = yield* findStreamByName(acct, name); + if (match) { + const attrs = toAttributes(match, acct); + return olds?.name !== undefined ? Unowned(attrs) : attrs; + } + return undefined; + }), + + reconcile: Effect.fn(function* ({ id, news = {}, output }) { + const { accountId } = yield* yield* CloudflareEnvironment; + const acct = output?.accountId ?? accountId; + const name = output?.streamName ?? (yield* streamName(id, news.name)); + const desired = desiredState(news); + + // Observe — the cached id is a hint; fall back to the name. + let observed = output?.streamId ? yield* getStream(acct, output.streamId) : undefined; + if (!observed) { + observed = yield* findStreamByName(acct, name); + } + + // Ensure — create when missing. A taken name is a race or a lost state + // write: adopt the stream with that name. + if (!observed) { + observed = yield* k2 + .createStream({ + accountId: acct, + name, + retentionSeconds: desired.retentionSeconds, + http: desired.http, + workerBinding: desired.workerBinding, + }) + .pipe( + Effect.catchTag("K2NameAlreadyExists", (error) => + findStreamByName(acct, name).pipe( + Effect.flatMap((match) => (match ? Effect.succeed(match) : Effect.fail(error))), + ), + ), + ); + } + + // Sync — diff observed cloud state against desired and PATCH only the + // delta. A provided input object replaces the existing one. + const patch: k2.UpdateStreamRequest = { accountId: acct, streamId: observed.id }; + let dirty = false; + if (observed.retentionSeconds !== desired.retentionSeconds) { + patch.retentionSeconds = desired.retentionSeconds; + dirty = true; + } + if (!sameHttp(desired.http, observed.http)) { + // An omitted `cors` keeps the existing origins; clear them explicitly. + patch.http = + desired.http.enabled && !desired.http.cors && observed.http.cors?.origins?.length + ? { ...desired.http, cors: { origins: [] } } + : desired.http; + dirty = true; + } + if (observed.workerBinding.enabled !== desired.workerBinding.enabled) { + patch.workerBinding = desired.workerBinding; + dirty = true; + } + if (dirty) { + observed = yield* k2.updateStream(patch); + } + + return toAttributes(observed, acct); + }), + + delete: Effect.fn(function* ({ output }) { + // Deleting a stream that does not exist succeeds. + yield* k2.deleteStream({ accountId: output.accountId, streamId: output.streamId }); + }), + + list: Effect.fn(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + return yield* k2.listStreams.items({ accountId }).pipe( + EffectStream.runCollect, + Effect.map((chunk) => Array.from(chunk).map((s) => toAttributes(s, accountId))), + ); + }), + }); + +/** The stream state shared by the get/list/create/update responses. */ +interface ObservedStream { + id: string; + name: string; + retentionSeconds: number; + endpoint: string; + http: k2.StreamHttpInput; + workerBinding: k2.StreamWorkerBindingInput; + createdAt: string; + modifiedAt: string; +} + +/** + * Stream names allow letters, numbers, and underscores only, so swap the + * default hyphen delimiter for underscores. + */ +const streamName = (id: string, name: string | undefined) => + Effect.gen(function* () { + if (name) return name; + const generated = yield* createPhysicalName({ + id, + lowercase: true, + delimiter: "_", + maxLength: 128, + }); + return generated.replaceAll(/[^a-zA-Z0-9_]/g, "_"); + }); + +const getStream = (accountId: string, streamId: string) => + k2.getStream({ accountId, streamId }).pipe( + Effect.map((s): ObservedStream | undefined => s), + Effect.catchTag("K2StreamNotFound", () => Effect.succeed(undefined)), + ); + +/** Names are unique per account and not case-sensitive. */ +const findStreamByName = (accountId: string, name: string) => + k2.listStreams.items({ accountId }).pipe( + EffectStream.filter((s) => s.name.toLowerCase() === name.toLowerCase()), + EffectStream.runHead, + Effect.map((match): ObservedStream | undefined => Option.getOrUndefined(match)), + ); + +const desiredState = (props: StreamProps) => { + const http: k2.StreamHttpInput = + props.http === undefined || props.http === false + ? { enabled: false } + : props.http === true + ? { enabled: true, authentication: true } + : { + enabled: true, + authentication: props.http.authentication ?? true, + ...(props.http.cors ? { cors: { origins: props.http.cors } } : {}), + }; + return { + retentionSeconds: toWireSeconds(props.retention) ?? DEFAULT_RETENTION_SECONDS, + http, + workerBinding: { enabled: props.workerBinding ?? true }, + }; +}; + +const sameHttp = (desired: k2.StreamHttpInput, observed: k2.StreamHttpInput) => { + if (!desired.enabled && !observed.enabled) return true; + if (desired.enabled !== observed.enabled) return false; + if ((desired.authentication ?? false) !== (observed.authentication ?? false)) return false; + const want = [...(desired.cors?.origins ?? [])].sort().join(","); + const have = [...(observed.cors?.origins ?? [])].sort().join(","); + return want === have; +}; + +const toAttributes = (observed: ObservedStream, accountId: string): StreamAttributes => ({ + streamId: observed.id, + streamName: observed.name, + endpoint: observed.endpoint, + retentionSeconds: observed.retentionSeconds, + httpEnabled: observed.http.enabled, + httpAuthentication: observed.http.authentication ?? false, + corsOrigins: observed.http.cors?.origins ? [...observed.http.cors.origins] : undefined, + workerBindingEnabled: observed.workerBinding.enabled, + accountId, + createdAt: observed.createdAt, + modifiedAt: observed.modifiedAt, +}); diff --git a/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts b/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts new file mode 100644 index 00000000000..cc1c63ad740 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts @@ -0,0 +1,247 @@ +import * as Context from "effect/Context"; +import type * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import type * as Stream from "effect/Stream"; +import * as Binding from "../../Binding.ts"; +import * as Namespace from "../../Namespace.ts"; +import { ServerHost } from "../../Server/Process.ts"; +import { decodeWithSchema } from "./K2Codec.ts"; +import type { ConsumedRecord, DecodedRecord, RecordSchema, StartAt } from "./K2Types.ts"; +import { ReadSubscription } from "./ReadSubscription.ts"; +import type { Stream as K2Stream } from "./Stream.ts"; +import { pollSubscription } from "./StreamPoller.ts"; +import { Subscription } from "./Subscription.ts"; + +/** Settings for {@link consumeStreamRecords}. */ +export interface ConsumeStreamRecordsProps { + /** + * Read through this subscription. Hosts sharing one subscription compete + * for batches. When omitted, a subscription owned by the consuming host is + * created, so every host reads every record. + */ + subscription?: Subscription; + /** + * Where the auto-created subscription starts reading. Ignored when + * `subscription` is passed. + * @default "latest" + */ + startAt?: StartAt; + /** + * At most this many records per batch, from 1 to 10000. + * @default 100 + */ + maxRecords?: number; + /** + * Number of concurrent pollers in this process, each with its own worker + * id (and lease). + * @default 1 + */ + concurrency?: number; + /** + * How long to wait after an empty batch (or a failed poll) before polling + * again. + * @default "1 second" + */ + idleBackoff?: Duration.Input; + /** + * Prefix of the pollers' worker ids. Must be unique per process. + * @default `${hostLogicalId}-${random}` + */ + workerId?: string; +} + +/** {@link ConsumeStreamRecordsProps} with a schema decoding JSON content into `record.value`. */ +export interface ConsumeStreamRecordsSchemaProps extends ConsumeStreamRecordsProps { + schema: RecordSchema; +} + +type Handler = (records: Stream.Stream) => Effect.Effect; + +export type StreamEventSourceService = ( + stream: K2Stream, + props: ConsumeStreamRecordsProps & { schema?: RecordSchema }, + process: Handler, +) => Effect.Effect; + +/** + * Event source connecting a K2 {@link Stream} to the hosting compute. The + * only implementation is {@link StreamEventSourcePolling}, a pull loop for + * long-running hosts. Consume it through {@link consumeStreamRecords}. + * + * :::note + * TODO: Workers cannot consume K2 yet — K2 has no push consumers (they are + * on Cloudflare's roadmap), so no Worker layer ships. + * ::: + */ +export class StreamEventSource extends Context.Service< + StreamEventSource, + StreamEventSourceService +>()("Cloudflare.K2.StreamEventSource") {} + +/** + * Subscribe a long-running host to a K2 stream with an Effect stream + * handler. + * + * - **Deploy-time**: creates a {@link Subscription} owned by the host + * (unless `subscription` is passed — hosts sharing one compete for + * batches). + * - **Runtime**: `concurrency` pollers lease batches of records, run the + * handler, and acknowledge each batch when the handler succeeds. A failed + * handler is logged and the batch is released for redelivery. Leases are + * extended every two minutes while the handler runs. + * + * Provide {@link StreamEventSourcePolling} together with a + * `ReadSubscription` implementation (`ReadSubscriptionHttp` on a deployed + * host). It runs on hosts with a long-lived process — Fly, ECS, EC2, + * Railway, Hetzner, Docker. + * + * :::note + * TODO: Workers cannot consume K2 yet — K2 has no push consumers (they are + * on Cloudflare's roadmap), so no Worker layer ships. + * ::: + * ### Consuming a Stream + * **Example:** Log every record + * ```typescript + * yield* Cloudflare.K2.consumeStreamRecords(Orders, (records) => + * records.pipe( + * Stream.runForEach((record) => + * Effect.log(new TextDecoder().decode(record.content)), + * ), + * ), + * ); + * ``` + * + * ### Typed Records + * **Example:** Decode JSON records with a Schema + * ```typescript + * yield* Cloudflare.K2.consumeStreamRecords( + * Orders, + * { schema: Order, startAt: "earliest", concurrency: 4 }, + * (records) => + * records.pipe(Stream.runForEach((record) => Effect.log(record.value.id))), + * ); + * ``` + * + * ### Providing the Layers + * **Example:** Poll from a container host + * ```typescript + * Effect.gen(function* () { + * yield* Cloudflare.K2.consumeStreamRecords(Orders, handler); + * }).pipe( + * Effect.provide( + * Layer.provideMerge( + * Cloudflare.K2.StreamEventSourcePolling, + * Cloudflare.K2.ReadSubscriptionHttp, + * ), + * ), + * ); + * ``` + * + * @binding + * @product K2 + * @category Storage & Databases + */ +export function consumeStreamRecords( + stream: K2Stream, + process: Handler, +): Effect.Effect; +export function consumeStreamRecords( + stream: K2Stream, + props: ConsumeStreamRecordsSchemaProps, + process: Handler, Req>, +): Effect.Effect; +export function consumeStreamRecords( + stream: K2Stream, + props: ConsumeStreamRecordsProps, + process: Handler, +): Effect.Effect; +export function consumeStreamRecords( + stream: K2Stream, + propsOrProcess: (ConsumeStreamRecordsProps & { schema?: RecordSchema }) | Handler, + maybeProcess?: Handler, +): Effect.Effect { + const [props, process] = + typeof propsOrProcess === "function" + ? [{} as ConsumeStreamRecordsProps, propsOrProcess] + : [propsOrProcess, maybeProcess!]; + return StreamEventSource.use((source) => source(stream, props, process)); +} + +/** + * Polling implementation of {@link StreamEventSource} for hosts with a + * long-lived process (any host providing `ServerHost`: Fly, ECS, EC2, + * Railway, Hetzner, Docker). Requires a `ReadSubscription` implementation. + * + * :::note + * TODO: Workers cannot consume K2 yet — K2 has no push consumers (they are + * on Cloudflare's roadmap), so no Worker layer ships. + * ::: + * ### Providing the Layer + * **Example:** Poll with a scoped token + * ```typescript + * Effect.gen(function* () { + * yield* Cloudflare.K2.consumeStreamRecords(Orders, handler); + * }).pipe( + * Effect.provide( + * Layer.provideMerge( + * Cloudflare.K2.StreamEventSourcePolling, + * Cloudflare.K2.ReadSubscriptionHttp, + * ), + * ), + * ); + * ``` + * + * @layer + * @provides Cloudflare.K2.StreamEventSource + * @product K2 + * @category Storage & Databases + */ +export const StreamEventSourcePolling = Layer.effect( + StreamEventSource, + Effect.gen(function* () { + const { run } = yield* ServerHost; + const readSubscription = yield* ReadSubscription; + + return Effect.fn(function* ( + stream: K2Stream, + props: ConsumeStreamRecordsProps & { schema?: RecordSchema }, + process: Handler, + ) { + const host = yield* Binding.Host; + const hostId = host?.LogicalId ?? "Host"; + // Declared in both phases: at runtime the declaration resolves to the + // deployed subscription's outputs, which the client reads. + const subscription = + props.subscription ?? + (yield* Namespace.push( + hostId, + Subscription(`${stream.LogicalId}Subscription`, { + streamId: stream.streamId, + startAt: props.startAt, + }), + )); + const client = yield* readSubscription(subscription); + const decode = props.schema + ? decodeWithSchema(props.schema) + : (records: ReadonlyArray) => Effect.succeed(records); + + yield* run( + Effect.gen(function* () { + const workerId = + props.workerId ?? + `${hostId}-${(yield* Effect.sync(() => crypto.randomUUID())).slice(0, 8)}`; + yield* pollSubscription({ + client, + workerId, + maxRecords: props.maxRecords, + idleBackoff: props.idleBackoff, + concurrency: props.concurrency, + decode, + process, + }); + }), + ); + }) as StreamEventSourceService; + }), +); diff --git a/packages/alchemy/src/Cloudflare/K2/StreamPoller.ts b/packages/alchemy/src/Cloudflare/K2/StreamPoller.ts new file mode 100644 index 00000000000..ca8961cb5df --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/StreamPoller.ts @@ -0,0 +1,134 @@ +import * as Cause from "effect/Cause"; +import type * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Option from "effect/Option"; +import * as Schedule from "effect/Schedule"; +import * as Stream from "effect/Stream"; +import type { RuntimeContext } from "../../RuntimeContext.ts"; +import type { Batch, ConsumedRecord, K2SchemaError } from "./K2Types.ts"; +import type { ReadSubscriptionClient } from "./ReadSubscription.ts"; + +/** + * The pull loop behind `consumeStreamRecords`. Internal — not exported from + * the K2 barrel (tests import it directly). + */ + +/** `consume` errors that mean "try again", not "give up". */ +const RETRYABLE_CONSUME_ERRORS: ReadonlyArray = [ + "K2Unavailable", + "K2ReadFailed", + "K2LeasesExhausted", + "K2BatchPending", +]; + +/** Bounded backoff for retryable `consume` errors: ~30 s in total. */ +const consumeRetry = Schedule.max([ + Schedule.min([Schedule.exponential("250 millis"), Schedule.spaced("5 seconds")]), + Schedule.recurs(8), +]); + +export interface PollOptions { + client: ReadSubscriptionClient; + /** Worker id holding the leases. Must be unique per concurrent poller. */ + workerId: string; + /** @default 100 */ + maxRecords?: number; + /** Sleep after an empty batch or a failed iteration. @default "1 second" */ + idleBackoff?: Duration.Input; + /** Extend the five-minute lease this often while the handler runs. @default "2 minutes" */ + extendEvery?: Duration.Input; + /** Turn consumed records into the handler's elements (e.g. schema decode). */ + decode: ( + records: ReadonlyArray, + ) => Effect.Effect, K2SchemaError>; + process: (records: Stream.Stream) => Effect.Effect; +} + +/** + * Keep a batch's lease alive while its handler runs. Stops on `K2LeaseLost` + * (the batch was acked, nacked, or redelivered elsewhere). + */ +const keepLeaseAlive = (client: ReadSubscriptionClient, batch: Batch, every: Duration.Input) => + Effect.sleep(every).pipe( + Effect.andThen(client.extend(batch)), + Effect.forever, + Effect.catchTag("K2LeaseLost", () => + Effect.logWarning(`K2 lease lost for batch ${batch.batchId}; it may be redelivered`), + ), + Effect.catchCause((cause) => + Effect.logWarning(`K2 lease extension failed for batch ${batch.batchId}`, cause), + ), + ); + +/** + * One poll: lease a batch, run the handler, then ack it on success or nack + * it on failure. Returns whether a batch was processed. + */ +export const pollOnce = ( + options: PollOptions, +): Effect.Effect => + Effect.gen(function* () { + const { client } = options; + const leased = yield* client + .consume({ workerId: options.workerId, maxRecords: options.maxRecords }) + .pipe( + Effect.retry({ + while: (e) => RETRYABLE_CONSUME_ERRORS.includes(e._tag), + schedule: consumeRetry, + }), + ); + if (Option.isNone(leased)) return false; + const batch = leased.value; + + const outcome = yield* Effect.scoped( + Effect.gen(function* () { + yield* Effect.forkScoped(keepLeaseAlive(client, batch, options.extendEvery ?? "2 minutes")); + return yield* Effect.exit( + options + .decode(batch.records) + .pipe(Effect.flatMap((values) => options.process(Stream.fromIterable(values)))), + ); + }), + ); + + if (Exit.isSuccess(outcome)) { + yield* client.ack(batch); + } else { + yield* Effect.logWarning( + `K2 handler failed for batch ${batch.batchId}; releasing it for redelivery`, + Cause.pretty(outcome.cause), + ); + yield* client.nack(batch); + } + return true; + }); + +/** + * Poll forever: an empty batch or a failed iteration sleeps `idleBackoff`; + * a processed batch polls again immediately. Never fails. + */ +export const pollForever = ( + options: PollOptions, +): Effect.Effect => + pollOnce(options).pipe( + Effect.catchCause((cause) => + Effect.logWarning(`K2 poll failed for worker ${options.workerId}`, cause).pipe( + Effect.as(false), + ), + ), + Effect.flatMap((processed) => + processed ? Effect.void : Effect.sleep(options.idleBackoff ?? "1 second"), + ), + Effect.forever, + ); + +/** Run `concurrency` pollers, each with its own worker id (`${workerId}-${i}`). */ +export const pollSubscription = ( + options: PollOptions & { concurrency?: number }, +): Effect.Effect => + Effect.forEach( + Array.from({ length: Math.max(1, options.concurrency ?? 1) }, (_, i) => i), + (i) => pollForever({ ...options, workerId: `${options.workerId}-${i}` }), + { concurrency: "unbounded", discard: true }, + ); diff --git a/packages/alchemy/src/Cloudflare/K2/StreamSink.ts b/packages/alchemy/src/Cloudflare/K2/StreamSink.ts new file mode 100644 index 00000000000..da76ba66926 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/StreamSink.ts @@ -0,0 +1,71 @@ +import type * as Effect from "effect/Effect"; +import type * as Sink from "effect/Sink"; +import * as Binding from "../../Binding.ts"; +import type { RuntimeContext } from "../../RuntimeContext.ts"; +import type { ProduceError, Record as K2Record } from "./K2Types.ts"; +import type { Stream } from "./Stream.ts"; +import type { WriteStreamOptions } from "./WriteStream.ts"; + +/** + * Binding service that exposes a K2 {@link Stream} as an Effect `Sink`: run + * a `Stream` of records into it and every element is appended. + * + * Each upstream chunk is packed, in order, into `send` calls of at most + * 5 MB; a record over the 1 MB limit ships alone so K2's rejection + * surfaces. A batch that failed with `K2Unavailable` (not stored) is + * retried a bounded number of times; `K2AppendOutcomeUnknown` is never + * retried because the batch may have been stored. + * + * Provide {@link StreamSinkBinding} (native Worker binding), + * {@link StreamSinkHttp} (scoped `K2 Produce` token) or + * {@link StreamSinkLocal} (current credentials, for Actions). + * ### Draining a Stream into K2 + * **Example:** Run a Stream into a K2 stream + * ```typescript + * export default Cloudflare.Worker( + * "Api", + * { main: import.meta.url }, + * Effect.gen(function* () { + * const sink = yield* Cloudflare.K2.StreamSink(Orders); + * return { + * fetch: Effect.gen(function* () { + * yield* Stream.fromIterable(events).pipe( + * Stream.map((event) => ({ content: JSON.stringify(event) })), + * Stream.run(sink), + * ); + * return HttpServerResponse.empty({ status: 202 }); + * }), + * }; + * }).pipe(Effect.provide(Cloudflare.K2.StreamSinkBinding)), + * ); + * ``` + * + * **Example:** Typed sink with a Schema + * ```typescript + * const sink = yield* Cloudflare.K2.StreamSink(Orders, { schema: Order }); + * yield* Stream.fromIterable(orders).pipe(Stream.run(sink)); + * ``` + * + * @binding + * @product K2 + * @category Storage & Databases + */ +export interface StreamSink extends Binding.Service< + StreamSink, + "Cloudflare.K2.StreamSink", + (stream: Stream, options?: WriteStreamOptions) => Effect.Effect> +> { + ( + stream: Stream, + options: WriteStreamOptions, + ): Effect.Effect, never, StreamSink>; + (stream: Stream): Effect.Effect, never, StreamSink>; +} + +export const StreamSink = Binding.Service("Cloudflare.K2.StreamSink"); + +/** + * A `Sink` that appends every element it receives as one K2 record. + * Runtime-only: it can only run inside the deployed host (or an Action). + */ +export type StreamSinkClient = Sink.Sink; diff --git a/packages/alchemy/src/Cloudflare/K2/StreamSinkBatch.ts b/packages/alchemy/src/Cloudflare/K2/StreamSinkBatch.ts new file mode 100644 index 00000000000..b2efd8ab6f2 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/StreamSinkBatch.ts @@ -0,0 +1,86 @@ +import type { NonEmptyReadonlyArray } from "effect/Array"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schedule from "effect/Schedule"; +import * as Sink from "effect/Sink"; +import { wireSize } from "./K2Codec.ts"; +import type { EncodedRecord } from "./K2Types.ts"; +import type { Stream } from "./Stream.ts"; +import { StreamSink, type StreamSinkClient } from "./StreamSink.ts"; +import { WriteStream, type WriteStreamClient, type WriteStreamOptions } from "./WriteStream.ts"; + +/** + * Batch packing for {@link StreamSink}. Internal — not exported from the K2 + * barrel. + */ + +/** K2's limit on one produce request (1 MB = 1,000,000 bytes). */ +export const MAX_REQUEST_BYTES = 5_000_000; + +/** + * Greedily pack records into requests of at most {@link MAX_REQUEST_BYTES} + * (wire size, base64 included), preserving order. A record larger than the + * limit ships alone so K2's rejection surfaces instead of the record being + * dropped. + */ +export const packStreamSinkBatches = ( + records: ReadonlyArray, + maxBytes: number = MAX_REQUEST_BYTES, +): ReadonlyArray> => { + const batches: EncodedRecord[][] = []; + let current: EncodedRecord[] = []; + let currentBytes = 0; + for (const record of records) { + const size = wireSize(record); + if (current.length > 0 && currentBytes + size > maxBytes) { + batches.push(current); + current = []; + currentBytes = 0; + } + current.push(record); + currentBytes += size; + } + if (current.length > 0) { + batches.push(current); + } + return batches; +}; + +/** Bounded retry for a batch K2 reported as not stored (`K2Unavailable`). */ +const unavailableRetry = Schedule.max([Schedule.exponential("200 millis"), Schedule.recurs(5)]); + +/** Build the sink over a `WriteStream` client: one `sendEncoded` per packed batch. */ +export const makeStreamSink = (client: WriteStreamClient): StreamSinkClient => + Sink.forEachArray((chunk: NonEmptyReadonlyArray) => + client.encode(chunk).pipe( + Effect.flatMap((encoded) => + Effect.forEach( + packStreamSinkBatches(encoded), + (batch) => + client.sendEncoded(batch).pipe( + Effect.retry({ + while: (e) => e._tag === "K2Unavailable", + schedule: unavailableRetry, + }), + ), + { discard: true }, + ), + ), + ), + ); + +/** + * {@link StreamSink} over whichever {@link WriteStream} implementation is + * provided. Composed with each `WriteStream` layer in `StreamSinkBinding`, + * `StreamSinkHttp` and `StreamSinkLocal`. + */ +export const StreamSinkFromWriteStream = Layer.effect( + StreamSink, + Effect.gen(function* () { + const writeStream = yield* WriteStream; + return Effect.fn(function* (stream: Stream, options?: WriteStreamOptions) { + const client = yield* writeStream(stream, options); + return makeStreamSink(client); + }); + }), +); diff --git a/packages/alchemy/src/Cloudflare/K2/StreamSinkBinding.ts b/packages/alchemy/src/Cloudflare/K2/StreamSinkBinding.ts new file mode 100644 index 00000000000..2058b4d93d1 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/StreamSinkBinding.ts @@ -0,0 +1,32 @@ +import * as Layer from "effect/Layer"; +import { StreamSinkFromWriteStream } from "./StreamSinkBatch.ts"; +import { WriteStreamBinding } from "./WriteStreamBinding.ts"; + +/** + * Implementation of the {@link StreamSink} service over the native Worker + * `k2` binding ({@link WriteStreamBinding}). Registers the same binding as + * `WriteStream`, so a Worker can use both on one stream. + * ### Providing the Layer + * **Example:** Drain a Stream from a Worker + * ```typescript + * export default Cloudflare.Worker( + * "Api", + * { main: import.meta.url }, + * Effect.gen(function* () { + * const sink = yield* Cloudflare.K2.StreamSink(Orders); + * return { + * fetch: Effect.gen(function* () { + * yield* Stream.make({ content: "a" }, { content: "b" }).pipe(Stream.run(sink)); + * return HttpServerResponse.empty({ status: 202 }); + * }), + * }; + * }).pipe(Effect.provide(Cloudflare.K2.StreamSinkBinding)), + * ); + * ``` + * + * @layer + * @provides Cloudflare.K2.StreamSink + * @product K2 + * @category Storage & Databases + */ +export const StreamSinkBinding = StreamSinkFromWriteStream.pipe(Layer.provide(WriteStreamBinding)); diff --git a/packages/alchemy/src/Cloudflare/K2/StreamSinkHttp.ts b/packages/alchemy/src/Cloudflare/K2/StreamSinkHttp.ts new file mode 100644 index 00000000000..c388589f5b9 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/StreamSinkHttp.ts @@ -0,0 +1,23 @@ +import * as Layer from "effect/Layer"; +import { StreamSinkFromWriteStream } from "./StreamSinkBatch.ts"; +import { WriteStreamHttp } from "./WriteStreamHttp.ts"; + +/** + * Implementation of the {@link StreamSink} service over the stream's HTTP + * input ({@link WriteStreamHttp}), authenticated with a scoped `K2 Produce` + * API token bound into the host. The stream needs `http: true`. + * ### Providing the Layer + * **Example:** Drain a Stream over HTTP + * ```typescript + * Effect.gen(function* () { + * const sink = yield* Cloudflare.K2.StreamSink(Orders); + * // ... + * }).pipe(Effect.provide(Cloudflare.K2.StreamSinkHttp)); + * ``` + * + * @layer + * @provides Cloudflare.K2.StreamSink + * @product K2 + * @category Storage & Databases + */ +export const StreamSinkHttp = StreamSinkFromWriteStream.pipe(Layer.provide(WriteStreamHttp)); diff --git a/packages/alchemy/src/Cloudflare/K2/StreamSinkLocal.ts b/packages/alchemy/src/Cloudflare/K2/StreamSinkLocal.ts new file mode 100644 index 00000000000..27b1b8d3659 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/StreamSinkLocal.ts @@ -0,0 +1,32 @@ +import * as Layer from "effect/Layer"; +import { StreamSinkFromWriteStream } from "./StreamSinkBatch.ts"; +import { WriteStreamLocal } from "./WriteStreamLocal.ts"; + +/** + * Implementation of the {@link StreamSink} service that appends over the + * stream's HTTP input with the **current credentials** + * ({@link WriteStreamLocal}) — for draining a Stream into K2 from an Action + * or other deploy-time Effect. The stream needs `http: true`. + * ### Providing the Layer + * **Example:** Seed a stream from an Action + * ```typescript + * const Seed = Alchemy.Action( + * "Seed", + * Effect.gen(function* () { + * const sink = yield* Cloudflare.K2.StreamSink(Orders); + * return Effect.fn(function* () { + * yield* Stream.range(1, 500).pipe( + * Stream.map((id) => ({ content: JSON.stringify({ id }) })), + * Stream.run(sink), + * ); + * }); + * }).pipe(Effect.provide(Cloudflare.K2.StreamSinkLocal)), + * ); + * ``` + * + * @layer + * @provides Cloudflare.K2.StreamSink + * @product K2 + * @category Storage & Databases + */ +export const StreamSinkLocal = StreamSinkFromWriteStream.pipe(Layer.provide(WriteStreamLocal)); diff --git a/packages/alchemy/src/Cloudflare/K2/Subscription.ts b/packages/alchemy/src/Cloudflare/K2/Subscription.ts new file mode 100644 index 00000000000..d51a9793821 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/Subscription.ts @@ -0,0 +1,229 @@ +import * as k2 from "@distilled.cloud/cloudflare/k2"; +import * as Effect from "effect/Effect"; +import * as Predicate from "effect/Predicate"; +import { Unowned } from "../../AdoptPolicy.ts"; +import { isResolved } from "../../Diff.ts"; +import { createPhysicalName } from "../../PhysicalName.ts"; +import * as Provider from "../../Provider.ts"; +import { Resource } from "../../Resource.ts"; +import type { Providers } from "../Providers.ts"; +import type { StartAt } from "./K2Types.ts"; + +const SubscriptionTypeId = "Cloudflare.K2.Subscription" as const; +type SubscriptionTypeId = typeof SubscriptionTypeId; + +export interface SubscriptionProps { + /** + * ID of the stream to read from, e.g. `stream.streamId`. Changing it + * replaces the subscription. + */ + streamId: string; + /** + * Name of the subscription: 1 to 128 letters, numbers, underscores, or + * hyphens, unique within the stream (not case-sensitive). If omitted, a + * unique name is generated from the app, stage, and logical ID. Changing + * it replaces the subscription. + * @default ${app}-${id}-${stage}-${suffix} + */ + name?: string; + /** + * Where the subscription starts reading: `earliest` from the oldest + * retained record, `latest` from after the newest one. Changing it + * replaces the subscription. + * @default "latest" + */ + startAt?: StartAt; +} + +export interface SubscriptionAttributes { + /** The subscription ID. */ + subscriptionId: string; + /** The subscription name (unique within the stream). */ + subscriptionName: string; + /** ID of the stream the subscription reads from. */ + streamId: string; + /** Where the subscription started reading. */ + startAt: StartAt; + /** When the subscription was created. */ + createdAt: string; + /** When the subscription was last modified. */ + modifiedAt: string; +} + +export type Subscription = Resource< + SubscriptionTypeId, + SubscriptionProps, + SubscriptionAttributes, + never, + Providers +>; + +/** + * A K2 subscription — a named read position on a {@link Stream}. Consumers + * lease batches of records from a subscription, then acknowledge them to + * advance it. Every subscription on a stream reads every record + * independently; consumers sharing one subscription compete for batches. + * + * Subscriptions are immutable: changing the stream, name, or start position + * replaces the subscription. + * ### Creating a Subscription + * **Example:** Read new records + * ```typescript + * const orders = yield* Cloudflare.K2.Stream("Orders"); + * const analytics = yield* Cloudflare.K2.Subscription("Analytics", { + * streamId: orders.streamId, + * }); + * ``` + * + * **Example:** Read from the oldest retained record + * ```typescript + * const backfill = yield* Cloudflare.K2.Subscription("Backfill", { + * streamId: orders.streamId, + * startAt: "earliest", + * }); + * ``` + * + * ### Pulling Records + * **Example:** Lease, process, and acknowledge a batch + * ```typescript + * const inbox = yield* Cloudflare.K2.ReadSubscription(analytics); + * const batch = yield* inbox.consume({ workerId: "worker-1", maxRecords: 100 }); + * if (Option.isSome(batch)) { + * yield* Effect.forEach(batch.value.records, handle); + * yield* inbox.ack(batch.value); + * } + * ``` + * + * @resource + * @product K2 + * @category Storage & Databases + */ +export const Subscription = Resource(SubscriptionTypeId); + +/** Returns true if the given value is a K2 Subscription resource. */ +export const isSubscription = (value: unknown): value is Subscription => + Predicate.hasProperty(value, "Type") && value.Type === SubscriptionTypeId; + +export const SubscriptionProvider = () => + Provider.succeed(Subscription, { + stables: ["subscriptionId", "subscriptionName", "streamId", "startAt", "createdAt"], + + diff: Effect.fn(function* ({ id, olds, news, output }) { + if (!isResolved(news)) return undefined; + const oldName = output?.subscriptionName ?? (yield* subscriptionName(id, olds?.name)); + const newName = news.name ?? oldName; + const oldStartAt = output?.startAt ?? olds?.startAt ?? "latest"; + const streamChanged = (output?.streamId ?? olds?.streamId) !== news.streamId; + if ( + streamChanged || + newName.toLowerCase() !== oldName.toLowerCase() || + (news.startAt ?? "latest") !== oldStartAt + ) { + // An explicit name is reused by the replacement; K2 rejects the same + // name with different settings, so delete the old one first. + return { + action: "replace", + deleteFirst: news.name !== undefined && !streamChanged, + } as const; + } + return undefined; + }), + + read: Effect.fn(function* ({ id, output, olds }) { + const streamId = output?.streamId ?? olds?.streamId; + if (!streamId) return undefined; + if (output?.subscriptionId) { + const observed = yield* getSubscription(streamId, output.subscriptionId); + if (observed) return toAttributes(observed, streamId); + } + const name = yield* subscriptionName(id, olds?.name); + const match = yield* findSubscriptionByName(streamId, name); + if (match) { + const attrs = toAttributes(match, streamId); + return olds?.name !== undefined ? Unowned(attrs) : attrs; + } + return undefined; + }), + + reconcile: Effect.fn(function* ({ id, news, output }) { + const streamId = news.streamId; + const name = + output?.streamId === streamId + ? output.subscriptionName + : yield* subscriptionName(id, news.name); + const startAt = news.startAt ?? "latest"; + + // Observe — the cached id is only valid on the same stream. + let observed = + output?.subscriptionId && output.streamId === streamId + ? yield* getSubscription(streamId, output.subscriptionId) + : undefined; + + // Ensure — creating with the same name and settings returns the + // existing subscription's id, so a lost state write converges here. + if (!observed) { + const created = yield* k2.createSubscription({ + streamId, + name, + startAt: { type: startAt }, + }); + observed = yield* k2.getSubscription({ streamId, subscriptionId: created.id }); + } + + // Subscriptions have no mutable settings; every change is a replace. + return toAttributes(observed, streamId); + }), + + delete: Effect.fn(function* ({ output }) { + yield* k2 + .deleteSubscription({ + streamId: output.streamId, + subscriptionId: output.subscriptionId, + }) + .pipe(Effect.catchTag(["K2SubscriptionNotFound", "K2StreamNotFound"], () => Effect.void)); + }), + }); + +interface ObservedSubscription { + id: string; + name: string; + startAt: k2.SubscriptionStartAt; + createdAt: string; + modifiedAt: string; +} + +const subscriptionName = (id: string, name: string | undefined) => + Effect.gen(function* () { + if (name) return name; + const generated = yield* createPhysicalName({ id, maxLength: 128 }); + return generated.replaceAll(/[^a-zA-Z0-9_-]/g, "-"); + }); + +const getSubscription = (streamId: string, subscriptionId: string) => + k2.getSubscription({ streamId, subscriptionId }).pipe( + Effect.map((s): ObservedSubscription | undefined => s), + Effect.catchTag(["K2SubscriptionNotFound", "K2StreamNotFound"], () => + Effect.succeed(undefined), + ), + ); + +/** `listSubscriptions` filters by name server-side (at most one match). */ +const findSubscriptionByName = (streamId: string, name: string) => + k2.listSubscriptions({ streamId, name }).pipe( + Effect.map((subs): ObservedSubscription | undefined => + subs.find((s) => s.name.toLowerCase() === name.toLowerCase()), + ), + Effect.catchTag("K2StreamNotFound", () => Effect.succeed(undefined)), + ); + +const toAttributes = ( + observed: ObservedSubscription, + streamId: string, +): SubscriptionAttributes => ({ + subscriptionId: observed.id, + subscriptionName: observed.name, + streamId, + startAt: observed.startAt.type === "earliest" ? "earliest" : "latest", + createdAt: observed.createdAt, + modifiedAt: observed.modifiedAt, +}); diff --git a/packages/alchemy/src/Cloudflare/K2/WriteStream.ts b/packages/alchemy/src/Cloudflare/K2/WriteStream.ts new file mode 100644 index 00000000000..f93aaf71865 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/WriteStream.ts @@ -0,0 +1,103 @@ +import type * as Effect from "effect/Effect"; +import * as Binding from "../../Binding.ts"; +import type { RuntimeContext } from "../../RuntimeContext.ts"; +import type { + EncodedRecord, + K2SchemaError, + ProduceError, + Record as K2Record, + RecordSchema, +} from "./K2Types.ts"; +import type { Stream } from "./Stream.ts"; + +/** Options accepted by `WriteStream(stream, options)` and `StreamSink(stream, options)`. */ +export interface WriteStreamOptions { + /** + * Encode every value with this schema and send it as JSON with a + * `content-type: application/json` header. Without a schema, `send` + * takes raw {@link K2Record}s. + */ + schema: RecordSchema; +} + +/** + * Binding service that turns a K2 {@link Stream} into a typed + * {@link WriteStreamClient} you can call from a Worker's (or any host's) + * runtime Effect. + * + * `send` appends a batch of records atomically: all of them are stored, or + * none. A batch can be at most 5 MB and each record at most 1 MB. + * ### Sending Records + * **Example:** Producer route + * ```typescript + * const orders = yield* Cloudflare.K2.WriteStream(Orders); + * + * return { + * fetch: Effect.gen(function* () { + * yield* orders.send([ + * { content: JSON.stringify({ id: 1 }), headers: { source: "api" } }, + * { content: new Uint8Array([1, 2, 3]) }, + * ]); + * return HttpServerResponse.empty({ status: 202 }); + * }), + * }; + * ``` + * + * ### Typed Records + * **Example:** Encode values with a Schema + * ```typescript + * const Order = Schema.Struct({ id: Schema.Number, total: Schema.Number }); + * const orders = yield* Cloudflare.K2.WriteStream(Orders, { schema: Order }); + * // JSON-encoded, sent with `content-type: application/json` + * yield* orders.send([{ id: 1, total: 42 }]); + * ``` + * + * ### Handling Errors + * **Example:** Retry only when the batch was not stored + * ```typescript + * yield* orders.send(records).pipe( + * // K2Unavailable: the batch was NOT stored. Never retry + * // K2AppendOutcomeUnknown — the batch may have been stored. + * Effect.retry({ + * while: (e) => e._tag === "K2Unavailable", + * times: 3, + * }), + * ); + * ``` + * + * Provide {@link WriteStreamBinding} (native `k2` Worker binding), + * {@link WriteStreamHttp} (scoped `K2 Produce` token over the stream's HTTP + * input) or {@link WriteStreamLocal} (current credentials, for Actions). + * + * @binding + * @product K2 + * @category Storage & Databases + */ +export interface WriteStream extends Binding.Service< + WriteStream, + "Cloudflare.K2.WriteStream", + (stream: Stream, options?: WriteStreamOptions) => Effect.Effect> +> { + ( + stream: Stream, + options: WriteStreamOptions, + ): Effect.Effect, never, WriteStream>; + (stream: Stream): Effect.Effect, never, WriteStream>; +} + +export const WriteStream = Binding.Service("Cloudflare.K2.WriteStream"); + +/** + * Producer client for a K2 stream. `A` is {@link K2Record} without a + * schema, or the schema's type with one. + */ +export interface WriteStreamClient { + /** Append a batch of records atomically. */ + send(records: ReadonlyArray): Effect.Effect; + /** Encode values into the bytes + headers shape `sendEncoded` accepts. */ + encode(records: ReadonlyArray): Effect.Effect, K2SchemaError>; + /** Append already-encoded records atomically. */ + sendEncoded( + records: ReadonlyArray, + ): Effect.Effect; +} diff --git a/packages/alchemy/src/Cloudflare/K2/WriteStreamBinding.ts b/packages/alchemy/src/Cloudflare/K2/WriteStreamBinding.ts new file mode 100644 index 00000000000..1e94ab6bd7b --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/WriteStreamBinding.ts @@ -0,0 +1,131 @@ +import { UnknownCloudflareError } from "@distilled.cloud/cloudflare/Errors"; +import * as k2 from "@distilled.cloud/cloudflare/k2"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { Worker, WorkerEnvironment } from "../Workers/Worker.ts"; +import type { EncodedRecord, ProduceError } from "./K2Types.ts"; +import type { Stream } from "./Stream.ts"; +import { WriteStream, type WriteStreamOptions } from "./WriteStream.ts"; +import { makeWriteStreamClient } from "./WriteStreamClient.ts"; + +/** + * The runtime shape of a `k2` Worker binding — `env.ORDERS` for a + * `K2.Stream` in a Worker's `env`. `@cloudflare/workers-types` does not + * declare it yet. `send` resolves (does not throw) on API errors; a + * rejected promise means the append outcome is unknown. + */ +export interface K2StreamBinding { + send( + records: Array<{ content: ArrayBuffer | Uint8Array; headers?: { [key: string]: string } }>, + ): Promise< + | { success: true } + | { success: false; error: { code: number; message: string; retryable: boolean } } + >; +} + +/** + * Implementation of the {@link WriteStream} service over the native Worker + * `k2` binding. The stream must have its Worker-binding input enabled (the + * default). + * + * K2 has no local simulation, so a Worker using this layer cannot run under + * `alchemy dev`. + * ### Providing the Layer + * **Example:** Produce from a Worker + * ```typescript + * export default Cloudflare.Worker( + * "Api", + * { main: import.meta.url }, + * Effect.gen(function* () { + * const orders = yield* Cloudflare.K2.WriteStream(Orders); + * return { + * fetch: Effect.gen(function* () { + * yield* orders.send([{ content: "hello" }]); + * return HttpServerResponse.empty({ status: 202 }); + * }), + * }; + * }).pipe(Effect.provide(Cloudflare.K2.WriteStreamBinding)), + * ); + * ``` + * + * @layer + * @provides Cloudflare.K2.WriteStream + * @product K2 + * @category Storage & Databases + */ +export const WriteStreamBinding = Layer.effect( + WriteStream, + Effect.gen(function* () { + const env = yield* WorkerEnvironment; + const host = yield* Worker; + + return Effect.fn(function* (stream: Stream, options?: WriteStreamOptions) { + if (!globalThis.__ALCHEMY_RUNTIME__) { + yield* host.bind`${stream}`({ + bindings: [ + { + type: "k2", + name: stream.LogicalId, + stream: stream.streamId, + }, + ], + }); + } + const raw = Effect.sync(() => (env as { [key: string]: K2StreamBinding })[stream.LogicalId]!); + return makeWriteStreamClient(options, (records) => + raw.pipe(Effect.flatMap((binding) => sendViaBinding(binding, records))), + ); + }); + }), +); + +/** + * Append through the native binding. API failures resolve (they do not + * throw), so map their code onto the same typed tags the HTTP path produces. + * A rejected promise means the append outcome is unknown. + */ +export const sendViaBinding = ( + binding: K2StreamBinding, + records: ReadonlyArray, +): Effect.Effect => + Effect.tryPromise({ + try: () => + binding.send( + records.map((record) => ({ + content: record.content, + ...(record.headers ? { headers: record.headers } : {}), + })), + ), + catch: (cause) => + new k2.K2AppendOutcomeUnknown({ + code: 10212, + message: `K2 binding send rejected; the batch may have been stored: ${ + cause instanceof Error ? cause.message : String(cause) + }`, + }), + }).pipe( + Effect.flatMap((result) => + result.success ? Effect.void : Effect.fail(toProduceError(result.error)), + ), + ); + +/** Map a binding error code onto the typed tag the HTTP API uses for it. */ +export const toProduceError = (error: { code: number; message: string }): ProduceError => { + const fields = { code: error.code, message: error.message }; + switch (error.code) { + case 10200: + return new k2.K2StreamNotFound(fields); + case 10204: + return new k2.K2InvalidRequest(fields); + case 10206: + return new k2.K2RequestTooLarge(fields); + case 10207: + return new k2.K2RecordTooLarge(fields); + case 10211: + return new k2.K2Unavailable(fields); + case 10212: + return new k2.K2AppendOutcomeUnknown(fields); + default: + return new UnknownCloudflareError(fields); + } +}; diff --git a/packages/alchemy/src/Cloudflare/K2/WriteStreamClient.ts b/packages/alchemy/src/Cloudflare/K2/WriteStreamClient.ts new file mode 100644 index 00000000000..b7ae8b77b7c --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/WriteStreamClient.ts @@ -0,0 +1,26 @@ +import * as Effect from "effect/Effect"; +import type { RuntimeContext } from "../../RuntimeContext.ts"; +import { encodeRecords, encodeWithSchema } from "./K2Codec.ts"; +import type { EncodedRecord, ProduceError, Record as K2Record } from "./K2Types.ts"; +import type { WriteStreamClient, WriteStreamOptions } from "./WriteStream.ts"; + +/** + * Build a {@link WriteStreamClient} over a transport that appends encoded + * records. Shared by the binding, HTTP, and local implementations — they + * differ only in `sendEncoded`. Internal — not exported from the K2 barrel. + */ +export const makeWriteStreamClient = ( + options: WriteStreamOptions | undefined, + sendEncoded: ( + records: ReadonlyArray, + ) => Effect.Effect, +): WriteStreamClient => { + const encode = options?.schema + ? encodeWithSchema(options.schema) + : (records: ReadonlyArray) => Effect.sync(() => encodeRecords(records)); + return { + encode, + sendEncoded, + send: (records) => encode(records).pipe(Effect.flatMap(sendEncoded)), + }; +}; diff --git a/packages/alchemy/src/Cloudflare/K2/WriteStreamHttp.ts b/packages/alchemy/src/Cloudflare/K2/WriteStreamHttp.ts new file mode 100644 index 00000000000..1d5bbf5184d --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/WriteStreamHttp.ts @@ -0,0 +1,62 @@ +import * as k2 from "@distilled.cloud/cloudflare/k2"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { toProduceRecords } from "./K2Codec.ts"; +import { type K2Auth, makeK2HttpAuth } from "./K2Http.ts"; +import type { EncodedRecord } from "./K2Types.ts"; +import type { Stream } from "./Stream.ts"; +import { WriteStream, type WriteStreamOptions } from "./WriteStream.ts"; +import { makeWriteStreamClient } from "./WriteStreamClient.ts"; + +/** + * HTTP-backed implementation of the {@link WriteStream} service. Mints a + * scoped `K2 Produce` {@link AccountApiToken}, binds it into the host, and + * appends records through the stream's HTTP input + * (`POST https://.k2.cloudflarestorage.com/produce`). + * + * The stream must have its HTTP input enabled (`http: true`). Works on any + * host — Workers, containers, Lambda. + * ### Providing the Layer + * **Example:** Produce over HTTP + * ```typescript + * const Orders = Cloudflare.K2.Stream("Orders", { http: true }); + * + * Effect.gen(function* () { + * const orders = yield* Cloudflare.K2.WriteStream(Orders); + * // ... + * }).pipe(Effect.provide(Cloudflare.K2.WriteStreamHttp)); + * ``` + * + * @layer + * @provides Cloudflare.K2.WriteStream + * @product K2 + * @category Storage & Databases + */ +export const WriteStreamHttp = Layer.effect( + WriteStream, + Effect.gen(function* () { + const auth = yield* makeK2HttpAuth; + return Effect.fn(function* (stream: Stream, options?: WriteStreamOptions) { + const streamAuth = yield* auth(stream, "K2 Produce"); + return makeWriteStreamHttpClient(streamAuth, yield* stream.streamId, options); + }); + }), +); + +/** + * Build the producer client over the K2 `produce` HTTP API. Shared by + * {@link WriteStreamHttp} and `WriteStreamLocal`. + */ +export const makeWriteStreamHttpClient = ( + auth: K2Auth, + streamId: Effect.Effect, + options: WriteStreamOptions | undefined, +) => + makeWriteStreamClient(options, (records: ReadonlyArray) => + streamId.pipe( + Effect.flatMap((id) => + auth.authorize(k2.produce({ streamId: id, records: toProduceRecords(records) })), + ), + Effect.asVoid, + ), + ); diff --git a/packages/alchemy/src/Cloudflare/K2/WriteStreamLocal.ts b/packages/alchemy/src/Cloudflare/K2/WriteStreamLocal.ts new file mode 100644 index 00000000000..14f173c2472 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/WriteStreamLocal.ts @@ -0,0 +1,46 @@ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { makeK2LocalAuth } from "./K2Http.ts"; +import type { Stream } from "./Stream.ts"; +import { WriteStream, type WriteStreamOptions } from "./WriteStream.ts"; +import { makeWriteStreamHttpClient } from "./WriteStreamHttp.ts"; + +/** + * Local implementation of the {@link WriteStream} service — appends records + * through the stream's HTTP input with the **current credentials** instead + * of a native binding or a scoped token. Provide it on an Action (or any + * deploy-time Effect) to seed a stream with the same client a Worker uses. + * + * The stream must have its HTTP input enabled (`http: true`). The stream id + * is resolved at apply time, so the stream can be created in the same + * deploy. + * ### Providing the Layer + * **Example:** Seed a stream from an Action + * ```typescript + * const Seed = Alchemy.Action( + * "Seed", + * Effect.gen(function* () { + * const orders = yield* Cloudflare.K2.WriteStream(Orders); + * return Effect.fn(function* () { + * yield* orders.send([{ content: "hello" }]); + * }); + * }).pipe(Effect.provide(Cloudflare.K2.WriteStreamLocal)), + * ); + * ``` + * + * @layer + * @provides Cloudflare.K2.WriteStream + * @product K2 + * @category Storage & Databases + */ +export const WriteStreamLocal = Layer.effect( + WriteStream, + Effect.gen(function* () { + const auth = yield* makeK2LocalAuth; + return Effect.fn(function* (stream: Stream, options?: WriteStreamOptions) { + // Deferred accessor — resolves the id at apply time. No `host.bind`: + // the local variant registers no binding. + return makeWriteStreamHttpClient(auth, yield* stream.streamId, options); + }); + }), +); diff --git a/packages/alchemy/src/Cloudflare/K2/index.ts b/packages/alchemy/src/Cloudflare/K2/index.ts new file mode 100644 index 00000000000..e7377d58388 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/K2/index.ts @@ -0,0 +1,15 @@ +export * from "./K2Types.ts"; +export * from "./ReadSubscription.ts"; +export * from "./ReadSubscriptionHttp.ts"; +export * from "./ReadSubscriptionLocal.ts"; +export * from "./Stream.ts"; +export * from "./StreamEventSource.ts"; +export * from "./StreamSink.ts"; +export * from "./StreamSinkBinding.ts"; +export * from "./StreamSinkHttp.ts"; +export * from "./StreamSinkLocal.ts"; +export * from "./Subscription.ts"; +export * from "./WriteStream.ts"; +export * from "./WriteStreamBinding.ts"; +export * from "./WriteStreamHttp.ts"; +export * from "./WriteStreamLocal.ts"; diff --git a/packages/alchemy/src/Cloudflare/Pipelines/Pipeline.ts b/packages/alchemy/src/Cloudflare/Pipelines/Pipeline.ts index 16bb2e0fc42..2d924836e93 100644 --- a/packages/alchemy/src/Cloudflare/Pipelines/Pipeline.ts +++ b/packages/alchemy/src/Cloudflare/Pipelines/Pipeline.ts @@ -1,4 +1,5 @@ import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; +import * as Data from "effect/Data"; import * as Effect from "effect/Effect"; import * as Predicate from "effect/Predicate"; import * as Schedule from "effect/Schedule"; @@ -32,11 +33,25 @@ export interface PipelineProps { * so the engine orders the pipeline after them on deploy (and before * them on destroy). * - * Immutable — changing the SQL triggers a replacement. + * Immutable — changing the SQL triggers a replacement. Whitespace-only + * changes are a no-op. A changed, fully-resolved statement is validated + * during plan, so invalid SQL fails before anything is replaced. */ sql: string; } +/** + * Raised during plan when a pipeline's SQL is rejected by Cloudflare's + * SQL validator (`InvalidSql`, code 1014). Nothing has been replaced or + * deleted when this fails. + */ +export class PipelineSqlInvalid extends Data.TaggedError( + "Cloudflare.Pipelines.PipelineSqlInvalid", +)<{ + message: string; + sql: string; +}> {} + export interface PipelineAttributes { /** Cloudflare-assigned pipeline identifier. */ pipelineId: string; @@ -62,26 +77,35 @@ export type Pipeline = Resource return { action: "replace" } as const; } const oldSql = typeof o.sql === "string" ? o.sql : output?.sql; - if (oldSql !== undefined && news.sql !== oldSql) { - return { action: "replace" } as const; + if (oldSql === undefined || canonicalSql(news.sql) === canonicalSql(oldSql)) { + return oldSql === undefined ? undefined : ({ action: "noop" } as const); } - return undefined; + // The replacement deletes the running pipeline — validate the new + // statement first so bad SQL fails the plan, not the deploy. Tables + // created in the same deploy are not visible yet, so a missing table + // is not a validation failure. + yield* pipelines.validateSqlPipeline({ accountId, sql: news.sql }).pipe( + Effect.catchTag("TableNotFound", () => Effect.void), + Effect.catchTag("InvalidSql", (error) => + Effect.fail( + new PipelineSqlInvalid({ + message: `Cloudflare.Pipelines.Pipeline '${id}': invalid SQL — ${error.message}`, + sql: news.sql, + }), + ), + ), + ); + return { action: "replace" } as const; }), read: Effect.fn(function* ({ id, output, olds }) { @@ -163,7 +202,10 @@ export const PipelineProvider = () => // replaced and `sql` was unresolved at diff time), delete the stale // pipeline and fall through to recreate it under the same name. // Nothing references a pipeline downstream, so this is safe. - if (observed && (observed.sql !== sql || observed.name !== name)) { + if ( + observed && + (canonicalSql(observed.sql) !== canonicalSql(sql) || observed.name !== name) + ) { yield* deletePipeline(accountId, observed.id); // Wait until the delete is visible so the recreate below does not // race an `PipelineAlreadyExists` against the dying pipeline. @@ -219,6 +261,31 @@ export const PipelineProvider = () => }), }); +/** + * Canonical SQL for change detection: whitespace runs outside quoted + * literals/identifiers collapse to one space, surrounding whitespace and + * a trailing `;` are dropped. + */ +const canonicalSql = (sql: string): string => { + let out = ""; + let quote: string | undefined; + let pendingSpace = false; + for (const ch of sql.trim().replace(/;\s*$/, "")) { + if (quote) { + out += ch; + if (ch === quote) quote = undefined; + } else if (/\s/.test(ch)) { + pendingSpace = true; + } else { + if (pendingSpace && out.length > 0) out += " "; + pendingSpace = false; + if (ch === "'" || ch === '"' || ch === "`") quote = ch; + out += ch; + } + } + return out; +}; + /** * The subset of pipeline state shared by get/list/create responses. */ diff --git a/packages/alchemy/src/Cloudflare/Pipelines/Sink.ts b/packages/alchemy/src/Cloudflare/Pipelines/Sink.ts index 44098c638d9..48339837189 100644 --- a/packages/alchemy/src/Cloudflare/Pipelines/Sink.ts +++ b/packages/alchemy/src/Cloudflare/Pipelines/Sink.ts @@ -45,7 +45,9 @@ export interface SinkR2Config { bucket: string; /** * R2 S3-compatible credentials the sink uses to write objects. - * Write-only — Cloudflare never echoes them back. + * Write-only — Cloudflare never echoes them back, and sinks cannot be + * updated, so changing them does NOT replace the sink (see the + * resource docs on rotating credentials). */ credentials: { /** R2 access key id (the API token id). */ @@ -92,13 +94,13 @@ export interface SinkR2Config { } /** - * Configuration of an `r2_data_catalog` sink writing Iceberg tables via - * the R2 Data Catalog. + * Configuration of a catalog sink writing an Iceberg table via the Basin + * (R2 Data) Catalog, spelled with the bucket name. */ export interface SinkR2DataCatalogConfig { /** * Name of the R2 bucket backing the catalog. The bucket must already - * exist and have the Data Catalog enabled. + * exist and have the catalog enabled. */ bucket: string; /** @@ -112,7 +114,8 @@ export interface SinkR2DataCatalogConfig { namespace?: string; /** * Cloudflare API token with R2 Data Catalog permissions. Write-only — - * Cloudflare never echoes it back. + * Cloudflare never echoes it back, and changing it does NOT replace + * the sink. */ token: Redacted.Redacted; /** @@ -121,6 +124,34 @@ export interface SinkR2DataCatalogConfig { rollingPolicy?: SinkRollingPolicy; } +/** + * The catalog a catalog sink writes to — pass a `Cloudflare.Basin.Catalog` + * (`Cloudflare.R2.DataCatalog`) resource so the sink is ordered after it. + */ +export interface SinkCatalogReference { + /** Name of the R2 bucket backing the catalog. */ + bucketName: string; +} + +/** + * The Iceberg table a catalog sink writes to. + */ +export interface SinkCatalogTable { + /** + * Catalog namespace the table lives in. + * @default "default" + */ + namespace?: string; + /** Table name. */ + name: string; +} + +/** + * Sink type of a catalog sink. `basin_catalog` and `r2_data_catalog` are + * the same destination; switching between them never replaces the sink. + */ +export type SinkCatalogType = "basin_catalog" | "r2_data_catalog"; + /** * Output file format written by the sink. */ @@ -147,14 +178,14 @@ interface SinkBaseProps { * underscores only (it is referenced as a SQL table name). If omitted, * a unique name is generated from the app, stage, and logical ID. * - * Sinks have no update API, so changing this (or any other) property - * triggers a replacement. + * Sinks have no update API, so changing this (or any destination + * property) triggers a replacement. * @default ${app}_${id}_${stage}_${suffix} */ name?: string; /** - * Output file format. - * @default { type: "json" } + * Output file format. Catalog sinks require Parquet. + * @default { type: "json" } for `r2` sinks, { type: "parquet" } for catalog sinks */ format?: SinkFormat; } @@ -172,14 +203,40 @@ export type SinkProps = }) | (SinkBaseProps & { /** - * Sink type — `r2_data_catalog` writes Iceberg tables via the R2 - * Data Catalog. + * Sink type — writes an Iceberg table via the Basin (R2 Data) + * Catalog. */ - type: "r2_data_catalog"; + type: SinkCatalogType; /** - * R2 Data Catalog destination configuration. + * Catalog destination configuration, spelled with the bucket name. + * Equivalent to the `catalog` / `table` / `token` form. */ config: SinkR2DataCatalogConfig; + }) + | (SinkBaseProps & { + /** + * Sink type — writes an Iceberg table via the Basin (R2 Data) + * Catalog. + */ + type: SinkCatalogType; + /** + * The catalog to write to — a `Cloudflare.Basin.Catalog` resource. + */ + catalog: SinkCatalogReference; + /** + * The Iceberg table to write. Cloudflare creates it with the + * sink — sinks cannot be created for existing Iceberg tables. + */ + table: SinkCatalogTable; + /** + * Cloudflare API token with R2 Data Catalog permissions. Write-only; + * changing it does NOT replace the sink. + */ + token: Redacted.Redacted; + /** + * When the sink rolls output files. + */ + rollingPolicy?: SinkRollingPolicy; }); export interface SinkAttributes { @@ -190,11 +247,15 @@ export interface SinkAttributes { /** Sink name (unique per account). */ name: string; /** Sink type. */ - type: "r2" | "r2_data_catalog"; + type: "r2" | SinkCatalogType; /** Destination R2 bucket name. */ bucket: string; /** Key prefix output objects are written under (r2 sinks). */ path: string | undefined; + /** Catalog namespace of the destination table (catalog sinks). */ + namespace: string | undefined; + /** Destination Iceberg table name (catalog sinks). */ + tableName: string | undefined; /** When the sink was created. */ createdAt: string; /** When the sink was last modified. */ @@ -204,17 +265,31 @@ export interface SinkAttributes { export type Sink = Resource; /** - * A Cloudflare Pipelines sink — the destination of the Pipelines product. - * A SQL {@link Pipeline} reads events from a {@link Stream} and - * writes them to a sink, which stores them in R2 either as raw files - * (`r2`) or as Iceberg tables via the R2 Data Catalog - * (`r2_data_catalog`). + * A Cloudflare Pipelines sink — the destination of the Pipelines product + * (also exported as `Cloudflare.Basin.Sink`). A SQL {@link Pipeline} + * reads events from a {@link Stream} and writes them to a sink, which + * stores them in R2 either as raw files (`r2`) or as an Iceberg table via + * the Basin Catalog (`basin_catalog`, a.k.a. `r2_data_catalog`). + * + * Sinks have no update API: changing the destination (bucket, path, + * table, format, partitioning, rolling policy) triggers a replacement. + * Equivalent spellings never replace — `basin_catalog` ⇄ + * `r2_data_catalog`, the `config` form ⇄ the `catalog` / `table` form, + * omitted ⇄ explicit defaults. With engine-generated names a replacement + * is seamless (the new sink gets a fresh name before the old one is + * deleted); with an explicit `name` it collides, so prefer generated + * names. + * + * Credentials (`credentials`, `token`) are write-only and are not part of + * the replacement diff, so rotating them never recreates the sink — a + * catalog sink could not be recreated anyway, because Cloudflare refuses + * to create sinks for existing Iceberg tables. The sink keeps the + * credentials it was created with; to move it onto new ones, rename it + * (which replaces it). + * + * Pipelines is ingest-only — to react to files the sink writes, subscribe + * to its bucket with `Cloudflare.R2.BucketEventNotification`. * - * Sinks have no update API: every property change triggers a - * replacement. With engine-generated names this is seamless (the new - * sink gets a fresh name before the old one is deleted); with an - * explicit `name` the create-before-delete replacement collides, so - * prefer generated names. * ### Creating a Sink * **Example:** R2 sink with JSON output * The S3-compatible credentials are derived from a Cloudflare API token: @@ -223,7 +298,7 @@ export type Sink = Resource * ```typescript * const bucket = yield* Cloudflare.R2.Bucket("events", {}); * - * const sink = yield* Cloudflare.Pipelines.Sink("events-sink", { + * const sink = yield* Cloudflare.Basin.Sink("events-sink", { * type: "r2", * config: { * bucket: bucket.bucketName, @@ -239,17 +314,32 @@ export type Sink = Resource * * **Example:** Parquet output * ```typescript - * const sink = yield* Cloudflare.Pipelines.Sink("parquet-sink", { + * const sink = yield* Cloudflare.Basin.Sink("parquet-sink", { * type: "r2", * config: { bucket: bucket.bucketName, credentials }, * format: { type: "parquet", compression: "zstd" }, * }); * ``` * - * ### R2 Data Catalog + * ### Basin Catalog * **Example:** Iceberg table sink * ```typescript - * const sink = yield* Cloudflare.Pipelines.Sink("iceberg-sink", { + * const bucket = yield* Cloudflare.R2.Bucket("Lakehouse", {}); + * const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { + * bucketName: bucket.bucketName, + * }); + * + * const sink = yield* Cloudflare.Basin.Sink("PageViewTable", { + * type: "basin_catalog", + * catalog, + * table: { namespace: "web", name: "page_views" }, + * token: yield* Config.Redacted("CATALOG_TOKEN"), + * }); + * ``` + * + * **Example:** Bucket-name form + * ```typescript + * const sink = yield* Cloudflare.Basin.Sink("iceberg-sink", { * type: "r2_data_catalog", * config: { * bucket: bucket.bucketName, @@ -286,17 +376,18 @@ export const SinkProvider = () => } const o = olds as SinkProps | undefined; if (o === undefined) return undefined; - // Sinks have no update API — any change is a replacement. + // Sinks have no update API — any destination change replaces. Compare + // the CANONICAL destination (credentials excluded) so equivalent + // spellings and credential rotation never replace. const newName = yield* sinkName(id, news.name); const oldName = output?.name ?? (yield* sinkName(id, o.name)); - if ( - newName !== oldName || - news.type !== o.type || - !stableEquals(normalizeProps(news), normalizeProps(o)) - ) { + if (newName !== oldName) { return { action: "replace" } as const; } - return undefined; + if (!stableEquals(canonicalSink(news), canonicalSink(o))) { + return { action: "replace" } as const; + } + return { action: "noop" } as const; }), read: Effect.fn(function* ({ id, output, olds }) { @@ -350,9 +441,10 @@ export const SinkProvider = () => } // 2. Ensure — create when missing. There is no sync step: sinks - // have no update API, so prop changes arrive as replacements - // (diff) rather than in-place updates. An AlreadyExists on - // create is a race or recovery — resolve it via the name lookup. + // have no update API, so destination changes arrive as + // replacements (diff) rather than in-place updates. An + // AlreadyExists on create is a race or recovery — resolve it via + // the name lookup. if (!observed) { observed = yield* pipelines .createSink({ @@ -360,7 +452,7 @@ export const SinkProvider = () => name, type: news.type, config: toRequestConfig(accountId, news), - format: news.format, + format: news.format ?? defaultFormat(news.type), }) .pipe( Effect.catchTag("SinkAlreadyExists", (error) => @@ -413,6 +505,34 @@ interface ObservedSink { modifiedAt: string; } +/** A catalog sink destination, whichever form it was declared in. */ +interface CatalogTarget { + bucket: string; + tableName: string; + namespace: string | undefined; + token: Redacted.Redacted; + rollingPolicy: SinkRollingPolicy | undefined; +} + +const isCatalogType = (type: string) => type === "r2_data_catalog" || type === "basin_catalog"; + +const catalogTarget = (props: Exclude): CatalogTarget => + "config" in props && props.config !== undefined + ? { + bucket: props.config.bucket, + tableName: props.config.tableName, + namespace: props.config.namespace, + token: props.config.token, + rollingPolicy: props.config.rollingPolicy, + } + : { + bucket: (props as { catalog: SinkCatalogReference }).catalog.bucketName, + tableName: (props as { table: SinkCatalogTable }).table.name, + namespace: (props as { table: SinkCatalogTable }).table.namespace, + token: (props as { token: Redacted.Redacted }).token, + rollingPolicy: (props as { rollingPolicy?: SinkRollingPolicy }).rollingPolicy, + }; + // Pipelines entity names must be alphanumeric/underscore only (they are // referenced as SQL table names), so swap the default hyphen delimiter // for underscores. @@ -461,21 +581,19 @@ const deleteSink = (accountId: string, sinkId: string) => * diffed so we don't fight server-side defaults. */ const sinkDrifted = (observed: ObservedSink, news: SinkProps): boolean => { - if (observed.type !== news.type) return true; + if (isCatalogType(observed.type) !== isCatalogType(news.type)) return true; const cfg = observed.config; if (!cfg) return false; - if (cfg.bucket !== (news.config.bucket as string)) return true; if (news.type === "r2") { + if (cfg.bucket !== news.config.bucket) return true; const observedPath = "path" in cfg ? (cfg.path ?? undefined) : undefined; - if (news.config.path !== undefined && news.config.path !== observedPath) { - return true; - } - } else if ("tableName" in cfg) { - if (cfg.tableName !== news.config.tableName) return true; - if ( - news.config.namespace !== undefined && - news.config.namespace !== (cfg.namespace ?? undefined) - ) { + return news.config.path !== undefined && news.config.path !== observedPath; + } + const target = catalogTarget(news); + if (cfg.bucket !== target.bucket) return true; + if ("tableName" in cfg) { + if (cfg.tableName !== target.tableName) return true; + if (target.namespace !== undefined && target.namespace !== (cfg.namespace ?? undefined)) { return true; } } @@ -497,7 +615,7 @@ const toRequestConfig = (accountId: string, news: SinkProps) => { const c = news.config; return { accountId, - bucket: c.bucket as string, + bucket: c.bucket, credentials: { accessKeyId: Redacted.value(c.credentials.accessKeyId), secretAccessKey: Redacted.value(c.credentials.secretAccessKey), @@ -509,42 +627,69 @@ const toRequestConfig = (accountId: string, news: SinkProps) => { jurisdiction: c.jurisdiction, }; } - const c = news.config; + const t = catalogTarget(news); return { accountId, - bucket: c.bucket as string, - tableName: c.tableName, - namespace: c.namespace, - token: Redacted.value(c.token), - rollingPolicy: c.rollingPolicy, + bucket: t.bucket, + tableName: t.tableName, + namespace: t.namespace, + token: Redacted.value(t.token), + rollingPolicy: t.rollingPolicy, }; }; +const canonicalRollingPolicy = (rp: SinkRollingPolicy | undefined) => ({ + fileSizeBytes: rp?.fileSizeBytes, + inactivitySeconds: rp?.inactivitySeconds, + intervalSeconds: rp?.intervalSeconds ?? 300, +}); + +/** Catalog (Iceberg) sinks only accept Parquet; raw R2 sinks default to JSON. */ +const defaultFormat = (type: SinkProps["type"]): SinkFormat => + isCatalogType(type) ? { type: "parquet" } : { type: "json" }; + +const canonicalFormat = (type: SinkProps["type"], format: SinkFormat | undefined) => { + const f = format ?? defaultFormat(type); + return f.type === "parquet" + ? { + type: "parquet", + compression: f.compression ?? "zstd", + rowGroupBytes: f.rowGroupBytes, + } + : { type: "json" }; +}; + /** - * Normalize props for change detection: unwrap redacted secrets so two - * `Redacted` wrappers holding the same value compare equal. + * The canonical destination of a sink, for change detection: catalog + * type spellings and declaration forms collapse, defaults are filled in, + * and write-only credentials are excluded (they cannot be updated, and + * rotating them must never recreate the sink). */ -const normalizeProps = (props: SinkProps): unknown => { +const canonicalSink = (props: SinkProps): unknown => { + const format = canonicalFormat(props.type, props.format); if (props.type === "r2") { + const c = props.config; return { - type: props.type, - format: props.format, - config: { - ...props.config, - credentials: { - accessKeyId: Redacted.value(props.config.credentials.accessKeyId), - secretAccessKey: Redacted.value(props.config.credentials.secretAccessKey), - }, - }, + kind: "r2", + format, + bucket: c.bucket, + path: c.path, + partitioning: c.partitioning, + fileNaming: c.fileNaming + ? { ...c.fileNaming, strategy: c.fileNaming.strategy ?? "uuid_v7" } + : undefined, + rollingPolicy: canonicalRollingPolicy(c.rollingPolicy), + jurisdiction: c.jurisdiction, }; } + const t = catalogTarget(props); return { - type: props.type, - format: props.format, - config: { - ...props.config, - token: Redacted.value(props.config.token), - }, + kind: "catalog", + format, + bucket: t.bucket, + namespace: t.namespace ?? "default", + tableName: t.tableName, + rollingPolicy: canonicalRollingPolicy(t.rollingPolicy), }; }; @@ -563,14 +708,18 @@ const stableStringify = (value: unknown): string => : v, ) ?? "undefined"; -const toAttributes = (observed: ObservedSink, accountId: string): SinkAttributes => ({ - sinkId: observed.id, - accountId, - name: observed.name, - type: observed.type as "r2" | "r2_data_catalog", - bucket: observed.config?.bucket ?? "", - path: - observed.config && "path" in observed.config ? (observed.config.path ?? undefined) : undefined, - createdAt: observed.createdAt, - modifiedAt: observed.modifiedAt, -}); +const toAttributes = (observed: ObservedSink, accountId: string): SinkAttributes => { + const cfg = observed.config ?? undefined; + return { + sinkId: observed.id, + accountId, + name: observed.name, + type: observed.type as SinkAttributes["type"], + bucket: cfg?.bucket ?? "", + path: cfg && "path" in cfg ? (cfg.path ?? undefined) : undefined, + namespace: cfg && "namespace" in cfg ? (cfg.namespace ?? undefined) : undefined, + tableName: cfg && "tableName" in cfg ? cfg.tableName : undefined, + createdAt: observed.createdAt, + modifiedAt: observed.modifiedAt, + }; +}; diff --git a/packages/alchemy/src/Cloudflare/Pipelines/Stream.ts b/packages/alchemy/src/Cloudflare/Pipelines/Stream.ts index 7dcc8e75a8b..7182d52b2c1 100644 --- a/packages/alchemy/src/Cloudflare/Pipelines/Stream.ts +++ b/packages/alchemy/src/Cloudflare/Pipelines/Stream.ts @@ -1,67 +1,29 @@ import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; import * as Effect from "effect/Effect"; +import * as Effectable from "effect/Effectable"; import * as Predicate from "effect/Predicate"; import * as Schedule from "effect/Schedule"; +import type * as Schema from "effect/Schema"; import * as EffectStream from "effect/Stream"; import { Unowned } from "../../AdoptPolicy.ts"; import { isResolved } from "../../Diff.ts"; +import type { PropsInput } from "../../Input.ts"; import { createPhysicalName } from "../../PhysicalName.ts"; import * as Provider from "../../Provider.ts"; -import { Resource } from "../../Resource.ts"; +import { Resource, type ResourceClass } from "../../Resource.ts"; import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; import type { Providers } from "../Providers.ts"; +import { + canonicalStreamFields, + isEffectSchema, + StreamSchemaUnsupported, + streamFieldsFromSchema, + type StreamFieldsSchema, +} from "./StreamSchema.ts"; const StreamTypeId = "Cloudflare.Pipelines.Stream" as const; type StreamTypeId = typeof StreamTypeId; -/** - * Scalar field types accepted by a structured stream schema. - */ -export type StreamFieldType = - | "int32" - | "int64" - | "float32" - | "float64" - | "bool" - | "string" - | "binary" - | "json"; - -/** - * A single field of a structured stream schema. `timestamp` fields - * additionally accept a `unit`. - */ -export type StreamField = - | { - /** Field type. */ - type: StreamFieldType; - /** Field name as it appears in ingested events. */ - name: string; - /** Whether the field must be present in every event. */ - required?: boolean; - /** Name to expose to SQL when it differs from `name`. */ - sqlName?: string; - /** Metadata key the field is populated from instead of the event body. */ - metadataKey?: string; - } - | { - /** Timestamp field type. */ - type: "timestamp"; - /** Field name as it appears in ingested events. */ - name: string; - /** Whether the field must be present in every event. */ - required?: boolean; - /** Name to expose to SQL when it differs from `name`. */ - sqlName?: string; - /** Metadata key the field is populated from instead of the event body. */ - metadataKey?: string; - /** - * Precision of the timestamp. - * @default "millisecond" - */ - unit?: "second" | "millisecond" | "microsecond" | "nanosecond"; - }; - /** * Input format of events ingested by the stream. */ @@ -92,7 +54,7 @@ export interface StreamHttp { enabled?: boolean; /** * Whether requests to the HTTP endpoint must carry a Cloudflare API - * token. + * token with the `Pipelines Send` permission. * @default false */ authentication?: boolean; @@ -113,14 +75,17 @@ export interface StreamProps { */ name?: string; /** - * Structured schema of ingested events. Immutable — changing the schema - * triggers a replacement. When omitted, the stream accepts unstructured - * JSON events. + * Structured schema of ingested events — either an explicit field list + * or an Effect Schema (`Schema.Struct` / `Schema.Class`), which is + * converted to the field list and also types and encodes the records + * sent through `WriteStream` / `StreamSink`. + * + * Immutable — changing the (derived) field list triggers a replacement, + * which drops buffered events and dependent pipelines. Equivalent + * spellings (a field list ⇄ the matching Effect Schema) are a no-op. + * When omitted, the stream accepts unstructured JSON events. */ - schema?: { - /** Fields of the structured schema. */ - fields: StreamField[]; - }; + schema?: StreamFieldsSchema | Schema.Top; /** * Input format configuration. Immutable — changing it triggers a * replacement. @@ -129,9 +94,13 @@ export interface StreamProps { format?: StreamFormat; /** * HTTP ingest endpoint configuration. Mutable in place. - * @default { enabled: true, authentication: false } + * + * - omitted / `false` — no public HTTP endpoint (`{ enabled: false }`) + * - `true` — endpoint enabled, API-token authentication required + * - an object — explicit settings + * @default false */ - http?: StreamHttp; + http?: boolean | StreamHttp; /** * Whether Workers can send events to this stream via a `pipelines` * binding. Mutable in place. @@ -168,52 +137,189 @@ export interface StreamAttributes { modifiedAt: string; } -export type Stream = Resource; +/** + * A Pipelines stream. `A` is the record type sent through `WriteStream` / + * `StreamSink` — the Effect Schema's `Type` when the stream was declared + * with one. + */ +export interface Stream extends Resource< + StreamTypeId, + StreamProps, + StreamAttributes, + never, + Providers +> { + /** + * The Effect Schema the stream was declared with, if any. Used by the + * producer clients to encode records; never persisted. + */ + readonly RecordSchema: Schema.Top | undefined; + /** Phantom record type. */ + readonly "~record"?: A; +} + +/** + * The record type a producer sends to a `Stream`: the declared Effect + * Schema's `Type`, or an untyped JSON object. + */ +export type StreamRecord = unknown extends A ? Record : A; + +/** + * Props of a stream declared with an Effect Schema. + */ +export type StreamSchemaProps = PropsInput> & { + schema: S; +}; + +const StreamResource = Resource(StreamTypeId); + +/** + * The Stream constructor: the plain resource constructor plus an overload + * that infers the record type from an Effect Schema `schema`. + */ +export type StreamClass = { + ( + id: string, + props: StreamSchemaProps, + ): Effect.Effect, never, Providers>; +} & ResourceClass; + +/** + * Convert an Effect Schema `schema` prop into the persisted field list and + * keep the schema aside for the producer clients. Dies with a clear + * {@link StreamSchemaUnsupported} when the schema has no stream + * representation — this runs while the program is constructed, before + * anything is planned. + */ +const normalizeStreamProps = ( + props: unknown, +): Effect.Effect<{ props: unknown; recordSchema: Schema.Top | undefined }> => + Effect.suspend(() => { + const schema = (props as { schema?: unknown } | undefined)?.schema; + if (!isEffectSchema(schema)) { + return Effect.succeed({ props, recordSchema: undefined }); + } + return resolveSchema(schema).pipe( + Effect.map((resolved) => ({ + props: { ...(props as object), schema: resolved }, + recordSchema: schema, + })), + Effect.catch((error) => Effect.die(error)), + ); + }); + +const constructStream = (id: string, props: unknown) => + Effect.gen(function* () { + let recordSchema: Schema.Top | undefined; + const capture = (n: { props: unknown; recordSchema: Schema.Top | undefined }) => { + recordSchema = n.recordSchema; + return n.props; + }; + const normalized = Effect.isEffect(props) + ? (props as Effect.Effect).pipe( + Effect.flatMap(normalizeStreamProps), + Effect.map(capture), + ) + : capture(yield* normalizeStreamProps(props)); + const construct = StreamResource as unknown as ( + id: string, + props: unknown, + ) => Effect.Effect; + const stream = yield* construct(id, normalized); + (stream as { RecordSchema: Schema.Top | undefined }).RecordSchema = recordSchema; + return stream; + }); /** * A Cloudflare Pipelines stream — the ingestion endpoint of the Pipelines - * product. Events are sent to a stream over HTTP (and/or from Workers via - * a binding), transformed by a SQL {@link Pipeline}, and written to a - * {@link Sink}. + * product (also exported as `Cloudflare.Basin.Stream`). Events are sent to + * a stream from Workers (`WriteStream` / `StreamSink`) or over HTTP, + * transformed by a SQL {@link Pipeline}, and written to a {@link Sink}. * * The stream's `schema` and `format` are fixed at creation (changing them - * triggers a replacement); the HTTP endpoint and Worker-binding toggles - * are mutable in place. + * triggers a replacement, which drops buffered events and dependent + * pipelines); the HTTP endpoint and Worker-binding toggles are mutable in + * place. + * + * Pipelines is ingest-only — there is no event source on a stream. To + * process data after it lands, subscribe to the sink's bucket with + * `Cloudflare.R2.BucketEventNotification`. + * + * Cloudflare accepts records that violate a structured stream's schema + * (they are dropped later, during processing). Declare the schema as an + * Effect Schema to have the producer clients validate and encode every + * record before it is sent. + * + * :::caution + * A stream declared without `http` has **no** public ingest endpoint + * (`http: { enabled: false }`). Earlier versions sent nothing, and + * Cloudflare's default is a public, unauthenticated endpoint. Existing + * streams are updated in place on the next deploy; set `http: true` + * (authenticated) or `http: { enabled: true }` to keep an endpoint. + * ::: + * * ### Creating a Stream * **Example:** Unstructured stream with default settings * ```typescript - * const stream = yield* Cloudflare.Pipelines.Stream("events", {}); + * const stream = yield* Cloudflare.Basin.Stream("events", {}); + * ``` + * + * **Example:** Structured stream from an Effect Schema + * ```typescript + * class PageView extends Schema.Class("PageView")({ + * url: Schema.String, + * at: Schema.Date, + * tags: Schema.Array(Schema.String), + * user: Schema.optional(Schema.Struct({ id: Schema.String })), + * }) {} + * + * // Stream: WriteStream / StreamSink are typed and encode records + * const stream = yield* Cloudflare.Basin.Stream("PageViews", { + * schema: PageView, + * }); * ``` * - * **Example:** Structured stream with a typed schema + * **Example:** Structured stream with an explicit field list * ```typescript - * const stream = yield* Cloudflare.Pipelines.Stream("clicks", { + * const stream = yield* Cloudflare.Basin.Stream("clicks", { * schema: { * fields: [ * { type: "string", name: "url", required: true }, * { type: "timestamp", name: "ts", unit: "millisecond" }, + * { type: "list", name: "tags", items: { type: "string" } }, + * { + * type: "struct", + * name: "user", + * fields: [{ type: "string", name: "id" }], + * }, * ], * }, * }); * ``` * * ### HTTP ingestion - * **Example:** Authenticated endpoint with CORS + * **Example:** Authenticated endpoint * ```typescript - * const stream = yield* Cloudflare.Pipelines.Stream("events", { + * const stream = yield* Cloudflare.Basin.Stream("events", { http: true }); + * // POST a JSON array to stream.endpoint with a `Pipelines Send` token, + * // or use Cloudflare.Basin.WriteStreamHttp / WriteStreamLocal. + * ``` + * + * **Example:** Public endpoint with CORS + * ```typescript + * const stream = yield* Cloudflare.Basin.Stream("beacons", { * http: { * enabled: true, - * authentication: true, + * authentication: false, * cors: { origins: ["https://app.example.com"] }, * }, * }); - * // POST events to stream.endpoint with an API token * ``` * * ### Wiring into a Pipeline - * **Example:** Stream → SQL Pipeline → R2 Sink + * **Example:** Stream → SQL Pipeline → Sink * ```typescript - * const pipeline = yield* Cloudflare.Pipelines.Pipeline("etl", { + * const pipeline = yield* Cloudflare.Basin.Pipeline("etl", { * sql: Output.interpolate`INSERT INTO ${sink.name} SELECT * FROM ${stream.name}`, * }); * ``` @@ -224,7 +330,17 @@ export type Stream = Resource(StreamTypeId); +export const Stream: StreamClass = Object.assign( + (...args: [id: string, props?: unknown] | [methods: object]) => + typeof args[0] === "object" + ? Object.assign(Stream, args[0]) + : constructStream(args[0], (args as [string, unknown])[1]), + StreamResource, + Effectable.Prototype({ + label: `Resource<${StreamTypeId}>`, + evaluate: () => Effect.succeed((id: string, props: unknown) => constructStream(id, props)), + }), +) as unknown as StreamClass; /** * Returns true if the given value is a Stream resource. @@ -248,15 +364,33 @@ export const StreamProvider = () => if (newName !== oldName) { return { action: "replace" } as const; } - // Schema and format are immutable — only http/workerBinding are - // patchable. - if (o !== undefined && !stableEquals(news.schema, o.schema)) { + if (o === undefined) return undefined; + // Schema and format are immutable. Compare CANONICAL derived values + // so equivalent spellings (field list ⇄ Effect Schema, omitted ⇄ + // explicit defaults) never replace. + const newSchema = yield* canonicalSchema(news.schema); + const oldSchema = yield* canonicalSchema(o.schema); + if (!stableEquals(newSchema, oldSchema)) { return { action: "replace" } as const; } - if (o !== undefined && !stableEquals(news.format, o.format)) { + if (!stableEquals(canonicalFormat(news.format), canonicalFormat(o.format))) { return { action: "replace" } as const; } - return undefined; + if (output === undefined) return undefined; + // http/workerBinding are patchable: diff desired against the last + // observed state, so the secure http default reaches streams created + // before it existed. + const httpConverged = httpMatches(desiredHttp(news.http), { + enabled: output.httpEnabled, + authentication: output.httpAuthentication, + origins: output.corsOrigins, + }); + const bindingConverged = + news.workerBinding === undefined || + news.workerBinding.enabled === output.workerBindingEnabled; + return httpConverged && bindingConverged + ? ({ action: "noop" } as const) + : ({ action: "update" } as const); }), read: Effect.fn(function* ({ id, output, olds }) { @@ -282,6 +416,7 @@ export const StreamProvider = () => reconcile: Effect.fn(function* ({ id, news, output }) { const { accountId } = yield* yield* CloudflareEnvironment; const name = yield* streamName(id, news.name); + const http = desiredHttp(news.http); // 1. Observe — the cached streamId is a hint, not a guarantee; a // missing stream falls through to a name lookup and then create. @@ -296,13 +431,16 @@ export const StreamProvider = () => // AlreadyExists is a race (or recovery from a lost state write): // fall back to the name lookup. if (!observed) { + const schema = yield* resolveSchema(news.schema); observed = yield* pipelines .createStream({ accountId, name, format: news.format, - schema: news.schema, - http: news.http ? desiredHttp(news.http) : undefined, + // The distilled field union does not model nested `items` / + // `fields`; they pass through to the wire verbatim. + schema: schema as pipelines.CreateStreamRequest["schema"], + http, workerBinding: news.workerBinding, }) .pipe( @@ -322,16 +460,15 @@ export const StreamProvider = () => streamId: observed.id, }; let dirty = false; - if (news.http !== undefined) { - const desired = desiredHttp(news.http); - if ( - desired.enabled !== observed.http.enabled || - desired.authentication !== observed.http.authentication || - !sameOrigins(desired.cors?.origins, observed.http.cors?.origins) - ) { - patch.http = desired; - dirty = true; - } + if ( + !httpMatches(http, { + enabled: observed.http.enabled, + authentication: observed.http.authentication, + origins: observed.http.cors?.origins, + }) + ) { + patch.http = http; + dirty = true; } if ( news.workerBinding !== undefined && @@ -435,11 +572,75 @@ const findStreamByName = (accountId: string, name: string) => ), ); -const desiredHttp = (http: StreamHttp) => ({ - enabled: http.enabled ?? true, - authentication: http.authentication ?? false, - cors: http.cors?.origins ? { origins: http.cors.origins } : undefined, -}); +/** Resolve a `schema` prop (field list or Effect Schema) to the field-list form. */ +const resolveSchema = ( + schema: StreamFieldsSchema | Schema.Top | undefined, +): Effect.Effect => + isEffectSchema(schema) + ? Effect.try({ + try: () => ({ fields: streamFieldsFromSchema(schema) }), + catch: (error) => + error instanceof StreamSchemaUnsupported + ? error + : new StreamSchemaUnsupported({ path: "", message: String(error) }), + }) + : Effect.succeed(schema as StreamFieldsSchema | undefined); + +const canonicalSchema = (schema: StreamFieldsSchema | Schema.Top | undefined) => + resolveSchema(schema).pipe( + Effect.map((resolved) => + resolved === undefined ? undefined : canonicalStreamFields(resolved.fields), + ), + ); + +const canonicalFormat = (format: StreamFormat | undefined) => { + const f = format ?? { type: "json" as const }; + return { + type: f.type, + unstructured: f.unstructured, + timestampFormat: f.timestampFormat ?? "rfc3339", + decimalEncoding: f.decimalEncoding, + }; +}; + +interface DesiredHttp { + enabled: boolean; + authentication: boolean; + cors?: { origins: string[] }; +} + +/** + * Desired HTTP ingest settings. Omitted (or `false`) means no endpoint — + * Cloudflare's own default is a public, unauthenticated endpoint. + */ +const desiredHttp = (http: boolean | StreamHttp | undefined): DesiredHttp => { + if (http === undefined || http === false) { + return { enabled: false, authentication: false }; + } + if (http === true) return { enabled: true, authentication: true }; + return { + enabled: http.enabled ?? true, + authentication: http.authentication ?? false, + cors: http.cors?.origins ? { origins: http.cors.origins } : undefined, + }; +}; + +const httpMatches = ( + desired: DesiredHttp, + observed: { + enabled: boolean; + authentication: boolean; + origins: readonly string[] | null | undefined; + }, +) => { + // A disabled endpoint's auth/CORS settings are inert. + if (!desired.enabled) return !observed.enabled; + return ( + observed.enabled && + desired.authentication === observed.authentication && + sameOrigins(desired.cors?.origins, observed.origins) + ); +}; const sameOrigins = ( desired: readonly string[] | undefined, diff --git a/packages/alchemy/src/Cloudflare/Pipelines/StreamCodec.ts b/packages/alchemy/src/Cloudflare/Pipelines/StreamCodec.ts new file mode 100644 index 00000000000..7ca793ca1ed --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Pipelines/StreamCodec.ts @@ -0,0 +1,101 @@ +import type { Pipeline, PipelineRecord } from "cloudflare:pipelines"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import type { RuntimeContext } from "../../RuntimeContext.ts"; +import { isLegacyPipeline, type LegacyPipeline } from "./LegacyPipeline.ts"; +import type { Stream } from "./Stream.ts"; +import { StreamSendError, type WriteStreamClient } from "./WriteStream.ts"; + +/** + * Shared record encoding + client assembly for the WriteStream / StreamSink + * implementations. Internal — not exported from the Pipelines barrel. + */ + +/** + * The Effect Schema a stream was declared with, if any. Legacy pipelines + * have none. + */ +export const recordSchemaOf = (stream: Stream | LegacyPipeline): Schema.Top | undefined => { + if (isLegacyPipeline(stream)) return undefined; + // References (`Stream.ref`) proxy unknown properties to Outputs — only a + // real Effect Schema counts. + const schema: unknown = stream.RecordSchema; + return Schema.isSchema(schema) ? schema : undefined; +}; + +/** + * Build the batch encoder: `Schema.toCodecJson(schema)` per record when + * the stream has an Effect Schema, identity otherwise. + */ +export const makeRecordEncoder = ( + schema: Schema.Top | undefined, +): (( + records: ReadonlyArray, +) => Effect.Effect, StreamSendError>) => { + if (schema === undefined) { + return (records) => Effect.succeed(records as ReadonlyArray); + } + const encode = Schema.encodeUnknownEffect( + Schema.toCodecJson(schema as unknown as Schema.Codec), + ); + return (records) => + Effect.forEach(records, (record, index) => + encode(record).pipe( + Effect.map((json) => json as PipelineRecord), + Effect.mapError( + (issue) => + new StreamSendError({ + message: `record ${index} does not match the stream schema: ${issue.message}`, + cause: issue, + reason: "InvalidRecord", + index, + }), + ), + ), + ); +}; + +/** Wrap any transport failure as a `SendFailed` {@link StreamSendError}. */ +export const toStreamSendError = (error: unknown): StreamSendError => + new StreamSendError({ + message: + typeof error === "object" && error !== null && "message" in error + ? String((error as { message: unknown }).message) + : "Unknown pipeline error", + cause: error, + reason: "SendFailed", + }); + +/** Assemble a {@link WriteStreamClient} from an encoder and a transport. */ +export const makeStreamClient = (options: { + schema: Schema.Top | undefined; + raw: Effect.Effect; + sendEncoded: ( + records: ReadonlyArray, + ) => Effect.Effect; +}): WriteStreamClient => { + const encode = makeRecordEncoder(options.schema); + return { + raw: options.raw, + encode, + sendEncoded: options.sendEncoded, + send: (records) => encode(records).pipe(Effect.flatMap(options.sendEncoded)), + }; +}; + +const encoder = new TextEncoder(); + +/** + * Approximate wire size of one encoded record in the JSON array body: + * its UTF-8 JSON plus the separating comma. Values JSON cannot encode + * count as 1 byte; the send rejects them and the error surfaces. + */ +export const recordSize = (record: PipelineRecord): number => { + let json: string | undefined; + try { + json = JSON.stringify(record); + } catch { + json = undefined; + } + return (json === undefined ? 0 : encoder.encode(json).length) + 1; +}; diff --git a/packages/alchemy/src/Cloudflare/Pipelines/StreamHttp.ts b/packages/alchemy/src/Cloudflare/Pipelines/StreamHttp.ts new file mode 100644 index 00000000000..ac5b2827148 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Pipelines/StreamHttp.ts @@ -0,0 +1,133 @@ +import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; +import * as Effect from "effect/Effect"; +import type * as HttpClient from "effect/http/HttpClient"; +import type { RuntimeContext } from "../../RuntimeContext.ts"; +import { Self } from "../../Self.ts"; +import { AccountApiToken } from "../ApiToken/AccountApiToken.ts"; +import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; +import type { Credentials } from "../Credentials.ts"; +import { authorizeWith } from "../HttpClientUtils.ts"; +import { isLegacyPipeline, type LegacyPipeline } from "./LegacyPipeline.ts"; +import type { Stream } from "./Stream.ts"; +import { makeStreamClient, recordSchemaOf, toStreamSendError } from "./StreamCodec.ts"; +import { StreamSendError, type WriteStreamClient } from "./WriteStream.ts"; + +/** + * Shared scaffolding for the HTTP-ingest Pipelines clients + * (`WriteStreamHttp`, `WriteStreamLocal` and the matching StreamSink + * layers). Internal — not exported from the Pipelines barrel. + */ + +/** + * Injectable auth: runs a raw distilled op (which needs + * `Credentials | HttpClient`) and discharges those requirements down to + * {@link RuntimeContext}. The HTTP variant provides a minted token; the + * Local variant provides the ambient current-credentials context. + */ +export interface StreamAuth { + authorize: ( + eff: Effect.Effect, + ) => Effect.Effect; +} + +/** Legacy pipelines have their own ingest endpoint; only streams are supported. */ +export const legacyPipelineUnsupported = (layer: string) => + Effect.die( + new StreamSendError({ + message: `${layer} supports Pipelines streams only; bind a LegacyPipeline with WriteStreamBinding.`, + reason: "SendFailed", + }), + ); + +/** + * Build the producer client over the stream's HTTP ingest endpoint + * (`POST https://{stream_id}.ingest.cloudflare.com`, at most 5 MB per + * request). The stream needs `http` enabled. + */ +export const makeWriteStreamHttpClient = ( + auth: StreamAuth, + stream: Stream, + streamId: Effect.Effect, +): WriteStreamClient => + makeStreamClient({ + schema: recordSchemaOf(stream), + raw: Effect.die( + new StreamSendError({ + message: "The Pipelines HTTP client has no native binding; use send/sendEncoded.", + reason: "SendFailed", + }), + ), + sendEncoded: (records) => + streamId.pipe( + Effect.flatMap((id) => + auth.authorize(pipelines.sendStreamRecords({ streamId: id, records: [...records] })), + ), + Effect.mapError(toStreamSendError), + Effect.asVoid, + ), + }); + +/** + * Mint (or reuse) the host's scoped {@link AccountApiToken}, grant it + * `Pipelines Send`, bind its value into the host at deploy time, and + * build the client with it. + */ +export const makeHttpStreamBinding = (options: { + layer: string; + makeClient: (client: WriteStreamClient) => Client; +}) => + Effect.gen(function* () { + const Token = yield* AccountApiToken; + const self = yield* Self; + const env = yield* CloudflareEnvironment; + + return Effect.fn(function* (stream: Stream | LegacyPipeline) { + if (isLegacyPipeline(stream)) return yield* legacyPipelineUnsupported(options.layer); + const { accountId } = yield* env; + const token = yield* Token(`${self.LogicalId}Token`); + if (!globalThis.__ALCHEMY_RUNTIME__) { + yield* token.bind`${stream.LogicalId}`({ + policies: [ + { + effect: "allow", + permissionGroups: ["Pipelines Send"], + resources: { + [`com.cloudflare.api.account.${accountId}`]: "*", + }, + }, + ], + }); + } + const value = yield* token.value; + const streamId = yield* stream.streamId; + return options.makeClient( + makeWriteStreamHttpClient({ authorize: authorizeWith({ value }) }, stream, streamId), + ); + }); + }); + +/** + * Build the client with the **current credentials** captured at layer + * build (no Worker host, no minted token). + */ +export const makeLocalStreamBinding = (options: { + layer: string; + makeClient: (client: WriteStreamClient) => Client; +}) => + Effect.gen(function* () { + const context = yield* Effect.context(); + + return Effect.fn(function* (stream: Stream | LegacyPipeline) { + if (isLegacyPipeline(stream)) return yield* legacyPipelineUnsupported(options.layer); + // Deferred accessor — resolves the stream id at apply time, so a + // stream created in the same deploy works. No `host.bind`. + const streamId = yield* stream.streamId; + return options.makeClient( + makeWriteStreamHttpClient( + { authorize: (eff) => eff.pipe(Effect.provideContext(context)) }, + stream, + streamId, + ), + ); + }); + }); diff --git a/packages/alchemy/src/Cloudflare/Pipelines/StreamSchema.ts b/packages/alchemy/src/Cloudflare/Pipelines/StreamSchema.ts new file mode 100644 index 00000000000..376a90163bd --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Pipelines/StreamSchema.ts @@ -0,0 +1,347 @@ +import * as Data from "effect/Data"; +import * as Schema from "effect/Schema"; +import * as AST from "effect/SchemaAST"; + +/** + * Scalar field types accepted by a structured stream schema. + */ +export type StreamFieldType = + | "int32" + | "int64" + | "float32" + | "float64" + | "bool" + | "string" + | "binary" + | "json"; + +/** + * Timestamp precision of a `timestamp` stream field. + */ +export type StreamTimestampUnit = "second" | "millisecond" | "microsecond" | "nanosecond"; + +/** + * The type of a stream field (or list element), without its name. + */ +export type StreamFieldItem = + | { + /** Scalar field type. */ + type: StreamFieldType; + /** Whether the value must be present. */ + required?: boolean; + } + | { + /** Timestamp field type. */ + type: "timestamp"; + /** Whether the value must be present. */ + required?: boolean; + /** + * Precision of numeric timestamps. + * @default "millisecond" + */ + unit?: StreamTimestampUnit; + } + | { + /** List (array) field type. */ + type: "list"; + /** Whether the value must be present. */ + required?: boolean; + /** Type of every element of the list. */ + items: StreamFieldItem; + } + | { + /** Nested object field type. */ + type: "struct"; + /** Whether the value must be present. */ + required?: boolean; + /** Fields of the nested object. */ + fields: StreamField[]; + }; + +/** + * A single named field of a structured stream schema. `timestamp` fields + * additionally accept a `unit`, `list` fields an `items` type and + * `struct` fields nested `fields`. + */ +export type StreamField = StreamFieldItem & { + /** Field name as it appears in ingested events. */ + name: string; + /** Whether the field must be present in every event. */ + required?: boolean; + /** Name to expose to SQL when it differs from `name`. */ + sqlName?: string; + /** Metadata key the field is populated from instead of the event body. */ + metadataKey?: string; +}; + +/** + * A structured stream schema as an explicit field list. + */ +export interface StreamFieldsSchema { + /** Fields of the structured schema. */ + fields: StreamField[]; +} + +/** + * Annotation key that overrides the stream field type derived from an + * Effect Schema, e.g. to store a number as `int32` or `float32`: + * + * ```typescript + * Schema.Number.annotate({ [StreamFieldTypeAnnotation]: "float32" }) + * ``` + */ +export const StreamFieldTypeAnnotation = "pipelinesFieldType" as const; + +/** + * Raised when an Effect Schema passed as a stream `schema` has no + * Pipelines representation (e.g. a union of different types). + */ +export class StreamSchemaUnsupported extends Data.TaggedError( + "Cloudflare.Pipelines.StreamSchemaUnsupported", +)<{ + message: string; + path: string; +}> {} + +const unsupported = (path: string, what: string) => + new StreamSchemaUnsupported({ + path, + message: + `Cloudflare.Pipelines.Stream: field '${path || ""}' is ${what}, ` + + "which has no Pipelines stream field type. Use a field list or annotate the field " + + `with { ${StreamFieldTypeAnnotation}: "json" }.`, + }); + +const INT32_MIN = -2147483648; +const INT32_MAX = 2147483647; + +interface CheckRep { + id: string; + payload: unknown; +} + +const checkReps = (ast: AST.AST): CheckRep[] => { + const out: CheckRep[] = []; + const visit = (check: unknown) => { + const c = check as { + annotations?: { representation?: CheckRep }; + checks?: readonly unknown[]; + }; + if (c.annotations?.representation?.id) out.push(c.annotations.representation); + c.checks?.forEach(visit); + }; + ((ast as { checks?: readonly unknown[] }).checks ?? []).forEach(visit); + return out; +}; + +const annotationOverride = (ast: AST.AST): StreamFieldType | "timestamp" | undefined => { + const read = (annotations: unknown) => + (annotations as Record | undefined)?.[StreamFieldTypeAnnotation]; + const candidates = [ + read((ast as { annotations?: unknown }).annotations), + ...((ast as { checks?: readonly { annotations?: unknown }[] }).checks ?? []).map((c) => + read(c.annotations), + ), + ]; + const found = candidates.find((v) => typeof v === "string"); + return found as StreamFieldType | "timestamp" | undefined; +}; + +const numberType = (ast: AST.AST): StreamFieldType => { + const reps = checkReps(ast); + if (!reps.some((r) => r.id === "effect/schema/isInt")) return "float64"; + const fitsInt32 = reps.some((r) => { + if (r.id !== "effect/schema/isBetween") return false; + const p = r.payload as { minimum?: number; maximum?: number } | null; + return ( + p?.minimum !== undefined && + p?.maximum !== undefined && + p.minimum >= INT32_MIN && + p.maximum <= INT32_MAX + ); + }); + return fitsInt32 ? "int32" : "int64"; +}; + +const isNullish = (ast: AST.AST) => + ast._tag === "Null" || ast._tag === "Undefined" || ast._tag === "Void"; + +const literalType = (value: unknown): StreamFieldType | undefined => + typeof value === "string" + ? "string" + : typeof value === "boolean" + ? "bool" + : typeof value === "number" + ? Number.isInteger(value) + ? "int64" + : "float64" + : undefined; + +/** + * Derive the stream field type of one (encoded-side) AST node. Returns the + * item plus whether the value may be absent (nullable union members). + */ +const toItem = (ast: AST.AST, path: string): { item: StreamFieldItem; nullable: boolean } => { + const override = annotationOverride(ast); + if (override) { + return { item: { type: override } as StreamFieldItem, nullable: false }; + } + switch (ast._tag) { + case "String": + case "TemplateLiteral": + return { item: { type: "string" }, nullable: false }; + case "Number": + return { item: { type: numberType(ast) }, nullable: false }; + case "Boolean": + return { item: { type: "bool" }, nullable: false }; + case "Literal": { + const type = literalType(ast.literal); + if (!type) throw unsupported(path, `the literal ${String(ast.literal)}`); + return { item: { type }, nullable: false }; + } + case "Enum": { + const types = new Set( + (ast as unknown as { enums: ReadonlyArray }).enums.map( + ([, v]) => literalType(v), + ), + ); + const [type] = [...types]; + if (types.size !== 1 || !type) throw unsupported(path, "a mixed enum"); + return { item: { type }, nullable: false }; + } + case "Unknown": + case "Any": + case "ObjectKeyword": + return { item: { type: "json" }, nullable: false }; + case "Declaration": { + const id = (ast.annotations?.representation as CheckRep | undefined)?.id; + if (id === "effect/schema/Date") return { item: { type: "timestamp" }, nullable: false }; + if (id === "effect/schema/Uint8Array") return { item: { type: "binary" }, nullable: false }; + if (id === "effect/schema/Json") return { item: { type: "json" }, nullable: false }; + throw unsupported(path, `a ${AST.resolveIdentifier(ast) ?? "declared"} value`); + } + case "Arrays": { + if (ast.elements.length > 0 || ast.rest.length !== 1) { + throw unsupported(path, "a tuple"); + } + const element = toItem(ast.rest[0]!, `${path}[]`); + return { + item: { + type: "list", + items: element.nullable ? { ...element.item, required: false } : element.item, + }, + nullable: false, + }; + } + case "Objects": { + if (ast.indexSignatures.length > 0 || ast.propertySignatures.length === 0) { + // Records and empty objects carry arbitrary keys — store as JSON. + return { item: { type: "json" }, nullable: false }; + } + return { + item: { type: "struct", fields: toFields(ast, path) }, + nullable: false, + }; + } + case "Union": { + const members = ast.types.filter((t) => !isNullish(t)); + const nullable = members.length !== ast.types.length; + if (members.length === 1) { + return { item: toItem(members[0]!, path).item, nullable }; + } + // A union of same-typed literals (e.g. `Schema.Literals(["a","b"])`). + if (members.length > 0 && members.every((m) => m._tag === "Literal")) { + const types = new Set(members.map((m) => literalType((m as AST.Literal).literal))); + const [type] = [...types]; + if (types.size === 1 && type) return { item: { type }, nullable }; + } + throw unsupported(path, "a union of different types"); + } + case "Suspend": + throw unsupported(path, "a recursive schema"); + default: + throw unsupported(path, `of type ${ast._tag}`); + } +}; + +const toFields = (ast: AST.Objects, path: string): StreamField[] => + ast.propertySignatures.map((ps) => { + const name = String(ps.name); + const fieldPath = path ? `${path}.${name}` : name; + const { item, nullable } = toItem(ps.type, fieldPath); + const optional = AST.isOptional(ps.type); + return { + ...item, + name, + required: !(optional || nullable), + } as StreamField; + }); + +/** + * Returns true if `value` is an Effect Schema (rather than a field list). + */ +export const isEffectSchema = (value: unknown): value is Schema.Top => Schema.isSchema(value); + +/** + * Convert an Effect Schema (`Schema.Struct` / `Schema.Class`) into the + * Pipelines stream field list. The schema's **encoded** side is used, so + * transformations (e.g. `NumberFromString`) map to their wire type, and + * records are sent with `Schema.toCodecJson(schema)`: + * + * - `String` → `string`, `Boolean` → `bool` + * - `Number` → `float64`, `Int` → `int64` (`int32` when bounded to 32 bits) + * - `Date` → `timestamp`, `Uint8Array` → `binary` + * - `Array` → `list`, `Struct` / `Class` → `struct` + * - `Record` / `Unknown` / `Json` → `json` + * - optional or nullable fields → `required: false` + * + * Unions of different types, tuples and recursive schemas have no stream + * representation and throw {@link StreamSchemaUnsupported}. Override any + * field's type with the {@link StreamFieldTypeAnnotation} annotation. + */ +export const streamFieldsFromSchema = (schema: Schema.Top): StreamField[] => { + const encoded = AST.toEncoded(schema.ast); + if (encoded._tag !== "Objects" || encoded.propertySignatures.length === 0) { + throw unsupported("", "not a Struct or Class with fields"); + } + return toFields(encoded, ""); +}; + +/** + * Resolve a stream `schema` prop (field list or Effect Schema) to the + * field-list form persisted in state and sent to Cloudflare. + */ +export const resolveStreamSchema = ( + schema: StreamFieldsSchema | Schema.Top | undefined, +): StreamFieldsSchema | undefined => { + if (schema === undefined) return undefined; + if (isEffectSchema(schema)) return { fields: streamFieldsFromSchema(schema) }; + return schema as StreamFieldsSchema; +}; + +/** + * Canonical (default-filled, key-ordered) form of a field item, so + * equivalent spellings compare equal. + */ +const canonicalItem = (item: StreamFieldItem): Record => { + const base: Record = { + type: item.type, + required: item.required ?? false, + }; + if (item.type === "timestamp") base.unit = item.unit ?? "millisecond"; + if (item.type === "list") base.items = canonicalItem(item.items); + if (item.type === "struct") base.fields = canonicalStreamFields(item.fields); + return base; +}; + +/** + * Canonical form of a field list: defaults filled in (`required: false`, + * `unit: "millisecond"`), absent optional keys dropped. + */ +export const canonicalStreamFields = (fields: ReadonlyArray): unknown[] => + fields.map((f) => { + const out: Record = { name: f.name, ...canonicalItem(f) }; + if (f.sqlName !== undefined) out.sqlName = f.sqlName; + if (f.metadataKey !== undefined) out.metadataKey = f.metadataKey; + return out; + }); diff --git a/packages/alchemy/src/Cloudflare/Pipelines/StreamSink.ts b/packages/alchemy/src/Cloudflare/Pipelines/StreamSink.ts new file mode 100644 index 00000000000..93f661b040c --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Pipelines/StreamSink.ts @@ -0,0 +1,72 @@ +import type { PipelineRecord } from "cloudflare:pipelines"; +import type * as Effect from "effect/Effect"; +import type * as Sink from "effect/Sink"; +import * as Binding from "../../Binding.ts"; +import type { RuntimeContext } from "../../RuntimeContext.ts"; +import type { Stream, StreamRecord } from "./Stream.ts"; +import type { StreamSendError } from "./WriteStream.ts"; + +/** + * Binding service that exposes a Pipelines {@link Stream} as an Effect + * `Sink`: run a `Stream` of records into it and every element is + * ingested. Also exported as `Cloudflare.Basin.StreamSink`. + * + * Records are typed, validated and encoded by the stream's Effect Schema + * (plain JSON objects when it has none). Each upstream chunk is packed + * greedily into requests of at most 5 MB, preserving order. + * + * Provide `StreamSinkBinding` (native Worker binding), `StreamSinkHttp` + * (scoped `Pipelines Send` token) or `StreamSinkLocal` (current + * credentials, for Actions) — each is the sink layered over the matching + * `WriteStream` implementation. + * ### Draining a Stream into a Pipelines Stream + * **Example:** Run an Effect Stream into a Pipelines stream + * ```typescript + * export default Cloudflare.Worker( + * "Api", + * { main: import.meta.url }, + * Effect.gen(function* () { + * const sink = yield* Cloudflare.Basin.StreamSink(PageViews); + * return { + * fetch: Effect.gen(function* () { + * yield* EffectStream.fromIterable(views).pipe(EffectStream.run(sink)); + * return HttpServerResponse.empty({ status: 202 }); + * }), + * }; + * }).pipe(Effect.provide(Cloudflare.Basin.StreamSinkBinding)), + * ); + * ``` + * + * **Example:** Controlling request size + * ```typescript + * // at most 500 records per send + * yield* events.pipe(EffectStream.rechunk(500), EffectStream.run(sink)); + * ``` + * + * @binding + * @product Pipelines + * @category Storage & Databases + */ +export interface StreamSink extends Binding.Service< + StreamSink, + "Cloudflare.Pipelines.StreamSink", + (stream: Stream) => Effect.Effect> +> { + ( + stream: Stream | Effect.Effect, never, Req>, + ): Effect.Effect>, never, StreamSink | Req>; +} + +export const StreamSink = Binding.Service("Cloudflare.Pipelines.StreamSink"); + +/** + * A `Sink` that ingests every element it receives as one record. + * Runtime-only: it can only run inside the deployed Worker (or Action). + */ +export type StreamSinkClient = Sink.Sink< + void, + A, + never, + StreamSendError, + RuntimeContext +>; diff --git a/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkBatch.ts b/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkBatch.ts new file mode 100644 index 00000000000..49caab1a76f --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkBatch.ts @@ -0,0 +1,72 @@ +import type { PipelineRecord } from "cloudflare:pipelines"; +import type { NonEmptyReadonlyArray } from "effect/Array"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Sink from "effect/Sink"; +import { makeBatchedSink } from "../../AWS/internal/BatchedSink.ts"; +import type { RuntimeContext } from "../../RuntimeContext.ts"; +import type { Stream } from "./Stream.ts"; +import { recordSize } from "./StreamCodec.ts"; +import { StreamSink, type StreamSinkClient } from "./StreamSink.ts"; +import { StreamSendError, WriteStream, type WriteStreamClient } from "./WriteStream.ts"; + +/** + * Batch packing for {@link StreamSink}. Internal — not exported from the + * Pipelines barrel. + */ + +/** Pipelines' limit on one ingest request (5 MB), minus the array brackets. */ +export const MAX_REQUEST_BYTES = 5_000_000 - 2; + +/** + * Build the sink over a `WriteStream` client: each chunk is encoded, then + * packed into `<= MAX_REQUEST_BYTES` requests (order preserved) and sent + * one `sendEncoded` per request. + */ +export const makeStreamSink = (client: WriteStreamClient): StreamSinkClient => + Sink.unwrap( + Effect.gen(function* () { + // BatchedSink sends without requirements; close over the runtime + // context the sink runs in. + const context = yield* Effect.context(); + return makeBatchedSink({ + maxRecords: Number.MAX_SAFE_INTEGER, + maxBytes: MAX_REQUEST_BYTES, + sizeOf: recordSize, + send: (batch) => client.sendEncoded(batch).pipe(Effect.provideContext(context)), + }).pipe( + Sink.mapInputArrayEffect((chunk: NonEmptyReadonlyArray) => + client + .encode(chunk) + .pipe(Effect.map((encoded) => encoded as NonEmptyReadonlyArray)), + ), + // No `unprocessed` extractor is configured, so retry exhaustion + // cannot occur; normalize the type anyway. + Sink.mapError((error) => + error._tag === "BatchRetryExhaustedError" + ? new StreamSendError({ + message: `${error.entries.length} records were not ingested`, + cause: error, + reason: "SendFailed", + }) + : error, + ), + ); + }), + ); + +/** + * {@link StreamSink} over whichever {@link WriteStream} implementation is + * provided. Composed with each `WriteStream` layer in + * `StreamSinkBinding`, `StreamSinkHttp` and `StreamSinkLocal`. + */ +export const StreamSinkFromWriteStream = Layer.effect( + StreamSink, + Effect.gen(function* () { + const writeStream = yield* WriteStream; + return Effect.fn(function* (stream: Stream) { + const client = yield* writeStream(stream); + return makeStreamSink(client); + }); + }), +); diff --git a/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkBinding.ts b/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkBinding.ts new file mode 100644 index 00000000000..dd42dee991f --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkBinding.ts @@ -0,0 +1,21 @@ +import * as Layer from "effect/Layer"; +import { StreamSinkFromWriteStream } from "./StreamSinkBatch.ts"; +import { WriteStreamBinding } from "./WriteStreamBinding.ts"; + +/** + * Implementation of the {@link StreamSink} service over the native Worker `pipelines` binding ({@link WriteStreamBinding}). Registers the same binding as `WriteStream`, so a Worker can use both on one stream. + * ### Providing the Layer + * **Example:** Provide the sink layer + * ```typescript + * Effect.gen(function* () { + * const sink = yield* Cloudflare.Basin.StreamSink(PageViews); + * // ... + * }).pipe(Effect.provide(Cloudflare.Basin.StreamSinkBinding)); + * ``` + * + * @layer + * @provides Cloudflare.Pipelines.StreamSink + * @product Pipelines + * @category Storage & Databases + */ +export const StreamSinkBinding = StreamSinkFromWriteStream.pipe(Layer.provide(WriteStreamBinding)); diff --git a/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkHttp.ts b/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkHttp.ts new file mode 100644 index 00000000000..63bfc1a6f84 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkHttp.ts @@ -0,0 +1,21 @@ +import * as Layer from "effect/Layer"; +import { StreamSinkFromWriteStream } from "./StreamSinkBatch.ts"; +import { WriteStreamHttp } from "./WriteStreamHttp.ts"; + +/** + * Implementation of the {@link StreamSink} service over the stream's HTTP ingest endpoint ({@link WriteStreamHttp}), authenticated with a scoped `Pipelines Send` API token bound into the host. The stream needs `http` enabled. + * ### Providing the Layer + * **Example:** Provide the sink layer + * ```typescript + * Effect.gen(function* () { + * const sink = yield* Cloudflare.Basin.StreamSink(PageViews); + * // ... + * }).pipe(Effect.provide(Cloudflare.Basin.StreamSinkHttp)); + * ``` + * + * @layer + * @provides Cloudflare.Pipelines.StreamSink + * @product Pipelines + * @category Storage & Databases + */ +export const StreamSinkHttp = StreamSinkFromWriteStream.pipe(Layer.provide(WriteStreamHttp)); diff --git a/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkLocal.ts b/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkLocal.ts new file mode 100644 index 00000000000..3f04d969130 --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Pipelines/StreamSinkLocal.ts @@ -0,0 +1,26 @@ +import * as Layer from "effect/Layer"; +import { StreamSinkFromWriteStream } from "./StreamSinkBatch.ts"; +import { WriteStreamLocal } from "./WriteStreamLocal.ts"; + +/** + * Implementation of the {@link StreamSink} service over the stream's HTTP ingest endpoint with the **current credentials** ({@link WriteStreamLocal}) — for draining an Effect Stream into a Pipelines stream from an Action, with no Worker host. The stream needs `http` enabled. + * ### Providing the Layer + * **Example:** Provide the sink layer + * ```typescript + * const Seed = Alchemy.Action( + * "Seed", + * Effect.gen(function* () { + * const sink = yield* Cloudflare.Basin.StreamSink(PageViews); + * return Effect.fn(function* () { + * yield* EffectStream.fromIterable(views).pipe(EffectStream.run(sink)); + * }); + * }).pipe(Effect.provide(Cloudflare.Basin.StreamSinkLocal)), + * ); + * ``` + * + * @layer + * @provides Cloudflare.Pipelines.StreamSink + * @product Pipelines + * @category Storage & Databases + */ +export const StreamSinkLocal = StreamSinkFromWriteStream.pipe(Layer.provide(WriteStreamLocal)); diff --git a/packages/alchemy/src/Cloudflare/Pipelines/WriteStream.ts b/packages/alchemy/src/Cloudflare/Pipelines/WriteStream.ts index 73e529ef5a8..62baef6a39e 100644 --- a/packages/alchemy/src/Cloudflare/Pipelines/WriteStream.ts +++ b/packages/alchemy/src/Cloudflare/Pipelines/WriteStream.ts @@ -4,34 +4,70 @@ import type * as Effect from "effect/Effect"; import * as Binding from "../../Binding.ts"; import type { RuntimeContext } from "../../RuntimeContext.ts"; import type { LegacyPipeline } from "./LegacyPipeline.ts"; -import type { Stream } from "./Stream.ts"; +import type { Stream, StreamRecord } from "./Stream.ts"; /** * Binding service that turns a Pipelines {@link Stream} (or a - * {@link LegacyPipeline}) into a typed {@link WriteStreamClient} you can - * call from a Worker's runtime Effect. + * {@link LegacyPipeline}) into a {@link WriteStreamClient} you can call + * from a Worker's (or Action's) runtime Effect. Also exported as + * `Cloudflare.Basin.WriteStream`. * - * The Cloudflare Worker `pipelines` binding is producer-only — `send` - * ingests a batch of JSON records into the stream. + * Producer-only — `send` ingests a batch of records into the stream. When + * the stream was declared with an Effect Schema, the client is typed by + * it (`send(records: PageView[])`) and every record is validated and + * encoded with `Schema.toCodecJson` before it is sent — Cloudflare + * accepts schema-violating records and silently drops them later, so + * this is the only validation. Without a schema, `send` takes plain JSON + * objects. + * + * Provide one implementation layer: + * - {@link WriteStreamBinding} — native Worker `pipelines` binding + * - `WriteStreamHttp` — the stream's HTTP ingest endpoint with a scoped + * `Pipelines Send` API token (the stream needs `http` enabled) + * - `WriteStreamLocal` — the HTTP ingest endpoint with the current + * credentials, for Actions and other deploy-time Effects * ### Sending Events - * **Example:** Producer route + * **Example:** Typed producer route * ```typescript - * const events = yield* Cloudflare.Pipelines.WriteStream(Stream); + * class PageView extends Schema.Class("PageView")({ + * url: Schema.String, + * at: Schema.Date, + * }) {} + * + * export const PageViews = Cloudflare.Basin.Stream("PageViews", { + * schema: PageView, + * }); * - * return { - * fetch: Effect.gen(function* () { - * yield* events.send([{ event: "click", at: new Date().toISOString() }]); - * return HttpServerResponse.empty({ status: 202 }); - * }), - * }; + * export default Cloudflare.Worker( + * "Api", + * { main: import.meta.url }, + * Effect.gen(function* () { + * const views = yield* Cloudflare.Basin.WriteStream(PageViews); + * return { + * fetch: Effect.gen(function* () { + * yield* views.send([new PageView({ url: "/", at: new Date() })]); + * return HttpServerResponse.empty({ status: 202 }); + * }), + * }; + * }).pipe(Effect.provide(Cloudflare.Basin.WriteStreamBinding)), + * ); * ``` * - * Provide {@link WriteStreamBinding} (native Worker binding) in the - * worker's runtime layer to resolve the underlying stream at request time. + * **Example:** Untyped stream + * ```typescript + * const events = yield* Cloudflare.Basin.WriteStream(Events); + * yield* events.send([{ event: "click", at: new Date().toISOString() }]); + * ``` * - * `WriteStream` is a single identifier that is simultaneously the binding's - * Context tag, its type, and the callable — - * `yield* Cloudflare.Pipelines.WriteStream(stream)`. + * ### Handling Errors + * **Example:** Recover from a failed send + * ```typescript + * yield* views.send(batch).pipe( + * Effect.catchTag("StreamSendError", (e) => + * Effect.logWarning(`dropped ${batch.length} events: ${e.message}`), + * ), + * ); + * ``` * * @binding * @product Pipelines @@ -40,19 +76,51 @@ import type { Stream } from "./Stream.ts"; export interface WriteStream extends Binding.Service< WriteStream, "Cloudflare.Pipelines.WriteStream", - (stream: Stream | LegacyPipeline) => Effect.Effect -> {} + (stream: Stream | LegacyPipeline) => Effect.Effect> +> { + ( + stream: Stream | Effect.Effect, never, Req>, + ): Effect.Effect>, never, WriteStream | Req>; + ( + stream: LegacyPipeline | Effect.Effect, + ): Effect.Effect, never, WriteStream | Req>; +} export const WriteStream = Binding.Service("Cloudflare.Pipelines.WriteStream"); -export interface WriteStreamClient { +/** + * Producer client for a Pipelines stream. `A` is the stream's record type + * (its Effect Schema's `Type`, or a plain JSON object). + */ +export interface WriteStreamClient { + /** + * The native Worker `pipelines` binding. Only available from + * `WriteStreamBinding`; the HTTP/local clients die. + */ raw: Effect.Effect; - send( + /** Validate, encode and ingest a batch of records. */ + send(records: ReadonlyArray): Effect.Effect; + /** + * Validate and encode records into the JSON objects sent on the wire + * (`Schema.toCodecJson` of the stream's schema; identity when untyped). + */ + encode(records: ReadonlyArray): Effect.Effect, StreamSendError>; + /** Ingest already-encoded records. */ + sendEncoded( records: ReadonlyArray, ): Effect.Effect; } +/** + * Raised when records could not be ingested: a record failed the stream's + * Effect Schema (`reason: "InvalidRecord"`), or the send itself failed + * (`reason: "SendFailed"`). + */ export class StreamSendError extends Data.TaggedError("StreamSendError")<{ message: string; cause?: unknown; + /** Why the send failed. Absent on errors raised by older clients. */ + reason?: "InvalidRecord" | "SendFailed"; + /** Index of the offending record within the batch, for `InvalidRecord`. */ + index?: number; }> {} diff --git a/packages/alchemy/src/Cloudflare/Pipelines/WriteStreamBinding.ts b/packages/alchemy/src/Cloudflare/Pipelines/WriteStreamBinding.ts index e0403c9acd9..f07ba284d5b 100644 --- a/packages/alchemy/src/Cloudflare/Pipelines/WriteStreamBinding.ts +++ b/packages/alchemy/src/Cloudflare/Pipelines/WriteStreamBinding.ts @@ -4,11 +4,27 @@ import * as Layer from "effect/Layer"; import { Worker, WorkerEnvironment } from "../Workers/Worker.ts"; import { isLegacyPipeline, type LegacyPipeline } from "./LegacyPipeline.ts"; import type { Stream } from "./Stream.ts"; -import { StreamSendError, WriteStream, type WriteStreamClient } from "./WriteStream.ts"; +import { makeStreamClient, recordSchemaOf, toStreamSendError } from "./StreamCodec.ts"; +import { WriteStream, type WriteStreamClient } from "./WriteStream.ts"; /** - * Implementation of the {@link WriteStream} service that uses a native - * Worker `pipelines` binding. + * Implementation of the {@link WriteStream} service over a native Worker + * `pipelines` binding. Registers the binding on the host Worker and, when + * the stream has an Effect Schema, encodes records with it before + * `send`. + * ### Providing the Layer + * **Example:** Send from a Worker + * ```typescript + * Effect.gen(function* () { + * const views = yield* Cloudflare.Basin.WriteStream(PageViews); + * // ... + * }).pipe(Effect.provide(Cloudflare.Basin.WriteStreamBinding)); + * ``` + * + * @layer + * @provides Cloudflare.Pipelines.WriteStream + * @product Pipelines + * @category Storage & Databases */ export const WriteStreamBinding = Layer.effect( WriteStream, @@ -16,7 +32,7 @@ export const WriteStreamBinding = Layer.effect( const env = yield* WorkerEnvironment; const host = yield* Worker; - return Effect.fn(function* (stream: Stream | LegacyPipeline) { + return Effect.fn(function* (stream: Stream | LegacyPipeline) { if (!globalThis.__ALCHEMY_RUNTIME__) { yield* host.bind`${stream}`({ bindings: [ @@ -39,23 +55,20 @@ export const WriteStreamBinding = Layer.effect( /** Build the producer client over a native Worker `pipelines` binding. */ export const makeWriteStreamClient = ( env: Record, - stream: Stream | LegacyPipeline, -): WriteStreamClient => { + stream: Stream | LegacyPipeline, +): WriteStreamClient => { const raw = Effect.sync(() => (env as Record)[stream.LogicalId]!); - return { + return makeStreamClient({ + schema: recordSchemaOf(stream), raw, - send: (records) => + sendEncoded: (records) => raw.pipe( Effect.flatMap((pipeline) => Effect.tryPromise({ try: () => pipeline.send([...records]), - catch: (error: any) => - new StreamSendError({ - message: error?.message ?? "Unknown pipeline error", - cause: error, - }), + catch: toStreamSendError, }), ), ), - }; + }); }; diff --git a/packages/alchemy/src/Cloudflare/Pipelines/WriteStreamHttp.ts b/packages/alchemy/src/Cloudflare/Pipelines/WriteStreamHttp.ts new file mode 100644 index 00000000000..d1a6a20804f --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Pipelines/WriteStreamHttp.ts @@ -0,0 +1,36 @@ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { makeHttpStreamBinding } from "./StreamHttp.ts"; +import { WriteStream } from "./WriteStream.ts"; + +/** + * HTTP-backed implementation of the {@link WriteStream} service. Creates + * a scoped `AccountApiToken` with the `Pipelines Send` permission, binds + * it into the host, and POSTs records to the stream's HTTP ingest + * endpoint (`https://{stream_id}.ingest.cloudflare.com`). + * + * The stream must have its HTTP endpoint enabled — declare it with + * `http: true` (authenticated). Records are validated and encoded with + * the stream's Effect Schema exactly as with `WriteStreamBinding`. + * ### Providing the Layer + * **Example:** Send over HTTP from any host + * ```typescript + * const Events = Cloudflare.Basin.Stream("Events", { http: true }); + * + * Effect.gen(function* () { + * const events = yield* Cloudflare.Basin.WriteStream(Events); + * // ... + * }).pipe(Effect.provide(Cloudflare.Basin.WriteStreamHttp)); + * ``` + * + * @layer + * @provides Cloudflare.Pipelines.WriteStream + * @product Pipelines + * @category Storage & Databases + */ +export const WriteStreamHttp = Layer.effect( + WriteStream, + Effect.suspend(() => + makeHttpStreamBinding({ layer: "WriteStreamHttp", makeClient: (client) => client }), + ), +); diff --git a/packages/alchemy/src/Cloudflare/Pipelines/WriteStreamLocal.ts b/packages/alchemy/src/Cloudflare/Pipelines/WriteStreamLocal.ts new file mode 100644 index 00000000000..b800952290f --- /dev/null +++ b/packages/alchemy/src/Cloudflare/Pipelines/WriteStreamLocal.ts @@ -0,0 +1,40 @@ +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import { makeLocalStreamBinding } from "./StreamHttp.ts"; +import { WriteStream } from "./WriteStream.ts"; + +/** + * Local implementation of the {@link WriteStream} service — POSTs records + * to the stream's HTTP ingest endpoint with the **current credentials** + * instead of a native Worker binding (`WriteStreamBinding`) or a scoped + * API token (`WriteStreamHttp`). + * + * Provide it on an Action (or any deploy-time Effect) to send events with + * the same typed client you use inside a Worker. The stream needs its HTTP + * endpoint enabled (`http: true`), and the current credentials need the + * `Pipelines Send` permission when it is authenticated. + * ### Providing the Layer + * **Example:** Seed a stream from an Action + * ```typescript + * const Seed = Alchemy.Action( + * "Seed", + * Effect.gen(function* () { + * const views = yield* Cloudflare.Basin.WriteStream(PageViews); + * return Effect.fn(function* () { + * yield* views.send([new PageView({ url: "/", at: new Date() })]); + * }); + * }).pipe(Effect.provide(Cloudflare.Basin.WriteStreamLocal)), + * ); + * ``` + * + * @layer + * @provides Cloudflare.Pipelines.WriteStream + * @product Pipelines + * @category Storage & Databases + */ +export const WriteStreamLocal = Layer.effect( + WriteStream, + Effect.suspend(() => + makeLocalStreamBinding({ layer: "WriteStreamLocal", makeClient: (client) => client }), + ), +); diff --git a/packages/alchemy/src/Cloudflare/Pipelines/index.ts b/packages/alchemy/src/Cloudflare/Pipelines/index.ts index 898bf7aa3ef..620745c7dae 100644 --- a/packages/alchemy/src/Cloudflare/Pipelines/index.ts +++ b/packages/alchemy/src/Cloudflare/Pipelines/index.ts @@ -2,5 +2,12 @@ export * from "./LegacyPipeline.ts"; export * from "./Pipeline.ts"; export * from "./Sink.ts"; export * from "./Stream.ts"; +export * from "./StreamSchema.ts"; +export * from "./StreamSink.ts"; +export * from "./StreamSinkBinding.ts"; +export * from "./StreamSinkHttp.ts"; +export * from "./StreamSinkLocal.ts"; export * from "./WriteStream.ts"; export * from "./WriteStreamBinding.ts"; +export * from "./WriteStreamHttp.ts"; +export * from "./WriteStreamLocal.ts"; diff --git a/packages/alchemy/src/Cloudflare/Providers.ts b/packages/alchemy/src/Cloudflare/Providers.ts index 7bd2511fb62..927e16bc537 100644 --- a/packages/alchemy/src/Cloudflare/Providers.ts +++ b/packages/alchemy/src/Cloudflare/Providers.ts @@ -31,6 +31,7 @@ import * as ApiShield from "./ApiShield/index.ts"; import * as ApiToken from "./ApiToken/index.ts"; import * as Argo from "./Argo/index.ts"; import { CloudflareAuth } from "./Auth/AuthProvider.ts"; +import * as Basin from "./Basin/index.ts"; import * as BotManagement from "./BotManagement/index.ts"; import * as Cache from "./Cache/index.ts"; import * as Calls from "./Calls/index.ts"; @@ -70,6 +71,7 @@ import * as Hyperdrive from "./Hyperdrive/index.ts"; import * as Iam from "./Iam/index.ts"; import * as Images from "./Images/index.ts"; import * as Intel from "./Intel/index.ts"; +import * as K2 from "./K2/index.ts"; import * as KeylessCertificate from "./KeylessCertificate/index.ts"; import * as KV from "./KV/index.ts"; import * as LeakedCredentialCheck from "./LeakedCredentialCheck/index.ts"; @@ -176,6 +178,7 @@ export const providers = () => ApiToken.UserApiToken, Argo.SmartRouting, Argo.TieredCaching, + Basin.Table, Bookmark.Bookmark, BotManagement.BotManagement, Cache.OriginCloudRegion, @@ -255,6 +258,8 @@ export const providers = () => Images.Variant, Intel.IndicatorFeed, Intel.IndicatorFeedPermission, + K2.Stream, + K2.Subscription, KeylessCertificate.KeylessCertificate, KeyPair, KV.Namespace, @@ -511,6 +516,8 @@ export const providers = () => Images.VariantProvider(), Intel.IndicatorFeedPermissionProvider(), Intel.IndicatorFeedProvider(), + K2.StreamProvider(), + K2.SubscriptionProvider(), KeylessCertificate.KeylessCertificateProvider(), KV.NamespaceProvider(), LeakedCredentialCheck.LeakedCredentialCheckProvider(), @@ -614,6 +621,7 @@ export const providers = () => Tunnel.WarpConnectorProvider(), ), Layer.mergeAll( + Basin.TableProvider(), Turnstile.WidgetProvider(), UrlNorm.UrlNormalizationProvider(), Vectorize.IndexProvider(), diff --git a/packages/alchemy/src/Cloudflare/R2/DataCatalog.ts b/packages/alchemy/src/Cloudflare/R2/DataCatalog.ts index 221a1cc96d9..ce2b80256c5 100644 --- a/packages/alchemy/src/Cloudflare/R2/DataCatalog.ts +++ b/packages/alchemy/src/Cloudflare/R2/DataCatalog.ts @@ -153,12 +153,19 @@ export type DataCatalog = Resource< * Iceberg tables stored in R2. The catalog is a singleton per bucket: this * resource enables it, keeps its maintenance configuration in sync, and * disables it on destroy (table data in the bucket is never deleted). + * + * Also exported as `Cloudflare.Basin.Catalog` — the catalog of Cloudflare's + * Basin product (Pipelines + Catalog + SQL). The warehouse is named + * `_` and served at + * `https://catalog.cloudflarestorage.com//`. Pass the + * catalog resource to a catalog `Cloudflare.Basin.Sink` (`catalog: Catalog`) + * so the sink is created after it. * ### Enabling a catalog * **Example:** Enable the catalog on an R2 bucket * ```typescript * const bucket = yield* Cloudflare.R2.Bucket("LakehouseBucket"); * - * const catalog = yield* Cloudflare.R2.R2DataCatalog("Lakehouse", { + * const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", { * bucketName: bucket.bucketName, * }); * @@ -170,7 +177,7 @@ export type DataCatalog = Resource< * ### Maintenance * **Example:** Configure compaction and snapshot expiration * ```typescript - * const catalog = yield* Cloudflare.R2.R2DataCatalog("Lakehouse", { + * const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", { * bucketName: bucket.bucketName, * compaction: { state: "enabled", targetSizeMb: "256" }, * snapshotExpiration: { @@ -184,7 +191,7 @@ export type DataCatalog = Resource< * **Example:** Register a maintenance credential * ```typescript * // Maintenance jobs need an API token with R2 read/write on the bucket. - * const catalog = yield* Cloudflare.R2.R2DataCatalog("Lakehouse", { + * const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", { * bucketName: bucket.bucketName, * compaction: { state: "enabled" }, * token: maintenanceToken, // Redacted @@ -200,7 +207,7 @@ export type DataCatalog = Resource< export const DataCatalog = Resource(TypeId); /** - * Returns true if the given value is an R2DataCatalog resource. + * Returns true if the given value is a DataCatalog (Basin Catalog) resource. */ export const isDataCatalog = (value: unknown): value is DataCatalog => Predicate.hasProperty(value, "Type") && value.Type === TypeId; diff --git a/packages/alchemy/src/Cloudflare/Workers/InferEnv.ts b/packages/alchemy/src/Cloudflare/Workers/InferEnv.ts index bbb0036f3c6..254dd7a3fe1 100644 --- a/packages/alchemy/src/Cloudflare/Workers/InferEnv.ts +++ b/packages/alchemy/src/Cloudflare/Workers/InferEnv.ts @@ -23,6 +23,8 @@ import type * as Email from "../Email/index.ts"; import type * as FlagshipNs from "../Flagship/index.ts"; import type * as HyperdriveNs from "../Hyperdrive/index.ts"; import type * as ImagesNs from "../Images/index.ts"; +import type * as K2Ns from "../K2/index.ts"; +import type { K2StreamBinding } from "../K2/WriteStreamBinding.ts"; import type * as KV from "../KV/index.ts"; import type * as PipelinesNs from "../Pipelines/index.ts"; import type * as Queues from "../Queues/index.ts"; @@ -149,11 +151,13 @@ export type GetBindingType = | PipelinesNs.Stream | PipelinesNs.LegacyPipeline ? Pipeline - : T extends Redacted - ? // redacteds are always stored as secret_text, so are always string - // we JSON.stringify when not a Redacted - string - : T; + : T extends K2Ns.Stream + ? K2StreamBinding + : T extends Redacted + ? // redacteds are always stored as secret_text, so are always string + // we JSON.stringify when not a Redacted + string + : T; /** * Cloudflare service-binding wire shape for an Effect-native Worker. diff --git a/packages/alchemy/src/Cloudflare/Workers/RuntimeBindings.ts b/packages/alchemy/src/Cloudflare/Workers/RuntimeBindings.ts index 8a241014987..ada8074170d 100644 --- a/packages/alchemy/src/Cloudflare/Workers/RuntimeBindings.ts +++ b/packages/alchemy/src/Cloudflare/Workers/RuntimeBindings.ts @@ -116,6 +116,13 @@ export const toRuntimeBinding = Effect.fn(function* ( return yield* unsupported(); case "json": return Json.local(b.name, b.json); + case "k2": + // Miniflare has no K2 simulation and no remote proxy for it yet. + return yield* new WorkerValidationError({ + message: `K2 binding "${b.name}" is not supported in local mode: K2 has no local simulation.`, + hint: "Run K2 producers against a deployed Worker (alchemy deploy).", + value: b, + }); case "kv_namespace": // A `dev:` id belongs to a locally-emulated namespace; a real id is // a live namespace the dev worker proxies to. diff --git a/packages/alchemy/src/Cloudflare/Workers/WorkerAsyncBindings.ts b/packages/alchemy/src/Cloudflare/Workers/WorkerAsyncBindings.ts index 62cfd752f5a..6c7e2ec0fa7 100644 --- a/packages/alchemy/src/Cloudflare/Workers/WorkerAsyncBindings.ts +++ b/packages/alchemy/src/Cloudflare/Workers/WorkerAsyncBindings.ts @@ -25,6 +25,7 @@ import { isApp } from "../Flagship/App.ts"; import { getHyperdriveDevOrigin } from "../Hyperdrive/ConnectBinding.ts"; import { isHyperdriveConnection } from "../Hyperdrive/Connection.ts"; import { isImages } from "../Images/Images.ts"; +import { isStream as isK2Stream } from "../K2/Stream.ts"; import { isNamespace as isKVNamespace } from "../KV/Namespace.ts"; import { isLegacyPipeline } from "../Pipelines/LegacyPipeline.ts"; import { isStream as isPipelinesStream } from "../Pipelines/Stream.ts"; @@ -660,6 +661,12 @@ const toBinding = ( name: bindingName, pipeline: binding.name, }; + } else if (isK2Stream(binding)) { + return { + type: "k2", + name: bindingName, + stream: binding.streamId, + }; } else if (Output.isOutput(binding)) { return Output.map(binding, (value: Json | Redacted.Redacted | VpcServiceLookup) => // A `VpcService.lookup(...)` data source resolves to the service's diff --git a/packages/alchemy/src/Cloudflare/Workers/WorkerBinding.ts b/packages/alchemy/src/Cloudflare/Workers/WorkerBinding.ts index e1362a5fcd5..e05b9c16c5e 100644 --- a/packages/alchemy/src/Cloudflare/Workers/WorkerBinding.ts +++ b/packages/alchemy/src/Cloudflare/Workers/WorkerBinding.ts @@ -17,6 +17,7 @@ import { SendEmail } from "../Email/SendEmail.ts"; import type { App as FlagshipApp } from "../Flagship/App.ts"; import type { Connection as Hyperdrive } from "../Hyperdrive/Connection.ts"; import type { ImagesBinding } from "../Images/ImagesBinding.ts"; +import type { Stream as K2Stream } from "../K2/Stream.ts"; import type { Namespace } from "../KV/Namespace.ts"; import type { LegacyPipeline } from "../Pipelines/LegacyPipeline.ts"; import type { Stream as PipelinesStream } from "../Pipelines/Stream.ts"; @@ -128,6 +129,12 @@ export type QueueWorkerBinding = Extract; + /** * The `service` metadata binding extended with workerd's `ctx.props`. * `props` is what a `Cloudflare.WorkerEntrypoint(worker, { props })` env @@ -214,6 +221,7 @@ export type WorkerBindingResource = | StreamBinding | PipelinesStream | LegacyPipeline + | K2Stream | Hyperdrive | VectorizeIndex | Secret diff --git a/packages/alchemy/src/Cloudflare/index.ts b/packages/alchemy/src/Cloudflare/index.ts index c8699d4dd16..f5d0abc91a5 100644 --- a/packages/alchemy/src/Cloudflare/index.ts +++ b/packages/alchemy/src/Cloudflare/index.ts @@ -18,6 +18,7 @@ export * as ApiShield from "./ApiShield/index.ts"; export * as ApiToken from "./ApiToken/index.ts"; export * as Argo from "./Argo/index.ts"; export * as Artifacts from "./Artifacts/index.ts"; +export * as Basin from "./Basin/index.ts"; export * as BotManagement from "./BotManagement/index.ts"; export * as Cache from "./Cache/index.ts"; export * as Calls from "./Calls/index.ts"; @@ -49,6 +50,7 @@ export * as Hyperdrive from "./Hyperdrive/index.ts"; export * as Iam from "./Iam/index.ts"; export * as Images from "./Images/index.ts"; export * as Intel from "./Intel/index.ts"; +export * as K2 from "./K2/index.ts"; export * as KeylessCertificate from "./KeylessCertificate/index.ts"; export * as KV from "./KV/index.ts"; export * as LeakedCredentialCheck from "./LeakedCredentialCheck/index.ts"; diff --git a/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts b/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts new file mode 100644 index 00000000000..1715ce2fa5c --- /dev/null +++ b/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts @@ -0,0 +1,99 @@ +import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; +import { expect } from "alchemy-test"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import { MinimumLogLevel } from "effect/References"; +import * as Schedule from "effect/Schedule"; +import * as Cloudflare from "@/Cloudflare"; +import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; +import * as Output from "@/Output"; +import * as Test from "@/Test/Alchemy"; +import { apiToken, PageView } from "./fixtures/pipelines-shared.ts"; + +const { test } = Test.make({ providers: Cloudflare.providers() }); + +const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); + +const getPipeline = (accountId: string, pipelineId: string) => + pipelines.getV1Pipeline({ accountId, pipelineId }).pipe( + Effect.retry({ + while: (e) => e._tag === "Forbidden", + schedule: Schedule.exponential("500 millis"), + times: 8, + }), + ); + +/** + * The whole Basin topology in one deploy: a typed stream, a catalog sink + * that takes the Catalog resource (so it is created after the catalog), + * and the SQL pipeline between them. + */ +const lakehouse = (sql: (stream: string, sink: string) => string) => + Effect.gen(function* () { + const token = yield* apiToken; + const bucket = yield* Cloudflare.R2.Bucket("LakeBucket", { forceDestroy: true }); + const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { + bucketName: bucket.bucketName, + }); + const stream = yield* Cloudflare.Basin.Stream("PageViews", { schema: PageView }); + const sink = yield* Cloudflare.Basin.Sink("PageViewTable", { + type: "basin_catalog", + catalog, + table: { namespace: "web", name: "page_views" }, + token, + }); + const pipeline = yield* Cloudflare.Basin.Pipeline("Etl", { + sql: Output.all(stream.name, sink.name).pipe( + Output.map(([streamName, sinkName]) => sql(streamName, sinkName)), + ), + }); + return { stream, sink, pipeline }; + }); + +const passthrough = (stream: string, sink: string) => `INSERT INTO ${sink} SELECT * FROM ${stream}`; + +test.provider( + "SQL is validated during plan; whitespace-only changes are a no-op", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + yield* stack.destroy(); + + const deployed = yield* stack.deploy(lakehouse(passthrough)); + expect(deployed.sink.type).toEqual("basin_catalog"); + expect(deployed.pipeline.pipelineId).toBeTruthy(); + + // Reformatted SQL is the same statement. + const reformatted = yield* stack.plan( + lakehouse((stream, sink) => ` INSERT INTO ${sink}\n SELECT *\n FROM ${stream}; `), + ); + expect(reformatted.resources.Etl!.action).toEqual("noop"); + + // Invalid SQL fails the plan — before the running pipeline is deleted. + const invalid = yield* stack + .plan(lakehouse((stream, sink) => `INSERT INTO ${sink} SELEC * FROM ${stream}`)) + .pipe(Effect.exit); + expect(Exit.isFailure(invalid)).toBe(true); + expect(String(Exit.isFailure(invalid) ? invalid.cause : "")).toContain("invalid SQL"); + + // A valid change is a replacement. + const filtered = yield* stack.plan( + lakehouse((stream, sink) => `INSERT INTO ${sink} SELECT * FROM ${stream} WHERE url != ''`), + ); + expect(filtered.resources.Etl!.action).toEqual("replace"); + + const live = yield* getPipeline(accountId, deployed.pipeline.pipelineId); + expect(live.id).toEqual(deployed.pipeline.pipelineId); + + yield* stack.destroy(); + }).pipe(logLevel), + { + tags: [ + "provider:cloudflare", + "provider:cloudflare:pipelines", + "provider:cloudflare:r2", + "live", + ], + timeout: 240_000, + }, +); diff --git a/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts b/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts new file mode 100644 index 00000000000..1467a72fc4a --- /dev/null +++ b/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts @@ -0,0 +1,163 @@ +import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; +import { expect } from "alchemy-test"; +import * as Effect from "effect/Effect"; +import * as Redacted from "effect/Redacted"; +import { MinimumLogLevel } from "effect/References"; +import * as Schedule from "effect/Schedule"; +import * as Cloudflare from "@/Cloudflare"; +import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; +import * as Test from "@/Test/Alchemy"; +import { apiToken, r2Credentials } from "./fixtures/pipelines-shared.ts"; + +const { test } = Test.make({ providers: Cloudflare.providers() }); + +const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); + +const getSink = (accountId: string, sinkId: string) => + pipelines.getSink({ accountId, sinkId }).pipe( + Effect.retry({ + while: (e) => e._tag === "Forbidden", + schedule: Schedule.exponential("500 millis"), + times: 8, + }), + ); + +const expectSinkGone = (accountId: string, sinkId: string) => + getSink(accountId, sinkId).pipe( + Effect.flatMap(() => Effect.fail({ _tag: "SinkNotDeleted" } as const)), + Effect.catchTag("SinkNotFound", () => Effect.void), + Effect.retry({ + while: (e) => e._tag === "SinkNotDeleted", + schedule: Schedule.max([Schedule.exponential("500 millis"), Schedule.recurs(10)]), + }), + ); + +const tags = [ + "provider:cloudflare", + "provider:cloudflare:pipelines", + "provider:cloudflare:r2", + "live", +]; + +const rotated = { + accessKeyId: Redacted.make("rotated-access-key-id"), + secretAccessKey: Redacted.make("rotated-secret-access-key"), +}; + +test.provider( + "r2 sink: credential rotation and equivalent defaults never replace", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + const credentials = yield* r2Credentials; + yield* stack.destroy(); + + const sink = (opts: { + credentials: typeof credentials; + explicitDefaults?: boolean; + path?: string; + }) => + Effect.gen(function* () { + const bucket = yield* Cloudflare.R2.Bucket("SinkBucket", { forceDestroy: true }); + return yield* Cloudflare.Basin.Sink("Sink", { + type: "r2", + format: opts.explicitDefaults ? { type: "json" } : undefined, + config: { + bucket: bucket.bucketName, + credentials: opts.credentials, + path: opts.path, + rollingPolicy: opts.explicitDefaults ? { intervalSeconds: 300 } : undefined, + }, + }); + }); + + const initial = yield* stack.deploy(sink({ credentials })); + + // Rotated credentials are write-only and cannot be updated: no replace. + const rotatePlan = yield* stack.plan(sink({ credentials: rotated })); + expect(rotatePlan.resources.Sink!.action).toEqual("noop"); + + // Spelling out the defaults is equivalent. + const defaultsPlan = yield* stack.plan(sink({ credentials, explicitDefaults: true })); + expect(defaultsPlan.resources.Sink!.action).toEqual("noop"); + + // A destination change still replaces. + const pathPlan = yield* stack.plan(sink({ credentials, path: "moved" })); + expect(pathPlan.resources.Sink!.action).toEqual("replace"); + + const after = yield* stack.deploy(sink({ credentials: rotated })); + expect(after.sinkId).toEqual(initial.sinkId); + + yield* stack.destroy(); + yield* expectSinkGone(accountId, initial.sinkId); + }).pipe(logLevel), + { tags, timeout: 240_000 }, +); + +test.provider( + "catalog sink: config form ⇄ catalog form and r2_data_catalog ⇄ basin_catalog are no-ops", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + const token = yield* apiToken; + yield* stack.destroy(); + + const lakehouse = Effect.gen(function* () { + const bucket = yield* Cloudflare.R2.Bucket("LakeBucket", { forceDestroy: true }); + const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { + bucketName: bucket.bucketName, + }); + return { bucket, catalog }; + }); + + // The catalog's attributes are all stable, so the old bucket-name + // spelling does not wait for it — enable it in a first deploy. + yield* stack.deploy(lakehouse); + + const initial = yield* stack.deploy( + Effect.gen(function* () { + // The old spelling. + const { catalog } = yield* lakehouse; + return yield* Cloudflare.Pipelines.Sink("Table", { + type: "r2_data_catalog", + config: { + bucket: catalog.bucketName, + namespace: "web", + tableName: "page_views", + token, + }, + }); + }), + ); + expect(initial.type).toEqual("r2_data_catalog"); + expect(initial.tableName).toEqual("page_views"); + const live = yield* getSink(accountId, initial.sinkId); + expect(live.config?.bucket).toEqual(initial.bucket); + + const catalogForm = (sinkToken: Redacted.Redacted) => + Effect.gen(function* () { + const { catalog } = yield* lakehouse; + return yield* Cloudflare.Basin.Sink("Table", { + type: "basin_catalog", + catalog, + table: { namespace: "web", name: "page_views" }, + token: sinkToken, + }); + }); + + const plan = yield* stack.plan(catalogForm(token)); + expect(plan.resources.Table!.action).toEqual("noop"); + + // A rotated token never replaces (the table already exists, so a + // catalog sink could not be recreated on it anyway). + const rotatePlan = yield* stack.plan(catalogForm(Redacted.make("rotated-token"))); + expect(rotatePlan.resources.Table!.action).toEqual("noop"); + + const switched = yield* stack.deploy(catalogForm(token)); + expect(switched.sinkId).toEqual(initial.sinkId); + + yield* stack.destroy(); + yield* expectSinkGone(accountId, initial.sinkId); + }).pipe(logLevel), + { tags, timeout: 240_000 }, +); diff --git a/packages/alchemy/test/Cloudflare/Basin/Stream.test.ts b/packages/alchemy/test/Cloudflare/Basin/Stream.test.ts new file mode 100644 index 00000000000..315cfa9614a --- /dev/null +++ b/packages/alchemy/test/Cloudflare/Basin/Stream.test.ts @@ -0,0 +1,123 @@ +import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; +import { expect } from "alchemy-test"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import { MinimumLogLevel } from "effect/References"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; +import * as Cloudflare from "@/Cloudflare"; +import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; +import * as Test from "@/Test/Alchemy"; +import { PageView, pageViewFields } from "./fixtures/pipelines-shared.ts"; + +const { test } = Test.make({ providers: Cloudflare.providers() }); + +const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); + +const getStream = (accountId: string, streamId: string) => + pipelines.getStream({ accountId, streamId }).pipe( + Effect.retry({ + while: (e) => e._tag === "Forbidden", + schedule: Schedule.exponential("500 millis"), + times: 8, + }), + ); + +const tags = ["provider:cloudflare", "provider:cloudflare:pipelines", "live"]; + +test.provider( + "http defaults to disabled and is applied in place to a public stream", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + yield* stack.destroy(); + + // The pre-upgrade shape: a public, unauthenticated endpoint. + const open = yield* stack.deploy( + Cloudflare.Basin.Stream("Stream", { http: { enabled: true } }), + ); + expect(open.httpEnabled).toBe(true); + expect(open.httpAuthentication).toBe(false); + + // Dropping `http` closes the endpoint — an in-place update. + const plan = yield* stack.plan(Cloudflare.Basin.Stream("Stream", {})); + expect(plan.resources.Stream!.action).toEqual("update"); + + const closed = yield* stack.deploy(Cloudflare.Basin.Stream("Stream", {})); + expect(closed.streamId).toEqual(open.streamId); + expect(closed.httpEnabled).toBe(false); + const live = yield* getStream(accountId, closed.streamId); + expect(live.http.enabled).toBe(false); + + // Converged — re-planning the same props is a no-op. + const again = yield* stack.plan(Cloudflare.Basin.Stream("Stream", {})); + expect(again.resources.Stream!.action).toEqual("noop"); + + // `http: true` re-enables the endpoint with authentication. + const authed = yield* stack.deploy(Cloudflare.Basin.Stream("Stream", { http: true })); + expect(authed.streamId).toEqual(open.streamId); + const liveAuthed = yield* getStream(accountId, authed.streamId); + expect(liveAuthed.http.enabled).toBe(true); + expect(liveAuthed.http.authentication).toBe(true); + + yield* stack.destroy(); + }).pipe(logLevel), + { tags, timeout: 180_000 }, +); + +test.provider( + "list/struct schema; field list ⇄ Effect Schema is a no-op", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + yield* stack.destroy(); + + const initial = yield* stack.deploy( + Cloudflare.Basin.Stream("Typed", { schema: { fields: pageViewFields } }), + ); + const live = yield* getStream(accountId, initial.streamId); + expect(live.schema?.fields?.map((f) => [f.name, f.type])).toEqual([ + ["url", "string"], + ["at", "timestamp"], + ["tags", "list"], + ["user", "struct"], + ]); + + // The equivalent Effect Schema derives the same field list. + const plan = yield* stack.plan(Cloudflare.Basin.Stream("Typed", { schema: PageView })); + expect(plan.resources.Typed!.action).toEqual("noop"); + + const switched = yield* stack.deploy(Cloudflare.Basin.Stream("Typed", { schema: PageView })); + expect(switched.streamId).toEqual(initial.streamId); + + // A real schema change still replaces. + const changed = yield* stack.plan( + Cloudflare.Basin.Stream("Typed", { + schema: Schema.Struct({ url: Schema.String }), + }), + ); + expect(changed.resources.Typed!.action).toEqual("replace"); + + yield* stack.destroy(); + }).pipe(logLevel), + { tags, timeout: 180_000 }, +); + +test.provider( + "an Effect Schema with no stream representation fails before planning", + (stack) => + Effect.gen(function* () { + const exit = yield* stack + .plan( + Cloudflare.Basin.Stream("Bad", { + schema: Schema.Struct({ value: Schema.Union([Schema.String, Schema.Number]) }), + }), + ) + .pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(String(Exit.isFailure(exit) ? exit.cause : "")).toContain( + "field 'value' is a union of different types", + ); + }).pipe(logLevel), + { tags: ["provider:cloudflare", "provider:cloudflare:pipelines"], timeout: 60_000 }, +); diff --git a/packages/alchemy/test/Cloudflare/Basin/Table.test.ts b/packages/alchemy/test/Cloudflare/Basin/Table.test.ts new file mode 100644 index 00000000000..76ed97c4c0a --- /dev/null +++ b/packages/alchemy/test/Cloudflare/Basin/Table.test.ts @@ -0,0 +1,337 @@ +import * as basin from "@distilled.cloud/cloudflare/basin_catalog"; +import * as Iceberg from "@distilled.cloud/iceberg"; +import { expect } from "alchemy-test"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import type * as Redacted from "effect/Redacted"; +import { MinimumLogLevel } from "effect/References"; +import * as Schedule from "effect/Schedule"; +import * as S from "effect/Schema"; +import * as Cloudflare from "@/Cloudflare"; +import * as Basin from "@/Cloudflare/Basin/index.ts"; +import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; +import * as Test from "@/Test/Alchemy"; + +const { test } = Test.make({ providers: Cloudflare.providers() }); + +const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); + +const tags = ["provider:cloudflare", "provider:cloudflare:r2", "live"]; + +const OrderV1 = S.Struct({ + id: S.String, + amount: S.Number, + quantity: S.Int, + at: S.Date, + note: S.optional(S.String), +}); + +const OrderV2 = S.Struct({ + ...OrderV1.fields, + coupon: S.optional(S.String), + lines: S.optional(S.Array(S.Struct({ sku: S.String, qty: S.Int }))), +}); + +// Drops `amount` and `quantity`. +const OrderDestructive = S.Struct({ + id: S.String, + at: S.Date, + note: S.optional(S.String), +}); + +type TableOpts = Omit & { + schema?: Basin.TableSchemaInput; +}; + +const apiToken = Effect.gen(function* () { + const creds = yield* yield* CloudflareEnvironment; + if (creds.type !== "apiToken") { + return yield* Effect.die(new Error("Basin Table tests require an API token profile")); + } + return creds.apiToken; +}); + +// Bucket + catalog + any number of tables, one stack program. +const program = ( + token: Redacted.Redacted, + tables: Record, + catalogMaintenance: Pick< + Cloudflare.R2.DataCatalogProps, + "compaction" | "snapshotExpiration" + > = {}, +) => + Effect.gen(function* () { + const bucket = yield* Cloudflare.R2.Bucket("TableBucket", { forceDestroy: true }); + const catalog = yield* Cloudflare.R2.DataCatalog("TableCatalog", { + bucketName: bucket.bucketName, + token, + ...catalogMaintenance, + }); + const out: Record = {}; + for (const [id, opts] of Object.entries(tables)) { + out[id] = yield* Basin.Table(id, { + catalog, + namespace: "sales", + schema: OrderV1, + ...opts, + }); + } + return { bucket, catalog, tables: out }; + }); + +// Out-of-band Iceberg client scoped to the deployed warehouse. +const iceberg = + (catalog: { catalogUri: string; name: string }, token: Redacted.Redacted) => + (eff: Effect.Effect) => + eff.pipe( + Effect.provide( + Layer.mergeAll( + Iceberg.IcebergProtocol, + Iceberg.fromCatalogConfig({ uri: catalog.catalogUri, warehouse: catalog.name, token }), + ), + ), + ); + +const currentFields = (metadata: Iceberg.TableMetadata) => + (metadata.schemas ?? []).find((s) => s.schema_id === metadata.current_schema_id)?.fields ?? []; + +const exists = (namespace: string, table: string) => + Iceberg.tableExists({ namespace, table }).pipe( + Effect.as(true), + Effect.catchTag("NotFound", () => Effect.succeed(false)), + ); + +test.provider( + "create, evolve the schema in place, sync properties, reject destructive changes", + (stack) => + Effect.gen(function* () { + yield* stack.destroy(); + const token = yield* apiToken; + + const v1 = yield* stack.deploy( + program(token, { + Orders: { + partitionBy: ["day(at)"], + properties: { "alchemy.test.a": "1", "alchemy.test.b": "2" }, + }, + }), + ); + const orders = v1.tables.Orders!; + expect(orders.namespace).toEqual("sales"); + expect(orders.identifier).toEqual(`sales.${orders.tableName}`); + expect(orders.tableName).toMatch(/^[a-z0-9_]+$/); + expect(orders.warehouse).toEqual(v1.catalog.name); + expect(orders.catalogUri).toEqual(v1.catalog.catalogUri); + + const withCatalog = iceberg(v1.catalog, token); + const live1 = yield* Iceberg.loadTable({ + namespace: "sales", + table: orders.tableName, + }).pipe(withCatalog); + expect(live1.metadata.table_uuid).toEqual(orders.tableUuid); + const fields1 = currentFields(live1.metadata); + expect(fields1.map((f) => f.name)).toEqual(["id", "amount", "quantity", "at", "note"]); + expect(fields1.find((f) => f.name === "at")?.type).toEqual("timestamptz"); + expect(fields1.find((f) => f.name === "quantity")?.type).toEqual("long"); + expect(fields1.find((f) => f.name === "note")?.required).toBe(false); + const spec = live1.metadata.partition_specs?.find( + (s) => s.spec_id === live1.metadata.default_spec_id, + ); + expect(spec?.fields.map((f) => f.transform)).toEqual(["day"]); + expect(live1.metadata.properties?.["alchemy.test.a"]).toEqual("1"); + expect(live1.metadata.properties?.["alchemy.test.b"]).toEqual("2"); + expect(live1.metadata.properties?.["alchemy::id"]).toEqual("Orders"); + + // Additive evolution + property sync: same table, new schema. + const v2 = yield* stack.deploy( + program(token, { + Orders: { + schema: OrderV2, + partitionBy: ["day(at)"], + properties: { "alchemy.test.a": "10" }, + }, + }), + ); + const orders2 = v2.tables.Orders!; + expect(orders2.tableUuid).toEqual(orders.tableUuid); + expect(orders2.currentSchemaId).not.toEqual(orders.currentSchemaId); + + const live2 = yield* Iceberg.loadTable({ + namespace: "sales", + table: orders.tableName, + }).pipe(withCatalog); + expect(live2.metadata.table_uuid).toEqual(orders.tableUuid); + const fields2 = currentFields(live2.metadata); + expect(fields2.map((f) => f.name)).toEqual([ + "id", + "amount", + "quantity", + "at", + "note", + "coupon", + "lines", + ]); + // Existing columns keep their field ids. + for (const f of fields1) { + expect(fields2.find((g) => g.name === f.name)?.id).toEqual(f.id); + } + expect(fields2.find((f) => f.name === "coupon")?.required).toBe(false); + expect(live2.metadata.properties?.["alchemy.test.a"]).toEqual("10"); + expect(live2.metadata.properties?.["alchemy.test.b"]).toBeUndefined(); + + // Re-deploying the same program is a no-op commit. + const v2b = yield* stack.deploy( + program(token, { + Orders: { + schema: OrderV2, + partitionBy: ["day(at)"], + properties: { "alchemy.test.a": "10" }, + }, + }), + ); + expect(v2b.tables.Orders!.metadataLocation).toEqual(orders2.metadataLocation); + + // Destructive schema change fails the plan. + const destructive = yield* stack + .deploy( + program(token, { + Orders: { schema: OrderDestructive, partitionBy: ["day(at)"] }, + }), + ) + .pipe(Effect.flip); + expect(destructive).toBeInstanceOf(Basin.BasinTableChangeRejected); + expect((destructive as Basin.BasinTableChangeRejected).reasons).toContain( + "drops column 'amount'", + ); + + // Partition specs are immutable. + const repartition = yield* stack + .deploy( + program(token, { + Orders: { schema: OrderV2, partitionBy: ["month(at)"] }, + }), + ) + .pipe(Effect.flip); + expect(repartition).toBeInstanceOf(Basin.BasinTableChangeRejected); + + // The table is untouched by the rejected plans. + const live3 = yield* Iceberg.loadTable({ + namespace: "sales", + table: orders.tableName, + }).pipe(withCatalog); + expect(live3.metadata.metadata_log?.length).toEqual(live2.metadata.metadata_log?.length); + expect(currentFields(live3.metadata).map((f) => f.name)).toContain("amount"); + + yield* stack.destroy(); + }).pipe(logLevel), + { tags, timeout: 120_000 }, +); + +// Table-level maintenance requires the job to be enabled on the catalog. +const catalogMaintenance = { + compaction: { state: "enabled" }, + snapshotExpiration: { state: "enabled" }, +} as const; + +test.provider( + "syncs per-table maintenance", + (stack) => + Effect.gen(function* () { + yield* stack.destroy(); + const token = yield* apiToken; + + const v1 = yield* stack.deploy( + program( + token, + { + Maintained: { + maintenance: { + compaction: { targetSizeMb: "256" }, + snapshotExpiration: { olderThan: "3 days", retainLast: 5 }, + }, + }, + }, + catalogMaintenance, + ), + ); + const table = v1.tables.Maintained!; + const key = { + accountId: table.accountId, + bucketName: table.bucketName, + namespace: table.namespace, + tableName: table.tableName, + }; + const live1 = yield* basin.getNamespacesTablesMaintenanceConfig(key); + expect(live1.maintenanceConfig.compaction?.state).toEqual("enabled"); + expect(live1.maintenanceConfig.compaction?.targetSizeMb).toEqual("256"); + expect(live1.maintenanceConfig.snapshotExpiration?.state).toEqual("enabled"); + expect(live1.maintenanceConfig.snapshotExpiration?.maxSnapshotAge).toEqual("3d"); + expect(live1.maintenanceConfig.snapshotExpiration?.minSnapshotsToKeep).toEqual(5); + + yield* stack.deploy( + program( + token, + { + Maintained: { + maintenance: { + compaction: { enabled: false }, + snapshotExpiration: { olderThan: "36 hours", retainLast: 7 }, + }, + }, + }, + catalogMaintenance, + ), + ); + const live2 = yield* basin.getNamespacesTablesMaintenanceConfig(key); + expect(live2.maintenanceConfig.compaction?.state).toEqual("disabled"); + expect(live2.maintenanceConfig.snapshotExpiration?.maxSnapshotAge).toEqual("36h"); + expect(live2.maintenanceConfig.snapshotExpiration?.minSnapshotsToKeep).toEqual(7); + + yield* stack.destroy(); + }).pipe(logLevel), + { tags, timeout: 120_000 }, +); + +test.provider( + "delete policies: retain leaves the table, drop and purge remove it", + (stack) => + Effect.gen(function* () { + yield* stack.destroy(); + const token = yield* apiToken; + + const deployed = yield* stack.deploy( + program(token, { + Retained: {}, + Dropped: { delete: "drop" }, + Purged: { delete: "purge" }, + }), + ); + const withCatalog = iceberg(deployed.catalog, token); + const { Retained, Dropped, Purged } = deployed.tables; + for (const t of [Retained!, Dropped!, Purged!]) { + expect(yield* exists("sales", t.tableName).pipe(withCatalog)).toBe(true); + } + + // Remove the tables from the stack; keep the bucket and catalog. + yield* stack.deploy(program(token, {})); + + // Catalog deletes are eventually consistent — poll briefly. + const settled = (table: string, expected: boolean) => + exists("sales", table).pipe( + withCatalog, + Effect.repeat({ + schedule: Schedule.spaced("1 second"), + until: (present) => present === expected, + times: 10, + }), + ); + expect(yield* settled(Retained!.tableName, true)).toBe(true); + expect(yield* settled(Dropped!.tableName, false)).toBe(false); + expect(yield* settled(Purged!.tableName, false)).toBe(false); + + // Destroy disables the catalog and empties + deletes the bucket, + // taking the retained table with it. + yield* stack.destroy(); + }).pipe(logLevel), + { tags, timeout: 120_000 }, +); diff --git a/packages/alchemy/test/Cloudflare/Basin/WriteStream.test.ts b/packages/alchemy/test/Cloudflare/Basin/WriteStream.test.ts new file mode 100644 index 00000000000..6795950641e --- /dev/null +++ b/packages/alchemy/test/Cloudflare/Basin/WriteStream.test.ts @@ -0,0 +1,160 @@ +import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; +import { expect } from "alchemy-test"; +import * as Data from "effect/Data"; +import * as Effect from "effect/Effect"; +import * as HttpClient from "effect/http/HttpClient"; +import { MinimumLogLevel } from "effect/References"; +import * as Schedule from "effect/Schedule"; +import * as EffectStream from "effect/Stream"; +import { Action } from "@/Action"; +import * as Cloudflare from "@/Cloudflare"; +import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; +import * as Test from "@/Test/Alchemy"; +import PipelinesHttpWorker from "./fixtures/pipelines-http-worker.ts"; +import { PageView, PageViews } from "./fixtures/pipelines-stream.ts"; +import PipelinesWorker from "./fixtures/pipelines-worker.ts"; + +const { test } = Test.make({ providers: Cloudflare.providers() }); + +const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); + +class WorkerNotReady extends Data.TaggedError("WorkerNotReady")<{ + status: number; + body: string; +}> {} + +// A fresh workers.dev URL takes a few seconds to serve; retry until 200. +const getJson = (url: string) => + HttpClient.get(url).pipe( + Effect.flatMap((res) => + res.status === 200 + ? res.json + : res.text.pipe( + Effect.flatMap((body) => Effect.fail(new WorkerNotReady({ status: res.status, body }))), + ), + ), + Effect.retry({ + while: (e): e is WorkerNotReady => e instanceof WorkerNotReady, + schedule: Schedule.max([ + Schedule.min([Schedule.exponential("500 millis"), Schedule.spaced("5 seconds")]), + Schedule.recurs(20), + ]), + }), + ); + +// A just-created stream's ingest host can take a moment to accept writes. +const ingestRetry = { + while: (e: { _tag: string; reason?: string }) => + e._tag === "StreamSendError" && e.reason === "SendFailed", + schedule: Schedule.exponential("1 second"), + times: 6, +} as const; + +test.provider( + "typed WriteStream and StreamSink: native binding (Worker) and HTTP ingest (Action)", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + yield* stack.destroy(); + + const deployed = yield* stack.deploy( + Effect.gen(function* () { + const stream = yield* PageViews; + const worker = yield* PipelinesWorker; + + // WriteStreamLocal / StreamSinkLocal POST to the stream's HTTP + // ingest endpoint (`http: true`) with the current credentials. + const Seed = Action( + "Seed", + Effect.gen(function* () { + const views = yield* Cloudflare.Basin.WriteStream(stream); + const sink = yield* Cloudflare.Basin.StreamSink(stream); + return Effect.fn(function* () { + yield* views + .send([new PageView({ url: "/seed", at: new Date(), tags: ["seed"] })]) + .pipe(Effect.retry(ingestRetry)); + yield* EffectStream.range(1, 50).pipe( + EffectStream.map( + (i) => new PageView({ url: `/seed/${i}`, at: new Date(), tags: [] }), + ), + EffectStream.run(sink), + Effect.retry(ingestRetry), + ); + const invalid = yield* views + .send([{ url: "/x", at: "nope", tags: [] } as unknown as PageView]) + .pipe( + Effect.as("none"), + Effect.catchTag("StreamSendError", (e) => + Effect.succeed(e.reason ?? "unknown"), + ), + ); + return { invalid }; + }); + }).pipe( + Effect.provide(Cloudflare.Basin.WriteStreamLocal), + Effect.provide(Cloudflare.Basin.StreamSinkLocal), + ), + ); + const seeded = yield* Seed({}); + + return { + streamId: stream.streamId, + url: worker.url.as(), + seeded, + }; + }), + ); + + expect(deployed.seeded).toEqual({ invalid: "InvalidRecord" }); + + const live = yield* pipelines.getStream({ accountId, streamId: deployed.streamId }); + expect(live.http.enabled).toBe(true); + expect(live.http.authentication).toBe(true); + + expect(yield* getJson(`${deployed.url}/send`)).toEqual({ sent: 2 }); + expect(yield* getJson(`${deployed.url}/sink?count=300`)).toEqual({ sent: 300 }); + expect(yield* getJson(`${deployed.url}/invalid`)).toEqual({ + reason: "InvalidRecord", + index: 0, + }); + + yield* stack.destroy(); + }).pipe(logLevel), + { + tags: [ + "provider:cloudflare", + "provider:cloudflare:pipelines", + "provider:cloudflare:worker", + "live", + ], + timeout: 240_000, + }, +); + +// WriteStreamHttp/StreamSinkHttp mint a scoped `Pipelines Send` API token; +// gate behind an env var for accounts whose credentials can mint tokens. +test.provider.skipIf(!process.env.CLOUDFLARE_TEST_PIPELINES_HTTP)( + "WriteStreamHttp and StreamSinkHttp send through a scoped token", + (stack) => + Effect.gen(function* () { + yield* stack.destroy(); + const { url } = yield* stack.deploy( + Effect.gen(function* () { + yield* PageViews; + const worker = yield* PipelinesHttpWorker; + return { url: worker.url.as() }; + }), + ); + expect(yield* getJson(`${url}/send`)).toEqual({ sent: 21 }); + yield* stack.destroy(); + }).pipe(logLevel), + { + tags: [ + "provider:cloudflare", + "provider:cloudflare:pipelines", + "provider:cloudflare:worker", + "live", + ], + timeout: 240_000, + }, +); diff --git a/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-http-worker.ts b/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-http-worker.ts new file mode 100644 index 00000000000..c566b6b8545 --- /dev/null +++ b/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-http-worker.ts @@ -0,0 +1,43 @@ +import * as Effect from "effect/Effect"; +import { HttpServerRequest } from "effect/http/HttpServerRequest"; +import * as HttpServerResponse from "effect/http/HttpServerResponse"; +import * as Layer from "effect/Layer"; +import * as EffectStream from "effect/Stream"; +import * as Cloudflare from "@/Cloudflare"; +import { PageView, PageViews } from "./pipelines-stream.ts"; + +/** + * Worker sending through the stream's HTTP ingest endpoint with a scoped + * `Pipelines Send` token (`WriteStreamHttp` / `StreamSinkHttp`). + */ +export default class PipelinesHttpWorker extends Cloudflare.Worker()( + "PipelinesHttpWorker", + { main: import.meta.url }, + Effect.gen(function* () { + const views = yield* Cloudflare.Basin.WriteStream(PageViews); + const sink = yield* Cloudflare.Basin.StreamSink(PageViews); + + return { + fetch: Effect.gen(function* () { + const request = yield* HttpServerRequest; + const url = new URL(request.originalUrl, "http://x"); + if (url.pathname === "/send") { + yield* views + .send([new PageView({ url: "/http", at: new Date(), tags: [] })]) + .pipe(Effect.orDie); + yield* EffectStream.range(1, 20).pipe( + EffectStream.map((i) => new PageView({ url: `/http/${i}`, at: new Date(), tags: [] })), + EffectStream.run(sink), + Effect.orDie, + ); + return yield* HttpServerResponse.json({ sent: 21 }); + } + return yield* HttpServerResponse.json({ ok: true }); + }), + }; + }).pipe( + Effect.provide( + Layer.mergeAll(Cloudflare.Basin.WriteStreamHttp, Cloudflare.Basin.StreamSinkHttp), + ), + ), +) {} diff --git a/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-shared.ts b/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-shared.ts new file mode 100644 index 00000000000..cbb577ae800 --- /dev/null +++ b/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-shared.ts @@ -0,0 +1,44 @@ +import crypto from "node:crypto"; +import * as accounts from "@distilled.cloud/cloudflare/accounts"; +import * as user from "@distilled.cloud/cloudflare/user"; +import * as Effect from "effect/Effect"; +import * as Redacted from "effect/Redacted"; +import * as Schedule from "effect/Schedule"; +import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; + +export { PageView, PageViews, pageViewFields } from "./pipelines-stream.ts"; + +/** The active API token (live tests run with an apiToken profile). */ +export const apiToken = Effect.gen(function* () { + const creds = yield* yield* CloudflareEnvironment; + if (creds.type !== "apiToken") { + return yield* Effect.die(new Error("Basin Pipelines tests require an apiToken profile")); + } + return creds.apiToken; +}); + +/** + * R2 S3-compatible credentials derived from the active API token: the + * access key id is the token id, the secret the SHA-256 hex of its value. + */ +export const r2Credentials = Effect.gen(function* () { + const token = yield* apiToken; + const { accountId } = yield* yield* CloudflareEnvironment; + // Account-owned tokens (`cfat_…`) verify against the account route. + const verified = yield* Redacted.value(token).startsWith("cfat_") + ? accounts.verifyToken({ accountId }) + : user.verifyToken({}).pipe( + Effect.retry({ + while: (e) => e._tag === "Forbidden", + schedule: Schedule.exponential("500 millis"), + times: 8, + }), + ); + const secretAccessKey = yield* Effect.sync(() => + crypto.createHash("sha256").update(Redacted.value(token)).digest("hex"), + ); + return { + accessKeyId: Redacted.make(verified.id), + secretAccessKey: Redacted.make(secretAccessKey), + }; +}); diff --git a/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-stream.ts b/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-stream.ts new file mode 100644 index 00000000000..bd9f54e1d68 --- /dev/null +++ b/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-stream.ts @@ -0,0 +1,29 @@ +import * as Schema from "effect/Schema"; +import * as Cloudflare from "@/Cloudflare"; + +/** Record schema shared by the Pipelines stream fixtures and tests. */ +export class PageView extends Schema.Class("PageView")({ + url: Schema.String, + at: Schema.Date, + tags: Schema.Array(Schema.String), + user: Schema.optional(Schema.Struct({ id: Schema.String })), +}) {} + +/** The field list `PageView` converts to — equivalent spelling. */ +export const pageViewFields: Cloudflare.Pipelines.StreamField[] = [ + { type: "string", name: "url", required: true }, + { type: "timestamp", name: "at", required: true }, + { type: "list", name: "tags", required: true, items: { type: "string" } }, + { + type: "struct", + name: "user", + required: false, + fields: [{ type: "string", name: "id", required: true }], + }, +]; + +/** Typed, authenticated-HTTP stream used by the producer fixtures. */ +export const PageViews = Cloudflare.Basin.Stream("PageViews", { + schema: PageView, + http: true, +}); diff --git a/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-worker.ts b/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-worker.ts new file mode 100644 index 00000000000..c104658a242 --- /dev/null +++ b/packages/alchemy/test/Cloudflare/Basin/fixtures/pipelines-worker.ts @@ -0,0 +1,62 @@ +import * as Effect from "effect/Effect"; +import { HttpServerRequest } from "effect/http/HttpServerRequest"; +import * as HttpServerResponse from "effect/http/HttpServerResponse"; +import * as Layer from "effect/Layer"; +import * as EffectStream from "effect/Stream"; +import * as Cloudflare from "@/Cloudflare"; +import { PageView, PageViews } from "./pipelines-stream.ts"; + +/** + * Effect-native Worker exercising the typed Pipelines producers over the + * native `pipelines` binding: `/send` (WriteStream), `/sink` (StreamSink) + * and `/invalid` (a record the stream's Effect Schema rejects). + */ +export default class PipelinesWorker extends Cloudflare.Worker()( + "PipelinesWorker", + { main: import.meta.url }, + Effect.gen(function* () { + const views = yield* Cloudflare.Basin.WriteStream(PageViews); + const sink = yield* Cloudflare.Basin.StreamSink(PageViews); + + return { + fetch: Effect.gen(function* () { + const request = yield* HttpServerRequest; + const url = new URL(request.originalUrl, "http://x"); + if (url.pathname === "/send") { + yield* views + .send([ + new PageView({ url: "/a", at: new Date(), tags: ["x"] }), + new PageView({ url: "/b", at: new Date(), tags: [], user: { id: "u1" } }), + ]) + .pipe(Effect.orDie); + return yield* HttpServerResponse.json({ sent: 2 }); + } + if (url.pathname === "/sink") { + const count = Number(url.searchParams.get("count") ?? "300"); + yield* EffectStream.range(1, count).pipe( + EffectStream.map((i) => new PageView({ url: `/p/${i}`, at: new Date(), tags: [] })), + EffectStream.run(sink), + Effect.orDie, + ); + return yield* HttpServerResponse.json({ sent: count }); + } + if (url.pathname === "/invalid") { + const result = yield* views + .send([{ url: "/c", at: "not-a-date", tags: [] } as unknown as PageView]) + .pipe( + Effect.as({ reason: "none" as string, index: -1 }), + Effect.catchTag("StreamSendError", (e) => + Effect.succeed({ reason: e.reason ?? "unknown", index: e.index ?? -1 }), + ), + ); + return yield* HttpServerResponse.json(result); + } + return yield* HttpServerResponse.json({ ok: true }); + }), + }; + }).pipe( + Effect.provide( + Layer.mergeAll(Cloudflare.Basin.WriteStreamBinding, Cloudflare.Basin.StreamSinkBinding), + ), + ), +) {} diff --git a/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts b/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts new file mode 100644 index 00000000000..26b455f5ebb --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts @@ -0,0 +1,246 @@ +import * as k2 from "@distilled.cloud/cloudflare/k2"; +import { expect } from "alchemy-test"; +import * as Data from "effect/Data"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { MinimumLogLevel } from "effect/References"; +import * as Schedule from "effect/Schedule"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import { Action } from "@/Action"; +import * as Cloudflare from "@/Cloudflare"; +import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; +import { makeK2LocalAuth } from "@/Cloudflare/K2/K2Http.ts"; +import { makeReadSubscriptionClient } from "@/Cloudflare/K2/ReadSubscriptionHttp.ts"; +import { RuntimeContext } from "@/RuntimeContext.ts"; +import { createHostRuntimeContext, ServerHost } from "@/Server/Process.ts"; +import * as Test from "@/Test/Alchemy"; +import { waitForStreamGone } from "./k2-test-utils.ts"; + +const { test } = Test.make({ providers: Cloudflare.providers() }); + +const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); + +const Event = Schema.Struct({ source: Schema.String, index: Schema.Number }); +type Event = typeof Event.Type; + +class NotConsumed extends Data.TaggedError("NotConsumed")<{ distinct: number }> {} + +const decoder = new TextDecoder(); + +/** + * The `*Local` layers end to end, from an Action: `WriteStreamLocal` and + * `StreamSinkLocal` produce typed records, and `consumeStreamRecords` + * (`StreamEventSourcePolling` over `ReadSubscriptionLocal`, on an in-process + * `ServerHost`) consumes them. The handler fails once to exercise + * nack → redelivery. The ReadSubscription client is then driven directly + * against a second subscription. + */ +test.provider( + "Local producers, consumeStreamRecords polling, and the ReadSubscription client", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + + yield* stack.destroy(); + + const out = yield* stack.deploy( + Effect.gen(function* () { + const events = yield* Cloudflare.K2.Stream("LocalEvents", { http: true }); + const poller = yield* Cloudflare.K2.Subscription("Poller", { + streamId: events.streamId, + startAt: "earliest", + }); + const reader = yield* Cloudflare.K2.Subscription("Reader", { + streamId: events.streamId, + startAt: "earliest", + }); + + const Roundtrip = Action( + "Roundtrip", + Effect.gen(function* () { + const writer = yield* Cloudflare.K2.WriteStream(events, { schema: Event }); + const sink = yield* Cloudflare.K2.StreamSink(events, { schema: Event }); + + const seen = new Set(); + let failures = 0; + const host = createHostRuntimeContext("Test.K2Host")("K2Host"); + yield* Cloudflare.K2.consumeStreamRecords( + events, + { + subscription: poller, + schema: Event, + workerId: "alchemy-k2-poller", + idleBackoff: "500 millis", + concurrency: 2, + }, + (records) => + Effect.gen(function* () { + if (failures === 0) { + failures++; + return yield* Effect.fail("first delivery fails" as const); + } + yield* records.pipe( + Stream.runForEach((record) => + Effect.sync(() => seen.add(`${record.value.source}:${record.value.index}`)), + ), + ); + }), + ).pipe( + Effect.provide( + Layer.provideMerge( + Cloudflare.K2.StreamEventSourcePolling, + Layer.mergeAll( + Cloudflare.K2.ReadSubscriptionLocal, + Layer.succeed(ServerHost, host), + ), + ), + ), + ); + + return Effect.fn(function* () { + yield* writer.send([0, 1, 2, 3, 4].map((index) => ({ source: "writer", index }))); + yield* Stream.range(0, 19).pipe( + Stream.map((index): Event => ({ source: "sink", index })), + Stream.run(sink), + ); + + const { program } = yield* host.exports; + const drained = Effect.suspend(() => + seen.size >= 25 + ? Effect.succeed(seen.size) + : Effect.fail(new NotConsumed({ distinct: seen.size })), + ).pipe( + Effect.retry({ + schedule: Schedule.max([Schedule.spaced("1 second"), Schedule.recurs(60)]), + }), + ); + const distinct = yield* Effect.raceFirst( + drained, + program.pipe(Effect.andThen(Effect.never)), + ); + return { distinct, failures, seen: [...seen].sort() }; + }); + }).pipe( + Effect.provide( + Layer.mergeAll(Cloudflare.K2.WriteStreamLocal, Cloudflare.K2.StreamSinkLocal), + ), + ), + ); + + return { + roundtrip: yield* Roundtrip({}), + streamId: events.streamId, + readerId: reader.subscriptionId, + }; + }), + ); + + expect(out.roundtrip.distinct).toEqual(25); + expect(out.roundtrip.failures).toEqual(1); + expect(out.roundtrip.seen).toContain("writer:4"); + expect(out.roundtrip.seen).toContain("sink:19"); + + // Drive the client directly (current credentials) on the reader. + const auth = yield* makeK2LocalAuth; + const client = makeReadSubscriptionClient(auth, { + streamId: Effect.succeed(out.streamId), + subscriptionId: Effect.succeed(out.readerId), + }); + const consume = (workerId: string, maxRecords: number) => + client.consume({ workerId, maxRecords }).pipe( + Effect.retry({ + while: (e) => e._tag === "K2BatchPending", + schedule: Schedule.max([Schedule.spaced("500 millis"), Schedule.recurs(10)]), + }), + ); + + const first = yield* consume("reader-a", 3); + expect(Option.isSome(first)).toBe(true); + const batch = Option.getOrThrow(first); + expect(batch.records.length).toEqual(3); + expect(JSON.parse(decoder.decode(batch.records[0]!.content))).toEqual({ + source: "writer", + index: 0, + }); + expect(batch.records[0]!.headers["content-type"]).toEqual("application/json"); + expect(batch.records[0]!.timestamp).toBeInstanceOf(Date); + + // The same worker gets the same batch back; extend refreshes the lease. + const again = Option.getOrThrow(yield* consume("reader-a", 3)); + expect(again.batchId).toEqual(batch.batchId); + const extended = yield* client.extend(batch); + expect(extended.leasedUntil.getTime()).toBeGreaterThan(0); + + // nack → redelivered under a new batch id. + yield* client.nack(batch); + const redelivered = Option.getOrThrow(yield* consume("reader-a", 3)); + expect(redelivered.batchId).not.toEqual(batch.batchId); + expect(redelivered.records.map((r) => decoder.decode(r.content))).toEqual( + batch.records.map((r) => decoder.decode(r.content)), + ); + + // ack advances the subscription; acking twice succeeds; extending an + // acknowledged batch is K2LeaseLost. + yield* client.ack(redelivered); + yield* client.ack(redelivered); + const lost = yield* client.extend(redelivered).pipe(Effect.flip); + expect(lost._tag).toEqual("K2LeaseLost"); + + // Drain the rest; an empty subscription consumes None. + const rest = Option.getOrThrow(yield* consume("reader-a", 1000)); + expect(rest.records.length).toEqual(22); + yield* client.ack(rest); + const empty = yield* consume("reader-a", 10); + expect(Option.isNone(empty)).toBe(true); + + yield* stack.destroy(); + yield* waitForStreamGone(accountId, out.streamId); + }).pipe(logLevel), + { tags: ["provider:cloudflare", "provider:cloudflare:k2", "live"], timeout: 120_000 }, +); + +// Deploy-time half of `consumeStreamRecords` without `subscription`: it +// declares a Subscription owned by the consuming host. +test.provider( + "consumeStreamRecords creates a host-owned subscription", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + + yield* stack.destroy(); + + const host = createHostRuntimeContext("Test.K2Host")("K2Host"); + const out = yield* stack.deploy( + Effect.gen(function* () { + const events = yield* Cloudflare.K2.Stream("AutoEvents"); + yield* Cloudflare.K2.consumeStreamRecords(events, { startAt: "earliest" }, (records) => + Stream.runDrain(records), + ).pipe( + Effect.provide( + Layer.provideMerge( + Cloudflare.K2.StreamEventSourcePolling, + Layer.mergeAll( + Cloudflare.K2.ReadSubscriptionLocal, + Layer.succeed(ServerHost, host), + // A real host provides its runtime context; outputs the + // client reads are bound into the host's env. + Layer.succeed(RuntimeContext, host), + ), + ), + ), + ); + return { streamId: events.streamId }; + }), + ); + + const subscriptions = yield* k2.listSubscriptions({ streamId: out.streamId }); + expect(subscriptions.length).toEqual(1); + expect(subscriptions[0]?.startAt.type).toEqual("earliest"); + + yield* stack.destroy(); + yield* waitForStreamGone(accountId, out.streamId); + }).pipe(logLevel), + { tags: ["provider:cloudflare", "provider:cloudflare:k2", "live"], timeout: 120_000 }, +); diff --git a/packages/alchemy/test/Cloudflare/K2/Stream.test.ts b/packages/alchemy/test/Cloudflare/K2/Stream.test.ts new file mode 100644 index 00000000000..f22d5743712 --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/Stream.test.ts @@ -0,0 +1,108 @@ +import * as k2 from "@distilled.cloud/cloudflare/k2"; +import { expect } from "alchemy-test"; +import * as Effect from "effect/Effect"; +import { MinimumLogLevel } from "effect/References"; +import * as Cloudflare from "@/Cloudflare"; +import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; +import * as Provider from "@/Provider"; +import * as Test from "@/Test/Alchemy"; +import { waitForStreamGone } from "./k2-test-utils.ts"; + +const { test } = Test.make({ providers: Cloudflare.providers() }); + +const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); + +test.provider( + "create, update in place, and delete a stream", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + + yield* stack.destroy(); + + // Defaults: worker binding on, HTTP off, seven-day retention. + const created = yield* stack.deploy(Cloudflare.K2.Stream("UpdateStream")); + expect(created.streamName).toMatch(/^[a-z0-9_]+$/); + expect(created.endpoint).toEqual(`https://${created.streamId}.k2.cloudflarestorage.com`); + expect(created.retentionSeconds).toEqual(604_800); + expect(created.httpEnabled).toBe(false); + expect(created.workerBindingEnabled).toBe(true); + + const observed = yield* k2.getStream({ accountId, streamId: created.streamId }); + expect(observed.name).toEqual(created.streamName); + expect(observed.http.enabled).toBe(false); + expect(observed.workerBinding.enabled).toBe(true); + + // Retention, HTTP input, and worker binding are patched in place. + const updated = yield* stack.deploy( + Cloudflare.K2.Stream("UpdateStream", { + retention: "1 day", + http: { authentication: false, cors: ["https://example.com"] }, + workerBinding: false, + }), + ); + expect(updated.streamId).toEqual(created.streamId); + expect(updated.retentionSeconds).toEqual(86_400); + expect(updated.httpEnabled).toBe(true); + expect(updated.httpAuthentication).toBe(false); + expect(updated.corsOrigins).toEqual(["https://example.com"]); + expect(updated.workerBindingEnabled).toBe(false); + + const patched = yield* k2.getStream({ accountId, streamId: created.streamId }); + expect(patched.retentionSeconds).toEqual(86_400); + expect(patched.http.enabled).toBe(true); + expect(patched.http.authentication ?? false).toBe(false); + expect(patched.http.cors?.origins).toEqual(["https://example.com"]); + expect(patched.workerBinding.enabled).toBe(false); + + // `http: true` turns authentication on; dropping cors clears it. + const authenticated = yield* stack.deploy( + Cloudflare.K2.Stream("UpdateStream", { retention: "1 day", http: true }), + ); + expect(authenticated.streamId).toEqual(created.streamId); + expect(authenticated.httpAuthentication).toBe(true); + expect(authenticated.corsOrigins ?? []).toEqual([]); + expect(authenticated.workerBindingEnabled).toBe(true); + + // The provider lists the deployed stream in the read Attributes shape. + const provider = yield* Provider.findProvider(Cloudflare.K2.Stream); + const all = yield* provider.list(); + const listed = all.find((s) => s.streamId === created.streamId); + expect(listed?.streamName).toEqual(created.streamName); + expect(listed?.httpAuthentication).toBe(true); + + yield* stack.destroy(); + yield* waitForStreamGone(accountId, created.streamId); + }).pipe(logLevel), + { tags: ["provider:cloudflare", "provider:cloudflare:k2", "live"], timeout: 120_000 }, +); + +test.provider( + "renaming a stream replaces it", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + + yield* stack.destroy(); + + const first = yield* stack.deploy( + Cloudflare.K2.Stream("RenamedStream", { name: "alchemy_k2_test_rename_a" }), + ); + expect(first.streamName).toEqual("alchemy_k2_test_rename_a"); + + const second = yield* stack.deploy( + Cloudflare.K2.Stream("RenamedStream", { name: "alchemy_k2_test_rename_b" }), + ); + expect(second.streamName).toEqual("alchemy_k2_test_rename_b"); + expect(second.streamId).not.toEqual(first.streamId); + + // The old generation is deleted after the replacement. + yield* waitForStreamGone(accountId, first.streamId); + const observed = yield* k2.getStream({ accountId, streamId: second.streamId }); + expect(observed.name).toEqual("alchemy_k2_test_rename_b"); + + yield* stack.destroy(); + yield* waitForStreamGone(accountId, second.streamId); + }).pipe(logLevel), + { tags: ["provider:cloudflare", "provider:cloudflare:k2", "live"], timeout: 120_000 }, +); diff --git a/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts b/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts new file mode 100644 index 00000000000..b2eeea20698 --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts @@ -0,0 +1,183 @@ +import * as k2 from "@distilled.cloud/cloudflare/k2"; +import { describe, expect, it } from "alchemy-test"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import { + decodeWithSchema, + encodeRecords, + fromBase64, + fromConsumedRecords, + toBase64, + toProduceRecords, + wireSize, +} from "@/Cloudflare/K2/K2Codec.ts"; +import type { EncodedRecord } from "@/Cloudflare/K2/K2Types.ts"; +import { + makeStreamSink, + MAX_REQUEST_BYTES, + packStreamSinkBatches, +} from "@/Cloudflare/K2/StreamSinkBatch.ts"; +import type { WriteStreamClient } from "@/Cloudflare/K2/WriteStream.ts"; +import { type K2StreamBinding, sendViaBinding } from "@/Cloudflare/K2/WriteStreamBinding.ts"; +import { makeWriteStreamClient } from "@/Cloudflare/K2/WriteStreamClient.ts"; +import { RuntimeContext } from "@/RuntimeContext.ts"; + +/** + * Pure tests of K2 record encoding, `StreamSink` packing and retries, and + * the native binding's error mapping. Live delivery is covered by + * `WriteStream.test.ts` and `ReadSubscription.test.ts`. + */ + +const Order = Schema.Struct({ id: Schema.Number }); + +const record = (bytes: number): EncodedRecord => ({ content: new Uint8Array(bytes) }); + +const fakeClient = (failures: ReadonlyArray = []) => { + const sent: EncodedRecord[][] = []; + let attempt = 0; + const client: WriteStreamClient<{ id: number }> = makeWriteStreamClient( + { schema: Order }, + (records) => + Effect.suspend(() => { + const tag = failures[attempt++]; + if (tag === "K2Unavailable") { + return Effect.fail(new k2.K2Unavailable({ code: 10211, message: "unavailable" })); + } + if (tag === "K2AppendOutcomeUnknown") { + return Effect.fail( + new k2.K2AppendOutcomeUnknown({ code: 10212, message: "maybe stored" }), + ); + } + sent.push([...records]); + return Effect.void; + }), + ); + return { client, sent, attempts: () => attempt }; +}; + +const run = (stream: Stream.Stream, client: WriteStreamClient) => + stream.pipe( + Stream.run(makeStreamSink(client)), + Effect.provideService(RuntimeContext, {} as never), + ); + +describe("Cloudflare K2 records and StreamSink", { tags: ["unit", "local"] }, () => { + it("round-trips bytes through standard base64", () => { + const bytes = Uint8Array.from({ length: 70_000 }, (_, i) => i % 256); + expect(fromBase64(toBase64(bytes))).toEqual(bytes); + expect(toBase64(new TextEncoder().encode("hi?"))).toEqual("aGk/"); + }); + + it("encodes strings as UTF-8 and drops empty headers", () => { + const [a, b] = encodeRecords([ + { content: "é", headers: {} }, + { content: new ArrayBuffer(2), headers: { k: "v" } }, + ]); + expect(a!.content).toEqual(new Uint8Array([0xc3, 0xa9])); + expect(a!.headers).toBeUndefined(); + expect(b!.content.byteLength).toEqual(2); + expect(toProduceRecords([b!])).toEqual([{ content: "AAA=", headers: { k: "v" } }]); + }); + + it.effect("decodes consumed JSON records with a schema", () => + Effect.gen(function* () { + const consumed = fromConsumedRecords([ + { timestampMs: 1000, content: toBase64(new TextEncoder().encode('{"id":7}')) }, + ]); + expect(consumed[0]!.timestamp.getTime()).toEqual(1000); + expect(consumed[0]!.headers).toEqual({}); + const [decoded] = yield* decodeWithSchema(Order)(consumed); + expect(decoded!.value).toEqual({ id: 7 }); + + const bad = fromConsumedRecords([ + { timestampMs: 0, content: toBase64(new TextEncoder().encode("not json")) }, + ]); + const error = yield* decodeWithSchema(Order)(bad).pipe(Effect.flip); + expect(error._tag).toEqual("K2SchemaError"); + }), + ); + + it("packs requests under 5 MB of wire size, preserving order", () => { + const records = Array.from({ length: 12 }, () => record(1_000_000)); + const batches = packStreamSinkBatches(records); + expect(batches.map((b) => b.length)).toEqual([3, 3, 3, 3]); + for (const batch of batches) { + expect(batch.reduce((sum, r) => sum + wireSize(r), 0)).toBeLessThanOrEqual(MAX_REQUEST_BYTES); + } + expect(batches.flat()).toEqual(records); + }); + + it("ships an oversized record alone", () => { + const huge = record(MAX_REQUEST_BYTES); + expect(packStreamSinkBatches([record(1), huge, record(1)]).map((b) => b.length)).toEqual([ + 1, 1, 1, + ]); + }); + + it.effect("sends one request per chunk with JSON headers", () => + Effect.gen(function* () { + const { client, sent } = fakeClient(); + yield* run( + Stream.fromIterable([1, 2, 3, 4, 5].map((id) => ({ id }))).pipe(Stream.rechunk(2)), + client, + ); + expect(sent.map((b) => b.length)).toEqual([2, 2, 1]); + expect(sent[0]![0]!.headers).toEqual({ "content-type": "application/json" }); + expect(new TextDecoder().decode(sent[2]![0]!.content)).toEqual('{"id":5}'); + }), + ); + + // `it.live`: the retry schedule sleeps on the real clock. + it.live("retries K2Unavailable (not stored) a bounded number of times", () => + Effect.gen(function* () { + const { client, sent, attempts } = fakeClient(["K2Unavailable", "K2Unavailable"]); + yield* run(Stream.make({ id: 1 }), client); + expect(attempts()).toEqual(3); + expect(sent.length).toEqual(1); + }), + ); + + it.effect("never retries K2AppendOutcomeUnknown (may have been stored)", () => + Effect.gen(function* () { + const { client, attempts } = fakeClient(["K2AppendOutcomeUnknown"]); + const error = yield* run(Stream.make({ id: 1 }), client).pipe(Effect.flip); + expect(error._tag).toEqual("K2AppendOutcomeUnknown"); + expect(attempts()).toEqual(1); + }), + ); + + it.effect("maps native binding results onto the HTTP API's typed tags", () => + Effect.gen(function* () { + const binding = ( + result: Awaited> | Error, + ): K2StreamBinding => ({ + send: () => (result instanceof Error ? Promise.reject(result) : Promise.resolve(result)), + }); + const records = [{ content: new Uint8Array([1]) }]; + yield* sendViaBinding(binding({ success: true }), records); + + const failure = (code: number) => + sendViaBinding( + binding({ success: false, error: { code, message: "nope", retryable: false } }), + records, + ).pipe( + Effect.flip, + Effect.map((e) => e._tag), + ); + expect(yield* failure(10200)).toEqual("K2StreamNotFound"); + expect(yield* failure(10204)).toEqual("K2InvalidRequest"); + expect(yield* failure(10206)).toEqual("K2RequestTooLarge"); + expect(yield* failure(10207)).toEqual("K2RecordTooLarge"); + expect(yield* failure(10211)).toEqual("K2Unavailable"); + expect(yield* failure(10212)).toEqual("K2AppendOutcomeUnknown"); + expect(yield* failure(99999)).toEqual("UnknownCloudflareError"); + + // A rejected RPC means the append outcome is unknown. + const rejected = yield* sendViaBinding(binding(new Error("rpc reset")), records).pipe( + Effect.flip, + ); + expect(rejected._tag).toEqual("K2AppendOutcomeUnknown"); + }), + ); +}); diff --git a/packages/alchemy/test/Cloudflare/K2/Subscription.test.ts b/packages/alchemy/test/Cloudflare/K2/Subscription.test.ts new file mode 100644 index 00000000000..99ec9914e9e --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/Subscription.test.ts @@ -0,0 +1,103 @@ +import * as k2 from "@distilled.cloud/cloudflare/k2"; +import { expect } from "alchemy-test"; +import * as Effect from "effect/Effect"; +import { MinimumLogLevel } from "effect/References"; +import * as Cloudflare from "@/Cloudflare"; +import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; +import * as Test from "@/Test/Alchemy"; +import { waitForStreamGone, waitForSubscriptionGone } from "./k2-test-utils.ts"; + +const { test } = Test.make({ providers: Cloudflare.providers() }); + +const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); + +const program = (startAt: "earliest" | "latest") => + Effect.gen(function* () { + const stream = yield* Cloudflare.K2.Stream("SubscriptionStream"); + const subscription = yield* Cloudflare.K2.Subscription("Analytics", { + streamId: stream.streamId, + startAt, + }); + return { stream, subscription }; + }); + +test.provider( + "create a subscription and replace it when startAt changes", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + + yield* stack.destroy(); + + const first = yield* stack.deploy(program("latest")); + const streamId = first.stream.streamId; + expect(first.subscription.streamId).toEqual(streamId); + expect(first.subscription.startAt).toEqual("latest"); + + const observed = yield* k2.getSubscription({ + streamId, + subscriptionId: first.subscription.subscriptionId, + }); + expect(observed.name).toEqual(first.subscription.subscriptionName); + expect(observed.startAt.type).toEqual("latest"); + + // A redeploy with the same props is a no-op. + const same = yield* stack.deploy(program("latest")); + expect(same.subscription.subscriptionId).toEqual(first.subscription.subscriptionId); + + // startAt is immutable — the subscription is replaced on the same stream. + const second = yield* stack.deploy(program("earliest")); + expect(second.stream.streamId).toEqual(streamId); + expect(second.subscription.startAt).toEqual("earliest"); + expect(second.subscription.subscriptionId).not.toEqual(first.subscription.subscriptionId); + yield* waitForSubscriptionGone(streamId, first.subscription.subscriptionId); + + const replaced = yield* k2.listSubscriptions({ streamId }); + expect(replaced.map((s) => s.id)).toEqual([second.subscription.subscriptionId]); + expect(replaced[0]?.startAt.type).toEqual("earliest"); + + yield* stack.destroy(); + yield* waitForSubscriptionGone(streamId, second.subscription.subscriptionId); + yield* waitForStreamGone(accountId, streamId); + }).pipe(logLevel), + { tags: ["provider:cloudflare", "provider:cloudflare:k2", "live"], timeout: 120_000 }, +); + +test.provider( + "an explicitly named subscription is replaced delete-first", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + + yield* stack.destroy(); + + const named = (startAt: "earliest" | "latest") => + Effect.gen(function* () { + const stream = yield* Cloudflare.K2.Stream("NamedSubscriptionStream"); + const subscription = yield* Cloudflare.K2.Subscription("Named", { + streamId: stream.streamId, + name: "alchemy-k2-named", + startAt, + }); + return { stream, subscription }; + }); + + const first = yield* stack.deploy(named("latest")); + expect(first.subscription.subscriptionName).toEqual("alchemy-k2-named"); + + // K2 rejects the same name with different settings, so the engine must + // delete the old subscription before creating the new one. + const second = yield* stack.deploy(named("earliest")); + expect(second.subscription.subscriptionName).toEqual("alchemy-k2-named"); + expect(second.subscription.startAt).toEqual("earliest"); + const observed = yield* k2.getSubscription({ + streamId: second.stream.streamId, + subscriptionId: second.subscription.subscriptionId, + }); + expect(observed.startAt.type).toEqual("earliest"); + + yield* stack.destroy(); + yield* waitForStreamGone(accountId, second.stream.streamId); + }).pipe(logLevel), + { tags: ["provider:cloudflare", "provider:cloudflare:k2", "live"], timeout: 120_000 }, +); diff --git a/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts b/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts new file mode 100644 index 00000000000..3c6d2452969 --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts @@ -0,0 +1,193 @@ +import { expect } from "alchemy-test"; +import * as Data from "effect/Data"; +import * as Effect from "effect/Effect"; +import * as HttpClient from "effect/http/HttpClient"; +import { MinimumLogLevel } from "effect/References"; +import * as Schedule from "effect/Schedule"; +import * as Cloudflare from "@/Cloudflare"; +import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; +import * as Test from "@/Test/Alchemy"; +import { AsyncOrders, K2AsyncWorker } from "./fixtures/k2-async-stack.ts"; +import K2BindingWorker from "./fixtures/k2-binding-worker.ts"; +import K2HttpWorker from "./fixtures/k2-http-worker.ts"; +import { BindingOrders, HttpOrders } from "./fixtures/k2-shared.ts"; +import { drainSubscription, type VerifiedRecord, waitForStreamGone } from "./k2-test-utils.ts"; + +const { test } = Test.make({ providers: Cloudflare.providers() }); + +const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); + +class NotProduced extends Data.TaggedError("NotProduced")<{ path: string; status: number }> {} + +/** + * `POST {route}`, retrying while a fresh workers.dev URL comes up (bounded + * to ~60 s). A 500 carries the typed K2 error tag in its JSON body. + */ +const produce = (url: string, path: string, run: string, count: number) => + HttpClient.post(`${url}${path}?run=${run}&count=${count}`).pipe( + Effect.flatMap((res) => + res.status === 202 + ? Effect.succeed(res) + : res.text.pipe( + Effect.flatMap((body) => { + if (res.status === 500) console.error(`${path} failed: ${body}`); + return Effect.fail(new NotProduced({ path, status: res.status })); + }), + ), + ), + Effect.retry({ + schedule: Schedule.max([ + Schedule.min([Schedule.exponential("500 millis"), Schedule.spaced("3 seconds")]), + Schedule.recurs(25), + ]), + }), + ); + +const byRun = (records: ReadonlyArray, run: string) => + records.filter( + (r) => + r.headers.run === run || r.text.startsWith(`${run}:`) || r.text.includes(`"run":"${run}"`), + ); + +const verifyRoutes = (url: string, streamId: string, subscriptionId: string) => + Effect.gen(function* () { + yield* produce(url, "/raw", "raw", 3); + yield* produce(url, "/typed", "typed", 4); + yield* produce(url, "/sink", "sink", 250); + + const records = yield* drainSubscription(streamId, subscriptionId, 3 + 4 + 250); + + // Raw records keep their bytes and headers. + const raw = byRun(records, "raw"); + expect(raw.map((r) => r.text).sort()).toEqual(["raw:0", "raw:1", "raw:2"]); + expect(raw.every((r) => r.headers.run === "raw")).toBe(true); + + // Typed records are JSON with a content-type header. + const typed = byRun(records, "typed"); + expect(typed.map((r) => JSON.parse(r.text).index).sort()).toEqual([0, 1, 2, 3]); + expect(typed.every((r) => r.headers["content-type"] === "application/json")).toBe(true); + + // The sink appends every element, in order. + const sink = byRun(records, "sink").map((r) => JSON.parse(r.text).index as number); + expect(new Set(sink).size).toEqual(250); + expect(sink).toEqual([...sink].sort((a, b) => a - b)); + }); + +test.provider( + "WriteStream and StreamSink produce through the k2 Worker binding", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + + yield* stack.destroy(); + + const deployed = yield* stack.deploy( + Effect.gen(function* () { + const orders = yield* BindingOrders; + const verifier = yield* Cloudflare.K2.Subscription("BindingVerifier", { + streamId: orders.streamId, + startAt: "earliest", + }); + const worker = yield* K2BindingWorker; + return { + url: worker.url.as(), + streamId: orders.streamId, + subscriptionId: verifier.subscriptionId, + }; + }), + ); + + yield* verifyRoutes(deployed.url, deployed.streamId, deployed.subscriptionId); + + yield* stack.destroy(); + yield* waitForStreamGone(accountId, deployed.streamId); + }).pipe(logLevel), + { + tags: ["provider:cloudflare", "provider:cloudflare:k2", "provider:cloudflare:worker", "live"], + timeout: 180_000, + }, +); + +test.provider( + "an async Worker produces through a K2 stream in env", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + + yield* stack.destroy(); + + const deployed = yield* stack.deploy( + Effect.gen(function* () { + const orders = yield* AsyncOrders; + const verifier = yield* Cloudflare.K2.Subscription("AsyncVerifier", { + streamId: orders.streamId, + startAt: "earliest", + }); + const worker = yield* K2AsyncWorker; + return { + url: worker.url.as(), + streamId: orders.streamId, + subscriptionId: verifier.subscriptionId, + }; + }), + ); + + yield* produce(deployed.url, "/send", "async", 5); + + const records = yield* drainSubscription(deployed.streamId, deployed.subscriptionId, 5); + expect(records.map((r) => r.text).sort()).toEqual([ + "async:0", + "async:1", + "async:2", + "async:3", + "async:4", + ]); + expect(records.every((r) => r.headers.run === "async")).toBe(true); + + yield* stack.destroy(); + yield* waitForStreamGone(accountId, deployed.streamId); + }).pipe(logLevel), + { + tags: ["provider:cloudflare", "provider:cloudflare:k2", "provider:cloudflare:worker", "live"], + timeout: 180_000, + }, +); + +// WriteStreamHttp / StreamSinkHttp mint a scoped `K2 Produce` API token; +// set CLOUDFLARE_TEST_K2_HTTP=1 with credentials that can create API tokens. +// The HTTP client path itself is covered through the `*Local` layers in +// ReadSubscription.test.ts. +test.provider.skipIf(!process.env.CLOUDFLARE_TEST_K2_HTTP)( + "WriteStream and StreamSink produce over HTTP with a scoped token", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + + yield* stack.destroy(); + + const deployed = yield* stack.deploy( + Effect.gen(function* () { + const orders = yield* HttpOrders; + const verifier = yield* Cloudflare.K2.Subscription("HttpVerifier", { + streamId: orders.streamId, + startAt: "earliest", + }); + const worker = yield* K2HttpWorker; + return { + url: worker.url.as(), + streamId: orders.streamId, + subscriptionId: verifier.subscriptionId, + }; + }), + ); + + yield* verifyRoutes(deployed.url, deployed.streamId, deployed.subscriptionId); + + yield* stack.destroy(); + yield* waitForStreamGone(accountId, deployed.streamId); + }).pipe(logLevel), + { + tags: ["provider:cloudflare", "provider:cloudflare:k2", "provider:cloudflare:worker", "live"], + timeout: 180_000, + }, +); diff --git a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-async-stack.ts b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-async-stack.ts new file mode 100644 index 00000000000..f2f333f38bc --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-async-stack.ts @@ -0,0 +1,13 @@ +import * as Cloudflare from "@/Cloudflare"; + +/** A K2 stream bound into an async (non-Effect) Worker through `env`. */ +export const AsyncOrders = Cloudflare.K2.Stream("K2AsyncOrders"); + +export const K2AsyncWorker = Cloudflare.Worker("K2AsyncWorker", { + main: `${import.meta.dirname}/k2-async-worker.ts`, + env: { + ORDERS: AsyncOrders, + }, +}); + +export type K2AsyncWorkerEnv = Cloudflare.InferEnv; diff --git a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-async-worker.ts b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-async-worker.ts new file mode 100644 index 00000000000..ad8506ba3ac --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-async-worker.ts @@ -0,0 +1,28 @@ +import type { K2AsyncWorkerEnv } from "./k2-async-stack.ts"; + +const encoder = new TextEncoder(); + +/** + * `POST /send?run=R&count=N` appends N records through `env.ORDERS.send` + * (the native `k2` binding) and answers 202, or 500 with the binding's + * error when the append failed. + */ +export default { + async fetch(request: Request, env: K2AsyncWorkerEnv): Promise { + const url = new URL(request.url); + if (request.method !== "POST" || url.pathname !== "/send") { + return new Response("Not Found", { status: 404 }); + } + const run = url.searchParams.get("run") ?? "default"; + const count = Number(url.searchParams.get("count") ?? "1"); + const result = await env.ORDERS.send( + Array.from({ length: count }, (_, index) => ({ + content: encoder.encode(`${run}:${index}`), + headers: { run }, + })), + ); + return result.success + ? Response.json({ produced: count }, { status: 202 }) + : Response.json({ error: result.error }, { status: 500 }); + }, +}; diff --git a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-binding-worker.ts b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-binding-worker.ts new file mode 100644 index 00000000000..ab6635475c4 --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-binding-worker.ts @@ -0,0 +1,33 @@ +import * as Effect from "effect/Effect"; +import { HttpServerRequest } from "effect/http/HttpServerRequest"; +import * as Layer from "effect/Layer"; +import * as Cloudflare from "@/Cloudflare/index.ts"; +import { BindingOrders, Order, produceRoutes } from "./k2-shared.ts"; + +/** + * Produces to a K2 stream through the native `k2` Worker binding + * (`WriteStreamBinding` / `StreamSinkBinding`). + */ +export default class K2BindingWorker extends Cloudflare.Worker()( + "K2BindingWorker", + { + main: import.meta.url, + }, + Effect.gen(function* () { + const orders = yield* BindingOrders; + const raw = yield* Cloudflare.K2.WriteStream(orders); + const typed = yield* Cloudflare.K2.WriteStream(orders, { schema: Order }); + const sink = yield* Cloudflare.K2.StreamSink(orders, { schema: Order }); + + return { + fetch: Effect.gen(function* () { + const request = yield* HttpServerRequest; + return yield* produceRoutes(new URL(request.url, "http://x"), { raw, typed, sink }); + }), + }; + }).pipe( + Effect.provide( + Layer.mergeAll(Cloudflare.K2.WriteStreamBinding, Cloudflare.K2.StreamSinkBinding), + ), + ), +) {} diff --git a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-http-worker.ts b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-http-worker.ts new file mode 100644 index 00000000000..75eb305adff --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-http-worker.ts @@ -0,0 +1,32 @@ +import * as Effect from "effect/Effect"; +import { HttpServerRequest } from "effect/http/HttpServerRequest"; +import * as Layer from "effect/Layer"; +import * as Cloudflare from "@/Cloudflare/index.ts"; +import { HttpOrders, Order, produceRoutes } from "./k2-shared.ts"; + +/** + * Produces to a K2 stream through its HTTP input with a scoped `K2 Produce` + * token (`WriteStreamHttp` / `StreamSinkHttp`). Live-only: the HTTP layers + * mint a real account API token. + */ +export default class K2HttpWorker extends Cloudflare.Worker()( + "K2HttpWorker", + { + main: import.meta.url, + }, + Effect.gen(function* () { + const orders = yield* HttpOrders; + const raw = yield* Cloudflare.K2.WriteStream(orders); + const typed = yield* Cloudflare.K2.WriteStream(orders, { schema: Order }); + const sink = yield* Cloudflare.K2.StreamSink(orders, { schema: Order }); + + return { + fetch: Effect.gen(function* () { + const request = yield* HttpServerRequest; + return yield* produceRoutes(new URL(request.url, "http://x"), { raw, typed, sink }); + }), + }; + }).pipe( + Effect.provide(Layer.mergeAll(Cloudflare.K2.WriteStreamHttp, Cloudflare.K2.StreamSinkHttp)), + ), +) {} diff --git a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-shared.ts b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-shared.ts new file mode 100644 index 00000000000..c2f3ca4ea46 --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-shared.ts @@ -0,0 +1,64 @@ +import * as Effect from "effect/Effect"; +import * as HttpServerResponse from "effect/http/HttpServerResponse"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as Cloudflare from "@/Cloudflare/index.ts"; + +/** Streams, schemas, and route helpers shared by the K2 Worker fixtures. */ + +export const Order = Schema.Struct({ + run: Schema.String, + index: Schema.Number, +}); +export type Order = typeof Order.Type; + +/** Worker-binding input only (the default). */ +export const BindingOrders = Cloudflare.K2.Stream("K2BindingOrders"); + +/** HTTP input enabled for the `*Http` producer layers. */ +export const HttpOrders = Cloudflare.K2.Stream("K2HttpOrders", { http: true }); + +const params = (url: URL) => ({ + run: url.searchParams.get("run") ?? "default", + count: Number(url.searchParams.get("count") ?? "1"), +}); + +const failed = (error: { _tag: string; message?: string }) => + HttpServerResponse.json({ error: error._tag, message: error.message }, { status: 500 }); + +/** + * Producer routes over a raw client, a schema-typed client, and a typed + * sink: `POST /raw|/typed|/sink?run=R&count=N`. + */ +export const produceRoutes = ( + url: URL, + clients: { + raw: Cloudflare.K2.WriteStreamClient; + typed: Cloudflare.K2.WriteStreamClient; + sink: Cloudflare.K2.StreamSinkClient; + }, +) => + Effect.gen(function* () { + const { run, count } = params(url); + const orders = Array.from({ length: count }, (_, index): Order => ({ run, index })); + const sent = + url.pathname === "/raw" + ? clients.raw.send( + orders.map((order) => ({ + content: `${order.run}:${order.index}`, + headers: { run: order.run }, + })), + ) + : url.pathname === "/typed" + ? clients.typed.send(orders) + : url.pathname === "/sink" + ? Stream.fromIterable(orders).pipe(Stream.run(clients.sink)) + : undefined; + if (sent === undefined) { + return HttpServerResponse.text("Not Found", { status: 404 }); + } + return yield* sent.pipe( + Effect.flatMap(() => HttpServerResponse.json({ produced: count }, { status: 202 })), + Effect.catch((error) => failed(error)), + ); + }); diff --git a/packages/alchemy/test/Cloudflare/K2/k2-test-utils.ts b/packages/alchemy/test/Cloudflare/K2/k2-test-utils.ts new file mode 100644 index 00000000000..168ae829d3f --- /dev/null +++ b/packages/alchemy/test/Cloudflare/K2/k2-test-utils.ts @@ -0,0 +1,82 @@ +import * as k2 from "@distilled.cloud/cloudflare/k2"; +import * as Data from "effect/Data"; +import * as Effect from "effect/Effect"; +import * as Schedule from "effect/Schedule"; + +/** Test-side helpers shared by the K2 suites. */ + +class StreamStillExists extends Data.TaggedError("StreamStillExists")<{ streamId: string }> {} + +class SubscriptionStillExists extends Data.TaggedError("SubscriptionStillExists")<{ + subscriptionId: string; +}> {} + +const gone = Schedule.max([Schedule.spaced("1 second"), Schedule.recurs(15)]); + +/** Wait until `getStream` reports `K2StreamNotFound`. */ +export const waitForStreamGone = (accountId: string, streamId: string) => + k2.getStream({ accountId, streamId }).pipe( + Effect.flatMap(() => Effect.fail(new StreamStillExists({ streamId }))), + Effect.retry({ while: (e) => e._tag === "StreamStillExists", schedule: gone }), + Effect.catchTag("K2StreamNotFound", () => Effect.void), + ); + +/** Wait until `getSubscription` reports the subscription (or its stream) gone. */ +export const waitForSubscriptionGone = (streamId: string, subscriptionId: string) => + k2.getSubscription({ streamId, subscriptionId }).pipe( + Effect.flatMap(() => Effect.fail(new SubscriptionStillExists({ subscriptionId }))), + Effect.retry({ while: (e) => e._tag === "SubscriptionStillExists", schedule: gone }), + Effect.catchTag(["K2SubscriptionNotFound", "K2StreamNotFound"], () => Effect.void), + ); + +const decoder = new TextDecoder(); + +export interface VerifiedRecord { + text: string; + headers: Record; +} + +class NotEnoughRecords extends Data.TaggedError("NotEnoughRecords")<{ + expected: number; + received: number; +}> {} + +/** + * Out-of-band verification: drain a subscription with distilled `consume` + * + `ackBatch` until `expected` records were read (bounded to ~60 s). + */ +export const drainSubscription = ( + streamId: string, + subscriptionId: string, + expected: number, + workerId = "alchemy-k2-verifier", +) => + Effect.gen(function* () { + const received: VerifiedRecord[] = []; + const once = Effect.gen(function* () { + const batch = yield* k2.consume({ streamId, subscriptionId, workerId, maxRecords: 1000 }); + if (batch.batchId) { + for (const record of batch.records) { + received.push({ + text: decoder.decode(Uint8Array.from(atob(record.content), (c) => c.charCodeAt(0))), + headers: Object.fromEntries( + Object.entries(record.headers ?? {}).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ), + }); + } + yield* k2.ackBatch({ streamId, subscriptionId, batchId: batch.batchId, workerId }); + } + if (received.length < expected) { + return yield* Effect.fail(new NotEnoughRecords({ expected, received: received.length })); + } + }); + yield* once.pipe( + Effect.retry({ + while: (e) => e._tag === "NotEnoughRecords" || e._tag === "K2BatchPending", + schedule: Schedule.max([Schedule.spaced("1 second"), Schedule.recurs(60)]), + }), + ); + return received; + }); diff --git a/packages/alchemy/test/Cloudflare/Pipelines/LegacyPipeline.test.ts b/packages/alchemy/test/Cloudflare/Pipelines/LegacyPipeline.test.ts index 387e22708c7..d549381c8fd 100644 --- a/packages/alchemy/test/Cloudflare/Pipelines/LegacyPipeline.test.ts +++ b/packages/alchemy/test/Cloudflare/Pipelines/LegacyPipeline.test.ts @@ -1,4 +1,5 @@ import crypto from "node:crypto"; +import * as accounts from "@distilled.cloud/cloudflare/accounts"; import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; import * as user from "@distilled.cloud/cloudflare/user"; import * as workers from "@distilled.cloud/cloudflare/workers"; @@ -50,7 +51,12 @@ const r2Credentials = Effect.gen(function* () { ); } const token = Redacted.value(creds.apiToken); - const verified = yield* retryAuthBlip(user.verifyToken({})); + const verified = yield* retryAuthBlip( + token.startsWith("cfat_") + ? // Account-owned tokens verify against the account route. + accounts.verifyToken({ accountId: creds.accountId }) + : user.verifyToken({}), + ); const secretAccessKey = yield* Effect.sync(() => crypto.createHash("sha256").update(token).digest("hex"), ); diff --git a/packages/alchemy/test/Cloudflare/Pipelines/Pipeline.test.ts b/packages/alchemy/test/Cloudflare/Pipelines/Pipeline.test.ts index fd21d94012f..415aadcb53b 100644 --- a/packages/alchemy/test/Cloudflare/Pipelines/Pipeline.test.ts +++ b/packages/alchemy/test/Cloudflare/Pipelines/Pipeline.test.ts @@ -1,4 +1,5 @@ import crypto from "node:crypto"; +import * as accounts from "@distilled.cloud/cloudflare/accounts"; import * as user from "@distilled.cloud/cloudflare/user"; import { expect } from "alchemy-test"; import * as Effect from "effect/Effect"; @@ -37,7 +38,12 @@ const r2Credentials = Effect.gen(function* () { ); } const token = Redacted.value(creds.apiToken); - const verified = yield* retryAuthBlip(user.verifyToken({})); + const verified = yield* retryAuthBlip( + token.startsWith("cfat_") + ? // Account-owned tokens verify against the account route. + accounts.verifyToken({ accountId: creds.accountId }) + : user.verifyToken({}), + ); const secretAccessKey = yield* Effect.sync(() => crypto.createHash("sha256").update(token).digest("hex"), ); diff --git a/packages/alchemy/test/Cloudflare/Pipelines/Pipelines.test.ts b/packages/alchemy/test/Cloudflare/Pipelines/Pipelines.test.ts index 8c9affba3e1..0ad95df2bb6 100644 --- a/packages/alchemy/test/Cloudflare/Pipelines/Pipelines.test.ts +++ b/packages/alchemy/test/Cloudflare/Pipelines/Pipelines.test.ts @@ -1,4 +1,5 @@ import crypto from "node:crypto"; +import * as accounts from "@distilled.cloud/cloudflare/accounts"; import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; import * as user from "@distilled.cloud/cloudflare/user"; import { expect } from "alchemy-test"; @@ -49,7 +50,12 @@ const r2Credentials = Effect.gen(function* () { ); } const token = Redacted.value(creds.apiToken); - const verified = yield* retryAuthBlip(user.verifyToken({})); + const verified = yield* retryAuthBlip( + token.startsWith("cfat_") + ? // Account-owned tokens verify against the account route. + accounts.verifyToken({ accountId: creds.accountId }) + : user.verifyToken({}), + ); const secretAccessKey = yield* Effect.sync(() => crypto.createHash("sha256").update(token).digest("hex"), ); @@ -106,19 +112,19 @@ test.provider( yield* stack.destroy(); - // Create — engine-generated name, default http/workerBinding. + // Create — engine-generated name, default http/workerBinding. The + // secure default is no public HTTP endpoint. const initial = yield* retryAuthBlip(stack.deploy(Cloudflare.Pipelines.Stream("Stream", {}))); expect(initial.streamId).toBeTruthy(); expect(initial.accountId).toEqual(accountId); - expect(initial.httpEnabled).toEqual(true); - expect(initial.httpAuthentication).toEqual(false); + expect(initial.httpEnabled).toEqual(false); expect(initial.workerBindingEnabled).toEqual(true); - expect(initial.endpoint).toBeTruthy(); const live = yield* getStream(accountId, initial.streamId); expect(live.id).toEqual(initial.streamId); expect(live.name).toEqual(initial.name); + expect(live.http.enabled).toEqual(false); // Patch http in place — same streamId. const updated = yield* retryAuthBlip( @@ -134,6 +140,8 @@ test.provider( ); expect(updated.streamId).toEqual(initial.streamId); + expect(updated.httpEnabled).toEqual(true); + expect(updated.endpoint).toBeTruthy(); expect(updated.httpAuthentication).toEqual(true); expect(updated.corsOrigins).toEqual(["https://example.com"]); diff --git a/packages/alchemy/test/Cloudflare/Pipelines/Sink.test.ts b/packages/alchemy/test/Cloudflare/Pipelines/Sink.test.ts index e5a909d16cd..f7de5573516 100644 --- a/packages/alchemy/test/Cloudflare/Pipelines/Sink.test.ts +++ b/packages/alchemy/test/Cloudflare/Pipelines/Sink.test.ts @@ -1,4 +1,5 @@ import crypto from "node:crypto"; +import * as accounts from "@distilled.cloud/cloudflare/accounts"; import * as user from "@distilled.cloud/cloudflare/user"; import { expect } from "alchemy-test"; import * as Effect from "effect/Effect"; @@ -39,7 +40,11 @@ const r2Credentials = Effect.gen(function* () { } const token = Redacted.value(creds.apiToken); const verified = yield* retryAuthBlip( - user.verifyToken({}).pipe( + (token.startsWith("cfat_") + ? // Account-owned tokens verify against the account route. + accounts.verifyToken({ accountId: creds.accountId }) + : user.verifyToken({}) + ).pipe( Effect.retry({ while: (e) => e._tag === "Forbidden", schedule: Schedule.exponential("500 millis"), From b34bb62aeb878b52db4716af048958bd610cf953 Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Mon, 5 Oct 2026 16:25:02 -0700 Subject: [PATCH 03/11] fix: reference @distilled.cloud/iceberg in tsconfig; fix test and suspend typings --- .../src/Cloudflare/Pipelines/Stream.ts | 2 +- .../test/Cloudflare/Basin/Pipeline.test.ts | 9 +------ .../test/Cloudflare/Basin/Sink.test.ts | 9 +------ .../test/Cloudflare/Basin/Stream.test.ts | 9 +------ .../test/Cloudflare/K2/StreamSink.test.ts | 2 +- .../Pipelines/LegacyPipeline.test.ts | 10 +++----- .../Cloudflare/Pipelines/Pipeline.test.ts | 10 +++----- .../Cloudflare/Pipelines/Pipelines.test.ts | 10 +++----- .../test/Cloudflare/Pipelines/Sink.test.ts | 25 +++++++++---------- packages/alchemy/tsconfig.json | 3 +++ tsconfig.json | 3 +++ 11 files changed, 35 insertions(+), 57 deletions(-) diff --git a/packages/alchemy/src/Cloudflare/Pipelines/Stream.ts b/packages/alchemy/src/Cloudflare/Pipelines/Stream.ts index 7182d52b2c1..4d35abf2372 100644 --- a/packages/alchemy/src/Cloudflare/Pipelines/Stream.ts +++ b/packages/alchemy/src/Cloudflare/Pipelines/Stream.ts @@ -194,7 +194,7 @@ export type StreamClass = { const normalizeStreamProps = ( props: unknown, ): Effect.Effect<{ props: unknown; recordSchema: Schema.Top | undefined }> => - Effect.suspend(() => { + Effect.suspend((): Effect.Effect<{ props: unknown; recordSchema: Schema.Top | undefined }> => { const schema = (props as { schema?: unknown } | undefined)?.schema; if (!isEffectSchema(schema)) { return Effect.succeed({ props, recordSchema: undefined }); diff --git a/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts b/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts index 1715ce2fa5c..5578f8e7dca 100644 --- a/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts +++ b/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts @@ -3,7 +3,6 @@ import { expect } from "alchemy-test"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import { MinimumLogLevel } from "effect/References"; -import * as Schedule from "effect/Schedule"; import * as Cloudflare from "@/Cloudflare"; import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; import * as Output from "@/Output"; @@ -15,13 +14,7 @@ const { test } = Test.make({ providers: Cloudflare.providers() }); const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); const getPipeline = (accountId: string, pipelineId: string) => - pipelines.getV1Pipeline({ accountId, pipelineId }).pipe( - Effect.retry({ - while: (e) => e._tag === "Forbidden", - schedule: Schedule.exponential("500 millis"), - times: 8, - }), - ); + pipelines.getV1Pipeline({ accountId, pipelineId }); /** * The whole Basin topology in one deploy: a typed stream, a catalog sink diff --git a/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts b/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts index 1467a72fc4a..8349147d9f4 100644 --- a/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts +++ b/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts @@ -13,14 +13,7 @@ const { test } = Test.make({ providers: Cloudflare.providers() }); const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); -const getSink = (accountId: string, sinkId: string) => - pipelines.getSink({ accountId, sinkId }).pipe( - Effect.retry({ - while: (e) => e._tag === "Forbidden", - schedule: Schedule.exponential("500 millis"), - times: 8, - }), - ); +const getSink = (accountId: string, sinkId: string) => pipelines.getSink({ accountId, sinkId }); const expectSinkGone = (accountId: string, sinkId: string) => getSink(accountId, sinkId).pipe( diff --git a/packages/alchemy/test/Cloudflare/Basin/Stream.test.ts b/packages/alchemy/test/Cloudflare/Basin/Stream.test.ts index 315cfa9614a..a92476feb2f 100644 --- a/packages/alchemy/test/Cloudflare/Basin/Stream.test.ts +++ b/packages/alchemy/test/Cloudflare/Basin/Stream.test.ts @@ -3,7 +3,6 @@ import { expect } from "alchemy-test"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import { MinimumLogLevel } from "effect/References"; -import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import * as Cloudflare from "@/Cloudflare"; import { CloudflareEnvironment } from "@/Cloudflare/CloudflareEnvironment"; @@ -15,13 +14,7 @@ const { test } = Test.make({ providers: Cloudflare.providers() }); const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); const getStream = (accountId: string, streamId: string) => - pipelines.getStream({ accountId, streamId }).pipe( - Effect.retry({ - while: (e) => e._tag === "Forbidden", - schedule: Schedule.exponential("500 millis"), - times: 8, - }), - ); + pipelines.getStream({ accountId, streamId }); const tags = ["provider:cloudflare", "provider:cloudflare:pipelines", "live"]; diff --git a/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts b/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts index b2eeea20698..afaf8ae8d2a 100644 --- a/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts +++ b/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts @@ -39,7 +39,7 @@ const fakeClient = (failures: ReadonlyArray = []) => { const client: WriteStreamClient<{ id: number }> = makeWriteStreamClient( { schema: Order }, (records) => - Effect.suspend(() => { + Effect.suspend((): Effect.Effect => { const tag = failures[attempt++]; if (tag === "K2Unavailable") { return Effect.fail(new k2.K2Unavailable({ code: 10211, message: "unavailable" })); diff --git a/packages/alchemy/test/Cloudflare/Pipelines/LegacyPipeline.test.ts b/packages/alchemy/test/Cloudflare/Pipelines/LegacyPipeline.test.ts index d549381c8fd..4b2efd988ec 100644 --- a/packages/alchemy/test/Cloudflare/Pipelines/LegacyPipeline.test.ts +++ b/packages/alchemy/test/Cloudflare/Pipelines/LegacyPipeline.test.ts @@ -51,12 +51,10 @@ const r2Credentials = Effect.gen(function* () { ); } const token = Redacted.value(creds.apiToken); - const verified = yield* retryAuthBlip( - token.startsWith("cfat_") - ? // Account-owned tokens verify against the account route. - accounts.verifyToken({ accountId: creds.accountId }) - : user.verifyToken({}), - ); + // Account-owned tokens verify against the account route. + const verified = token.startsWith("cfat_") + ? yield* retryAuthBlip(accounts.verifyToken({ accountId: creds.accountId })) + : yield* retryAuthBlip(user.verifyToken({})); const secretAccessKey = yield* Effect.sync(() => crypto.createHash("sha256").update(token).digest("hex"), ); diff --git a/packages/alchemy/test/Cloudflare/Pipelines/Pipeline.test.ts b/packages/alchemy/test/Cloudflare/Pipelines/Pipeline.test.ts index 415aadcb53b..cfb7c5f196b 100644 --- a/packages/alchemy/test/Cloudflare/Pipelines/Pipeline.test.ts +++ b/packages/alchemy/test/Cloudflare/Pipelines/Pipeline.test.ts @@ -38,12 +38,10 @@ const r2Credentials = Effect.gen(function* () { ); } const token = Redacted.value(creds.apiToken); - const verified = yield* retryAuthBlip( - token.startsWith("cfat_") - ? // Account-owned tokens verify against the account route. - accounts.verifyToken({ accountId: creds.accountId }) - : user.verifyToken({}), - ); + // Account-owned tokens verify against the account route. + const verified = token.startsWith("cfat_") + ? yield* retryAuthBlip(accounts.verifyToken({ accountId: creds.accountId })) + : yield* retryAuthBlip(user.verifyToken({})); const secretAccessKey = yield* Effect.sync(() => crypto.createHash("sha256").update(token).digest("hex"), ); diff --git a/packages/alchemy/test/Cloudflare/Pipelines/Pipelines.test.ts b/packages/alchemy/test/Cloudflare/Pipelines/Pipelines.test.ts index 0ad95df2bb6..21b93666531 100644 --- a/packages/alchemy/test/Cloudflare/Pipelines/Pipelines.test.ts +++ b/packages/alchemy/test/Cloudflare/Pipelines/Pipelines.test.ts @@ -50,12 +50,10 @@ const r2Credentials = Effect.gen(function* () { ); } const token = Redacted.value(creds.apiToken); - const verified = yield* retryAuthBlip( - token.startsWith("cfat_") - ? // Account-owned tokens verify against the account route. - accounts.verifyToken({ accountId: creds.accountId }) - : user.verifyToken({}), - ); + // Account-owned tokens verify against the account route. + const verified = token.startsWith("cfat_") + ? yield* retryAuthBlip(accounts.verifyToken({ accountId: creds.accountId })) + : yield* retryAuthBlip(user.verifyToken({})); const secretAccessKey = yield* Effect.sync(() => crypto.createHash("sha256").update(token).digest("hex"), ); diff --git a/packages/alchemy/test/Cloudflare/Pipelines/Sink.test.ts b/packages/alchemy/test/Cloudflare/Pipelines/Sink.test.ts index f7de5573516..3d637cea879 100644 --- a/packages/alchemy/test/Cloudflare/Pipelines/Sink.test.ts +++ b/packages/alchemy/test/Cloudflare/Pipelines/Sink.test.ts @@ -39,19 +39,18 @@ const r2Credentials = Effect.gen(function* () { ); } const token = Redacted.value(creds.apiToken); - const verified = yield* retryAuthBlip( - (token.startsWith("cfat_") - ? // Account-owned tokens verify against the account route. - accounts.verifyToken({ accountId: creds.accountId }) - : user.verifyToken({}) - ).pipe( - Effect.retry({ - while: (e) => e._tag === "Forbidden", - schedule: Schedule.exponential("500 millis"), - times: 8, - }), - ), - ); + // Account-owned tokens verify against the account route. + const verified = token.startsWith("cfat_") + ? yield* retryAuthBlip(accounts.verifyToken({ accountId: creds.accountId })) + : yield* retryAuthBlip( + user.verifyToken({}).pipe( + Effect.retry({ + while: (e) => e._tag === "Forbidden", + schedule: Schedule.exponential("500 millis"), + times: 8, + }), + ), + ); const secretAccessKey = yield* Effect.sync(() => crypto.createHash("sha256").update(token).digest("hex"), ); diff --git a/packages/alchemy/tsconfig.json b/packages/alchemy/tsconfig.json index e411b7cb15b..beec47b8eeb 100644 --- a/packages/alchemy/tsconfig.json +++ b/packages/alchemy/tsconfig.json @@ -45,6 +45,9 @@ { "path": "../../submodules/distilled/packages/hetzner/tsconfig.json" }, + { + "path": "../../submodules/distilled/packages/iceberg/tsconfig.json" + }, { "path": "../../submodules/distilled/packages/fly-io/tsconfig.json" }, diff --git a/tsconfig.json b/tsconfig.json index 6171365c04b..de84f2f057a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,6 +26,9 @@ { "path": "./submodules/distilled/packages/hetzner/tsconfig.json" }, + { + "path": "./submodules/distilled/packages/iceberg/tsconfig.json" + }, { "path": "./submodules/distilled/packages/fly-io/tsconfig.json" }, From 44a6d9bf8543402714f4805f1e800ed913d7e507 Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Mon, 5 Oct 2026 16:30:15 -0700 Subject: [PATCH 04/11] fix(cloudflare/basin): narrow Table diff props with isResolved --- packages/alchemy/src/Cloudflare/Basin/Table.ts | 1 + packages/alchemy/src/Cloudflare/Basin/internal/schema.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/alchemy/src/Cloudflare/Basin/Table.ts b/packages/alchemy/src/Cloudflare/Basin/Table.ts index 6e5b7310b5b..f1c4595676a 100644 --- a/packages/alchemy/src/Cloudflare/Basin/Table.ts +++ b/packages/alchemy/src/Cloudflare/Basin/Table.ts @@ -774,6 +774,7 @@ export const TableProvider = () => ], diff: Effect.fn(function* ({ olds, news, output }) { + if (!isResolved(news)) return undefined; const table = output?.identifier ?? ""; const reasons: string[] = []; if (output !== undefined) { diff --git a/packages/alchemy/src/Cloudflare/Basin/internal/schema.ts b/packages/alchemy/src/Cloudflare/Basin/internal/schema.ts index 643c44e2c60..5e48960d84b 100644 --- a/packages/alchemy/src/Cloudflare/Basin/internal/schema.ts +++ b/packages/alchemy/src/Cloudflare/Basin/internal/schema.ts @@ -217,7 +217,7 @@ const structFieldsOf = (ast: AST.Objects, path: string): TableColumn[] => * descriptor persisted in Table props. Plain descriptors pass through. */ export const toTableSchema = (schema: TableSchemaInput): TableSchema => { - if (!S.isSchema(schema)) return schema; + if (!S.isSchema(schema)) return schema as TableSchema; const root = columnTypeOf(schema.ast, "").type; if (typeof root === "string" || root.type !== "struct") { throw unsupported("", "not a struct (a table schema must be a Struct or Class)"); From 54c9710e2841e75c7b6d8d41d4610ddd91d0555f Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Tue, 6 Oct 2026 09:33:47 -0700 Subject: [PATCH 05/11] refactor(cloudflare/k2): move the polling layer into StreamEventSourcePoller.ts StreamEventSource.ts keeps the contract and consumeStreamRecords; the StreamEventSourcePolling layer and its pull loop (formerly StreamPoller.ts) live together in StreamEventSourcePoller.ts. --- .../src/Cloudflare/K2/StreamEventSource.ts | 105 ++---------------- ...amPoller.ts => StreamEventSourcePoller.ts} | 104 +++++++++++++++-- packages/alchemy/src/Cloudflare/K2/index.ts | 1 + 3 files changed, 109 insertions(+), 101 deletions(-) rename packages/alchemy/src/Cloudflare/K2/{StreamPoller.ts => StreamEventSourcePoller.ts} (56%) diff --git a/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts b/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts index cc1c63ad740..587e814f266 100644 --- a/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts +++ b/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts @@ -1,17 +1,10 @@ import * as Context from "effect/Context"; import type * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; -import * as Layer from "effect/Layer"; import type * as Stream from "effect/Stream"; -import * as Binding from "../../Binding.ts"; -import * as Namespace from "../../Namespace.ts"; -import { ServerHost } from "../../Server/Process.ts"; -import { decodeWithSchema } from "./K2Codec.ts"; import type { ConsumedRecord, DecodedRecord, RecordSchema, StartAt } from "./K2Types.ts"; -import { ReadSubscription } from "./ReadSubscription.ts"; import type { Stream as K2Stream } from "./Stream.ts"; -import { pollSubscription } from "./StreamPoller.ts"; -import { Subscription } from "./Subscription.ts"; +import type { Subscription } from "./Subscription.ts"; /** Settings for {@link consumeStreamRecords}. */ export interface ConsumeStreamRecordsProps { @@ -56,12 +49,14 @@ export interface ConsumeStreamRecordsSchemaProps extends ConsumeStreamRecords schema: RecordSchema; } -type Handler = (records: Stream.Stream) => Effect.Effect; +export type ConsumeStreamRecordsHandler = ( + records: Stream.Stream, +) => Effect.Effect; export type StreamEventSourceService = ( stream: K2Stream, props: ConsumeStreamRecordsProps & { schema?: RecordSchema }, - process: Handler, + process: ConsumeStreamRecordsHandler, ) => Effect.Effect; /** @@ -144,22 +139,24 @@ export class StreamEventSource extends Context.Service< */ export function consumeStreamRecords( stream: K2Stream, - process: Handler, + process: ConsumeStreamRecordsHandler, ): Effect.Effect; export function consumeStreamRecords( stream: K2Stream, props: ConsumeStreamRecordsSchemaProps, - process: Handler, Req>, + process: ConsumeStreamRecordsHandler, Req>, ): Effect.Effect; export function consumeStreamRecords( stream: K2Stream, props: ConsumeStreamRecordsProps, - process: Handler, + process: ConsumeStreamRecordsHandler, ): Effect.Effect; export function consumeStreamRecords( stream: K2Stream, - propsOrProcess: (ConsumeStreamRecordsProps & { schema?: RecordSchema }) | Handler, - maybeProcess?: Handler, + propsOrProcess: + | (ConsumeStreamRecordsProps & { schema?: RecordSchema }) + | ConsumeStreamRecordsHandler, + maybeProcess?: ConsumeStreamRecordsHandler, ): Effect.Effect { const [props, process] = typeof propsOrProcess === "function" @@ -167,81 +164,3 @@ export function consumeStreamRecords( : [propsOrProcess, maybeProcess!]; return StreamEventSource.use((source) => source(stream, props, process)); } - -/** - * Polling implementation of {@link StreamEventSource} for hosts with a - * long-lived process (any host providing `ServerHost`: Fly, ECS, EC2, - * Railway, Hetzner, Docker). Requires a `ReadSubscription` implementation. - * - * :::note - * TODO: Workers cannot consume K2 yet — K2 has no push consumers (they are - * on Cloudflare's roadmap), so no Worker layer ships. - * ::: - * ### Providing the Layer - * **Example:** Poll with a scoped token - * ```typescript - * Effect.gen(function* () { - * yield* Cloudflare.K2.consumeStreamRecords(Orders, handler); - * }).pipe( - * Effect.provide( - * Layer.provideMerge( - * Cloudflare.K2.StreamEventSourcePolling, - * Cloudflare.K2.ReadSubscriptionHttp, - * ), - * ), - * ); - * ``` - * - * @layer - * @provides Cloudflare.K2.StreamEventSource - * @product K2 - * @category Storage & Databases - */ -export const StreamEventSourcePolling = Layer.effect( - StreamEventSource, - Effect.gen(function* () { - const { run } = yield* ServerHost; - const readSubscription = yield* ReadSubscription; - - return Effect.fn(function* ( - stream: K2Stream, - props: ConsumeStreamRecordsProps & { schema?: RecordSchema }, - process: Handler, - ) { - const host = yield* Binding.Host; - const hostId = host?.LogicalId ?? "Host"; - // Declared in both phases: at runtime the declaration resolves to the - // deployed subscription's outputs, which the client reads. - const subscription = - props.subscription ?? - (yield* Namespace.push( - hostId, - Subscription(`${stream.LogicalId}Subscription`, { - streamId: stream.streamId, - startAt: props.startAt, - }), - )); - const client = yield* readSubscription(subscription); - const decode = props.schema - ? decodeWithSchema(props.schema) - : (records: ReadonlyArray) => Effect.succeed(records); - - yield* run( - Effect.gen(function* () { - const workerId = - props.workerId ?? - `${hostId}-${(yield* Effect.sync(() => crypto.randomUUID())).slice(0, 8)}`; - yield* pollSubscription({ - client, - workerId, - maxRecords: props.maxRecords, - idleBackoff: props.idleBackoff, - concurrency: props.concurrency, - decode, - process, - }); - }), - ); - }) as StreamEventSourceService; - }), -); diff --git a/packages/alchemy/src/Cloudflare/K2/StreamPoller.ts b/packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts similarity index 56% rename from packages/alchemy/src/Cloudflare/K2/StreamPoller.ts rename to packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts index ca8961cb5df..eaaac0732b4 100644 --- a/packages/alchemy/src/Cloudflare/K2/StreamPoller.ts +++ b/packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts @@ -2,17 +2,105 @@ import * as Cause from "effect/Cause"; import type * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; import * as Stream from "effect/Stream"; +import * as Binding from "../../Binding.ts"; +import * as Namespace from "../../Namespace.ts"; import type { RuntimeContext } from "../../RuntimeContext.ts"; -import type { Batch, ConsumedRecord, K2SchemaError } from "./K2Types.ts"; -import type { ReadSubscriptionClient } from "./ReadSubscription.ts"; +import { ServerHost } from "../../Server/Process.ts"; +import { decodeWithSchema } from "./K2Codec.ts"; +import type { Batch, ConsumedRecord, K2SchemaError, RecordSchema } from "./K2Types.ts"; +import { ReadSubscription, type ReadSubscriptionClient } from "./ReadSubscription.ts"; +import type { Stream as K2Stream } from "./Stream.ts"; +import { + StreamEventSource, + type ConsumeStreamRecordsProps, + type ConsumeStreamRecordsHandler, + type StreamEventSourceService, +} from "./StreamEventSource.ts"; +import { Subscription } from "./Subscription.ts"; /** - * The pull loop behind `consumeStreamRecords`. Internal — not exported from - * the K2 barrel (tests import it directly). + * Polling implementation of {@link StreamEventSource} for hosts with a + * long-lived process (any host providing `ServerHost`: Fly, ECS, EC2, + * Railway, Hetzner, Docker). Requires a `ReadSubscription` implementation. + * + * :::note + * TODO: Workers cannot consume K2 yet — K2 has no push consumers (they are + * on Cloudflare's roadmap), so no Worker layer ships. + * ::: + * ### Providing the Layer + * **Example:** Poll with a scoped token + * ```typescript + * Effect.gen(function* () { + * yield* Cloudflare.K2.consumeStreamRecords(Orders, handler); + * }).pipe( + * Effect.provide( + * Layer.provideMerge( + * Cloudflare.K2.StreamEventSourcePolling, + * Cloudflare.K2.ReadSubscriptionHttp, + * ), + * ), + * ); + * ``` + * + * @layer + * @provides Cloudflare.K2.StreamEventSource + * @product K2 + * @category Storage & Databases */ +export const StreamEventSourcePolling = Layer.effect( + StreamEventSource, + Effect.gen(function* () { + const { run } = yield* ServerHost; + const readSubscription = yield* ReadSubscription; + + return Effect.fn(function* ( + stream: K2Stream, + props: ConsumeStreamRecordsProps & { schema?: RecordSchema }, + process: ConsumeStreamRecordsHandler, + ) { + const host = yield* Binding.Host; + const hostId = host?.LogicalId ?? "Host"; + // Declared in both phases: at runtime the declaration resolves to the + // deployed subscription's outputs, which the client reads. + const subscription = + props.subscription ?? + (yield* Namespace.push( + hostId, + Subscription(`${stream.LogicalId}Subscription`, { + streamId: stream.streamId, + startAt: props.startAt, + }), + )); + const client = yield* readSubscription(subscription); + const decode = props.schema + ? decodeWithSchema(props.schema) + : (records: ReadonlyArray) => Effect.succeed(records); + + yield* run( + Effect.gen(function* () { + const workerId = + props.workerId ?? + `${hostId}-${(yield* Effect.sync(() => crypto.randomUUID())).slice(0, 8)}`; + yield* pollSubscription({ + client, + workerId, + maxRecords: props.maxRecords, + idleBackoff: props.idleBackoff, + concurrency: props.concurrency, + decode, + process, + }); + }), + ); + }) as StreamEventSourceService; + }), +); + +// ---- the pull loop ----------------------------------------------------------- /** `consume` errors that mean "try again", not "give up". */ const RETRYABLE_CONSUME_ERRORS: ReadonlyArray = [ @@ -28,7 +116,7 @@ const consumeRetry = Schedule.max([ Schedule.recurs(8), ]); -export interface PollOptions { +interface PollOptions { client: ReadSubscriptionClient; /** Worker id holding the leases. Must be unique per concurrent poller. */ workerId: string; @@ -65,7 +153,7 @@ const keepLeaseAlive = (client: ReadSubscriptionClient, batch: Batch, every: Dur * One poll: lease a batch, run the handler, then ack it on success or nack * it on failure. Returns whether a batch was processed. */ -export const pollOnce = ( +const pollOnce = ( options: PollOptions, ): Effect.Effect => Effect.gen(function* () { @@ -108,7 +196,7 @@ export const pollOnce = ( * Poll forever: an empty batch or a failed iteration sleeps `idleBackoff`; * a processed batch polls again immediately. Never fails. */ -export const pollForever = ( +const pollForever = ( options: PollOptions, ): Effect.Effect => pollOnce(options).pipe( @@ -124,7 +212,7 @@ export const pollForever = ( ); /** Run `concurrency` pollers, each with its own worker id (`${workerId}-${i}`). */ -export const pollSubscription = ( +const pollSubscription = ( options: PollOptions & { concurrency?: number }, ): Effect.Effect => Effect.forEach( diff --git a/packages/alchemy/src/Cloudflare/K2/index.ts b/packages/alchemy/src/Cloudflare/K2/index.ts index e7377d58388..fb065919475 100644 --- a/packages/alchemy/src/Cloudflare/K2/index.ts +++ b/packages/alchemy/src/Cloudflare/K2/index.ts @@ -4,6 +4,7 @@ export * from "./ReadSubscriptionHttp.ts"; export * from "./ReadSubscriptionLocal.ts"; export * from "./Stream.ts"; export * from "./StreamEventSource.ts"; +export * from "./StreamEventSourcePoller.ts"; export * from "./StreamSink.ts"; export * from "./StreamSinkBinding.ts"; export * from "./StreamSinkHttp.ts"; From 3f4c22fcccaa6bf9b7d6b3001c0cd969d3dc09ea Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Tue, 6 Oct 2026 09:39:59 -0700 Subject: [PATCH 06/11] docs(cloudflare): add the K2 streams guide under Messaging & events --- website/astro.config.ts | 1 + website/src/content/docs/cloudflare/index.mdx | 2 + .../content/docs/cloudflare/messaging/k2.mdx | 321 ++++++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 website/src/content/docs/cloudflare/messaging/k2.mdx diff --git a/website/astro.config.ts b/website/astro.config.ts index d797334e7b2..9d053fe9ac7 100644 --- a/website/astro.config.ts +++ b/website/astro.config.ts @@ -640,6 +640,7 @@ export default defineConfig({ label: "Messaging & events", items: [ { label: "Queues", link: "/cloudflare/messaging/queues" }, + { label: "K2 streams", link: "/cloudflare/messaging/k2" }, { label: "Cron triggers", link: "/cloudflare/messaging/cron" }, { label: "GitHub events", diff --git a/website/src/content/docs/cloudflare/index.mdx b/website/src/content/docs/cloudflare/index.mdx index cd58c2c9206..c5bf5effe4c 100644 --- a/website/src/content/docs/cloudflare/index.mdx +++ b/website/src/content/docs/cloudflare/index.mdx @@ -47,6 +47,8 @@ New here? [Set up your account](/cloudflare/setup), then start the - [Queues](/cloudflare/messaging/queues) — at-least-once message delivery between Workers. +- [K2 streams](/cloudflare/messaging/k2) — a durable, replayable log with + independent subscriptions. ## Networking diff --git a/website/src/content/docs/cloudflare/messaging/k2.mdx b/website/src/content/docs/cloudflare/messaging/k2.mdx new file mode 100644 index 00000000000..533a5597d8b --- /dev/null +++ b/website/src/content/docs/cloudflare/messaging/k2.mdx @@ -0,0 +1,321 @@ +--- +title: K2 streams +description: Cloudflare K2 is a durable, replayable log. Produce records from Workers with the WriteStream binding, then consume them from a long-running host with consumeStreamRecords. +--- + +A K2 **stream** is a durable, ordered log of records stored on R2. Producers +append records; readers consume them through **subscriptions**. Each +subscription tracks its own position, so many readers can read the same +records independently, and records stay in the stream for its retention +period whether or not anyone has read them. + +Reach for K2 when several consumers need the same events, when you need to +replay history, or when you want a log you can read at your own pace. Reach +for [Queues](/cloudflare/messaging/queues) when each message is a unit of work +that one consumer handles once, with retries and a dead-letter queue. + +| | Queues | K2 | +| --- | --- | --- | +| Model | Work queue: a message is removed once it is acknowledged | Log: records stay for the retention period | +| Readers | One consumer | Any number of subscriptions, each reading every record | +| Delivery | Pushed to a consumer Worker | Pulled in leased batches over HTTP | +| Consume from a Worker | Yes | Not yet | +| Replay | No | From the start of retention (`startAt: "earliest"`) | + +:::note +K2 cannot push records to a Worker yet; push consumers are on Cloudflare's +roadmap. Produce from Workers, and consume from a long-running host such as +a Fly, ECS, EC2, Railway, Hetzner, or Docker service. Alchemy will add a +Worker consumer once Cloudflare supports one. +::: + +## Create a stream + +```typescript +// src/Orders.ts +import * as Cloudflare from "alchemy/Cloudflare"; + +export const Orders = Cloudflare.K2.Stream("Orders", { + retention: "7 days", +}); +``` + +The stream name is generated from the stack, stage, and logical ID. Changing +`retention` updates the stream in place. It accepts any `Duration.Input` +between one hour and 30 days, and defaults to seven days. + +A stream accepts records from Workers by default. Its HTTP input is off unless +you turn it on: + +```typescript +Cloudflare.K2.Stream("Orders", { http: true }); // authenticated +Cloudflare.K2.Stream("Orders", { http: { authentication: false } }); // public +Cloudflare.K2.Stream("Orders", { http: { cors: ["https://shop.dev"] } }); +``` + +Turn on the HTTP input when a host outside Cloudflare produces to the stream, +or when you produce with `WriteStreamHttp`. + +## Produce from a Worker + +In the Worker's init phase, ask `Cloudflare.K2.WriteStream` for a client. Its +`send` appends a batch of records atomically. + +```typescript +// src/Api.ts +import * as Cloudflare from "alchemy/Cloudflare"; +import * as Effect from "effect/Effect"; +import { HttpServerRequest } from "effect/http/HttpServerRequest"; +import * as HttpServerResponse from "effect/http/HttpServerResponse"; +import { Orders } from "./Orders.ts"; + +export default class Api extends Cloudflare.Worker()( + "Api", + { main: import.meta.url }, + Effect.gen(function* () { + const orders = yield* Cloudflare.K2.WriteStream(Orders); + + return { + fetch: Effect.gen(function* () { + const request = yield* HttpServerRequest; + const body = yield* request.text; + yield* orders.send([{ content: body, headers: { source: "api" } }]); + return HttpServerResponse.empty({ status: 202 }); + }), + }; + }).pipe(Effect.provide(Cloudflare.K2.WriteStreamBinding)), +) {} +``` + +A record's `content` is a string (sent as UTF-8), a `Uint8Array`, or an +`ArrayBuffer`. `headers` is an optional map of up to 32 string pairs. A batch +can be up to 5 MB, and each record up to 1 MB. + +`WriteStreamBinding` sends through the Worker's native `k2` binding, so it +needs no API token. + +## Type records with a schema + +Pass an Effect Schema, and `send` takes values of that type. The client +encodes each value as JSON and sets a `content-type: application/json` +header. + +```diff lang="typescript" ++import * as Schema from "effect/Schema"; ++ ++export const Order = Schema.Struct({ ++ orderId: Schema.String, ++ amountCents: Schema.Number, ++}); + +- const orders = yield* Cloudflare.K2.WriteStream(Orders); ++ const orders = yield* Cloudflare.K2.WriteStream(Orders, { schema: Order }); +``` + +```typescript +yield* orders.send([{ orderId: "o_1", amountCents: 4200 }]); +``` + +## Handle produce errors + +Failures are typed. Two of them tell you whether the batch was stored: + +```typescript +yield* orders.send(batch).pipe( + // The batch was not stored. Retrying it is safe. + Effect.catchTag("K2Unavailable", () => retryLater(batch)), + // The batch may or may not have been stored, and K2 does not deduplicate. + // Alchemy never retries this one for you. + Effect.catchTag("K2AppendOutcomeUnknown", () => reconcile(batch)), +); +``` + +The native binding and the HTTP path report the same error tags, so this code +works with either layer. + +## Drain a Stream into K2 + +`Cloudflare.K2.StreamSink` exposes the stream as an Effect +[Sink](/infrastructure-as-effects/sinks). It packs records into requests of +up to 5 MB and retries only the errors that mean the batch was not stored. + +```typescript +const sink = yield* Cloudflare.K2.StreamSink(Orders, { schema: Order }); +yield* Stream.fromIterable(pending).pipe(Stream.run(sink)); +``` + +Provide `Cloudflare.K2.StreamSinkBinding` alongside `WriteStreamBinding`. + +## Produce from outside a Worker + +The producer code stays the same on every host. Only the layer changes: + +- `WriteStreamBinding` and `StreamSinkBinding` run in Cloudflare Workers and + send through the native `k2` binding. +- `WriteStreamHttp` and `StreamSinkHttp` run on any host, such as Lambda or a + container. They send through the stream's HTTP input with a scoped "K2 + Produce" token that Alchemy mints at deploy. +- `WriteStreamLocal` and `StreamSinkLocal` run in scripts and Actions. They + send through the stream's HTTP input with your current Cloudflare + credentials. + +The HTTP and local layers need the stream's HTTP input turned on +(`http: true`). + +## Use K2 from an async Worker + +Put the stream in a Worker's `env` and Alchemy adds the `k2` binding: + +```typescript +// alchemy.run.ts +export const Ingest = Cloudflare.Worker("Ingest", { + main: "./src/ingest.ts", + env: { ORDERS: Orders }, +}); + +export type IngestEnv = Cloudflare.InferEnv; +``` + +`env.ORDERS.send` returns a result object instead of throwing on an append +failure: + +```typescript +// src/ingest.ts +import type { IngestEnv } from "../alchemy.run.ts"; + +export default { + async fetch(request: Request, env: IngestEnv) { + const result = await env.ORDERS.send([ + { content: new TextEncoder().encode(await request.text()) }, + ]); + if (!result.success) { + const status = result.error.retryable ? 503 : 500; + return new Response(result.error.message, { status }); + } + return new Response(null, { status: 202 }); + }, +}; +``` + +## Consume from a long-running host + +`Cloudflare.K2.consumeStreamRecords(stream, handler)` subscribes a host to a +stream. The handler receives each batch as an Effect `Stream`: + +```typescript +// src/Fraud.ts +import * as Cloudflare from "alchemy/Cloudflare"; +import * as Fly from "alchemy/Fly"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import { Order, Orders } from "./Orders.ts"; + +export default class Fraud extends Fly.Service()( + "Fraud", + { main: import.meta.url, region: "iad" }, + Effect.gen(function* () { + yield* Cloudflare.K2.consumeStreamRecords( + Orders, + { schema: Order, startAt: "earliest", concurrency: 4 }, + (records) => + records.pipe( + Stream.map((record) => record.value), + Stream.filter((order) => order.amountCents > 100_000), + Stream.runForEach((order) => Effect.log(`review ${order.orderId}`)), + ), + ); + + return {}; + }).pipe( + Effect.provide( + Layer.provideMerge( + Cloudflare.K2.StreamEventSourcePolling, + Cloudflare.K2.ReadSubscriptionHttp, + ), + ), + ), +) {} +``` + +At deploy time, `consumeStreamRecords` creates a subscription owned by the +host. At runtime, `StreamEventSourcePolling` runs `concurrency` pollers. Each +poller leases a batch of up to `maxRecords` (default 100) and runs the +handler: + +- When the handler succeeds, the batch is acknowledged. +- When it fails, the failure is logged and the batch is released for + redelivery. +- While the handler runs, the five-minute lease is extended every two + minutes. +- When a batch comes back empty, the poller waits `idleBackoff` (default one + second) before it polls again. + +`ReadSubscriptionHttp` reads with a scoped "K2 Consume" token that Alchemy +mints at deploy. `StreamEventSourcePolling` needs a host with a long-lived +process (Fly, ECS, EC2, Railway, Hetzner, Docker), so it doesn't work in a +Worker. + +Without `schema`, each record is `{ content: Uint8Array, headers, timestamp }`. +With `schema`, the JSON content is also decoded into `record.value`. + +## Share a subscription between hosts + +By default every consuming host gets its own subscription, so each host sees +every record. To split records between hosts instead, declare a +`Subscription` and pass it to each one: + +```typescript +export const FraudWorkers = Cloudflare.K2.Subscription("FraudWorkers", { + streamId: Orders.streamId, + startAt: "latest", +}); +``` + +```diff lang="typescript" + yield* Cloudflare.K2.consumeStreamRecords( + Orders, +- { schema: Order, startAt: "earliest", concurrency: 4 }, ++ { schema: Order, subscription: FraudWorkers, concurrency: 4 }, +``` + +Hosts sharing a subscription compete for batches. A subscription allows up +to 128 leases at a time. Subscriptions can't be changed after they are +created, so changing `startAt` replaces the subscription. + +## Control leases yourself + +`Cloudflare.K2.ReadSubscription` is the client `consumeStreamRecords` uses +internally. Use it when you need your own loop: + +```typescript +const inbox = yield* Cloudflare.K2.ReadSubscription(FraudWorkers); + +const batch = yield* inbox.consume({ workerId: "worker-1", maxRecords: 100 }); +if (Option.isSome(batch)) { + yield* handle(batch.value.records).pipe( + Effect.andThen(inbox.ack(batch.value)), + Effect.catchCause(() => inbox.nack(batch.value)), + ); +} +``` + +`consume` returns `Option.none()` when there are no records. A worker that +already holds a lease gets the same batch back. `extend(batch)` renews a +lease for another five minutes, and fails with `K2LeaseLost` once the lease +is gone. + +## Where next + +Related: + +- [Queues](/cloudflare/messaging/queues): a work queue with push delivery to + Workers, retries, and dead-letter queues. +- [Workers](/cloudflare/compute/workers): the runtime that produces records. +- [Sinks](/infrastructure-as-effects/sinks): the Effect `Sink` model behind + `StreamSink`. + +Reference: + +- [K2 API reference](/providers/cloudflare/k2): `Stream`, `Subscription`, + `WriteStream`, `StreamSink`, `ReadSubscription`, and the polling layer. From bbb85a287c0b955f6233eaac9e78cc817c50663c Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Tue, 6 Oct 2026 10:25:38 -0700 Subject: [PATCH 07/11] refactor(cloudflare/k2): declare the record schema on the Stream K2.Stream("Orders", { schema: Order }) returns a Stream; WriteStream, StreamSink and consumeStreamRecords read the schema from the stream instead of taking it per call, matching Pipelines.Stream. The schema is a client-side codec and is never persisted, so changing it never updates the stream. --- packages/alchemy/src/Cloudflare/K2/Stream.ts | 111 +++++++++++++++++- .../src/Cloudflare/K2/StreamEventSource.ts | 44 +++---- .../Cloudflare/K2/StreamEventSourcePoller.ts | 10 +- .../alchemy/src/Cloudflare/K2/StreamSink.ts | 13 +- .../src/Cloudflare/K2/StreamSinkBatch.ts | 6 +- .../alchemy/src/Cloudflare/K2/WriteStream.ts | 38 ++---- .../src/Cloudflare/K2/WriteStreamBinding.ts | 6 +- .../src/Cloudflare/K2/WriteStreamClient.ts | 10 +- .../src/Cloudflare/K2/WriteStreamHttp.ts | 12 +- .../src/Cloudflare/K2/WriteStreamLocal.ts | 6 +- .../Cloudflare/K2/ReadSubscription.test.ts | 7 +- .../test/Cloudflare/K2/StreamSink.test.ts | 28 ++--- .../test/Cloudflare/K2/WriteStream.test.ts | 8 +- .../K2/fixtures/k2-binding-worker.ts | 9 +- .../Cloudflare/K2/fixtures/k2-http-worker.ts | 9 +- .../test/Cloudflare/K2/fixtures/k2-shared.ts | 30 ++--- 16 files changed, 200 insertions(+), 147 deletions(-) diff --git a/packages/alchemy/src/Cloudflare/K2/Stream.ts b/packages/alchemy/src/Cloudflare/K2/Stream.ts index 2e6b54650af..c69579872ff 100644 --- a/packages/alchemy/src/Cloudflare/K2/Stream.ts +++ b/packages/alchemy/src/Cloudflare/K2/Stream.ts @@ -1,17 +1,20 @@ import * as k2 from "@distilled.cloud/cloudflare/k2"; import type * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Effectable from "effect/Effectable"; import * as Option from "effect/Option"; import * as Predicate from "effect/Predicate"; import * as EffectStream from "effect/Stream"; import { Unowned } from "../../AdoptPolicy.ts"; import { isResolved } from "../../Diff.ts"; +import type { PropsInput } from "../../Input.ts"; import { createPhysicalName } from "../../PhysicalName.ts"; import * as Provider from "../../Provider.ts"; -import { Resource } from "../../Resource.ts"; +import { Resource, type ResourceClass } from "../../Resource.ts"; import { toWireSeconds } from "../../Util/Duration.ts"; import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; import type { Providers } from "../Providers.ts"; +import type { ConsumedRecord, DecodedRecord, Record as K2Record, RecordSchema } from "./K2Types.ts"; const StreamTypeId = "Cloudflare.K2.Stream" as const; type StreamTypeId = typeof StreamTypeId; @@ -92,7 +95,90 @@ export interface StreamAttributes { modifiedAt: string; } -export type Stream = Resource; +/** + * A K2 stream. `A` is the record type its producers send and its consumers + * decode — the Effect Schema's `Type` when the stream was declared with + * `schema`, unknown otherwise. + */ +export interface Stream extends Resource< + StreamTypeId, + StreamProps, + StreamAttributes, + never, + Providers +> { + /** + * The Effect Schema the stream was declared with, if any. Producers encode + * records with it and consumers decode them; it is never persisted, because + * K2 itself stores bytes. + */ + readonly RecordSchema: RecordSchema | undefined; + /** Phantom record type. */ + readonly "~record"?: A; +} + +/** + * The record a producer sends to a `Stream`: the schema's type, or raw + * bytes plus headers ({@link K2Record}) for a stream without a schema. + */ +export type StreamRecord = unknown extends A ? K2Record : A; + +/** + * The record a consumer receives from a `Stream`: decoded into `value` for + * a stream with a schema, raw bytes otherwise. + */ +export type StreamConsumedRecord = unknown extends A ? ConsumedRecord : DecodedRecord; + +/** Props of a stream declared with an Effect Schema. */ +export type StreamSchemaProps> = PropsInput & { + /** + * Encode every record as JSON with this schema on the way in, and decode + * it on the way out. A client-side codec only: K2 stores bytes, so + * changing the schema never updates or replaces the stream. + */ + schema: S; +}; + +const StreamResource = Resource(StreamTypeId); + +/** + * The Stream constructor: the plain resource constructor plus an overload + * that types the stream's records from an Effect Schema `schema`. + */ +export type StreamClass = { + >( + id: string, + props: StreamSchemaProps, + ): Effect.Effect, never, Providers>; +} & ResourceClass; + +/** Split the client-side `schema` off the props the engine persists. */ +const splitSchema = (props: unknown): { props: unknown; schema: RecordSchema | undefined } => { + if (props === null || typeof props !== "object" || !("schema" in props)) { + return { props, schema: undefined }; + } + const { schema, ...rest } = props as { schema?: RecordSchema }; + return { props: rest, schema }; +}; + +const constructStream = (id: string, props: unknown) => + Effect.gen(function* () { + let schema: RecordSchema | undefined; + const capture = (split: { props: unknown; schema: RecordSchema | undefined }) => { + schema = split.schema; + return split.props; + }; + const persisted = Effect.isEffect(props) + ? (props as Effect.Effect).pipe(Effect.map((p) => capture(splitSchema(p)))) + : capture(splitSchema(props)); + const construct = StreamResource as unknown as ( + id: string, + props: unknown, + ) => Effect.Effect; + const stream = yield* construct(id, persisted); + (stream as { RecordSchema: RecordSchema | undefined }).RecordSchema = schema; + return stream; + }); /** * A Cloudflare K2 stream — a durable, ordered log of records. Producers @@ -116,6 +202,15 @@ export type Stream = Resource(StreamTypeId); +export const Stream: StreamClass = Object.assign( + (...args: [id: string, props?: unknown] | [methods: object]) => + typeof args[0] === "object" + ? Object.assign(Stream, args[0]) + : constructStream(args[0], (args as [string, unknown])[1]), + StreamResource, + Effectable.Prototype({ + label: `Resource<${StreamTypeId}>`, + evaluate: () => Effect.succeed((id: string, props: unknown) => constructStream(id, props)), + }), +) as unknown as StreamClass; /** Returns true if the given value is a K2 Stream resource. */ export const isStream = (value: unknown): value is Stream => diff --git a/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts b/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts index 587e814f266..43664920b7d 100644 --- a/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts +++ b/packages/alchemy/src/Cloudflare/K2/StreamEventSource.ts @@ -2,8 +2,8 @@ import * as Context from "effect/Context"; import type * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import type * as Stream from "effect/Stream"; -import type { ConsumedRecord, DecodedRecord, RecordSchema, StartAt } from "./K2Types.ts"; -import type { Stream as K2Stream } from "./Stream.ts"; +import type { StartAt } from "./K2Types.ts"; +import type { Stream as K2Stream, StreamConsumedRecord } from "./Stream.ts"; import type { Subscription } from "./Subscription.ts"; /** Settings for {@link consumeStreamRecords}. */ @@ -44,18 +44,13 @@ export interface ConsumeStreamRecordsProps { workerId?: string; } -/** {@link ConsumeStreamRecordsProps} with a schema decoding JSON content into `record.value`. */ -export interface ConsumeStreamRecordsSchemaProps extends ConsumeStreamRecordsProps { - schema: RecordSchema; -} - export type ConsumeStreamRecordsHandler = ( records: Stream.Stream, ) => Effect.Effect; export type StreamEventSourceService = ( - stream: K2Stream, - props: ConsumeStreamRecordsProps & { schema?: RecordSchema }, + stream: K2Stream, + props: ConsumeStreamRecordsProps, process: ConsumeStreamRecordsHandler, ) => Effect.Effect; @@ -82,7 +77,9 @@ export class StreamEventSource extends Context.Service< * (unless `subscription` is passed — hosts sharing one compete for * batches). * - **Runtime**: `concurrency` pollers lease batches of records, run the - * handler, and acknowledge each batch when the handler succeeds. A failed + * handler, and acknowledge each batch when the handler succeeds. When the + * stream was declared with a `schema`, each record's JSON content is + * decoded into `record.value`. A failed * handler is logged and the batch is released for redelivery. Leases are * extended every two minutes while the handler runs. * @@ -108,11 +105,13 @@ export class StreamEventSource extends Context.Service< * ``` * * ### Typed Records - * **Example:** Decode JSON records with a Schema + * **Example:** Records decoded with the stream's schema * ```typescript + * const Orders = Cloudflare.K2.Stream("Orders", { schema: Order }); + * * yield* Cloudflare.K2.consumeStreamRecords( * Orders, - * { schema: Order, startAt: "earliest", concurrency: 4 }, + * { startAt: "earliest", concurrency: 4 }, * (records) => * records.pipe(Stream.runForEach((record) => Effect.log(record.value.id))), * ); @@ -137,25 +136,18 @@ export class StreamEventSource extends Context.Service< * @product K2 * @category Storage & Databases */ -export function consumeStreamRecords( - stream: K2Stream, - process: ConsumeStreamRecordsHandler, -): Effect.Effect; export function consumeStreamRecords( - stream: K2Stream, - props: ConsumeStreamRecordsSchemaProps, - process: ConsumeStreamRecordsHandler, Req>, + stream: K2Stream, + process: ConsumeStreamRecordsHandler, Req>, ): Effect.Effect; -export function consumeStreamRecords( - stream: K2Stream, +export function consumeStreamRecords( + stream: K2Stream, props: ConsumeStreamRecordsProps, - process: ConsumeStreamRecordsHandler, + process: ConsumeStreamRecordsHandler, Req>, ): Effect.Effect; export function consumeStreamRecords( - stream: K2Stream, - propsOrProcess: - | (ConsumeStreamRecordsProps & { schema?: RecordSchema }) - | ConsumeStreamRecordsHandler, + stream: K2Stream, + propsOrProcess: ConsumeStreamRecordsProps | ConsumeStreamRecordsHandler, maybeProcess?: ConsumeStreamRecordsHandler, ): Effect.Effect { const [props, process] = diff --git a/packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts b/packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts index eaaac0732b4..5c1803a047c 100644 --- a/packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts +++ b/packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts @@ -11,7 +11,7 @@ import * as Namespace from "../../Namespace.ts"; import type { RuntimeContext } from "../../RuntimeContext.ts"; import { ServerHost } from "../../Server/Process.ts"; import { decodeWithSchema } from "./K2Codec.ts"; -import type { Batch, ConsumedRecord, K2SchemaError, RecordSchema } from "./K2Types.ts"; +import type { Batch, ConsumedRecord, K2SchemaError } from "./K2Types.ts"; import { ReadSubscription, type ReadSubscriptionClient } from "./ReadSubscription.ts"; import type { Stream as K2Stream } from "./Stream.ts"; import { @@ -58,8 +58,8 @@ export const StreamEventSourcePolling = Layer.effect( const readSubscription = yield* ReadSubscription; return Effect.fn(function* ( - stream: K2Stream, - props: ConsumeStreamRecordsProps & { schema?: RecordSchema }, + stream: K2Stream, + props: ConsumeStreamRecordsProps, process: ConsumeStreamRecordsHandler, ) { const host = yield* Binding.Host; @@ -76,8 +76,8 @@ export const StreamEventSourcePolling = Layer.effect( }), )); const client = yield* readSubscription(subscription); - const decode = props.schema - ? decodeWithSchema(props.schema) + const decode = stream.RecordSchema + ? decodeWithSchema(stream.RecordSchema) : (records: ReadonlyArray) => Effect.succeed(records); yield* run( diff --git a/packages/alchemy/src/Cloudflare/K2/StreamSink.ts b/packages/alchemy/src/Cloudflare/K2/StreamSink.ts index da76ba66926..904cb9f531a 100644 --- a/packages/alchemy/src/Cloudflare/K2/StreamSink.ts +++ b/packages/alchemy/src/Cloudflare/K2/StreamSink.ts @@ -2,9 +2,8 @@ import type * as Effect from "effect/Effect"; import type * as Sink from "effect/Sink"; import * as Binding from "../../Binding.ts"; import type { RuntimeContext } from "../../RuntimeContext.ts"; -import type { ProduceError, Record as K2Record } from "./K2Types.ts"; -import type { Stream } from "./Stream.ts"; -import type { WriteStreamOptions } from "./WriteStream.ts"; +import type { ProduceError } from "./K2Types.ts"; +import type { Stream, StreamRecord } from "./Stream.ts"; /** * Binding service that exposes a K2 {@link Stream} as an Effect `Sink`: run @@ -53,13 +52,9 @@ import type { WriteStreamOptions } from "./WriteStream.ts"; export interface StreamSink extends Binding.Service< StreamSink, "Cloudflare.K2.StreamSink", - (stream: Stream, options?: WriteStreamOptions) => Effect.Effect> + (stream: Stream) => Effect.Effect> > { - ( - stream: Stream, - options: WriteStreamOptions, - ): Effect.Effect, never, StreamSink>; - (stream: Stream): Effect.Effect, never, StreamSink>; + (stream: Stream): Effect.Effect>, never, StreamSink>; } export const StreamSink = Binding.Service("Cloudflare.K2.StreamSink"); diff --git a/packages/alchemy/src/Cloudflare/K2/StreamSinkBatch.ts b/packages/alchemy/src/Cloudflare/K2/StreamSinkBatch.ts index b2efd8ab6f2..96ccf4f9245 100644 --- a/packages/alchemy/src/Cloudflare/K2/StreamSinkBatch.ts +++ b/packages/alchemy/src/Cloudflare/K2/StreamSinkBatch.ts @@ -7,7 +7,7 @@ import { wireSize } from "./K2Codec.ts"; import type { EncodedRecord } from "./K2Types.ts"; import type { Stream } from "./Stream.ts"; import { StreamSink, type StreamSinkClient } from "./StreamSink.ts"; -import { WriteStream, type WriteStreamClient, type WriteStreamOptions } from "./WriteStream.ts"; +import { WriteStream, type WriteStreamClient } from "./WriteStream.ts"; /** * Batch packing for {@link StreamSink}. Internal — not exported from the K2 @@ -78,8 +78,8 @@ export const StreamSinkFromWriteStream = Layer.effect( StreamSink, Effect.gen(function* () { const writeStream = yield* WriteStream; - return Effect.fn(function* (stream: Stream, options?: WriteStreamOptions) { - const client = yield* writeStream(stream, options); + return Effect.fn(function* (stream: Stream) { + const client = yield* writeStream(stream); return makeStreamSink(client); }); }), diff --git a/packages/alchemy/src/Cloudflare/K2/WriteStream.ts b/packages/alchemy/src/Cloudflare/K2/WriteStream.ts index f93aaf71865..8feca94b00f 100644 --- a/packages/alchemy/src/Cloudflare/K2/WriteStream.ts +++ b/packages/alchemy/src/Cloudflare/K2/WriteStream.ts @@ -1,24 +1,8 @@ import type * as Effect from "effect/Effect"; import * as Binding from "../../Binding.ts"; import type { RuntimeContext } from "../../RuntimeContext.ts"; -import type { - EncodedRecord, - K2SchemaError, - ProduceError, - Record as K2Record, - RecordSchema, -} from "./K2Types.ts"; -import type { Stream } from "./Stream.ts"; - -/** Options accepted by `WriteStream(stream, options)` and `StreamSink(stream, options)`. */ -export interface WriteStreamOptions { - /** - * Encode every value with this schema and send it as JSON with a - * `content-type: application/json` header. Without a schema, `send` - * takes raw {@link K2Record}s. - */ - schema: RecordSchema; -} +import type { EncodedRecord, K2SchemaError, ProduceError } from "./K2Types.ts"; +import type { Stream, StreamRecord } from "./Stream.ts"; /** * Binding service that turns a K2 {@link Stream} into a typed @@ -44,10 +28,12 @@ export interface WriteStreamOptions { * ``` * * ### Typed Records - * **Example:** Encode values with a Schema + * **Example:** Send values of the stream's schema * ```typescript * const Order = Schema.Struct({ id: Schema.Number, total: Schema.Number }); - * const orders = yield* Cloudflare.K2.WriteStream(Orders, { schema: Order }); + * const Orders = Cloudflare.K2.Stream("Orders", { schema: Order }); + * + * const orders = yield* Cloudflare.K2.WriteStream(Orders); * // JSON-encoded, sent with `content-type: application/json` * yield* orders.send([{ id: 1, total: 42 }]); * ``` @@ -76,20 +62,16 @@ export interface WriteStreamOptions { export interface WriteStream extends Binding.Service< WriteStream, "Cloudflare.K2.WriteStream", - (stream: Stream, options?: WriteStreamOptions) => Effect.Effect> + (stream: Stream) => Effect.Effect> > { - ( - stream: Stream, - options: WriteStreamOptions, - ): Effect.Effect, never, WriteStream>; - (stream: Stream): Effect.Effect, never, WriteStream>; + (stream: Stream): Effect.Effect>, never, WriteStream>; } export const WriteStream = Binding.Service("Cloudflare.K2.WriteStream"); /** - * Producer client for a K2 stream. `A` is {@link K2Record} without a - * schema, or the schema's type with one. + * Producer client for a K2 stream. `A` is the stream's schema type, or raw + * bytes plus headers (`Cloudflare.K2.Record`) for a stream without one. */ export interface WriteStreamClient { /** Append a batch of records atomically. */ diff --git a/packages/alchemy/src/Cloudflare/K2/WriteStreamBinding.ts b/packages/alchemy/src/Cloudflare/K2/WriteStreamBinding.ts index 1e94ab6bd7b..331cb573826 100644 --- a/packages/alchemy/src/Cloudflare/K2/WriteStreamBinding.ts +++ b/packages/alchemy/src/Cloudflare/K2/WriteStreamBinding.ts @@ -5,7 +5,7 @@ import * as Layer from "effect/Layer"; import { Worker, WorkerEnvironment } from "../Workers/Worker.ts"; import type { EncodedRecord, ProduceError } from "./K2Types.ts"; import type { Stream } from "./Stream.ts"; -import { WriteStream, type WriteStreamOptions } from "./WriteStream.ts"; +import { WriteStream } from "./WriteStream.ts"; import { makeWriteStreamClient } from "./WriteStreamClient.ts"; /** @@ -59,7 +59,7 @@ export const WriteStreamBinding = Layer.effect( const env = yield* WorkerEnvironment; const host = yield* Worker; - return Effect.fn(function* (stream: Stream, options?: WriteStreamOptions) { + return Effect.fn(function* (stream: Stream) { if (!globalThis.__ALCHEMY_RUNTIME__) { yield* host.bind`${stream}`({ bindings: [ @@ -72,7 +72,7 @@ export const WriteStreamBinding = Layer.effect( }); } const raw = Effect.sync(() => (env as { [key: string]: K2StreamBinding })[stream.LogicalId]!); - return makeWriteStreamClient(options, (records) => + return makeWriteStreamClient(stream.RecordSchema, (records) => raw.pipe(Effect.flatMap((binding) => sendViaBinding(binding, records))), ); }); diff --git a/packages/alchemy/src/Cloudflare/K2/WriteStreamClient.ts b/packages/alchemy/src/Cloudflare/K2/WriteStreamClient.ts index b7ae8b77b7c..d36a6634420 100644 --- a/packages/alchemy/src/Cloudflare/K2/WriteStreamClient.ts +++ b/packages/alchemy/src/Cloudflare/K2/WriteStreamClient.ts @@ -1,8 +1,8 @@ import * as Effect from "effect/Effect"; import type { RuntimeContext } from "../../RuntimeContext.ts"; import { encodeRecords, encodeWithSchema } from "./K2Codec.ts"; -import type { EncodedRecord, ProduceError, Record as K2Record } from "./K2Types.ts"; -import type { WriteStreamClient, WriteStreamOptions } from "./WriteStream.ts"; +import type { EncodedRecord, ProduceError, Record as K2Record, RecordSchema } from "./K2Types.ts"; +import type { WriteStreamClient } from "./WriteStream.ts"; /** * Build a {@link WriteStreamClient} over a transport that appends encoded @@ -10,13 +10,13 @@ import type { WriteStreamClient, WriteStreamOptions } from "./WriteStream.ts"; * differ only in `sendEncoded`. Internal — not exported from the K2 barrel. */ export const makeWriteStreamClient = ( - options: WriteStreamOptions | undefined, + schema: RecordSchema | undefined, sendEncoded: ( records: ReadonlyArray, ) => Effect.Effect, ): WriteStreamClient => { - const encode = options?.schema - ? encodeWithSchema(options.schema) + const encode = schema + ? encodeWithSchema(schema) : (records: ReadonlyArray) => Effect.sync(() => encodeRecords(records)); return { encode, diff --git a/packages/alchemy/src/Cloudflare/K2/WriteStreamHttp.ts b/packages/alchemy/src/Cloudflare/K2/WriteStreamHttp.ts index 1d5bbf5184d..07e68f749c5 100644 --- a/packages/alchemy/src/Cloudflare/K2/WriteStreamHttp.ts +++ b/packages/alchemy/src/Cloudflare/K2/WriteStreamHttp.ts @@ -3,9 +3,9 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import { toProduceRecords } from "./K2Codec.ts"; import { type K2Auth, makeK2HttpAuth } from "./K2Http.ts"; -import type { EncodedRecord } from "./K2Types.ts"; +import type { EncodedRecord, RecordSchema } from "./K2Types.ts"; import type { Stream } from "./Stream.ts"; -import { WriteStream, type WriteStreamOptions } from "./WriteStream.ts"; +import { WriteStream } from "./WriteStream.ts"; import { makeWriteStreamClient } from "./WriteStreamClient.ts"; /** @@ -36,9 +36,9 @@ export const WriteStreamHttp = Layer.effect( WriteStream, Effect.gen(function* () { const auth = yield* makeK2HttpAuth; - return Effect.fn(function* (stream: Stream, options?: WriteStreamOptions) { + return Effect.fn(function* (stream: Stream) { const streamAuth = yield* auth(stream, "K2 Produce"); - return makeWriteStreamHttpClient(streamAuth, yield* stream.streamId, options); + return makeWriteStreamHttpClient(streamAuth, yield* stream.streamId, stream.RecordSchema); }); }), ); @@ -50,9 +50,9 @@ export const WriteStreamHttp = Layer.effect( export const makeWriteStreamHttpClient = ( auth: K2Auth, streamId: Effect.Effect, - options: WriteStreamOptions | undefined, + schema: RecordSchema | undefined, ) => - makeWriteStreamClient(options, (records: ReadonlyArray) => + makeWriteStreamClient(schema, (records: ReadonlyArray) => streamId.pipe( Effect.flatMap((id) => auth.authorize(k2.produce({ streamId: id, records: toProduceRecords(records) })), diff --git a/packages/alchemy/src/Cloudflare/K2/WriteStreamLocal.ts b/packages/alchemy/src/Cloudflare/K2/WriteStreamLocal.ts index 14f173c2472..7690153f005 100644 --- a/packages/alchemy/src/Cloudflare/K2/WriteStreamLocal.ts +++ b/packages/alchemy/src/Cloudflare/K2/WriteStreamLocal.ts @@ -2,7 +2,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import { makeK2LocalAuth } from "./K2Http.ts"; import type { Stream } from "./Stream.ts"; -import { WriteStream, type WriteStreamOptions } from "./WriteStream.ts"; +import { WriteStream } from "./WriteStream.ts"; import { makeWriteStreamHttpClient } from "./WriteStreamHttp.ts"; /** @@ -37,10 +37,10 @@ export const WriteStreamLocal = Layer.effect( WriteStream, Effect.gen(function* () { const auth = yield* makeK2LocalAuth; - return Effect.fn(function* (stream: Stream, options?: WriteStreamOptions) { + return Effect.fn(function* (stream: Stream) { // Deferred accessor — resolves the id at apply time. No `host.bind`: // the local variant registers no binding. - return makeWriteStreamHttpClient(auth, yield* stream.streamId, options); + return makeWriteStreamHttpClient(auth, yield* stream.streamId, stream.RecordSchema); }); }), ); diff --git a/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts b/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts index 26b455f5ebb..d0a2fe823cb 100644 --- a/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts +++ b/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts @@ -47,7 +47,7 @@ test.provider( const out = yield* stack.deploy( Effect.gen(function* () { - const events = yield* Cloudflare.K2.Stream("LocalEvents", { http: true }); + const events = yield* Cloudflare.K2.Stream("LocalEvents", { http: true, schema: Event }); const poller = yield* Cloudflare.K2.Subscription("Poller", { streamId: events.streamId, startAt: "earliest", @@ -60,8 +60,8 @@ test.provider( const Roundtrip = Action( "Roundtrip", Effect.gen(function* () { - const writer = yield* Cloudflare.K2.WriteStream(events, { schema: Event }); - const sink = yield* Cloudflare.K2.StreamSink(events, { schema: Event }); + const writer = yield* Cloudflare.K2.WriteStream(events); + const sink = yield* Cloudflare.K2.StreamSink(events); const seen = new Set(); let failures = 0; @@ -70,7 +70,6 @@ test.provider( events, { subscription: poller, - schema: Event, workerId: "alchemy-k2-poller", idleBackoff: "500 millis", concurrency: 2, diff --git a/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts b/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts index afaf8ae8d2a..9b6fab75d11 100644 --- a/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts +++ b/packages/alchemy/test/Cloudflare/K2/StreamSink.test.ts @@ -36,22 +36,18 @@ const record = (bytes: number): EncodedRecord => ({ content: new Uint8Array(byte const fakeClient = (failures: ReadonlyArray = []) => { const sent: EncodedRecord[][] = []; let attempt = 0; - const client: WriteStreamClient<{ id: number }> = makeWriteStreamClient( - { schema: Order }, - (records) => - Effect.suspend((): Effect.Effect => { - const tag = failures[attempt++]; - if (tag === "K2Unavailable") { - return Effect.fail(new k2.K2Unavailable({ code: 10211, message: "unavailable" })); - } - if (tag === "K2AppendOutcomeUnknown") { - return Effect.fail( - new k2.K2AppendOutcomeUnknown({ code: 10212, message: "maybe stored" }), - ); - } - sent.push([...records]); - return Effect.void; - }), + const client: WriteStreamClient<{ id: number }> = makeWriteStreamClient(Order, (records) => + Effect.suspend((): Effect.Effect => { + const tag = failures[attempt++]; + if (tag === "K2Unavailable") { + return Effect.fail(new k2.K2Unavailable({ code: 10211, message: "unavailable" })); + } + if (tag === "K2AppendOutcomeUnknown") { + return Effect.fail(new k2.K2AppendOutcomeUnknown({ code: 10212, message: "maybe stored" })); + } + sent.push([...records]); + return Effect.void; + }), ); return { client, sent, attempts: () => attempt }; }; diff --git a/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts b/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts index 3c6d2452969..fde0209ec90 100644 --- a/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts +++ b/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts @@ -51,16 +51,10 @@ const byRun = (records: ReadonlyArray, run: string) => const verifyRoutes = (url: string, streamId: string, subscriptionId: string) => Effect.gen(function* () { - yield* produce(url, "/raw", "raw", 3); yield* produce(url, "/typed", "typed", 4); yield* produce(url, "/sink", "sink", 250); - const records = yield* drainSubscription(streamId, subscriptionId, 3 + 4 + 250); - - // Raw records keep their bytes and headers. - const raw = byRun(records, "raw"); - expect(raw.map((r) => r.text).sort()).toEqual(["raw:0", "raw:1", "raw:2"]); - expect(raw.every((r) => r.headers.run === "raw")).toBe(true); + const records = yield* drainSubscription(streamId, subscriptionId, 4 + 250); // Typed records are JSON with a content-type header. const typed = byRun(records, "typed"); diff --git a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-binding-worker.ts b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-binding-worker.ts index ab6635475c4..6bd57f7e43e 100644 --- a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-binding-worker.ts +++ b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-binding-worker.ts @@ -2,7 +2,7 @@ import * as Effect from "effect/Effect"; import { HttpServerRequest } from "effect/http/HttpServerRequest"; import * as Layer from "effect/Layer"; import * as Cloudflare from "@/Cloudflare/index.ts"; -import { BindingOrders, Order, produceRoutes } from "./k2-shared.ts"; +import { BindingOrders, produceRoutes } from "./k2-shared.ts"; /** * Produces to a K2 stream through the native `k2` Worker binding @@ -15,14 +15,13 @@ export default class K2BindingWorker extends Cloudflare.Worker( }, Effect.gen(function* () { const orders = yield* BindingOrders; - const raw = yield* Cloudflare.K2.WriteStream(orders); - const typed = yield* Cloudflare.K2.WriteStream(orders, { schema: Order }); - const sink = yield* Cloudflare.K2.StreamSink(orders, { schema: Order }); + const typed = yield* Cloudflare.K2.WriteStream(orders); + const sink = yield* Cloudflare.K2.StreamSink(orders); return { fetch: Effect.gen(function* () { const request = yield* HttpServerRequest; - return yield* produceRoutes(new URL(request.url, "http://x"), { raw, typed, sink }); + return yield* produceRoutes(new URL(request.url, "http://x"), { typed, sink }); }), }; }).pipe( diff --git a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-http-worker.ts b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-http-worker.ts index 75eb305adff..e1315bc3a1d 100644 --- a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-http-worker.ts +++ b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-http-worker.ts @@ -2,7 +2,7 @@ import * as Effect from "effect/Effect"; import { HttpServerRequest } from "effect/http/HttpServerRequest"; import * as Layer from "effect/Layer"; import * as Cloudflare from "@/Cloudflare/index.ts"; -import { HttpOrders, Order, produceRoutes } from "./k2-shared.ts"; +import { HttpOrders, produceRoutes } from "./k2-shared.ts"; /** * Produces to a K2 stream through its HTTP input with a scoped `K2 Produce` @@ -16,14 +16,13 @@ export default class K2HttpWorker extends Cloudflare.Worker()( }, Effect.gen(function* () { const orders = yield* HttpOrders; - const raw = yield* Cloudflare.K2.WriteStream(orders); - const typed = yield* Cloudflare.K2.WriteStream(orders, { schema: Order }); - const sink = yield* Cloudflare.K2.StreamSink(orders, { schema: Order }); + const typed = yield* Cloudflare.K2.WriteStream(orders); + const sink = yield* Cloudflare.K2.StreamSink(orders); return { fetch: Effect.gen(function* () { const request = yield* HttpServerRequest; - return yield* produceRoutes(new URL(request.url, "http://x"), { raw, typed, sink }); + return yield* produceRoutes(new URL(request.url, "http://x"), { typed, sink }); }), }; }).pipe( diff --git a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-shared.ts b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-shared.ts index c2f3ca4ea46..a098f3edc52 100644 --- a/packages/alchemy/test/Cloudflare/K2/fixtures/k2-shared.ts +++ b/packages/alchemy/test/Cloudflare/K2/fixtures/k2-shared.ts @@ -12,11 +12,11 @@ export const Order = Schema.Struct({ }); export type Order = typeof Order.Type; -/** Worker-binding input only (the default). */ -export const BindingOrders = Cloudflare.K2.Stream("K2BindingOrders"); +/** Worker-binding input only (the default), typed by `Order`. */ +export const BindingOrders = Cloudflare.K2.Stream("K2BindingOrders", { schema: Order }); -/** HTTP input enabled for the `*Http` producer layers. */ -export const HttpOrders = Cloudflare.K2.Stream("K2HttpOrders", { http: true }); +/** HTTP input enabled for the `*Http` producer layers, typed by `Order`. */ +export const HttpOrders = Cloudflare.K2.Stream("K2HttpOrders", { http: true, schema: Order }); const params = (url: URL) => ({ run: url.searchParams.get("run") ?? "default", @@ -27,13 +27,12 @@ const failed = (error: { _tag: string; message?: string }) => HttpServerResponse.json({ error: error._tag, message: error.message }, { status: 500 }); /** - * Producer routes over a raw client, a schema-typed client, and a typed - * sink: `POST /raw|/typed|/sink?run=R&count=N`. + * Producer routes over the stream's typed client and typed sink: + * `POST /typed|/sink?run=R&count=N`. */ export const produceRoutes = ( url: URL, clients: { - raw: Cloudflare.K2.WriteStreamClient; typed: Cloudflare.K2.WriteStreamClient; sink: Cloudflare.K2.StreamSinkClient; }, @@ -42,18 +41,11 @@ export const produceRoutes = ( const { run, count } = params(url); const orders = Array.from({ length: count }, (_, index): Order => ({ run, index })); const sent = - url.pathname === "/raw" - ? clients.raw.send( - orders.map((order) => ({ - content: `${order.run}:${order.index}`, - headers: { run: order.run }, - })), - ) - : url.pathname === "/typed" - ? clients.typed.send(orders) - : url.pathname === "/sink" - ? Stream.fromIterable(orders).pipe(Stream.run(clients.sink)) - : undefined; + url.pathname === "/typed" + ? clients.typed.send(orders) + : url.pathname === "/sink" + ? Stream.fromIterable(orders).pipe(Stream.run(clients.sink)) + : undefined; if (sent === undefined) { return HttpServerResponse.text("Not Found", { status: 404 }); } From 09de8b2bfe7d0e1c1c106317effe373915615932 Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Tue, 6 Oct 2026 10:25:38 -0700 Subject: [PATCH 08/11] docs(cloudflare): add the Basin guide; K2 guide uses stream-level schemas --- website/astro.config.ts | 1 + .../content/docs/cloudflare/data/basin.mdx | 263 ++++++++++++++++++ website/src/content/docs/cloudflare/index.mdx | 4 + .../content/docs/cloudflare/messaging/k2.mdx | 37 ++- 4 files changed, 293 insertions(+), 12 deletions(-) create mode 100644 website/src/content/docs/cloudflare/data/basin.mdx diff --git a/website/astro.config.ts b/website/astro.config.ts index 9d053fe9ac7..8366b957813 100644 --- a/website/astro.config.ts +++ b/website/astro.config.ts @@ -634,6 +634,7 @@ export default defineConfig({ link: "/cloudflare/data/branch-from-shared-database", }, { label: "Artifacts", link: "/cloudflare/data/artifacts" }, + { label: "Basin (Pipelines & Iceberg)", link: "/cloudflare/data/basin" }, ], }, { diff --git a/website/src/content/docs/cloudflare/data/basin.mdx b/website/src/content/docs/cloudflare/data/basin.mdx new file mode 100644 index 00000000000..47cc698c240 --- /dev/null +++ b/website/src/content/docs/cloudflare/data/basin.mdx @@ -0,0 +1,263 @@ +--- +title: Basin +description: Land events as Apache Iceberg tables on R2 with Cloudflare Basin — a typed Pipelines stream, a SQL pipeline, a catalog sink, and Iceberg tables you own directly. +--- + +**Basin** is Cloudflare's analytics stack on R2. It has three parts: + +- **Pipelines** ingests JSON events through a **stream**, transforms them with + SQL, and writes them to a **sink**. +- **Catalog** (formerly R2 Data Catalog) turns an R2 bucket into an Apache + Iceberg warehouse. +- **SQL** queries the tables in that warehouse. + +In Alchemy these are the `Cloudflare.Basin` resources. They are the same +resources as `Cloudflare.Pipelines.*` and `Cloudflare.R2.DataCatalog`, which +keep working under their old names. + +Reach for Basin when events should end up as tables you query. Reach for +[K2](/cloudflare/messaging/k2) or [Queues](/cloudflare/messaging/queues) when +your own code consumes them. + +By the end of this page, a Worker sends page views to a stream, and a SQL +pipeline writes them to an Iceberg table in your bucket. + +## Enable the catalog on a bucket + +```typescript +// src/Lake.ts +import * as Cloudflare from "alchemy/Cloudflare"; + +export const Lake = Cloudflare.R2.Bucket("Lake"); + +export const Catalog = Cloudflare.Basin.Catalog("Catalog", { + bucketName: Lake.bucketName, +}); +``` + +`Basin.Catalog` enables the Iceberg catalog on the bucket. It outputs the +catalog URI and the warehouse name that Iceberg clients connect with. + +## Create a typed stream + +Declare the stream with an Effect Schema. Alchemy converts it into the stream's +field list, which Cloudflare enforces. + +```typescript +// src/Lake.ts +import * as Schema from "effect/Schema"; + +export const PageView = Schema.Struct({ + userId: Schema.String, + path: Schema.String, + at: Schema.Date, + tags: Schema.optional(Schema.Array(Schema.String)), +}); + +export const PageViews = Cloudflare.Basin.Stream("PageViews", { schema: PageView }); +``` + +Arrays become `list` fields, nested structs become `struct` fields, and +`Schema.optional` fields are not required. A schema with no stream +representation, such as a union, fails when the program is built. + +A stream's schema can't change after it is created. Changing it replaces the +stream, which drops events it hasn't processed yet and deletes the pipelines +that read from it. Writing the same schema as an Effect Schema or as an +explicit `{ fields }` list is the same stream and causes no change. + +## Write to an Iceberg table + +A catalog sink writes the pipeline's output to an Iceberg table. Declare it in +your stack's Effect, and pass the catalog resource so the sink is created after +the catalog is enabled: + +```typescript +// alchemy.run.ts +import * as Alchemy from "alchemy"; +import * as Cloudflare from "alchemy/Cloudflare"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import { Catalog, PageViews } from "./src/Lake.ts"; + +export default Alchemy.Stack( + "Analytics", + { + providers: Cloudflare.providers(), + state: Cloudflare.state(), + }, + Effect.gen(function* () { + const catalog = yield* Catalog; + const views = yield* PageViews; + + const table = yield* Cloudflare.Basin.Sink("PageViewTable", { + type: "basin_catalog", + catalog, + table: { namespace: "web", name: "page_views" }, + token: yield* Config.Redacted("CATALOG_TOKEN"), + }); + }), +); +``` + +The sink creates the namespace and table on its first write. The `token` is a +Cloudflare API token with access to R2 and the catalog. Rotating it updates the +sink without replacing it. + +:::caution +A catalog sink can't be created for a table that already exists. Changing the +sink's other settings replaces the sink, and the replacement fails once the +table exists. Choose the table and format before the first deploy. +::: + +## Connect them with SQL + +Add the pipeline in the same Effect: + +```diff lang="typescript" +// alchemy.run.ts ++import * as Output from "alchemy/Output"; + + const table = yield* Cloudflare.Basin.Sink("PageViewTable", { /* ... */ }); ++ ++ yield* Cloudflare.Basin.Pipeline("PageViewEtl", { ++ sql: Output.interpolate` ++ INSERT INTO ${table.name} ++ SELECT userId, path, at FROM ${views.name} ++ WHERE path NOT LIKE '/health%'`, ++ }); +``` + +Alchemy validates the SQL with Cloudflare during plan, so a bad statement fails +before anything is replaced. Reformatting the SQL is not a change. + +## Send events from a Worker + +`Cloudflare.Basin.WriteStream` is typed by the stream's schema. The client +encodes each value with the schema before sending it. + +```typescript +// src/Api.ts +import * as Cloudflare from "alchemy/Cloudflare"; +import * as Effect from "effect/Effect"; +import { HttpServerRequest } from "effect/http/HttpServerRequest"; +import * as HttpServerResponse from "effect/http/HttpServerResponse"; +import { PageViews } from "./Lake.ts"; + +export default class Api extends Cloudflare.Worker()( + "Api", + { main: import.meta.url }, + Effect.gen(function* () { + const views = yield* Cloudflare.Basin.WriteStream(PageViews); + + return { + fetch: Effect.gen(function* () { + const request = yield* HttpServerRequest; + yield* views.send([{ userId: "anon", path: request.url, at: new Date() }]); + return HttpServerResponse.text("ok"); + }), + }; + }).pipe(Effect.provide(Cloudflare.Basin.WriteStreamBinding)), +) {} +``` + +Cloudflare accepts events that don't match a stream's schema and drops them +later during processing, so the client-side encoding is the only validation. + +To drain an Effect `Stream` into the pipeline, use `Cloudflare.Basin.StreamSink`. +It packs events into requests of up to 5 MB: + +```typescript +const sink = yield* Cloudflare.Basin.StreamSink(PageViews); +yield* Stream.fromIterable(views).pipe(Stream.run(sink)); +``` + +Provide `Cloudflare.Basin.StreamSinkBinding` alongside `WriteStreamBinding`. + +## Send events from outside a Worker + +The producer code stays the same on every host. Only the layer changes: + +- `WriteStreamBinding` and `StreamSinkBinding` run in Cloudflare Workers and + send through the native `pipelines` binding. +- `WriteStreamHttp` and `StreamSinkHttp` run on any host. They send to the + stream's HTTP endpoint with a scoped "Pipelines Send" token that Alchemy mints + at deploy. +- `WriteStreamLocal` and `StreamSinkLocal` run in scripts and Actions. They + send to the stream's HTTP endpoint with your current Cloudflare credentials. + +The HTTP and local layers need the stream's HTTP endpoint turned on. + +## The HTTP endpoint is off by default + +A stream gets an HTTP ingest endpoint only when you ask for one: + +```typescript +Cloudflare.Basin.Stream("PageViews", { http: true }); // authenticated +Cloudflare.Basin.Stream("PageViews", { http: { enabled: true, authentication: false } }); // public +``` + +:::caution +Earlier versions of Alchemy left a stream declared without `http` with +Cloudflare's default, a public endpoint that anyone can write to. The next +deploy now closes that endpoint in place, without replacing the stream. If +external producers post to it, keep it public with +`http: { enabled: true, authentication: false }`. +::: + +## Own a table directly + +`Cloudflare.Basin.Table` creates and manages an Iceberg table through the +catalog's Iceberg REST API. Use it for tables your own code writes, or when you +want table settings in your program: + +```typescript +const Order = Schema.Struct({ + orderId: Schema.String, + amountCents: Schema.Number, + at: Schema.Date, +}); + +const orders = yield* Cloudflare.Basin.Table("Orders", { + catalog, + namespace: "sales", + schema: Order, + partitionBy: ["day(at)"], + properties: { "write.format.default": "parquet" }, + maintenance: { + compaction: { targetSizeMb: "256" }, + }, +}); +``` + +Adding an optional column to `schema` evolves the table in place. A change that +would lose data, such as dropping or narrowing a column, fails the plan. +Changing `namespace`, `name`, or `partitionBy` also fails the plan, because each +would mean dropping the table. + +`maintenance.snapshotExpiration` works only after snapshot expiration is enabled +on the catalog. + +Destroying the resource leaves the table and its data in place. Set +`delete: "drop"` to drop the table, or `delete: "purge"` to also delete its data +files. + +## React to new files + +Pipelines has no event source of its own, because its consumer is the sink. To +process the files a sink writes, listen for object events on the bucket with +`Cloudflare.R2.BucketEventNotification`. + +## Where next + +Related: + +- [R2](/cloudflare/data/r2): the bucket under the catalog. +- [K2 streams](/cloudflare/messaging/k2): a durable log your own code consumes. +- [Queues](/cloudflare/messaging/queues): per-message work with push delivery + to Workers. + +Reference: + +- [Basin API reference](/providers/cloudflare/basin): `Catalog`, `Stream`, + `Sink`, `Pipeline`, `Table`, `WriteStream`, and `StreamSink`. diff --git a/website/src/content/docs/cloudflare/index.mdx b/website/src/content/docs/cloudflare/index.mdx index c5bf5effe4c..d7351120b2a 100644 --- a/website/src/content/docs/cloudflare/index.mdx +++ b/website/src/content/docs/cloudflare/index.mdx @@ -42,6 +42,8 @@ New here? [Set up your account](/cloudflare/setup), then start the bindings. - [Hyperdrive](/cloudflare/data/hyperdrive) — edge connection pooling for external Postgres and MySQL. +- [Basin](/cloudflare/data/basin) — ingest events into Apache Iceberg tables on + R2 with Pipelines. ## Messaging @@ -78,6 +80,8 @@ New here? [Set up your account](/cloudflare/setup), then start the | Real-time / WebSockets | [Durable Objects](/cloudflare/compute/durable-objects) — see [Accept WebSockets](/cloudflare/compute/hibernatable-websockets) | | Full-stack app | [Worker](/cloudflare/compute/workers) + a React SPA — see [Add a React SPA](/cloudflare/frontend/vite-spa) or [Frontend frameworks](/cloudflare/frontend/frontends) | | Background jobs | [Queues](/cloudflare/messaging/queues) | +| Event log with replay | [K2 streams](/cloudflare/messaging/k2) | +| Analytics events as tables | [Basin](/cloudflare/data/basin) | | Postgres-backed API | [Hyperdrive](/cloudflare/data/hyperdrive) + [Drizzle](/cloudflare/data/drizzle) | | Durable multi-step jobs | [Workflows](/cloudflare/compute/workflows) | | AI apps | [AI Gateway](/cloudflare/ai/ai-gateway), [Effect AI](/cloudflare/ai/effect-ai), [AI Search](/cloudflare/ai/ai-search) | diff --git a/website/src/content/docs/cloudflare/messaging/k2.mdx b/website/src/content/docs/cloudflare/messaging/k2.mdx index 533a5597d8b..62f9b50fedf 100644 --- a/website/src/content/docs/cloudflare/messaging/k2.mdx +++ b/website/src/content/docs/cloudflare/messaging/k2.mdx @@ -96,11 +96,12 @@ needs no API token. ## Type records with a schema -Pass an Effect Schema, and `send` takes values of that type. The client -encodes each value as JSON and sets a `content-type: application/json` -header. +Declare the stream with an Effect Schema, and every binding on it is typed by +that schema: ```diff lang="typescript" +// src/Orders.ts +import * as Cloudflare from "alchemy/Cloudflare"; +import * as Schema from "effect/Schema"; + +export const Order = Schema.Struct({ @@ -108,14 +109,23 @@ header. + amountCents: Schema.Number, +}); -- const orders = yield* Cloudflare.K2.WriteStream(Orders); -+ const orders = yield* Cloudflare.K2.WriteStream(Orders, { schema: Order }); +export const Orders = Cloudflare.K2.Stream("Orders", { + retention: "7 days", ++ schema: Order, +}); ``` +Now `orders.send` takes `Order` values. The client encodes each one as JSON +and sets a `content-type: application/json` header: + ```typescript yield* orders.send([{ orderId: "o_1", amountCents: 4200 }]); ``` +K2 itself stores bytes, so the schema is a client-side codec. Changing it +never updates or replaces the stream. Producers and consumers read the schema +from the same stream, so they can't disagree about the record type. + ## Handle produce errors Failures are typed. Two of them tell you whether the batch was stored: @@ -140,7 +150,7 @@ works with either layer. up to 5 MB and retries only the errors that mean the batch was not stored. ```typescript -const sink = yield* Cloudflare.K2.StreamSink(Orders, { schema: Order }); +const sink = yield* Cloudflare.K2.StreamSink(Orders); yield* Stream.fromIterable(pending).pipe(Stream.run(sink)); ``` @@ -209,7 +219,7 @@ import * as Fly from "alchemy/Fly"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; -import { Order, Orders } from "./Orders.ts"; +import { Orders } from "./Orders.ts"; export default class Fraud extends Fly.Service()( "Fraud", @@ -217,7 +227,7 @@ export default class Fraud extends Fly.Service()( Effect.gen(function* () { yield* Cloudflare.K2.consumeStreamRecords( Orders, - { schema: Order, startAt: "earliest", concurrency: 4 }, + { startAt: "earliest", concurrency: 4 }, (records) => records.pipe( Stream.map((record) => record.value), @@ -256,8 +266,9 @@ mints at deploy. `StreamEventSourcePolling` needs a host with a long-lived process (Fly, ECS, EC2, Railway, Hetzner, Docker), so it doesn't work in a Worker. -Without `schema`, each record is `{ content: Uint8Array, headers, timestamp }`. -With `schema`, the JSON content is also decoded into `record.value`. +For a stream with a schema, each record's JSON content is decoded into +`record.value`. For a stream without one, each record is +`{ content: Uint8Array, headers, timestamp }`. ## Share a subscription between hosts @@ -275,8 +286,8 @@ export const FraudWorkers = Cloudflare.K2.Subscription("FraudWorkers", { ```diff lang="typescript" yield* Cloudflare.K2.consumeStreamRecords( Orders, -- { schema: Order, startAt: "earliest", concurrency: 4 }, -+ { schema: Order, subscription: FraudWorkers, concurrency: 4 }, +- { startAt: "earliest", concurrency: 4 }, ++ { subscription: FraudWorkers, concurrency: 4 }, ``` Hosts sharing a subscription compete for batches. A subscription allows up @@ -311,6 +322,8 @@ Related: - [Queues](/cloudflare/messaging/queues): a work queue with push delivery to Workers, retries, and dead-letter queues. +- [Basin](/cloudflare/data/basin): land events as Iceberg tables with + Pipelines instead of consuming them yourself. - [Workers](/cloudflare/compute/workers): the runtime that produces records. - [Sinks](/infrastructure-as-effects/sinks): the Effect `Sink` model behind `StreamSink`. From 93d4f267c4b0b41e877f0e259002ce4ecbb5df44 Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Tue, 6 Oct 2026 10:57:54 -0700 Subject: [PATCH 09/11] docs(cloudflare): split the Basin guide into Pipelines and Iceberg tables Pipelines covers stream -> SQL -> sink (Iceberg and R2-file sinks, producers, the HTTP endpoint default). Iceberg tables covers the catalog, Basin.Table, maintenance, delete policies, and querying from other engines. --- website/astro.config.ts | 3 +- .../docs/cloudflare/data/iceberg-tables.mdx | 194 +++++++++++++ .../data/{basin.mdx => pipelines.mdx} | 265 ++++++++---------- website/src/content/docs/cloudflare/index.mdx | 8 +- .../content/docs/cloudflare/messaging/k2.mdx | 4 +- 5 files changed, 324 insertions(+), 150 deletions(-) create mode 100644 website/src/content/docs/cloudflare/data/iceberg-tables.mdx rename website/src/content/docs/cloudflare/data/{basin.mdx => pipelines.mdx} (54%) diff --git a/website/astro.config.ts b/website/astro.config.ts index 8366b957813..33fd78fbc17 100644 --- a/website/astro.config.ts +++ b/website/astro.config.ts @@ -634,7 +634,8 @@ export default defineConfig({ link: "/cloudflare/data/branch-from-shared-database", }, { label: "Artifacts", link: "/cloudflare/data/artifacts" }, - { label: "Basin (Pipelines & Iceberg)", link: "/cloudflare/data/basin" }, + { label: "Pipelines", link: "/cloudflare/data/pipelines" }, + { label: "Iceberg tables", link: "/cloudflare/data/iceberg-tables" }, ], }, { diff --git a/website/src/content/docs/cloudflare/data/iceberg-tables.mdx b/website/src/content/docs/cloudflare/data/iceberg-tables.mdx new file mode 100644 index 00000000000..904cca46fc7 --- /dev/null +++ b/website/src/content/docs/cloudflare/data/iceberg-tables.mdx @@ -0,0 +1,194 @@ +--- +title: Iceberg tables +description: Turn an R2 bucket into an Apache Iceberg warehouse with Cloudflare's catalog, manage tables with schema evolution and maintenance, and query them from Spark, DuckDB, or PyIceberg. +--- + +Cloudflare's **catalog** (formerly R2 Data Catalog) turns an R2 bucket into an +Apache Iceberg warehouse. Iceberg stores a table as data files plus metadata, +so any engine that speaks the Iceberg REST protocol can read and write it: +Spark, DuckDB, PyIceberg, Snowflake, Trino, and others. + +The catalog is part of **Basin**, Cloudflare's analytics stack on R2. Tables +get into it in two ways: + +- A [Pipelines](/cloudflare/data/pipelines) catalog sink creates a table and + writes events into it. +- `Cloudflare.Basin.Table` creates and manages a table directly, for tables + your own code or engines write. + +This page covers enabling the catalog, managing tables, and connecting query +engines. + +## Enable the catalog on a bucket + +```typescript +// alchemy.run.ts +const lake = yield* Cloudflare.R2.Bucket("Lake"); + +const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { + bucketName: lake.bucketName, +}); +``` + +`Basin.Catalog` is the same resource as `Cloudflare.R2.DataCatalog`. It +outputs: + +- `catalogUri`: the Iceberg REST endpoint, for example + `https://catalog.cloudflarestorage.com//`. +- `name`: the warehouse name, `_`. + +## Turn on catalog maintenance + +Iceberg tables accumulate small files and old snapshots. The catalog can +compact files and expire snapshots for every table in the warehouse: + +```typescript +const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { + bucketName: lake.bucketName, + compaction: { state: "enabled", targetSizeMb: "128" }, + snapshotExpiration: { state: "enabled", maxSnapshotAge: "7d", minSnapshotsToKeep: 10 }, + token: yield* Config.Redacted("CATALOG_MAINTENANCE_TOKEN"), +}); +``` + +The maintenance jobs run with `token`, a Cloudflare API token with access to R2 +and the catalog. They stay pending until a token is set. Only the fields you +set are enforced; the rest keep Cloudflare's defaults. + +## Create a table + +`Cloudflare.Basin.Table` creates a namespace and table through the catalog's +Iceberg REST API, with columns from an Effect Schema: + +```typescript +import * as Schema from "effect/Schema"; + +const Order = Schema.Struct({ + orderId: Schema.String, + amountCents: Schema.Number, + at: Schema.Date, + note: Schema.optional(Schema.String), +}); + +const orders = yield* Cloudflare.Basin.Table("Orders", { + catalog, + namespace: "sales", + schema: Order, + partitionBy: ["day(at)"], + properties: { "write.format.default": "parquet" }, +}); +``` + +The table name is generated from the stack, stage, and logical ID unless you +set `name`. The table outputs `identifier` (for example `sales.orders`), +`tableUuid`, `location`, and `metadataLocation`. + +`partitionBy` takes Iceberg transforms: a column name, `day(at)`, `month(at)`, +`bucket[16](orderId)`, and so on. `properties` are Iceberg table properties, +updated in place. + +The table authenticates with your deploying Cloudflare credentials. Pass `token` +to use a different one. + +## Evolve the schema + +Adding an optional column evolves the table in place. Existing data stays, and +the new column reads as empty for old rows: + +```diff lang="typescript" + const Order = Schema.Struct({ + orderId: Schema.String, + amountCents: Schema.Number, + at: Schema.Date, + note: Schema.optional(Schema.String), ++ coupon: Schema.optional(Schema.String), + }); +``` + +A change that would lose data fails the plan before anything is applied: +dropping or narrowing a column, or adding a required one. Changing `namespace`, +`name`, or `partitionBy` also fails the plan, because each would mean dropping +the table. + +## Tune maintenance per table + +`maintenance` overrides the catalog's settings for one table: + +```typescript +const orders = yield* Cloudflare.Basin.Table("Orders", { + catalog, + namespace: "sales", + schema: Order, + maintenance: { + compaction: { targetSizeMb: "256" }, + snapshotExpiration: { olderThan: "3 days", retainLast: 5 }, + }, +}); +``` + +Per-table snapshot expiration only works after snapshot expiration is enabled +on the catalog. + +## Choose what happens on destroy + +Destroying a `Table` resource leaves the table and its data in place by +default, so removing it from your program never deletes data: + +```typescript +Cloudflare.Basin.Table("Orders", { catalog, namespace: "sales", schema: Order, delete: "retain" }); // default +Cloudflare.Basin.Table("Orders", { catalog, namespace: "sales", schema: Order, delete: "drop" }); // drop the table, keep its files +Cloudflare.Basin.Table("Orders", { catalog, namespace: "sales", schema: Order, delete: "purge" }); // drop the table and its files +``` + +## Query from other engines + +Any Iceberg REST client connects with the catalog's URI, the warehouse name, +and a Cloudflare API token with access to R2 and the catalog. Export the +first two from your stack: + +```typescript +return { + catalogUri: catalog.catalogUri, + warehouse: catalog.name, +}; +``` + +Then point the client at them. With PyIceberg: + +```python +from pyiceberg.catalog.rest import RestCatalog + +catalog = RestCatalog( + name="lake", + uri=CATALOG_URI, + warehouse=WAREHOUSE, + token=CLOUDFLARE_API_TOKEN, +) +orders = catalog.load_table(("sales", "orders")) +print(orders.scan().to_pandas()) +``` + +From Effect code, `@distilled.cloud/iceberg` is a typed client for the same +API: + +```typescript +import * as Iceberg from "@distilled.cloud/iceberg"; + +const table = yield* Iceberg.loadTable({ namespace: "sales", table: "orders" }).pipe( + Effect.provide(Iceberg.IcebergProtocol), + Effect.provide(Iceberg.fromCatalogConfig({ uri: catalogUri, warehouse, token })), +); +``` + +## Where next + +Related: + +- [Pipelines](/cloudflare/data/pipelines): ingest events into a table with a + catalog sink. +- [R2](/cloudflare/data/r2): the bucket under the warehouse. + +Reference: + +- [Basin API reference](/providers/cloudflare/basin): `Catalog` and `Table`. +- [R2 API reference](/providers/cloudflare/r2): `DataCatalog`. diff --git a/website/src/content/docs/cloudflare/data/basin.mdx b/website/src/content/docs/cloudflare/data/pipelines.mdx similarity index 54% rename from website/src/content/docs/cloudflare/data/basin.mdx rename to website/src/content/docs/cloudflare/data/pipelines.mdx index 47cc698c240..1b1e576a184 100644 --- a/website/src/content/docs/cloudflare/data/basin.mdx +++ b/website/src/content/docs/cloudflare/data/pipelines.mdx @@ -1,42 +1,21 @@ --- -title: Basin -description: Land events as Apache Iceberg tables on R2 with Cloudflare Basin — a typed Pipelines stream, a SQL pipeline, a catalog sink, and Iceberg tables you own directly. +title: Pipelines +description: Ingest events into R2 with Cloudflare Pipelines — a typed stream, a SQL pipeline, and a sink that writes JSON or Parquet files, or Apache Iceberg tables. --- -**Basin** is Cloudflare's analytics stack on R2. It has three parts: +**Cloudflare Pipelines** ingests events into R2. Producers send JSON events to a +**stream**. A **pipeline** filters or reshapes them with SQL and writes them to +a **sink**: plain JSON or Parquet files in a bucket, or rows in an Apache +Iceberg table. -- **Pipelines** ingests JSON events through a **stream**, transforms them with - SQL, and writes them to a **sink**. -- **Catalog** (formerly R2 Data Catalog) turns an R2 bucket into an Apache - Iceberg warehouse. -- **SQL** queries the tables in that warehouse. - -In Alchemy these are the `Cloudflare.Basin` resources. They are the same -resources as `Cloudflare.Pipelines.*` and `Cloudflare.R2.DataCatalog`, which -keep working under their old names. - -Reach for Basin when events should end up as tables you query. Reach for +Pipelines is part of **Basin**, Cloudflare's analytics stack on R2, and its +resources are also exported as `Cloudflare.Basin.*`. Reach for Pipelines when +events should land as files or tables you query later. Reach for [K2](/cloudflare/messaging/k2) or [Queues](/cloudflare/messaging/queues) when -your own code consumes them. - -By the end of this page, a Worker sends page views to a stream, and a SQL -pipeline writes them to an Iceberg table in your bucket. - -## Enable the catalog on a bucket - -```typescript -// src/Lake.ts -import * as Cloudflare from "alchemy/Cloudflare"; - -export const Lake = Cloudflare.R2.Bucket("Lake"); +your own code consumes the events. -export const Catalog = Cloudflare.Basin.Catalog("Catalog", { - bucketName: Lake.bucketName, -}); -``` - -`Basin.Catalog` enables the Iceberg catalog on the bucket. It outputs the -catalog URI and the warehouse name that Iceberg clients connect with. +By the end of this page, a Worker sends page views to a stream, and a pipeline +writes them to an Iceberg table. ## Create a typed stream @@ -44,7 +23,8 @@ Declare the stream with an Effect Schema. Alchemy converts it into the stream's field list, which Cloudflare enforces. ```typescript -// src/Lake.ts +// src/Events.ts +import * as Cloudflare from "alchemy/Cloudflare"; import * as Schema from "effect/Schema"; export const PageView = Schema.Struct({ @@ -54,7 +34,9 @@ export const PageView = Schema.Struct({ tags: Schema.optional(Schema.Array(Schema.String)), }); -export const PageViews = Cloudflare.Basin.Stream("PageViews", { schema: PageView }); +export const PageViews = Cloudflare.Pipelines.Stream("PageViews", { + schema: PageView, +}); ``` Arrays become `list` fields, nested structs become `struct` fields, and @@ -66,74 +48,9 @@ stream, which drops events it hasn't processed yet and deletes the pipelines that read from it. Writing the same schema as an Effect Schema or as an explicit `{ fields }` list is the same stream and causes no change. -## Write to an Iceberg table - -A catalog sink writes the pipeline's output to an Iceberg table. Declare it in -your stack's Effect, and pass the catalog resource so the sink is created after -the catalog is enabled: - -```typescript -// alchemy.run.ts -import * as Alchemy from "alchemy"; -import * as Cloudflare from "alchemy/Cloudflare"; -import * as Config from "effect/Config"; -import * as Effect from "effect/Effect"; -import { Catalog, PageViews } from "./src/Lake.ts"; - -export default Alchemy.Stack( - "Analytics", - { - providers: Cloudflare.providers(), - state: Cloudflare.state(), - }, - Effect.gen(function* () { - const catalog = yield* Catalog; - const views = yield* PageViews; - - const table = yield* Cloudflare.Basin.Sink("PageViewTable", { - type: "basin_catalog", - catalog, - table: { namespace: "web", name: "page_views" }, - token: yield* Config.Redacted("CATALOG_TOKEN"), - }); - }), -); -``` - -The sink creates the namespace and table on its first write. The `token` is a -Cloudflare API token with access to R2 and the catalog. Rotating it updates the -sink without replacing it. - -:::caution -A catalog sink can't be created for a table that already exists. Changing the -sink's other settings replaces the sink, and the replacement fails once the -table exists. Choose the table and format before the first deploy. -::: - -## Connect them with SQL - -Add the pipeline in the same Effect: - -```diff lang="typescript" -// alchemy.run.ts -+import * as Output from "alchemy/Output"; - - const table = yield* Cloudflare.Basin.Sink("PageViewTable", { /* ... */ }); -+ -+ yield* Cloudflare.Basin.Pipeline("PageViewEtl", { -+ sql: Output.interpolate` -+ INSERT INTO ${table.name} -+ SELECT userId, path, at FROM ${views.name} -+ WHERE path NOT LIKE '/health%'`, -+ }); -``` - -Alchemy validates the SQL with Cloudflare during plan, so a bad statement fails -before anything is replaced. Reformatting the SQL is not a change. - ## Send events from a Worker -`Cloudflare.Basin.WriteStream` is typed by the stream's schema. The client +`Cloudflare.Pipelines.WriteStream` is typed by the stream's schema. The client encodes each value with the schema before sending it. ```typescript @@ -142,13 +59,13 @@ import * as Cloudflare from "alchemy/Cloudflare"; import * as Effect from "effect/Effect"; import { HttpServerRequest } from "effect/http/HttpServerRequest"; import * as HttpServerResponse from "effect/http/HttpServerResponse"; -import { PageViews } from "./Lake.ts"; +import { PageViews } from "./Events.ts"; export default class Api extends Cloudflare.Worker()( "Api", { main: import.meta.url }, Effect.gen(function* () { - const views = yield* Cloudflare.Basin.WriteStream(PageViews); + const views = yield* Cloudflare.Pipelines.WriteStream(PageViews); return { fetch: Effect.gen(function* () { @@ -157,22 +74,22 @@ export default class Api extends Cloudflare.Worker()( return HttpServerResponse.text("ok"); }), }; - }).pipe(Effect.provide(Cloudflare.Basin.WriteStreamBinding)), + }).pipe(Effect.provide(Cloudflare.Pipelines.WriteStreamBinding)), ) {} ``` Cloudflare accepts events that don't match a stream's schema and drops them later during processing, so the client-side encoding is the only validation. -To drain an Effect `Stream` into the pipeline, use `Cloudflare.Basin.StreamSink`. -It packs events into requests of up to 5 MB: +To drain an Effect `Stream` into the pipeline, use +`Cloudflare.Pipelines.StreamSink`. It packs events into requests of up to 5 MB: ```typescript -const sink = yield* Cloudflare.Basin.StreamSink(PageViews); -yield* Stream.fromIterable(views).pipe(Stream.run(sink)); +const sink = yield* Cloudflare.Pipelines.StreamSink(PageViews); +yield* Stream.fromIterable(events).pipe(Stream.run(sink)); ``` -Provide `Cloudflare.Basin.StreamSinkBinding` alongside `WriteStreamBinding`. +Provide `Cloudflare.Pipelines.StreamSinkBinding` alongside `WriteStreamBinding`. ## Send events from outside a Worker @@ -188,13 +105,15 @@ The producer code stays the same on every host. Only the layer changes: The HTTP and local layers need the stream's HTTP endpoint turned on. -## The HTTP endpoint is off by default +## Turn on the HTTP endpoint A stream gets an HTTP ingest endpoint only when you ask for one: ```typescript -Cloudflare.Basin.Stream("PageViews", { http: true }); // authenticated -Cloudflare.Basin.Stream("PageViews", { http: { enabled: true, authentication: false } }); // public +Cloudflare.Pipelines.Stream("PageViews", { http: true }); // authenticated +Cloudflare.Pipelines.Stream("PageViews", { + http: { enabled: true, authentication: false }, // public, e.g. for browsers +}); ``` :::caution @@ -205,42 +124,100 @@ external producers post to it, keep it public with `http: { enabled: true, authentication: false }`. ::: -## Own a table directly +## Write events to an Iceberg table -`Cloudflare.Basin.Table` creates and manages an Iceberg table through the -catalog's Iceberg REST API. Use it for tables your own code writes, or when you -want table settings in your program: +A catalog sink writes the pipeline's output to an +[Iceberg table](/cloudflare/data/iceberg-tables). It needs a bucket with the +catalog enabled. Declare them in your stack's Effect, and pass the catalog +resource to the sink so it is created after the catalog is enabled: ```typescript -const Order = Schema.Struct({ - orderId: Schema.String, - amountCents: Schema.Number, - at: Schema.Date, -}); +// alchemy.run.ts +import * as Alchemy from "alchemy"; +import * as Cloudflare from "alchemy/Cloudflare"; +import * as Config from "effect/Config"; +import * as Effect from "effect/Effect"; +import { PageViews } from "./src/Events.ts"; -const orders = yield* Cloudflare.Basin.Table("Orders", { - catalog, - namespace: "sales", - schema: Order, - partitionBy: ["day(at)"], - properties: { "write.format.default": "parquet" }, - maintenance: { - compaction: { targetSizeMb: "256" }, +export default Alchemy.Stack( + "Analytics", + { + providers: Cloudflare.providers(), + state: Cloudflare.state(), }, -}); + Effect.gen(function* () { + const lake = yield* Cloudflare.R2.Bucket("Lake"); + const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", { + bucketName: lake.bucketName, + }); + const views = yield* PageViews; + + const table = yield* Cloudflare.Pipelines.Sink("PageViewTable", { + type: "basin_catalog", + catalog, + table: { namespace: "web", name: "page_views" }, + token: yield* Config.Redacted("CATALOG_TOKEN"), + }); + }), +); ``` -Adding an optional column to `schema` evolves the table in place. A change that -would lose data, such as dropping or narrowing a column, fails the plan. -Changing `namespace`, `name`, or `partitionBy` also fails the plan, because each -would mean dropping the table. +The sink creates the namespace and table on its first write. The `token` is a +Cloudflare API token with access to R2 and the catalog. Rotating it updates the +sink without replacing it. + +:::caution +A catalog sink can't be created for a table that already exists. Changing the +sink's other settings replaces the sink, and the replacement fails once the +table exists. Choose the table and format before the first deploy. +::: + +## Connect the stream and sink with SQL -`maintenance.snapshotExpiration` works only after snapshot expiration is enabled -on the catalog. +Add the pipeline in the same Effect: + +```diff lang="typescript" +// alchemy.run.ts ++import * as Output from "alchemy/Output"; + + const table = yield* Cloudflare.Pipelines.Sink("PageViewTable", { /* ... */ }); ++ ++ yield* Cloudflare.Pipelines.Pipeline("PageViewEtl", { ++ sql: Output.interpolate` ++ INSERT INTO ${table.name} ++ SELECT userId, path, at FROM ${views.name} ++ WHERE path NOT LIKE '/health%'`, ++ }); +``` + +Alchemy validates the SQL with Cloudflare during plan, so a bad statement fails +before anything is replaced. Reformatting the SQL is not a change. + +## Write files instead of a table + +An `r2` sink writes events to a bucket as JSON or Parquet files, without a +catalog: + +```typescript +const archive = yield* Cloudflare.Pipelines.Sink("EventArchive", { + type: "r2", + config: { + bucket: lake.bucketName, + credentials: { + accessKeyId: yield* Config.Redacted("R2_ACCESS_KEY_ID"), + secretAccessKey: yield* Config.Redacted("R2_SECRET_ACCESS_KEY"), + }, + path: "events", + rollingPolicy: { intervalSeconds: 60 }, + }, + format: { type: "parquet", compression: "zstd" }, +}); +``` -Destroying the resource leaves the table and its data in place. Set -`delete: "drop"` to drop the table, or `delete: "purge"` to also delete its data -files. +`rollingPolicy` decides when the sink closes a file and starts the next one: +after `intervalSeconds`, after `inactivitySeconds` without new events, or when +the file reaches `fileSizeBytes`. Point a pipeline at it the same way, with `${archive.name}` in +the SQL. Unlike a catalog sink, an `r2` sink can be replaced freely. ## React to new files @@ -252,12 +229,12 @@ process the files a sink writes, listen for object events on the bucket with Related: -- [R2](/cloudflare/data/r2): the bucket under the catalog. +- [Iceberg tables](/cloudflare/data/iceberg-tables): manage the tables a + catalog sink writes, and query them. +- [R2](/cloudflare/data/r2): the bucket a sink writes to. - [K2 streams](/cloudflare/messaging/k2): a durable log your own code consumes. -- [Queues](/cloudflare/messaging/queues): per-message work with push delivery - to Workers. Reference: -- [Basin API reference](/providers/cloudflare/basin): `Catalog`, `Stream`, - `Sink`, `Pipeline`, `Table`, `WriteStream`, and `StreamSink`. +- [Pipelines API reference](/providers/cloudflare/pipelines): `Stream`, `Sink`, + `Pipeline`, `WriteStream`, and `StreamSink`. diff --git a/website/src/content/docs/cloudflare/index.mdx b/website/src/content/docs/cloudflare/index.mdx index d7351120b2a..863e3fc11b8 100644 --- a/website/src/content/docs/cloudflare/index.mdx +++ b/website/src/content/docs/cloudflare/index.mdx @@ -42,8 +42,10 @@ New here? [Set up your account](/cloudflare/setup), then start the bindings. - [Hyperdrive](/cloudflare/data/hyperdrive) — edge connection pooling for external Postgres and MySQL. -- [Basin](/cloudflare/data/basin) — ingest events into Apache Iceberg tables on - R2 with Pipelines. +- [Pipelines](/cloudflare/data/pipelines) — ingest events into R2 as files or + Iceberg tables. +- [Iceberg tables](/cloudflare/data/iceberg-tables) — an Apache Iceberg + warehouse on R2 that Spark, DuckDB, and PyIceberg can query. ## Messaging @@ -81,7 +83,7 @@ New here? [Set up your account](/cloudflare/setup), then start the | Full-stack app | [Worker](/cloudflare/compute/workers) + a React SPA — see [Add a React SPA](/cloudflare/frontend/vite-spa) or [Frontend frameworks](/cloudflare/frontend/frontends) | | Background jobs | [Queues](/cloudflare/messaging/queues) | | Event log with replay | [K2 streams](/cloudflare/messaging/k2) | -| Analytics events as tables | [Basin](/cloudflare/data/basin) | +| Analytics events as tables | [Pipelines](/cloudflare/data/pipelines) → [Iceberg tables](/cloudflare/data/iceberg-tables) | | Postgres-backed API | [Hyperdrive](/cloudflare/data/hyperdrive) + [Drizzle](/cloudflare/data/drizzle) | | Durable multi-step jobs | [Workflows](/cloudflare/compute/workflows) | | AI apps | [AI Gateway](/cloudflare/ai/ai-gateway), [Effect AI](/cloudflare/ai/effect-ai), [AI Search](/cloudflare/ai/ai-search) | diff --git a/website/src/content/docs/cloudflare/messaging/k2.mdx b/website/src/content/docs/cloudflare/messaging/k2.mdx index 62f9b50fedf..20c0ec8a17c 100644 --- a/website/src/content/docs/cloudflare/messaging/k2.mdx +++ b/website/src/content/docs/cloudflare/messaging/k2.mdx @@ -322,8 +322,8 @@ Related: - [Queues](/cloudflare/messaging/queues): a work queue with push delivery to Workers, retries, and dead-letter queues. -- [Basin](/cloudflare/data/basin): land events as Iceberg tables with - Pipelines instead of consuming them yourself. +- [Pipelines](/cloudflare/data/pipelines): land events as files or Iceberg + tables instead of consuming them yourself. - [Workers](/cloudflare/compute/workers): the runtime that produces records. - [Sinks](/infrastructure-as-effects/sinks): the Effect `Sink` model behind `StreamSink`. From 8e659845c70b22c071ae11b3781163225bf57cc1 Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Tue, 6 Oct 2026 15:08:02 -0700 Subject: [PATCH 10/11] docs(cloudflare/k2): drop the async Worker section --- .../content/docs/cloudflare/messaging/k2.mdx | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/website/src/content/docs/cloudflare/messaging/k2.mdx b/website/src/content/docs/cloudflare/messaging/k2.mdx index 20c0ec8a17c..b0ea619a3b0 100644 --- a/website/src/content/docs/cloudflare/messaging/k2.mdx +++ b/website/src/content/docs/cloudflare/messaging/k2.mdx @@ -172,41 +172,6 @@ The producer code stays the same on every host. Only the layer changes: The HTTP and local layers need the stream's HTTP input turned on (`http: true`). -## Use K2 from an async Worker - -Put the stream in a Worker's `env` and Alchemy adds the `k2` binding: - -```typescript -// alchemy.run.ts -export const Ingest = Cloudflare.Worker("Ingest", { - main: "./src/ingest.ts", - env: { ORDERS: Orders }, -}); - -export type IngestEnv = Cloudflare.InferEnv; -``` - -`env.ORDERS.send` returns a result object instead of throwing on an append -failure: - -```typescript -// src/ingest.ts -import type { IngestEnv } from "../alchemy.run.ts"; - -export default { - async fetch(request: Request, env: IngestEnv) { - const result = await env.ORDERS.send([ - { content: new TextEncoder().encode(await request.text()) }, - ]); - if (!result.success) { - const status = result.error.retryable ? 503 : 500; - return new Response(result.error.message, { status }); - } - return new Response(null, { status: 202 }); - }, -}; -``` - ## Consume from a long-running host `Cloudflare.K2.consumeStreamRecords(stream, handler)` subscribes a host to a From 8d73988507ce90c82ef2bee43c79a9076541aaa7 Mon Sep 17 00:00:00 2001 From: Sam Goodwin Date: Tue, 6 Oct 2026 15:19:40 -0700 Subject: [PATCH 11/11] feat(cloudflare): reference props take the resource or its identifier - R2.DataCatalog / Basin.Catalog: bucket: string | R2.Bucket (bucketName kept as a deprecated alias; switching forms is not a change) - K2.Subscription: stream: string | K2.Stream (was streamId) - Basin.Table: catalog: string | Basin.Catalog (a string is the bucket name) - Pipelines.Sink: catalog: string | Basin.Catalog; config.bucket: string | R2.Bucket --- .../alchemy/src/Cloudflare/Basin/Table.ts | 40 +++++++--- packages/alchemy/src/Cloudflare/K2/Stream.ts | 2 +- .../Cloudflare/K2/StreamEventSourcePoller.ts | 2 +- .../alchemy/src/Cloudflare/K2/Subscription.ts | 43 +++++++++-- .../alchemy/src/Cloudflare/Pipelines/Sink.ts | 58 +++++++++------ .../alchemy/src/Cloudflare/R2/DataCatalog.ts | 74 +++++++++++++------ .../test/Cloudflare/Basin/Pipeline.test.ts | 2 +- .../test/Cloudflare/Basin/Sink.test.ts | 2 +- .../test/Cloudflare/Basin/Table.test.ts | 2 +- .../Cloudflare/K2/ReadSubscription.test.ts | 4 +- .../test/Cloudflare/K2/Subscription.test.ts | 4 +- .../test/Cloudflare/K2/WriteStream.test.ts | 6 +- .../R2DataCatalog/R2DataCatalog.test.ts | 51 ++++++++++++- .../docs/cloudflare/data/iceberg-tables.mdx | 4 +- .../docs/cloudflare/data/pipelines.mdx | 2 +- .../content/docs/cloudflare/messaging/k2.mdx | 2 +- 16 files changed, 216 insertions(+), 82 deletions(-) diff --git a/packages/alchemy/src/Cloudflare/Basin/Table.ts b/packages/alchemy/src/Cloudflare/Basin/Table.ts index f1c4595676a..de36ad18998 100644 --- a/packages/alchemy/src/Cloudflare/Basin/Table.ts +++ b/packages/alchemy/src/Cloudflare/Basin/Table.ts @@ -177,11 +177,12 @@ export interface TableMaintenance { export interface TableProps { /** - * The Basin (R2 Data) catalog that owns the table. Pass the catalog - * resource; the table is created in its warehouse. Changing the catalog - * fails the plan. + * The catalog that owns the table: a `Cloudflare.Basin.Catalog` resource + * (orders the table after the catalog) or the name of the bucket the + * catalog is enabled on. The table is created in its warehouse. Changing + * the catalog fails the plan. */ - catalog: DataCatalog; + catalog: string | DataCatalog; /** * Iceberg namespace of the table. Created if it does not exist; never * deleted (other tables may share it). Changing the namespace fails the @@ -325,7 +326,7 @@ const toProps =

(props: P): Effect.Effect

=> * * const bucket = yield* Cloudflare.R2.Bucket("Analytics"); * const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", { - * bucketName: bucket.bucketName, + * bucket, * }); * const orders = yield* Cloudflare.Basin.Table("Orders", { * catalog, @@ -369,7 +370,7 @@ const toProps =

(props: P): Effect.Effect

=> * ```typescript * // Table maintenance needs the jobs enabled on the catalog first. * const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", { - * bucketName: bucket.bucketName, + * bucket, * compaction: { state: "enabled" }, * snapshotExpiration: { state: "enabled" }, * token: maintenanceToken, @@ -436,12 +437,21 @@ const CATALOG_HOST = "https://catalog.cloudflarestorage.com"; * The catalog prop is resolved to the DataCatalog's attributes before a * lifecycle operation runs. During plan it may still be unresolved. */ -const catalogRefOf = (catalog: unknown): CatalogRef | undefined => { +const catalogRefOf = (catalog: unknown, defaultAccountId?: string): CatalogRef | undefined => { + if (typeof catalog === "string") { + if (catalog.length === 0 || defaultAccountId === undefined) return undefined; + return { + accountId: defaultAccountId, + bucketName: catalog, + catalogUri: `${CATALOG_HOST}/${defaultAccountId}/${catalog}`, + warehouse: `${defaultAccountId}_${catalog}`, + }; + } if (catalog === null || typeof catalog !== "object") return undefined; const c = catalog as Record; const str = (v: unknown) => (typeof v === "string" && v.length > 0 ? v : undefined); const fromUri = str(c.catalogUri)?.match(/\/([^/]+)\/([^/]+)\/?$/); - const accountId = str(c.accountId) ?? fromUri?.[1]; + const accountId = str(c.accountId) ?? fromUri?.[1] ?? defaultAccountId; const bucketName = str(c.bucketName) ?? str(c.bucket) ?? fromUri?.[2]; if (!accountId || !bucketName) return undefined; return { @@ -784,7 +794,8 @@ export const TableProvider = () => if (typeof news.name === "string" && news.name !== output.tableName) { reasons.push(`renames the table from '${output.tableName}' to '${news.name}'`); } - const ref = isResolved(news.catalog) ? catalogRefOf(news.catalog) : undefined; + const { accountId } = yield* yield* CloudflareEnvironment; + const ref = isResolved(news.catalog) ? catalogRefOf(news.catalog, accountId) : undefined; if (ref !== undefined && ref.warehouse !== output.warehouse) { reasons.push( `moves the table from warehouse '${output.warehouse}' to '${ref.warehouse}'`, @@ -812,7 +823,8 @@ export const TableProvider = () => }), read: Effect.fn(function* ({ id, olds, output }) { - const ref = output ? refFromOutput(output) : catalogRefOf(olds?.catalog); + const { accountId } = yield* yield* CloudflareEnvironment; + const ref = output ? refFromOutput(output) : catalogRefOf(olds?.catalog, accountId); const namespace = output?.namespace ?? olds?.namespace; if (ref === undefined || typeof namespace !== "string") return undefined; const tableName = output?.tableName ?? (yield* tableNameFor(id, olds?.name)); @@ -833,13 +845,17 @@ export const TableProvider = () => }), reconcile: Effect.fn(function* ({ id, news, olds, output }) { - const ref = catalogRefOf(news.catalog) ?? (output ? refFromOutput(output) : undefined); + const { accountId } = yield* yield* CloudflareEnvironment; + const ref = + catalogRefOf(news.catalog, accountId) ?? (output ? refFromOutput(output) : undefined); const namespace = news.namespace; const tableName = yield* tableNameFor(id, news.name ?? output?.tableName); const identifier = `${namespace}.${tableName}`; if (ref === undefined) { return yield* Effect.fail( - rejected(identifier, ["has no resolved catalog (pass a Cloudflare.R2.DataCatalog)"]), + rejected(identifier, [ + "has no resolved catalog (pass a Cloudflare.Basin.Catalog or a bucket name)", + ]), ); } if (output !== undefined && output.identifier !== identifier) { diff --git a/packages/alchemy/src/Cloudflare/K2/Stream.ts b/packages/alchemy/src/Cloudflare/K2/Stream.ts index c69579872ff..488055eac31 100644 --- a/packages/alchemy/src/Cloudflare/K2/Stream.ts +++ b/packages/alchemy/src/Cloudflare/K2/Stream.ts @@ -250,7 +250,7 @@ const constructStream = (id: string, props: unknown) => * **Example:** A subscription reading from the start of the stream * ```typescript * const analytics = yield* Cloudflare.K2.Subscription("Analytics", { - * streamId: orders.streamId, + * stream: orders, * startAt: "earliest", * }); * ``` diff --git a/packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts b/packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts index 5c1803a047c..2adfb981775 100644 --- a/packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts +++ b/packages/alchemy/src/Cloudflare/K2/StreamEventSourcePoller.ts @@ -71,7 +71,7 @@ export const StreamEventSourcePolling = Layer.effect( (yield* Namespace.push( hostId, Subscription(`${stream.LogicalId}Subscription`, { - streamId: stream.streamId, + stream, startAt: props.startAt, }), )); diff --git a/packages/alchemy/src/Cloudflare/K2/Subscription.ts b/packages/alchemy/src/Cloudflare/K2/Subscription.ts index d51a9793821..7df479d059b 100644 --- a/packages/alchemy/src/Cloudflare/K2/Subscription.ts +++ b/packages/alchemy/src/Cloudflare/K2/Subscription.ts @@ -1,4 +1,5 @@ import * as k2 from "@distilled.cloud/cloudflare/k2"; +import * as Data from "effect/Data"; import * as Effect from "effect/Effect"; import * as Predicate from "effect/Predicate"; import { Unowned } from "../../AdoptPolicy.ts"; @@ -8,16 +9,18 @@ import * as Provider from "../../Provider.ts"; import { Resource } from "../../Resource.ts"; import type { Providers } from "../Providers.ts"; import type { StartAt } from "./K2Types.ts"; +import type { Stream } from "./Stream.ts"; const SubscriptionTypeId = "Cloudflare.K2.Subscription" as const; type SubscriptionTypeId = typeof SubscriptionTypeId; export interface SubscriptionProps { /** - * ID of the stream to read from, e.g. `stream.streamId`. Changing it - * replaces the subscription. + * The stream to read from: a `Cloudflare.K2.Stream` resource (orders the + * subscription after the stream) or a stream ID. Changing it replaces the + * subscription. */ - streamId: string; + stream: string | Stream; /** * Name of the subscription: 1 to 128 letters, numbers, underscores, or * hyphens, unique within the stream (not case-sensitive). If omitted, a @@ -71,14 +74,14 @@ export type Subscription = Resource< * ```typescript * const orders = yield* Cloudflare.K2.Stream("Orders"); * const analytics = yield* Cloudflare.K2.Subscription("Analytics", { - * streamId: orders.streamId, + * stream: orders, * }); * ``` * * **Example:** Read from the oldest retained record * ```typescript * const backfill = yield* Cloudflare.K2.Subscription("Backfill", { - * streamId: orders.streamId, + * stream: orders, * startAt: "earliest", * }); * ``` @@ -113,7 +116,8 @@ export const SubscriptionProvider = () => const oldName = output?.subscriptionName ?? (yield* subscriptionName(id, olds?.name)); const newName = news.name ?? oldName; const oldStartAt = output?.startAt ?? olds?.startAt ?? "latest"; - const streamChanged = (output?.streamId ?? olds?.streamId) !== news.streamId; + const streamChanged = + (output?.streamId ?? streamIdOf(olds?.stream)) !== streamIdOf(news.stream); if ( streamChanged || newName.toLowerCase() !== oldName.toLowerCase() || @@ -130,7 +134,7 @@ export const SubscriptionProvider = () => }), read: Effect.fn(function* ({ id, output, olds }) { - const streamId = output?.streamId ?? olds?.streamId; + const streamId = output?.streamId ?? streamIdOf(olds?.stream); if (!streamId) return undefined; if (output?.subscriptionId) { const observed = yield* getSubscription(streamId, output.subscriptionId); @@ -146,7 +150,12 @@ export const SubscriptionProvider = () => }), reconcile: Effect.fn(function* ({ id, news, output }) { - const streamId = news.streamId; + const streamId = streamIdOf(news.stream); + if (streamId === undefined) { + return yield* Effect.fail( + new SubscriptionStreamUnresolved({ message: "`stream` did not resolve to a stream ID" }), + ); + } const name = output?.streamId === streamId ? output.subscriptionName @@ -199,6 +208,24 @@ const subscriptionName = (id: string, name: string | undefined) => return generated.replaceAll(/[^a-zA-Z0-9_-]/g, "-"); }); +/** The `stream` prop did not resolve to a stream ID. */ +export class SubscriptionStreamUnresolved extends Data.TaggedError( + "K2SubscriptionStreamUnresolved", +)<{ message: string }> {} + +/** + * The stream ID behind a `stream` prop: the string itself, or the `streamId` + * attribute of a resolved `Cloudflare.K2.Stream`. + */ +const streamIdOf = (stream: unknown): string | undefined => + typeof stream === "string" + ? stream + : stream !== null && + typeof stream === "object" && + typeof (stream as { streamId?: unknown }).streamId === "string" + ? (stream as { streamId: string }).streamId + : undefined; + const getSubscription = (streamId: string, subscriptionId: string) => k2.getSubscription({ streamId, subscriptionId }).pipe( Effect.map((s): ObservedSubscription | undefined => s), diff --git a/packages/alchemy/src/Cloudflare/Pipelines/Sink.ts b/packages/alchemy/src/Cloudflare/Pipelines/Sink.ts index 48339837189..ea9526f69af 100644 --- a/packages/alchemy/src/Cloudflare/Pipelines/Sink.ts +++ b/packages/alchemy/src/Cloudflare/Pipelines/Sink.ts @@ -11,6 +11,8 @@ import * as Provider from "../../Provider.ts"; import { Resource } from "../../Resource.ts"; import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; import type { Providers } from "../Providers.ts"; +import type { Bucket } from "../R2/Bucket.ts"; +import type { DataCatalog } from "../R2/DataCatalog.ts"; const TypeId = "Cloudflare.Pipelines.Sink" as const; type TypeId = typeof TypeId; @@ -40,9 +42,10 @@ export interface SinkRollingPolicy { */ export interface SinkR2Config { /** - * Name of the destination R2 bucket. The bucket must already exist. + * The destination R2 bucket: a `Cloudflare.R2.Bucket` resource (orders the + * sink after the bucket) or a bucket name. The bucket must already exist. */ - bucket: string; + bucket: string | Bucket; /** * R2 S3-compatible credentials the sink uses to write objects. * Write-only — Cloudflare never echoes them back, and sinks cannot be @@ -99,10 +102,10 @@ export interface SinkR2Config { */ export interface SinkR2DataCatalogConfig { /** - * Name of the R2 bucket backing the catalog. The bucket must already - * exist and have the catalog enabled. + * The R2 bucket backing the catalog: a `Cloudflare.R2.Bucket` resource or a + * bucket name. The bucket must already exist and have the catalog enabled. */ - bucket: string; + bucket: string | Bucket; /** * Name of the Iceberg table to write. */ @@ -125,13 +128,25 @@ export interface SinkR2DataCatalogConfig { } /** - * The catalog a catalog sink writes to — pass a `Cloudflare.Basin.Catalog` - * (`Cloudflare.R2.DataCatalog`) resource so the sink is ordered after it. + * The catalog a catalog sink writes to: a `Cloudflare.Basin.Catalog` + * (`Cloudflare.R2.DataCatalog`) resource, which orders the sink after the + * catalog, or the name of the bucket the catalog is enabled on. */ -export interface SinkCatalogReference { - /** Name of the R2 bucket backing the catalog. */ - bucketName: string; -} +export type SinkCatalogReference = string | DataCatalog; + +/** + * The bucket name behind a bucket or catalog reference: the string itself, + * or the `bucketName` attribute of a resolved `Cloudflare.R2.Bucket` or + * `Cloudflare.Basin.Catalog`. + */ +const bucketNameOf = (ref: unknown): string => + typeof ref === "string" + ? ref + : ref !== null && + typeof ref === "object" && + typeof (ref as { bucketName?: unknown }).bucketName === "string" + ? (ref as { bucketName: string }).bucketName + : ""; /** * The Iceberg table a catalog sink writes to. @@ -220,7 +235,8 @@ export type SinkProps = */ type: SinkCatalogType; /** - * The catalog to write to — a `Cloudflare.Basin.Catalog` resource. + * The catalog to write to: a `Cloudflare.Basin.Catalog` resource or + * the name of the bucket the catalog is enabled on. */ catalog: SinkCatalogReference; /** @@ -301,7 +317,7 @@ export type Sink = Resource * const sink = yield* Cloudflare.Basin.Sink("events-sink", { * type: "r2", * config: { - * bucket: bucket.bucketName, + * bucket, * credentials: { * accessKeyId: yield* Config.Redacted("R2_ACCESS_KEY_ID"), * secretAccessKey: yield* Config.Redacted("R2_SECRET_ACCESS_KEY"), @@ -316,7 +332,7 @@ export type Sink = Resource * ```typescript * const sink = yield* Cloudflare.Basin.Sink("parquet-sink", { * type: "r2", - * config: { bucket: bucket.bucketName, credentials }, + * config: { bucket, credentials }, * format: { type: "parquet", compression: "zstd" }, * }); * ``` @@ -326,7 +342,7 @@ export type Sink = Resource * ```typescript * const bucket = yield* Cloudflare.R2.Bucket("Lakehouse", {}); * const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { - * bucketName: bucket.bucketName, + * bucket, * }); * * const sink = yield* Cloudflare.Basin.Sink("PageViewTable", { @@ -342,7 +358,7 @@ export type Sink = Resource * const sink = yield* Cloudflare.Basin.Sink("iceberg-sink", { * type: "r2_data_catalog", * config: { - * bucket: bucket.bucketName, + * bucket, * tableName: "events", * namespace: "default", * token: yield* Config.Redacted("CATALOG_TOKEN"), @@ -519,14 +535,14 @@ const isCatalogType = (type: string) => type === "r2_data_catalog" || type === " const catalogTarget = (props: Exclude): CatalogTarget => "config" in props && props.config !== undefined ? { - bucket: props.config.bucket, + bucket: bucketNameOf(props.config.bucket), tableName: props.config.tableName, namespace: props.config.namespace, token: props.config.token, rollingPolicy: props.config.rollingPolicy, } : { - bucket: (props as { catalog: SinkCatalogReference }).catalog.bucketName, + bucket: bucketNameOf((props as { catalog: SinkCatalogReference }).catalog), tableName: (props as { table: SinkCatalogTable }).table.name, namespace: (props as { table: SinkCatalogTable }).table.namespace, token: (props as { token: Redacted.Redacted }).token, @@ -585,7 +601,7 @@ const sinkDrifted = (observed: ObservedSink, news: SinkProps): boolean => { const cfg = observed.config; if (!cfg) return false; if (news.type === "r2") { - if (cfg.bucket !== news.config.bucket) return true; + if (cfg.bucket !== bucketNameOf(news.config.bucket)) return true; const observedPath = "path" in cfg ? (cfg.path ?? undefined) : undefined; return news.config.path !== undefined && news.config.path !== observedPath; } @@ -615,7 +631,7 @@ const toRequestConfig = (accountId: string, news: SinkProps) => { const c = news.config; return { accountId, - bucket: c.bucket, + bucket: bucketNameOf(c.bucket), credentials: { accessKeyId: Redacted.value(c.credentials.accessKeyId), secretAccessKey: Redacted.value(c.credentials.secretAccessKey), @@ -672,7 +688,7 @@ const canonicalSink = (props: SinkProps): unknown => { return { kind: "r2", format, - bucket: c.bucket, + bucket: bucketNameOf(c.bucket), path: c.path, partitioning: c.partitioning, fileNaming: c.fileNaming diff --git a/packages/alchemy/src/Cloudflare/R2/DataCatalog.ts b/packages/alchemy/src/Cloudflare/R2/DataCatalog.ts index ce2b80256c5..1073f2e7574 100644 --- a/packages/alchemy/src/Cloudflare/R2/DataCatalog.ts +++ b/packages/alchemy/src/Cloudflare/R2/DataCatalog.ts @@ -9,6 +9,7 @@ import * as Provider from "../../Provider.ts"; import { Resource } from "../../Resource.ts"; import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; import type { Providers } from "../Providers.ts"; +import type { Bucket } from "./Bucket.ts"; const TypeId = "Cloudflare.R2.DataCatalog" as const; type TypeId = typeof TypeId; @@ -58,14 +59,7 @@ export type SnapshotExpiration = { minSnapshotsToKeep?: number; }; -export type DataCatalogProps = { - /** - * Name of the R2 bucket to enable the Iceberg data catalog on. The bucket - * must already exist — pass `bucket.bucketName` from a `Cloudflare.R2.Bucket` - * resource to order catalog-after-bucket. Changing the bucket replaces the - * catalog (the old bucket's catalog is disabled; table data is untouched). - */ - bucketName: string; +export type DataCatalogProps = DataCatalogBucket & { /** * Compaction maintenance configuration. Only the fields you specify are * enforced; omitted fields keep Cloudflare's defaults. @@ -85,6 +79,46 @@ export type DataCatalogProps = { token?: Redacted.Redacted; }; +/** + * The bucket a catalog is enabled on. `bucketName` is the deprecated spelling + * of `bucket`; both name the same bucket and switching between them is not a + * change. + */ +export type DataCatalogBucket = + | { + /** + * The R2 bucket to enable the Iceberg catalog on: a + * `Cloudflare.R2.Bucket` resource (orders the catalog after the bucket) + * or a bucket name. Changing the bucket replaces the catalog (the old + * bucket's catalog is disabled; table data is untouched). + */ + bucket: string | Bucket; + bucketName?: never; + } + | { + /** + * Name of the R2 bucket to enable the catalog on. + * @deprecated Use `bucket`, which also accepts the bucket resource. + */ + bucketName: string; + bucket?: never; + }; + +/** + * The bucket name behind a catalog's props: `bucket` as a name or a resolved + * `Cloudflare.R2.Bucket`, or the deprecated `bucketName`. + */ +export const catalogBucketName = (props: unknown): string | undefined => { + if (props === null || typeof props !== "object") return undefined; + const { bucket, bucketName } = props as { bucket?: unknown; bucketName?: unknown }; + if (typeof bucket === "string") return bucket; + if (bucket !== null && typeof bucket === "object") { + const name = (bucket as { bucketName?: unknown }).bucketName; + if (typeof name === "string") return name; + } + return typeof bucketName === "string" ? bucketName : undefined; +}; + export type DataCatalogAttributes = { /** * Unique identifier of the catalog (stable across disable/enable cycles). @@ -166,7 +200,7 @@ export type DataCatalog = Resource< * const bucket = yield* Cloudflare.R2.Bucket("LakehouseBucket"); * * const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", { - * bucketName: bucket.bucketName, + * bucket, * }); * * // Point any Iceberg REST client at the warehouse: @@ -178,7 +212,7 @@ export type DataCatalog = Resource< * **Example:** Configure compaction and snapshot expiration * ```typescript * const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", { - * bucketName: bucket.bucketName, + * bucket, * compaction: { state: "enabled", targetSizeMb: "256" }, * snapshotExpiration: { * state: "enabled", @@ -192,7 +226,7 @@ export type DataCatalog = Resource< * ```typescript * // Maintenance jobs need an API token with R2 read/write on the bucket. * const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", { - * bucketName: bucket.bucketName, + * bucket, * compaction: { state: "enabled" }, * token: maintenanceToken, // Redacted * }); @@ -224,13 +258,9 @@ export const DataCatalogProvider = () => // The catalog is keyed to its bucket — moving buckets is a replacement // (disable on the old bucket, enable on the new one). bucketName is an // Input; compare only once both sides are concrete strings. - const oldBucket = - output?.bucketName ?? (typeof olds?.bucketName === "string" ? olds.bucketName : undefined); - if ( - oldBucket !== undefined && - typeof news.bucketName === "string" && - oldBucket !== news.bucketName - ) { + const oldBucket = output?.bucketName ?? catalogBucketName(olds); + const newBucket = catalogBucketName(news); + if (oldBucket !== undefined && newBucket !== undefined && oldBucket !== newBucket) { return { action: "replace" } as const; } return undefined; @@ -238,8 +268,7 @@ export const DataCatalogProvider = () => read: Effect.fn(function* ({ output, olds }) { const { accountId } = yield* yield* CloudflareEnvironment; const acct = output?.accountId ?? accountId; - const bucketName = - output?.bucketName ?? (typeof olds?.bucketName === "string" ? olds.bucketName : undefined); + const bucketName = output?.bucketName ?? catalogBucketName(olds); if (bucketName === undefined) return undefined; const observed = yield* getCatalog(acct, bucketName); @@ -268,8 +297,9 @@ export const DataCatalogProvider = () => reconcile: Effect.fn(function* ({ news, olds, output }) { const { accountId } = yield* yield* CloudflareEnvironment; const acct = output?.accountId ?? accountId; - // Inputs have been resolved to concrete strings by Plan. - const bucketName = news.bucketName as string; + // Inputs have been resolved by Plan: `bucket` is a name or the + // bucket's attributes. + const bucketName = catalogBucketName(news) as string; // Observe — `output` is only a cache; a disabled or never-enabled // catalog observes as missing/inactive and falls through to ensure. diff --git a/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts b/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts index 5578f8e7dca..3fe841fba78 100644 --- a/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts +++ b/packages/alchemy/test/Cloudflare/Basin/Pipeline.test.ts @@ -26,7 +26,7 @@ const lakehouse = (sql: (stream: string, sink: string) => string) => const token = yield* apiToken; const bucket = yield* Cloudflare.R2.Bucket("LakeBucket", { forceDestroy: true }); const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { - bucketName: bucket.bucketName, + bucket, }); const stream = yield* Cloudflare.Basin.Stream("PageViews", { schema: PageView }); const sink = yield* Cloudflare.Basin.Sink("PageViewTable", { diff --git a/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts b/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts index 8349147d9f4..50db354c6a2 100644 --- a/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts +++ b/packages/alchemy/test/Cloudflare/Basin/Sink.test.ts @@ -98,7 +98,7 @@ test.provider( const lakehouse = Effect.gen(function* () { const bucket = yield* Cloudflare.R2.Bucket("LakeBucket", { forceDestroy: true }); const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { - bucketName: bucket.bucketName, + bucket, }); return { bucket, catalog }; }); diff --git a/packages/alchemy/test/Cloudflare/Basin/Table.test.ts b/packages/alchemy/test/Cloudflare/Basin/Table.test.ts index 76ed97c4c0a..4c89864b033 100644 --- a/packages/alchemy/test/Cloudflare/Basin/Table.test.ts +++ b/packages/alchemy/test/Cloudflare/Basin/Table.test.ts @@ -63,7 +63,7 @@ const program = ( Effect.gen(function* () { const bucket = yield* Cloudflare.R2.Bucket("TableBucket", { forceDestroy: true }); const catalog = yield* Cloudflare.R2.DataCatalog("TableCatalog", { - bucketName: bucket.bucketName, + bucket, token, ...catalogMaintenance, }); diff --git a/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts b/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts index d0a2fe823cb..1ad0ee59e23 100644 --- a/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts +++ b/packages/alchemy/test/Cloudflare/K2/ReadSubscription.test.ts @@ -49,11 +49,11 @@ test.provider( Effect.gen(function* () { const events = yield* Cloudflare.K2.Stream("LocalEvents", { http: true, schema: Event }); const poller = yield* Cloudflare.K2.Subscription("Poller", { - streamId: events.streamId, + stream: events, startAt: "earliest", }); const reader = yield* Cloudflare.K2.Subscription("Reader", { - streamId: events.streamId, + stream: events, startAt: "earliest", }); diff --git a/packages/alchemy/test/Cloudflare/K2/Subscription.test.ts b/packages/alchemy/test/Cloudflare/K2/Subscription.test.ts index 99ec9914e9e..af141998c20 100644 --- a/packages/alchemy/test/Cloudflare/K2/Subscription.test.ts +++ b/packages/alchemy/test/Cloudflare/K2/Subscription.test.ts @@ -15,7 +15,7 @@ const program = (startAt: "earliest" | "latest") => Effect.gen(function* () { const stream = yield* Cloudflare.K2.Stream("SubscriptionStream"); const subscription = yield* Cloudflare.K2.Subscription("Analytics", { - streamId: stream.streamId, + stream, startAt, }); return { stream, subscription }; @@ -75,7 +75,7 @@ test.provider( Effect.gen(function* () { const stream = yield* Cloudflare.K2.Stream("NamedSubscriptionStream"); const subscription = yield* Cloudflare.K2.Subscription("Named", { - streamId: stream.streamId, + stream, name: "alchemy-k2-named", startAt, }); diff --git a/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts b/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts index fde0209ec90..c3e7e68048f 100644 --- a/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts +++ b/packages/alchemy/test/Cloudflare/K2/WriteStream.test.ts @@ -79,7 +79,7 @@ test.provider( Effect.gen(function* () { const orders = yield* BindingOrders; const verifier = yield* Cloudflare.K2.Subscription("BindingVerifier", { - streamId: orders.streamId, + stream: orders, startAt: "earliest", }); const worker = yield* K2BindingWorker; @@ -114,7 +114,7 @@ test.provider( Effect.gen(function* () { const orders = yield* AsyncOrders; const verifier = yield* Cloudflare.K2.Subscription("AsyncVerifier", { - streamId: orders.streamId, + stream: orders, startAt: "earliest", }); const worker = yield* K2AsyncWorker; @@ -163,7 +163,7 @@ test.provider.skipIf(!process.env.CLOUDFLARE_TEST_K2_HTTP)( Effect.gen(function* () { const orders = yield* HttpOrders; const verifier = yield* Cloudflare.K2.Subscription("HttpVerifier", { - streamId: orders.streamId, + stream: orders, startAt: "earliest", }); const worker = yield* K2HttpWorker; diff --git a/packages/alchemy/test/Cloudflare/R2DataCatalog/R2DataCatalog.test.ts b/packages/alchemy/test/Cloudflare/R2DataCatalog/R2DataCatalog.test.ts index 04019a38383..1eb10cec9d7 100644 --- a/packages/alchemy/test/Cloudflare/R2DataCatalog/R2DataCatalog.test.ts +++ b/packages/alchemy/test/Cloudflare/R2DataCatalog/R2DataCatalog.test.ts @@ -20,15 +20,15 @@ interface CatalogOpts { } // One program deploying both the R2 bucket and the catalog enabled on it. -// `bucketName` references the bucket's output attribute, so the engine -// orders catalog-after-bucket on deploy (and the reverse on destroy). +// `bucket` references the bucket resource, so the engine orders +// catalog-after-bucket on deploy (and the reverse on destroy). const program = (opts: CatalogOpts = {}) => Effect.gen(function* () { const bucket = yield* Cloudflare.R2.Bucket("CatalogBucket", { forceDestroy: true, }); const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", { - bucketName: bucket.bucketName, + bucket, ...opts, }); return { bucket, catalog }; @@ -161,6 +161,51 @@ test.provider( }, ); +test.provider( + "switching between bucket forms keeps the same catalog", + (stack) => + Effect.gen(function* () { + const { accountId } = yield* yield* CloudflareEnvironment; + + yield* stack.destroy(); + + // The deprecated `bucketName` string form. + const legacy = yield* stack.deploy( + Effect.gen(function* () { + const bucket = yield* Cloudflare.R2.Bucket("CatalogBucket", { forceDestroy: true }); + const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", { + bucketName: bucket.bucketName, + }); + return { bucket, catalog }; + }), + ); + + // The bucket resource, then a plain bucket name: neither is a change. + const viaResource = yield* stack.deploy(program()); + expect(viaResource.catalog.catalogId).toEqual(legacy.catalog.catalogId); + + const viaName = yield* stack.deploy( + Effect.gen(function* () { + const bucket = yield* Cloudflare.R2.Bucket("CatalogBucket", { forceDestroy: true }); + const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", { + bucket: legacy.bucket.bucketName, + }); + return { bucket, catalog }; + }), + ); + expect(viaName.catalog.catalogId).toEqual(legacy.catalog.catalogId); + expect(viaName.catalog.bucketName).toEqual(legacy.bucket.bucketName); + + yield* stack.destroy(); + + yield* expectGone(accountId, legacy.bucket.bucketName); + }).pipe(logLevel), + { + tags: ["provider:cloudflare", "provider:cloudflare:r2", "live"], + timeout: 240_000, + }, +); + test.provider( "list enumerates the deployed catalog", (stack) => diff --git a/website/src/content/docs/cloudflare/data/iceberg-tables.mdx b/website/src/content/docs/cloudflare/data/iceberg-tables.mdx index 904cca46fc7..46a32098539 100644 --- a/website/src/content/docs/cloudflare/data/iceberg-tables.mdx +++ b/website/src/content/docs/cloudflare/data/iceberg-tables.mdx @@ -26,7 +26,7 @@ engines. const lake = yield* Cloudflare.R2.Bucket("Lake"); const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { - bucketName: lake.bucketName, + bucket: lake, }); ``` @@ -44,7 +44,7 @@ compact files and expire snapshots for every table in the warehouse: ```typescript const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { - bucketName: lake.bucketName, + bucket: lake, compaction: { state: "enabled", targetSizeMb: "128" }, snapshotExpiration: { state: "enabled", maxSnapshotAge: "7d", minSnapshotsToKeep: 10 }, token: yield* Config.Redacted("CATALOG_MAINTENANCE_TOKEN"), diff --git a/website/src/content/docs/cloudflare/data/pipelines.mdx b/website/src/content/docs/cloudflare/data/pipelines.mdx index 1b1e576a184..ed6b74eef95 100644 --- a/website/src/content/docs/cloudflare/data/pipelines.mdx +++ b/website/src/content/docs/cloudflare/data/pipelines.mdx @@ -148,7 +148,7 @@ export default Alchemy.Stack( Effect.gen(function* () { const lake = yield* Cloudflare.R2.Bucket("Lake"); const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", { - bucketName: lake.bucketName, + bucket: lake, }); const views = yield* PageViews; diff --git a/website/src/content/docs/cloudflare/messaging/k2.mdx b/website/src/content/docs/cloudflare/messaging/k2.mdx index b0ea619a3b0..68b84c9b730 100644 --- a/website/src/content/docs/cloudflare/messaging/k2.mdx +++ b/website/src/content/docs/cloudflare/messaging/k2.mdx @@ -243,7 +243,7 @@ every record. To split records between hosts instead, declare a ```typescript export const FraudWorkers = Cloudflare.K2.Subscription("FraudWorkers", { - streamId: Orders.streamId, + stream: Orders, startAt: "latest", }); ```