deps(storage): bump visulima storage to 2.0.33, storage-client to 1.0.8

apps/docs/src/content/docs/concepts/file-storage.mdx

@@ -174,9 +174,8 @@ One handler speaks one protocol:
 
 - `"tus"` (the default): resumable. The client can pause, resume, and continue
   after a dropped connection from the last acknowledged byte.
-- `"chunked-rest"`: plain REST chunks, one at a time and in order, over
-  `createR2BindingUploadStorage` only. The bundled client sends four chunks in
-  parallel, so with it this protocol works only for a single chunk. Prefer TUS.
+- `"chunked-rest"`: plain REST chunks, one at a time and in order, which is how
+  the bundled client sends them. Not resumable mid-chunk the way TUS is.
 - `"multipart"`: one `multipart/form-data` request per file. Not resumable.
 
 #### Mount the handler
@@ -279,8 +278,8 @@ pair a type-based cap with `allowMIME` on the provider
 (`createR2BindingUploadStorage(env.UPLOADS, { allowMIME: ["image/*", "video/*"] })`).
 
 `createR2UploadStorage` is the other R2 provider: it writes over R2's S3 API
-with S3 credentials and no binding. Use it when the Worker that mounts the
-route has no binding to the bucket.
+with S3 credentials and no binding. Prefer `createR2BindingUploadStorage`: it
+needs no credentials and runs under `wrangler dev`.
 
 #### Upload from the client
 
@@ -358,16 +357,20 @@ the binding of that same bucket.
 - **R2 part size.** R2 requires every multipart part but the last to be at
   least 5 MiB and all of them the same size. `createR2BindingUploadStorage`
   coalesces chunks of any size into 5 MiB parts, so the client's `chunkSize` is
-  free. Over `createR2UploadStorage` each chunk becomes one part, so set
-  `chunkSize` to the same 5 MiB or more on every chunk (TUS defaults to 1 MiB).
+  free. Over `createR2UploadStorage` each chunk becomes one part, so
+  `chunkSize` has to be the same 5 MiB or more on every chunk (the TUS client's
+  default is 5 MiB).
 - **Unsupported TUS extensions.** The binding provider does not offer
-  `creation-defer-length`, `concatenation` (parallel uploads) or `checksum`.
-- **Chunked REST** runs over `createR2BindingUploadStorage` only, one chunk at a
-  time and in order: a chunk at any other offset, or one sent while another is
-  streaming, is a `409` and is not stored. The bundled client sends four chunks
-  in parallel, so it can upload only as a single chunk (`chunkSize` at least the
-  file size). `createR2UploadStorage` does not check chunk offsets and can
-  corrupt a chunked-REST upload. Use TUS for resumable uploads;
+  `creation-defer-length`, `concatenation` (parallel uploads) or `checksum`. A
+  request that sends `Upload-Checksum` anyway gets a `400` before its body is
+  read.
+- **Method overrides.** A request carrying `X-HTTP-Method-Override` (or
+  `X-HTTP-Method`, `X-Method-Override`) gets a `405`, so no request can pass the
+  write gate as one method and run as another.
+- **Chunked REST** over either R2 provider takes one chunk at a time and in
+  order: a chunk at any other offset is a `409` and is not stored, and over the
+  binding provider so is one sent while another is streaming. The bundled client
+  sends one chunk at a time, so its uploads complete;
   [@lunora/storage](/docs/packages/storage#chunked-rest) has the details.
 - **`maxFileSize`** defaults to 100 MiB, has no unlimited setting, and must be a
   finite, non-negative number. Uploads over it get a `413`.

packages/storage/__tests__/chunked-rest-driver.ts

@@ -78,5 +78,5 @@ const routedFetch = (route: ChunkedRestRoute, origin = "https://test.local"): Ro
     return { fetch, requests };
 };
 
-export type { ChunkedRestDriver, ChunkedRestRoute, RoutedFetch };
+export type { ChunkBody, ChunkedRestDriver, ChunkedRestRoute, RoutedFetch };
 export { CHUNKED_REST_ENDPOINT, chunkedRest, routedFetch, uploadId };

packages/storage/__tests__/fake-r2-s3.ts

@@ -0,0 +1,194 @@
+/**
+ * An in-memory R2 S3 API, path-style (`https://<account>.r2.cloudflarestorage.com/<bucket>/<key>`) or virtual-hosted,
+ * for the `createR2UploadStorage` tests: the object and multipart calls
+ * `@visulima/storage`'s aws-light provider makes, served from a `fetch` stub.
+ * Like R2, it refuses a multipart completion that breaks R2's part rules, and
+ * it answers a request whose path names no bucket it holds with `404 NoSuchBucket`.
+ */
+import { concatParts, validParts } from "./r2-multipart-rules";
+
+interface StoredObject {
+    body: Uint8Array;
+    etag: string;
+    headers: Record<string, string>;
+}
+
+interface MultipartUpload {
+    headers: Record<string, string>;
+    key: string;
+    parts: Map<number, StoredObject>;
+}
+
+interface FakeR2S3 {
+    /** Serve one request; install it as `globalThis.fetch`. */
+    fetch: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
+    /** The stored object under `key`, if any. */
+    object: (key: string) => Uint8Array | undefined;
+    /** `METHOD url` of every request, in order. */
+    requests: string[];
+}
+
+const xml = (body: string, status = 200): Response =>
+    new Response(`<?xml version="1.0" encoding="UTF-8"?>${body}`, { headers: { "content-type": "application/xml" }, status });
+
+/** One XML element. */
+const tag = (name: string, content: string): string => `<${name}>${content}</${name}>`;
+
+const s3Error = (code: string, status: number): Response => xml(tag("Error", tag("Code", code)), status);
+
+const withEtag = (etag: string): Response => new Response(undefined, { headers: { ETag: `"${etag}"` }, status: 200 });
+
+/** The headers an object keeps: its type and its `x-amz-meta-*` metadata. */
+const metaHeaders = (request: Request): Record<string, string> => {
+    const headers: Record<string, string> = {};
+
+    for (const [name, value] of request.headers) {
+        if (name.startsWith("x-amz-meta-") || name === "content-type") {
+            headers[name] = value;
+        }
+    }
+
+    return headers;
+};
+
+/**
+ * `partsPerPage` is how many parts one ListParts answer holds before it is
+ * truncated and points at the next page (S3's own maximum is 1,000).
+ */
+const createFakeR2S3 = (bucket: string, { partsPerPage = 1000 }: { partsPerPage?: number } = {}): FakeR2S3 => {
+    const objects = new Map<string, StoredObject>();
+    const uploads = new Map<string, MultipartUpload>();
+    const requests: string[] = [];
+    let etags = 0;
+    let uploadIds = 0;
+
+    const nextEtag = (): string => {
+        etags += 1;
+
+        return `etag-${String(etags)}`;
+    };
+
+    const sortedParts = (upload: MultipartUpload): [number, StoredObject][] => [...upload.parts.entries()].toSorted(([a], [b]) => a - b);
+
+    /** Create, part upload, ListParts, complete and abort of one multipart upload. */
+    const serveMultipart = async (request: Request, url: URL, key: string): Promise<Response> => {
+        if (url.searchParams.has("uploads")) {
+            uploadIds += 1;
+
+            const id = `upload-${String(uploadIds)}`;
+
+            uploads.set(id, { headers: metaHeaders(request), key, parts: new Map() });
+
+            return xml(tag("InitiateMultipartUploadResult", tag("Key", key) + tag("UploadId", id)));
+        }
+
+        const id = url.searchParams.get("uploadId") ?? "";
+        const upload = uploads.get(id);
+
+        if (upload === undefined) {
+            return s3Error("NoSuchUpload", 404);
+        }
+
+        if (request.method === "PUT") {
+            const etag = nextEtag();
+
+            upload.parts.set(Number(url.searchParams.get("partNumber")), { body: new Uint8Array(await request.arrayBuffer()), etag, headers: {} });
+
+            return withEtag(etag);
+        }
+
+        if (request.method === "GET") {
+            const marker = Number(url.searchParams.get("part-number-marker") ?? "0");
+            const after = sortedParts(upload).filter(([number]) => number > marker);
+            const page = after.slice(0, partsPerPage);
+            const parts = page.map(([number, part]) =>
+                tag("Part", tag("PartNumber", String(number)) + tag("ETag", `"${part.etag}"`) + tag("Size", String(part.body.byteLength))),
+            );
+            const truncated = after.length > page.length;
+            const paging = tag("IsTruncated", String(truncated)) + (truncated ? tag("NextPartNumberMarker", String(page.at(-1)?.[0] ?? marker)) : "");
+
+            return xml(tag("ListPartsResult", paging + parts.join("")));
+        }
+
+        uploads.delete(id);
+
+        if (request.method !== "POST") {
+            return new Response(undefined, { status: 204 });
+        }
+
+        const parts = sortedParts(upload).map(([, part]) => part.body);
+
+        if (!validParts(parts)) {
+            return s3Error("EntityTooSmall", 400);
+        }
+
+        const etag = nextEtag();
+
+        objects.set(upload.key, { body: concatParts(parts), etag, headers: upload.headers });
+
+        return xml(tag("CompleteMultipartUploadResult", tag("Key", upload.key) + tag("ETag", `"${etag}"`)));
+    };
+
+    /** A plain object: conditional `PUT`, `DELETE`, `HEAD` and `GET`. */
+    const serveObject = async (request: Request, key: string): Promise<Response> => {
+        const stored = objects.get(key);
+
+        if (request.method === "PUT") {
+            const ifMatch = request.headers.get("if-match");
+
+            if (ifMatch !== null && `"${stored?.etag ?? ""}"` !== ifMatch) {
+                return s3Error("PreconditionFailed", 412);
+            }
+
+            const etag = nextEtag();
+
+            objects.set(key, { body: new Uint8Array(await request.arrayBuffer()), etag, headers: metaHeaders(request) });
+
+            return withEtag(etag);
+        }
+
+        if (request.method === "DELETE") {
+            objects.delete(key);
+
+            return new Response(undefined, { status: 204 });
+        }
+
+        if (stored === undefined) {
+            return s3Error("NoSuchKey", 404);
+        }
+
+        const headers = { ...stored.headers, "content-length": String(stored.body.byteLength), etag: `"${stored.etag}"` };
+
+        return new Response(request.method === "HEAD" ? undefined : new Uint8Array(stored.body), { headers, status: 200 });
+    };
+
+    const serve = async (request: Request): Promise<Response> => {
+        const url = new URL(request.url);
+        // Path-style, or virtual-hosted (`<bucket>.<account>.r2…`).
+        const prefix = url.hostname.startsWith(`${bucket}.`) ? "/" : `/${bucket}/`;
+
+        requests.push(`${request.method} ${url.href}`);
+
+        if (!url.pathname.startsWith(prefix)) {
+            return s3Error("NoSuchBucket", 404);
+        }
+
+        const key = decodeURIComponent(url.pathname.slice(prefix.length));
+
+        if (key === "") {
+            // HeadBucket, the provider's readiness probe.
+            return request.method === "HEAD" ? new Response(undefined, { status: 200 }) : s3Error("NotImplemented", 501);
+        }
+
+        return url.searchParams.has("uploads") || url.searchParams.has("uploadId") ? serveMultipart(request, url, key) : serveObject(request, key);
+    };
+
+    return {
+        fetch: async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => serve(input instanceof Request ? input : new Request(input, init)),
+        object: (key: string): Uint8Array | undefined => objects.get(key)?.body,
+        requests,
+    };
+};
+
+export type { FakeR2S3 };
+export { createFakeR2S3 };

packages/storage/__tests__/fake-r2-upload-bucket.ts

@@ -12,8 +12,7 @@ import type { R2ConditionalLike, R2MultipartUploadLike, R2ObjectBodyLike, R2Obje
 
 import { toBytes } from "../src/byte-queue";
 import type { R2UploadBucket } from "../src/r2-binding-upload-storage";
-
-const MIN_PART = 5 * 1024 * 1024;
+import { concatParts, validParts } from "./r2-multipart-rules";
 
 interface StoredObject {
     bytes: Uint8Array;
@@ -86,40 +85,24 @@ const createFakeR2UploadBucket = (): R2UploadBucket & {
             },
             complete: async (uploadedParts) => {
                 const upload = open();
-                const chunks: Uint8Array[] = [];
-
-                uploadedParts.forEach(({ etag, partNumber }, index) => {
+                const chunks = uploadedParts.map(({ etag, partNumber }) => {
                     const part = upload.parts.get(partNumber);
 
                     if (part?.etag !== etag) {
                         throw new Error(`InvalidPart: ${String(partNumber)}`);
                     }
 
-                    const isLast = index === uploadedParts.length - 1;
-
-                    if (!isLast && part.bytes.byteLength < MIN_PART) {
-                        throw new Error("EntityTooSmall: every part but the last must be at least 5 MiB");
-                    }
-
-                    if (!isLast && index > 0 && part.bytes.byteLength !== chunks[0]?.byteLength) {
-                        throw new Error("InvalidPart: all non-trailing parts must have the same length");
-                    }
-
-                    chunks.push(part.bytes);
+                    return part.bytes;
                 });
 
-                const bytes = new Uint8Array(chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0));
-                let offset = 0;
-
-                for (const chunk of chunks) {
-                    bytes.set(chunk, offset);
-                    offset += chunk.byteLength;
+                if (!validParts(chunks)) {
+                    throw new Error("InvalidPart: every part but the last must be at least 5 MiB, and all of them the same size");
                 }
 
                 partSizes.push(chunks.map((chunk) => chunk.byteLength));
                 openUploads.delete(uploadId);
 
-                return store(key, bytes, upload.contentType);
+                return store(key, concatParts(chunks), upload.contentType);
             },
             key,
             uploadId,