feat: harden self-managed reconciliation

server/materialize.ts

@@ -27,7 +27,30 @@ import {
 
 export interface MaterializeCas {
   get(digest: Sha256Digest): Promise<Uint8Array>;
-  has(digest: Sha256Digest): Promise<boolean>;
+}
+
+const MATERIALIZE_CONCURRENCY = 20;
+
+async function mapConcurrent<T, R>(
+  values: T[],
+  run: (value: T) => Promise<R>,
+): Promise<R[]> {
+  const results = new Array<R>(values.length);
+  let index = 0;
+  const worker = async () => {
+    for (;;) {
+      const current = index++;
+      if (current >= values.length) return;
+      results[current] = await run(values[current]!);
+    }
+  };
+  await Promise.all(
+    Array.from(
+      { length: Math.min(MATERIALIZE_CONCURRENCY, values.length) },
+      worker,
+    ),
+  );
+  return results;
 }
 
 function safePath(path: string): boolean {
@@ -105,11 +128,6 @@ export async function materializeWorkspace(
 ): Promise<WorkspaceManifest> {
   assertSha256Digest(manifestDigest);
   const manifest = parseWorkspaceManifest(await cas.get(manifestDigest));
-  const missing: Sha256Digest[] = [];
-  for (const file of manifest.files)
-    if (!(await cas.has(file.digest))) missing.push(file.digest);
-  if (missing.length)
-    throw new Error(`Workspace blobs are missing: ${missing.join(", ")}`);
 
   await mkdir(dirname(destination), { recursive: true });
   try {
@@ -124,7 +142,7 @@ export async function materializeWorkspace(
   );
   await mkdir(temporary, { mode: 0o755 });
   try {
-    for (const file of manifest.files) {
+    await mapConcurrent(manifest.files, async (file) => {
       const target = join(temporary, file.path);
       if (relative(temporary, target).startsWith(".."))
         throw new Error("Unsafe workspace path");
@@ -151,7 +169,7 @@ export async function materializeWorkspace(
         }
         await chmod(target, file.mode);
       }
-    }
+    });
     await rename(temporary, destination);
   } catch (error) {
     await rm(temporary, { recursive: true, force: true });