Release 2.6.2-rc1

server/materialize.ts

@@ -29,6 +29,16 @@ export interface MaterializeCas {
   get(digest: Sha256Digest): Promise<Uint8Array>;
 }
 
+export interface MaterializeTiming {
+  fileCount?: number;
+  manifestBytes?: number;
+  fileBytes?: number;
+  // Sum of operation durations; concurrent operations can exceed wall time.
+  // CAS get includes its content hash verification.
+  casReadMs: number;
+  fsWriteMs: number;
+}
+
 const MATERIALIZE_CONCURRENCY = 20;
 
 async function mapConcurrent<T, R>(
@@ -101,8 +111,18 @@ export function parseWorkspaceManifest(data: Uint8Array): WorkspaceManifest {
     }
     paths.add(file.path);
   }
+  const ancestors = new Set<string>();
   for (const path of paths) {
-    if ([...paths].some((other) => other.startsWith(`${path}/`))) {
+    for (
+      let separator = path.indexOf("/");
+      separator !== -1;
+      separator = path.indexOf("/", separator + 1)
+    ) {
+      ancestors.add(path.slice(0, separator));
+    }
+  }
+  for (const path of paths) {
+    if (ancestors.has(path)) {
       throw new Error(`Workspace path conflicts with a directory: ${path}`);
     }
   }
@@ -125,13 +145,43 @@ export async function materializeWorkspace(
   cas: MaterializeCas,
   manifestDigest: Sha256Digest,
   destination: string,
+  timing?: MaterializeTiming,
 ): Promise<WorkspaceManifest> {
+  const read = timing
+    ? async (digest: Sha256Digest) => {
+        const start = performance.now();
+        try {
+          return await cas.get(digest);
+        } finally {
+          timing.casReadMs += performance.now() - start;
+        }
+      }
+    : (digest: Sha256Digest) => cas.get(digest);
+  const write = timing
+    ? async <T>(operation: () => Promise<T>): Promise<T> => {
+        const start = performance.now();
+        try {
+          return await operation();
+        } finally {
+          timing.fsWriteMs += performance.now() - start;
+        }
+      }
+    : <T>(operation: () => Promise<T>) => operation();
   assertSha256Digest(manifestDigest);
-  const manifest = parseWorkspaceManifest(await cas.get(manifestDigest));
+  const manifestData = await read(manifestDigest);
+  if (timing) timing.manifestBytes = manifestData.byteLength;
+  const manifest = parseWorkspaceManifest(manifestData);
+  if (timing) {
+    timing.fileCount = manifest.files.length;
+    timing.fileBytes = manifest.files.reduce(
+      (total, file) => total + file.size,
+      0,
+    );
+  }
 
-  await mkdir(dirname(destination), { recursive: true });
+  await write(() => mkdir(dirname(destination), { recursive: true }));
   try {
-    await lstat(destination);
+    await write(() => lstat(destination));
     throw new Error(`Workspace destination already exists: ${destination}`);
   } catch (error) {
     if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
@@ -140,14 +190,16 @@ export async function materializeWorkspace(
     dirname(destination),
     `.${basename(destination)}.${process.pid}.${randomUUID()}.tmp`,
   );
-  await mkdir(temporary, { mode: 0o755 });
+  await write(() => mkdir(temporary, { mode: 0o755 }));
   try {
     await mapConcurrent(manifest.files, async (file) => {
       const target = join(temporary, file.path);
       if (relative(temporary, target).startsWith(".."))
         throw new Error("Unsafe workspace path");
-      await mkdir(dirname(target), { recursive: true, mode: 0o755 });
-      const data = await cas.get(file.digest);
+      await write(() =>
+        mkdir(dirname(target), { recursive: true, mode: 0o755 }),
+      );
+      const data = await read(file.digest);
       if (data.byteLength !== file.size) {
         throw new Error(`Workspace blob size mismatch for ${file.path}`);
       }
@@ -155,24 +207,26 @@ export async function materializeWorkspace(
         const link = new TextDecoder("utf-8", { fatal: true }).decode(data);
         if (!safeSymlinkTarget(file.path, link))
           throw new Error(`Unsafe symlink target for ${file.path}`);
-        await symlink(link, target);
+        await write(() => symlink(link, target));
       } else {
-        const handle = await open(
-          target,
-          constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,
-          file.mode,
+        const handle = await write(() =>
+          open(
+            target,
+            constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY,
+            file.mode,
+          ),
         );
         try {
-          await handle.writeFile(data);
+          await write(() => handle.writeFile(data));
         } finally {
-          await handle.close();
+          await write(() => handle.close());
         }
-        await chmod(target, file.mode);
+        await write(() => chmod(target, file.mode));
       }
     });
-    await rename(temporary, destination);
+    await write(() => rename(temporary, destination));
   } catch (error) {
-    await rm(temporary, { recursive: true, force: true });
+    await write(() => rm(temporary, { recursive: true, force: true }));
     throw error;
   }
   return manifest;