feat: improve resource reconciliation and observability

server/build-store.ts

@@ -1,3 +1,4 @@
+import { randomUUID } from "node:crypto";
 import type {
   BuildEvent,
   BuildRequest,
@@ -29,9 +30,17 @@ export interface BuildRecord {
     logOffset: number;
     nextSequence: number;
     events: BuildEvent[];
+    reconcileLease?: BuildReconciliationLease;
   };
 }
 
+/** A record-local fencing token for distributed build reconciliation. */
+export interface BuildReconciliationLease {
+  holder: string;
+  token: string;
+  expiresAt: string;
+}
+
 export interface UploadRecord {
   apiVersion: typeof BUILD_RECORD_API_VERSION;
   kind: "BuildUpload";
@@ -58,7 +67,19 @@ export interface BuildStore {
   replaceBuild(
     record: BuildRecord,
     expectedResourceVersion: string,
+    reconcileLeaseToken?: string,
   ): Promise<void>;
+  acquireBuildReconciliationLease(
+    id: string,
+    holder: string,
+    ttlMs: number,
+  ): Promise<BuildReconciliationLease | undefined>;
+  renewBuildReconciliationLease(
+    id: string,
+    token: string,
+    ttlMs: number,
+  ): Promise<boolean>;
+  releaseBuildReconciliationLease(id: string, token: string): Promise<void>;
   ownsBuild?(imageKey: string, buildId: string): Promise<boolean>;
   getUpload(digest: Sha256Digest): Promise<UploadRecord | undefined>;
   createUpload(record: UploadRecord): Promise<UploadRecord>;
@@ -237,7 +258,11 @@ export class MemoryBuildStore implements BuildStore {
     return [...this.builds.values()].sort(compareBuildRecords).map(clone);
   }
 
-  async replaceBuild(record: BuildRecord, expectedResourceVersion: string) {
+  async replaceBuild(
+    record: BuildRecord,
+    expectedResourceVersion: string,
+    reconcileLeaseToken?: string,
+  ) {
     const current = this.builds.get(record.metadata.name);
     if (
       !current ||
@@ -250,6 +275,14 @@ export class MemoryBuildStore implements BuildStore {
     if (!sameSpec(current, record)) {
       throw new BuildStoreConflictError("Build specification is immutable");
     }
+    if (
+      reconcileLeaseToken &&
+      current.status.reconcileLease?.token !== reconcileLeaseToken
+    ) {
+      throw new BuildStoreConflictError(
+        "Build reconciliation lease ownership was lost",
+      );
+    }
     if (
       current.status.state === "succeeded" &&
       (record.status.state !== "succeeded" ||
@@ -268,6 +301,79 @@ export class MemoryBuildStore implements BuildStore {
     }
   }
 
+  async acquireBuildReconciliationLease(
+    id: string,
+    holder: string,
+    ttlMs: number,
+  ): Promise<BuildReconciliationLease | undefined> {
+    if (!holder || !Number.isFinite(ttlMs) || ttlMs <= 0)
+      throw new BuildStoreConflictError("Invalid build reconciliation lease");
+    const current = this.builds.get(id);
+    if (!current) return;
+    if (
+      current.status.reconcileLease &&
+      Date.parse(current.status.reconcileLease.expiresAt) > Date.now()
+    ) {
+      return;
+    }
+    const lease: BuildReconciliationLease = {
+      holder,
+      token: randomUUID(),
+      expiresAt: new Date(Date.now() + ttlMs).toISOString(),
+    };
+    const next = clone(current);
+    next.metadata.resourceVersion = String(
+      Number(current.metadata.resourceVersion) + 1,
+    );
+    next.status.reconcileLease = lease;
+    await this.replaceBuild(next, current.metadata.resourceVersion);
+    return clone(lease);
+  }
+
+  async renewBuildReconciliationLease(
+    id: string,
+    token: string,
+    ttlMs: number,
+  ): Promise<boolean> {
+    if (!token || !Number.isFinite(ttlMs) || ttlMs <= 0) return false;
+    const current = this.builds.get(id);
+    if (
+      !current ||
+      current.status.reconcileLease?.token !== token ||
+      Date.parse(current.status.reconcileLease.expiresAt) <= Date.now()
+    ) {
+      return false;
+    }
+    const next = clone(current);
+    next.metadata.resourceVersion = String(
+      Number(current.metadata.resourceVersion) + 1,
+    );
+    next.status.reconcileLease!.expiresAt = new Date(
+      Date.now() + ttlMs,
+    ).toISOString();
+    try {
+      await this.replaceBuild(next, current.metadata.resourceVersion, token);
+      return true;
+    } catch (error) {
+      if (error instanceof BuildStoreConflictError) return false;
+      throw error;
+    }
+  }
+
+  async releaseBuildReconciliationLease(
+    id: string,
+    token: string,
+  ): Promise<void> {
+    const current = this.builds.get(id);
+    if (current?.status.reconcileLease?.token !== token) return;
+    const next = clone(current);
+    next.metadata.resourceVersion = String(
+      Number(current.metadata.resourceVersion) + 1,
+    );
+    delete next.status.reconcileLease;
+    await this.replaceBuild(next, current.metadata.resourceVersion, token);
+  }
+
   async ownsBuild(imageKey: string, buildId: string): Promise<boolean> {
     return this.active.get(imageKey) === buildId;
   }