feat: harden self-managed reconciliation

server/build-store.ts

@@ -44,7 +44,11 @@ export interface UploadRecord {
   status: { offset: number; data: Uint8Array };
 }
 
-export type CreateBuildResult = { record: BuildRecord; created: boolean };
+export type CreateBuildResult = {
+  record: BuildRecord;
+  created: boolean;
+  superseded?: BuildRecord[];
+};
 
 /** Implementations must make createBuild and replace operations atomic. */
 export interface BuildStore {
@@ -55,6 +59,7 @@ export interface BuildStore {
     record: BuildRecord,
     expectedResourceVersion: string,
   ): Promise<void>;
+  ownsBuild?(imageKey: string, buildId: string): Promise<boolean>;
   getUpload(digest: Sha256Digest): Promise<UploadRecord | undefined>;
   createUpload(record: UploadRecord): Promise<UploadRecord>;
   replaceUpload(
@@ -82,10 +87,63 @@ function sameSpec(left: BuildRecord, right: BuildRecord): boolean {
   return JSON.stringify(left.spec) === JSON.stringify(right.spec);
 }
 
+export function buildTimestamp(record: BuildRecord): string {
+  const creationTimestamp = record.metadata?.creationTimestamp;
+  if (typeof creationTimestamp === "string") return creationTimestamp;
+  const createdAt = record.status?.createdAt;
+  return typeof createdAt === "string" ? createdAt : "";
+}
+
+export function compareBuildRecords(
+  left: BuildRecord,
+  right: BuildRecord,
+): number {
+  const timestamp = buildTimestamp(left).localeCompare(buildTimestamp(right));
+  if (timestamp !== 0) return timestamp;
+  return left.metadata.name.localeCompare(right.metadata.name);
+}
+
+function newer(left: BuildRecord, right: BuildRecord): boolean {
+  return compareBuildRecords(left, right) > 0;
+}
+
+function supersede(record: BuildRecord, finishedAt: string): BuildRecord {
+  const reason = "Superseded by newer build";
+  record.status = {
+    ...record.status,
+    state: "failed",
+    finishedAt,
+    error: reason,
+    cancelled: true,
+    events: [
+      ...record.status.events,
+      {
+        type: "status",
+        status: {
+          version: record.status.version,
+          id: record.status.id,
+          state: "failed",
+          createdAt: record.status.createdAt,
+          ...(record.status.startedAt && {
+            startedAt: record.status.startedAt,
+          }),
+          finishedAt,
+          error: reason,
+        },
+      },
+    ],
+  };
+  record.metadata.resourceVersion = String(
+    Number(record.metadata.resourceVersion) + 1,
+  );
+  return record;
+}
+
 export class MemoryBuildStore implements BuildStore {
   private readonly builds = new Map<string, BuildRecord>();
   private readonly uploads = new Map<Sha256Digest, UploadRecord>();
   private readonly active = new Map<string, string>();
+  private readonly byImage = new Map<string, Set<string>>();
 
   async createBuild(record: BuildRecord): Promise<CreateBuildResult> {
     const existing = this.builds.get(record.metadata.name);
@@ -95,20 +153,79 @@ export class MemoryBuildStore implements BuildStore {
           "Build ID was already used for a different request",
         );
       }
-      return { record: clone(existing), created: false };
-    }
-    const activeId = this.active.get(record.spec.imageKey);
-    if (activeId) {
-      const active = this.builds.get(activeId);
-      if (active && !terminal(active)) {
-        throw new BuildStoreConflictError(
-          `Build '${activeId}' is already active for ${record.spec.imageKey}`,
+      if (terminal(existing))
+        return { record: clone(existing), created: false };
+      const candidates = [...(this.byImage.get(existing.spec.imageKey) ?? [])]
+        .map((id) => this.builds.get(id))
+        .filter(
+          (value): value is BuildRecord =>
+            !!value &&
+            !terminal(value) &&
+            value.metadata.name !== existing.metadata.name,
         );
+      const newerCandidate = candidates.find((candidate) =>
+        newer(candidate, existing),
+      );
+      if (newerCandidate) {
+        const superseded = supersede(existing, buildTimestamp(existing));
+        this.builds.set(existing.metadata.name, clone(superseded));
+        if (this.active.get(existing.spec.imageKey) === existing.metadata.name)
+          this.active.delete(existing.spec.imageKey);
+        return {
+          record: clone(superseded),
+          created: false,
+          superseded: [clone(superseded)],
+        };
       }
+      const superseded: BuildRecord[] = [];
+      for (const old of candidates) {
+        const next = supersede(old, buildTimestamp(existing));
+        this.builds.set(old.metadata.name, clone(next));
+        superseded.push(clone(next));
+      }
+      this.active.set(existing.spec.imageKey, existing.metadata.name);
+      return {
+        record: clone(existing),
+        created: false,
+        ...(superseded.length && { superseded }),
+      };
+    }
+    const candidates = [...(this.byImage.get(record.spec.imageKey) ?? [])]
+      .map((id) => this.builds.get(id))
+      .filter((value): value is BuildRecord => !!value && !terminal(value));
+    const active = candidates.sort(compareBuildRecords).at(-1);
+    if (active && newer(active, record)) {
+      const superseded = supersede(record, record.status.createdAt);
+      this.builds.set(record.metadata.name, clone(superseded));
+      this.byImage.get(record.spec.imageKey)?.add(record.metadata.name);
+      return {
+        record: clone(superseded),
+        created: false,
+        superseded: [clone(superseded)],
+      };
+    }
+    if (active) {
+      const old = supersede(active, record.status.createdAt);
+      this.builds.set(active.metadata.name, clone(old));
+      if (this.active.get(record.spec.imageKey) === active.metadata.name)
+        this.active.delete(record.spec.imageKey);
     }
     this.builds.set(record.metadata.name, clone(record));
+    this.byImage.set(
+      record.spec.imageKey,
+      new Set([
+        ...(this.byImage.get(record.spec.imageKey) ?? []),
+        record.metadata.name,
+      ]),
+    );
     this.active.set(record.spec.imageKey, record.metadata.name);
-    return { record: clone(record), created: true };
+    return {
+      record: clone(record),
+      created: true,
+      superseded: active
+        ? [clone(this.builds.get(active.metadata.name)!)]
+        : undefined,
+    };
   }
 
   async getBuild(id: string) {
@@ -117,13 +234,7 @@ export class MemoryBuildStore implements BuildStore {
   }
 
   async listBuilds() {
-    return [...this.builds.values()]
-      .sort((a, b) =>
-        a.metadata.creationTimestamp.localeCompare(
-          b.metadata.creationTimestamp,
-        ),
-      )
-      .map(clone);
+    return [...this.builds.values()].sort(compareBuildRecords).map(clone);
   }
 
   async replaceBuild(record: BuildRecord, expectedResourceVersion: string) {
@@ -157,6 +268,10 @@ export class MemoryBuildStore implements BuildStore {
     }
   }
 
+  async ownsBuild(imageKey: string, buildId: string): Promise<boolean> {
+    return this.active.get(imageKey) === buildId;
+  }
+
   async getUpload(digest: Sha256Digest) {
     const value = this.uploads.get(digest);
     return value && clone(value);