feat: harden self-managed reconciliation

server/build-kubernetes.ts

@@ -2,6 +2,7 @@ import {
   BatchV1Api,
   CoreV1Api,
   KubernetesObjectApi,
+  type V1DeleteOptions,
   type V1Job,
 } from "@kubernetes/client-node";
 import { createHash } from "node:crypto";
@@ -15,6 +16,8 @@ import type {
 import type { KubernetesJob } from "./build-job";
 import {
   BuildStoreConflictError,
+  buildTimestamp,
+  compareBuildRecords,
   type BuildRecord,
   type BuildStore,
   type CreateBuildResult,
@@ -22,6 +25,7 @@ import {
 } from "./build-store";
 
 const TYPE_LABEL = "kuber.astrxl.dev/type";
+const IMAGE_LABEL = "kuber.astrxl.dev/image";
 
 type ConfigMap = {
   apiVersion: "v1";
@@ -30,6 +34,7 @@ type ConfigMap = {
     name: string;
     namespace: string;
     resourceVersion?: string;
+    creationTimestamp?: string;
     labels?: Record<string, string>;
   };
   data?: Record<string, string>;
@@ -39,7 +44,7 @@ export interface BuildObjectApi {
   create(value: ConfigMap): Promise<unknown>;
   read(value: ConfigMap): Promise<unknown>;
   replace(value: ConfigMap): Promise<unknown>;
-  delete(value: ConfigMap): Promise<unknown>;
+  delete(value: ConfigMap, options?: V1DeleteOptions): Promise<unknown>;
   list(
     apiVersion: string,
     kind: string,
@@ -69,10 +74,12 @@ function payload<T>(value: unknown): T | undefined {
   if (!raw) return;
   try {
     const parsed = JSON.parse(raw) as T & {
-      metadata?: { resourceVersion?: string };
+      metadata?: { resourceVersion?: string; creationTimestamp?: string };
     };
     if (parsed.metadata && object.metadata.resourceVersion)
       parsed.metadata.resourceVersion = object.metadata.resourceVersion;
+    if (parsed.metadata && object.metadata.creationTimestamp)
+      parsed.metadata.creationTimestamp = object.metadata.creationTimestamp;
     return parsed;
   } catch {
     return;
@@ -85,6 +92,7 @@ function map(
   type: "build" | "build-upload" | "build-lock",
   value?: unknown,
   resourceVersion?: string,
+  labels?: Record<string, string>,
 ): ConfigMap {
   return {
     apiVersion: "v1",
@@ -93,20 +101,59 @@ function map(
       name,
       namespace,
       ...(resourceVersion && { resourceVersion }),
-      labels: { [TYPE_LABEL]: type },
+      labels: { [TYPE_LABEL]: type, ...labels },
     },
     ...(value !== undefined && { data: { payload: JSON.stringify(value) } }),
   };
 }
 
 function terminal(record: BuildRecord): boolean {
-  return record.status.state === "succeeded" || record.status.state === "failed";
+  return (
+    record.status.state === "succeeded" || record.status.state === "failed"
+  );
 }
 
 function sameSpec(left: BuildRecord, right: BuildRecord): boolean {
   return JSON.stringify(left.spec) === JSON.stringify(right.spec);
 }
 
+function newer(left: BuildRecord, right: BuildRecord): boolean {
+  return compareBuildRecords(left, right) > 0;
+}
+
+function supersede(record: BuildRecord, finishedAt: string): BuildRecord {
+  const reason = "Superseded by newer build";
+  const next = structuredClone(record);
+  next.metadata.resourceVersion = String(
+    Number(record.metadata.resourceVersion) + 1,
+  );
+  next.status = {
+    ...record.status,
+    state: "failed",
+    finishedAt,
+    error: reason,
+    cancelled: true,
+    events: [
+      ...record.status.events,
+      {
+        type: "status",
+        status: {
+          version: record.status.version,
+          id: record.status.id,
+          state: "failed",
+          createdAt: record.status.createdAt,
+          ...(record.status.startedAt && {
+            startedAt: record.status.startedAt,
+          }),
+          finishedAt,
+          error: reason,
+        },
+      },
+    ],
+  };
+  return next;
+}
+
 /** Build metadata lives in ConfigMaps; resumable upload bytes live only on the RWX volume. */
 export class KubernetesBuildStore implements BuildStore {
   constructor(
@@ -131,6 +178,68 @@ export class KubernetesBuildStore implements BuildStore {
     return hashName("build-lock", imageKey);
   }
 
+  private imageLabel(imageKey: string): string {
+    return createHash("sha256").update(imageKey).digest("hex").slice(0, 63);
+  }
+
+  private buildLabels(record: BuildRecord): Record<string, string> {
+    return {
+      ...record.metadata.labels,
+      [IMAGE_LABEL]: this.imageLabel(record.spec.imageKey),
+    };
+  }
+
+  private async listImageBuilds(imageKey: string): Promise<BuildRecord[]> {
+    const result = await this.objects.list(
+      "v1",
+      "ConfigMap",
+      this.namespace,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      `${TYPE_LABEL}=build,${IMAGE_LABEL}=${this.imageLabel(imageKey)}`,
+    );
+    const records = result.items
+      .map((item) => payload<BuildRecord>(item))
+      .filter(
+        (item): item is BuildRecord =>
+          item?.kind === "BuildRecord" && item.spec?.imageKey === imageKey,
+      );
+    // Legacy records lack the image index label. Keep this compatibility scan
+    // bounded to build records, then match all identifying payload fields.
+    const [project, service] = imageKey.split("\0");
+    if (!project || !service) return records;
+    const legacy = await this.objects.list(
+      "v1",
+      "ConfigMap",
+      this.namespace,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      `${TYPE_LABEL}=build`,
+    );
+    const legacyRecords = legacy.items
+      .map((item) => payload<BuildRecord>(item))
+      .filter(
+        (item): item is BuildRecord =>
+          item?.kind === "BuildRecord" &&
+          item.spec?.request?.project === project &&
+          item.spec?.request?.service === service &&
+          item.spec?.imageKey === imageKey,
+      );
+    return [
+      ...records,
+      ...legacyRecords.filter(
+        (legacyRecord) =>
+          !records.some(
+            (record) => record.metadata.name === legacyRecord.metadata.name,
+          ),
+      ),
+    ];
+  }
+
   private async read<T>(value: ConfigMap): Promise<T | undefined> {
     try {
       return payload<T>(await this.objects.read(value));
@@ -140,14 +249,73 @@ export class KubernetesBuildStore implements BuildStore {
     }
   }
 
-  private async delete(value: ConfigMap): Promise<void> {
+  private async readConfigMap(
+    value: ConfigMap,
+  ): Promise<ConfigMap | undefined> {
     try {
-      await this.objects.delete(value);
+      return (await this.objects.read(value)) as ConfigMap;
     } catch (error) {
-      if (statusCode(error) !== 404) throw error;
+      if (statusCode(error) === 404) return;
+      throw error;
     }
   }
 
+  private async delete(
+    value: ConfigMap,
+    expectedResourceVersion?: string,
+  ): Promise<void> {
+    try {
+      await this.objects.delete(
+        value,
+        expectedResourceVersion
+          ? { preconditions: { resourceVersion: expectedResourceVersion } }
+          : undefined,
+      );
+    } catch (error) {
+      if (statusCode(error) !== 404 && statusCode(error) !== 409) throw error;
+    }
+  }
+
+  private async supersedeBuild(
+    record: BuildRecord,
+    finishedAt: string,
+    releaseLock = false,
+  ): Promise<BuildRecord> {
+    const next = supersede(record, finishedAt);
+    await this.replaceBuildRecord(
+      next,
+      record.metadata.resourceVersion,
+      releaseLock,
+    );
+    return next;
+  }
+
+  /**
+   * Build records are indexed by image label, rather than scanning all records.
+   * This repairs record-first crashes and makes the record ordering authoritative
+   * even when one of the records never acquired a lock.
+   */
+  private async reconcileOlderBuilds(
+    candidate: BuildRecord,
+  ): Promise<BuildRecord[]> {
+    const records = await this.listImageBuilds(candidate.spec.imageKey);
+    const superseded: BuildRecord[] = [];
+    for (const other of records) {
+      if (other.metadata.name === candidate.metadata.name || terminal(other))
+        continue;
+      if (newer(other, candidate)) {
+        superseded.push(
+          await this.supersedeBuild(candidate, buildTimestamp(candidate)),
+        );
+        return superseded;
+      }
+      superseded.push(
+        await this.supersedeBuild(other, buildTimestamp(candidate)),
+      );
+    }
+    return superseded;
+  }
+
   async createBuild(record: BuildRecord): Promise<CreateBuildResult> {
     const existing = await this.getBuild(record.metadata.name);
     if (existing) {
@@ -155,48 +323,195 @@ export class KubernetesBuildStore implements BuildStore {
         throw new BuildStoreConflictError(
           "Build ID was already used for a different request",
         );
-      return { record: existing, created: false };
+      if (terminal(existing)) return { record: existing, created: false };
+      record = existing;
     }
 
-    const lockName = this.lockName(record.spec.imageKey);
-    try {
-      await this.objects.create(
-        map(this.namespace, lockName, "build-lock", {
-          buildId: record.metadata.name,
-        }),
-      );
-    } catch (error) {
-      if (statusCode(error) !== 409) throw error;
-      const lock = await this.read<{ buildId: string }>(
-        map(this.namespace, lockName, "build-lock"),
-      );
-      const active = lock && (await this.getBuild(lock.buildId));
-      if (!active || terminal(active)) {
-        await this.delete(map(this.namespace, lockName, "build-lock"));
-        return this.createBuild(record);
-      }
-      throw new BuildStoreConflictError(
-        `Build '${active.metadata.name}' is already active for ${record.spec.imageKey}`,
-      );
-    }
-
-    try {
-      const created = (await this.objects.create(
-        map(this.namespace, this.buildName(record.metadata.name), "build", record),
-      )) as ConfigMap;
-      return { record: payload<BuildRecord>(created) ?? record, created: true };
-    } catch (error) {
-      await this.delete(map(this.namespace, lockName, "build-lock"));
-      if (statusCode(error) === 409) {
+    if (!existing) {
+      // Persist before locking so contenders can find and supersede queued records.
+      try {
+        await this.objects.create(
+          map(
+            this.namespace,
+            this.buildName(record.metadata.name),
+            "build",
+            record,
+            undefined,
+            this.buildLabels(record),
+          ),
+        );
+      } catch (error) {
+        if (statusCode(error) !== 409) throw error;
         const concurrent = await this.getBuild(record.metadata.name);
         if (concurrent && sameSpec(concurrent, record))
-          return { record: concurrent, created: false };
+          return this.createBuild(concurrent);
         throw new BuildStoreConflictError(
           "Build ID was already used for a different request",
         );
       }
+      const stored = await this.getBuild(record.metadata.name);
+      if (!stored)
+        throw new BuildStoreConflictError("Build record disappeared");
+      record = stored;
+    }
+
+    const reconciled = await this.reconcileOlderBuilds(record);
+    if (
+      reconciled.some((value) => value.metadata.name === record.metadata.name)
+    )
+      return {
+        record: (await this.getBuild(record.metadata.name))!,
+        created: false,
+        superseded: reconciled,
+      };
+
+    const lockName = this.lockName(record.spec.imageKey);
+    for (let attempt = 0; attempt < 8; attempt++) {
+      try {
+        await this.objects.create(
+          map(this.namespace, lockName, "build-lock", {
+            buildId: record.metadata.name,
+          }),
+        );
+        return {
+          record: (await this.getBuild(record.metadata.name))!,
+          created: true,
+          ...(reconciled.length && { superseded: reconciled }),
+        };
+      } catch (error) {
+        if (statusCode(error) !== 409) {
+          await this.failCreatedBuild(record, error);
+          throw error;
+        }
+        const lockObject = await this.readConfigMap(
+          map(this.namespace, lockName, "build-lock"),
+        );
+        if (!lockObject) continue;
+        const lock = payload<{ buildId: string }>(lockObject);
+        if (!lock?.buildId)
+          throw new BuildStoreConflictError("Invalid build lock");
+        if (lock.buildId === record.metadata.name)
+          return {
+            record: (await this.getBuild(record.metadata.name))!,
+            created: false,
+            ...(reconciled.length && { superseded: reconciled }),
+          };
+        const active = await this.getBuild(lock.buildId);
+        if (!active) {
+          await this.delete(lockObject, lockObject.metadata.resourceVersion);
+          continue;
+        }
+        if (terminal(active)) {
+          await this.delete(lockObject, lockObject.metadata.resourceVersion);
+          continue;
+        }
+        if (newer(active, record)) {
+          const superseded = await this.supersedeBuild(
+            record,
+            buildTimestamp(record),
+          );
+          return { record: superseded, created: false };
+        }
+        try {
+          const nextOld = await this.supersedeBuild(
+            active,
+            buildTimestamp(record),
+          );
+          await this.objects.replace(
+            map(
+              this.namespace,
+              lockName,
+              "build-lock",
+              { buildId: record.metadata.name },
+              lockObject.metadata.resourceVersion,
+            ),
+          );
+          return {
+            record: (await this.getBuild(record.metadata.name))!,
+            created: true,
+            superseded: [...reconciled, nextOld],
+          };
+        } catch (takeoverError) {
+          if (statusCode(takeoverError) === 409) continue;
+          await this.failCreatedBuild(record, takeoverError);
+          throw takeoverError;
+        }
+      }
+    }
+    await this.failCreatedBuild(
+      record,
+      new BuildStoreConflictError("Build lock acquisition timed out"),
+    );
+    throw new BuildStoreConflictError("Build lock acquisition timed out");
+  }
+
+  private async failCreatedBuild(
+    record: BuildRecord,
+    error: unknown,
+  ): Promise<void> {
+    const current = await this.getBuild(record.metadata.name);
+    if (!current || terminal(current)) return;
+    const next = structuredClone(current);
+    next.metadata.resourceVersion = String(
+      Number(current.metadata.resourceVersion) + 1,
+    );
+    next.status = {
+      ...current.status,
+      state: "failed",
+      finishedAt: new Date().toISOString(),
+      error: error instanceof Error ? error.message : String(error),
+      events: [
+        ...current.status.events,
+        {
+          type: "status",
+          status: {
+            version: current.status.version,
+            id: current.status.id,
+            state: "failed",
+            createdAt: current.status.createdAt,
+            finishedAt: next.status.finishedAt,
+            error: next.status.error,
+          },
+        },
+      ],
+    };
+    await this.replaceBuildRecord(next, current.metadata.resourceVersion);
+  }
+
+  private async replaceBuildRecord(
+    record: BuildRecord,
+    expectedResourceVersion: string,
+    releaseLock = true,
+  ): Promise<void> {
+    const current = await this.getBuild(record.metadata.name);
+    if (
+      !current ||
+      current.metadata.resourceVersion !== expectedResourceVersion ||
+      !sameSpec(current, record)
+    )
+      throw new BuildStoreConflictError(
+        "Build record was concurrently modified",
+      );
+    try {
+      await this.objects.replace(
+        map(
+          this.namespace,
+          this.buildName(record.metadata.name),
+          "build",
+          record,
+          expectedResourceVersion,
+          this.buildLabels(record),
+        ),
+      );
+    } catch (error) {
+      if (statusCode(error) === 409)
+        throw new BuildStoreConflictError(
+          "Build record was concurrently modified",
+        );
       throw error;
     }
+    if (terminal(record) && releaseLock)
+      await this.releaseLock(record.spec.imageKey, record.metadata.name);
   }
 
   async getBuild(id: string): Promise<BuildRecord | undefined> {
@@ -220,9 +535,7 @@ export class KubernetesBuildStore implements BuildStore {
     return result.items
       .map((item) => payload<BuildRecord>(item))
       .filter((item): item is BuildRecord => item?.kind === "BuildRecord")
-      .sort((a, b) =>
-        a.metadata.creationTimestamp.localeCompare(b.metadata.creationTimestamp),
-      );
+      .sort(compareBuildRecords);
   }
 
   async replaceBuild(
@@ -230,8 +543,13 @@ export class KubernetesBuildStore implements BuildStore {
     expectedResourceVersion: string,
   ): Promise<void> {
     const current = await this.getBuild(record.metadata.name);
-    if (!current || current.metadata.resourceVersion !== expectedResourceVersion)
-      throw new BuildStoreConflictError("Build record was concurrently modified");
+    if (
+      !current ||
+      current.metadata.resourceVersion !== expectedResourceVersion
+    )
+      throw new BuildStoreConflictError(
+        "Build record was concurrently modified",
+      );
     if (!sameSpec(current, record))
       throw new BuildStoreConflictError("Build specification is immutable");
     if (
@@ -251,17 +569,46 @@ export class KubernetesBuildStore implements BuildStore {
           "build",
           record,
           expectedResourceVersion,
+          this.buildLabels(record),
         ),
       );
     } catch (error) {
       if (statusCode(error) === 409)
-        throw new BuildStoreConflictError("Build record was concurrently modified");
+        throw new BuildStoreConflictError(
+          "Build record was concurrently modified",
+        );
       throw error;
     }
-    if (terminal(record))
-      await this.delete(
-        map(this.namespace, this.lockName(record.spec.imageKey), "build-lock"),
-      );
+    if (terminal(record)) {
+      await this.releaseLock(record.spec.imageKey, record.metadata.name);
+    }
+  }
+
+  async ownsBuild(imageKey: string, buildId: string): Promise<boolean> {
+    const lock = await this.read<{ buildId: string }>(
+      map(this.namespace, this.lockName(imageKey), "build-lock"),
+    );
+    return lock?.buildId === buildId;
+  }
+
+  private async releaseLock(imageKey: string, buildId: string): Promise<void> {
+    const name = this.lockName(imageKey);
+    const object = await this.readConfigMap(
+      map(this.namespace, name, "build-lock"),
+    );
+    if (!object || payload<{ buildId: string }>(object)?.buildId !== buildId)
+      return;
+    if (!object.metadata.resourceVersion) return;
+    await this.delete(
+      map(
+        this.namespace,
+        name,
+        "build-lock",
+        undefined,
+        object.metadata.resourceVersion,
+      ),
+      object.metadata.resourceVersion,
+    );
   }
 
   async getUpload(digest: Sha256Digest): Promise<UploadRecord | undefined> {
@@ -270,7 +617,9 @@ export class KubernetesBuildStore implements BuildStore {
     );
     if (!record || record.spec.digest !== digest) return;
     try {
-      record.status.data = new Uint8Array(await readFile(this.uploadPath(digest)));
+      record.status.data = new Uint8Array(
+        await readFile(this.uploadPath(digest)),
+      );
     } catch (error) {
       if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
       record.status.data = new Uint8Array();
@@ -305,7 +654,8 @@ export class KubernetesBuildStore implements BuildStore {
       result.status.data = new Uint8Array();
       return result;
     } catch (error) {
-      if (statusCode(error) === 409) return (await this.getUpload(record.spec.digest))!;
+      if (statusCode(error) === 409)
+        return (await this.getUpload(record.spec.digest))!;
       await rm(path, { force: true });
       throw error;
     }
@@ -332,7 +682,9 @@ export class KubernetesBuildStore implements BuildStore {
       });
     } catch (error) {
       if (statusCode(error) === 409)
-        throw new BuildStoreConflictError("Upload record was concurrently modified");
+        throw new BuildStoreConflictError(
+          "Upload record was concurrently modified",
+        );
       throw error;
     }
   }
@@ -375,7 +727,8 @@ export class KubernetesBuildOperations implements BuildKubernetesOperations {
       (condition) => condition.type === "Failed" && condition.status === "True",
     );
     const complete = job.status?.conditions?.find(
-      (condition) => condition.type === "Complete" && condition.status === "True",
+      (condition) =>
+        condition.type === "Complete" && condition.status === "True",
     );
     const phase = failed
       ? "failed"
@@ -398,9 +751,10 @@ export class KubernetesBuildOperations implements BuildKubernetesOperations {
       labelSelector: `job-name=${name}`,
     });
     const pod = pods.items
-      .sort((a, b) =>
-        (a.metadata?.creationTimestamp?.getTime() ?? 0) -
-        (b.metadata?.creationTimestamp?.getTime() ?? 0),
+      .sort(
+        (a, b) =>
+          (a.metadata?.creationTimestamp?.getTime() ?? 0) -
+          (b.metadata?.creationTimestamp?.getTime() ?? 0),
       )
       .at(-1);
     if (!pod?.metadata?.name) return "";