feat: improve resource reconciliation and observability

tests/server/build-kubernetes.test.ts

@@ -78,7 +78,9 @@ class FakeObjects implements BuildObjectApi {
   readonly selectors: string[] = [];
   readonly deletes: Array<{ name: string; options?: DeleteOptions }> = [];
   onLockRead?: () => void;
+  onReplace?: (value: ConfigMap) => Promise<void> | void;
   private lockRead = false;
+  private replacing = false;
 
   async create(value: ConfigMap): Promise<unknown> {
     if (
@@ -115,6 +117,14 @@ class FakeObjects implements BuildObjectApi {
       )
     )
       throw { code: 422 };
+    if (!this.replacing && this.onReplace) {
+      this.replacing = true;
+      try {
+        await this.onReplace(value);
+      } finally {
+        this.replacing = false;
+      }
+    }
     const current = this.maps.get(value.metadata.name);
     if (
       !current ||
@@ -346,4 +356,59 @@ describe("KubernetesBuildStore", () => {
       "1",
     );
   });
+
+  test("fences a stale replace when an expired lease is taken over", async () => {
+    const fake = new FakeObjects();
+    const kuber = store(fake);
+    const record = build("fenced", "2026-09-02T00:00:00.000Z");
+    record.status.reconcileLease = {
+      holder: "stale-replica",
+      token: "stale-token",
+      expiresAt: new Date(0).toISOString(),
+    };
+    fake.maps.set(name("build", record.metadata.name), configMap(record));
+
+    const stale = (await kuber.getBuild(record.metadata.name))!;
+    stale.metadata.resourceVersion = "2";
+    stale.status.logBytes = 99;
+    stale.status.reconcileLease!.expiresAt = "2100-01-01T00:00:00.000Z";
+    let successorToken = "";
+    fake.onReplace = async () => {
+      const successor = await kuber.acquireBuildReconciliationLease(
+        record.metadata.name,
+        "current-replica",
+        60_000,
+      );
+      successorToken = successor!.token;
+    };
+
+    await expect(kuber.replaceBuild(stale, "1", "stale-token")).rejects.toThrow(
+      "concurrently modified",
+    );
+    fake.onReplace = undefined;
+
+    const takenOver = (await kuber.getBuild(record.metadata.name))!;
+    expect(takenOver.metadata.resourceVersion).toBe("2");
+    expect(takenOver.status.logBytes).toBe(0);
+    expect(takenOver.status.reconcileLease).toMatchObject({
+      holder: "current-replica",
+      token: successorToken,
+    });
+    expect(takenOver.status.reconcileLease?.expiresAt).not.toBe(
+      "2100-01-01T00:00:00.000Z",
+    );
+
+    const current = structuredClone(takenOver);
+    current.metadata.resourceVersion = "3";
+    current.status.logBytes = 7;
+    await kuber.replaceBuild(
+      current,
+      takenOver.metadata.resourceVersion,
+      successorToken,
+    );
+    expect((await kuber.getBuild(record.metadata.name))?.status).toMatchObject({
+      logBytes: 7,
+      reconcileLease: { token: successorToken },
+    });
+  });
 });