import { createHash } from "node:crypto"; import { describe, expect, test } from "bun:test"; import type { V1DeleteOptions } from "@kubernetes/client-node"; import { BUILD_RECORD_API_VERSION, type BuildRecord, } from "../../server/build-store"; import { KubernetesBuildStore, type BuildObjectApi, } from "../../server/build-kubernetes"; import { BUILD_PROTOCOL_VERSION, type BuildRequest, type Sha256Digest, } from "../../shared/build-protocol"; const namespace = "kuber-test"; const imageKey = "project\0service\0registry.test/app:latest"; const workspace = `sha256:${"a".repeat(64)}` as Sha256Digest; function validLabelValue(value: string): boolean { return ( value.length <= 63 && /^[A-Za-z0-9](?:[-_.A-Za-z0-9]*[A-Za-z0-9])?$/.test(value) ); } function name(prefix: string, value: string): string { return `${prefix}-${createHash("sha256").update(value).digest("hex").slice(0, 48)}`; } function build(id: string, createdAt: string): BuildRecord { const request: BuildRequest = { version: BUILD_PROTOCOL_VERSION, id, project: "project", service: "service", spec: { architecture: "amd64", image: "registry.test/app:latest", context: ".", buildArgs: [], workspace, }, }; const status = { version: BUILD_PROTOCOL_VERSION, id, state: "queued" as const, createdAt, }; return { apiVersion: BUILD_RECORD_API_VERSION, kind: "BuildRecord", metadata: { name: id, resourceVersion: "1", creationTimestamp: createdAt, labels: { project: "project", service: "service" }, }, spec: { request, imageKey, jobName: `job-${id}`, workspaceSubPath: id }, status: { ...status, logBytes: 0, logOffset: 0, nextSequence: 1, events: [{ type: "status", status }], }, }; } type ConfigMap = Parameters[0]; type DeleteOptions = Pick; class FakeObjects implements BuildObjectApi { readonly maps = new Map(); readonly selectors: string[] = []; readonly deletes: Array<{ name: string; options?: DeleteOptions }> = []; onLockRead?: () => void; onReplace?: (value: ConfigMap) => Promise | void; private lockRead = false; private replacing = false; async create(value: ConfigMap): Promise { if ( Object.values(value.metadata.labels ?? {}).some( (label) => !validLabelValue(label), ) ) throw { code: 422 }; if (this.maps.has(value.metadata.name)) throw { code: 409 }; this.maps.set( value.metadata.name, structuredClone({ ...value, metadata: { ...value.metadata, resourceVersion: "1" }, }), ); return value; } async read(value: ConfigMap): Promise { const found = this.maps.get(value.metadata.name); if (!found) throw { code: 404 }; if (value.metadata.name.startsWith("build-lock-") && !this.lockRead) { this.lockRead = true; this.onLockRead?.(); } return structuredClone(found); } async replace(value: ConfigMap): Promise { if ( Object.values(value.metadata.labels ?? {}).some( (label) => !validLabelValue(label), ) ) throw { code: 422 }; if (!this.replacing && this.onReplace) { this.replacing = true; try { await this.onReplace(value); } finally { this.replacing = false; } } const current = this.maps.get(value.metadata.name); if ( !current || current.metadata.resourceVersion !== value.metadata.resourceVersion ) throw { code: 409 }; const next = structuredClone({ ...value, metadata: { ...value.metadata, resourceVersion: String(Number(current.metadata.resourceVersion) + 1), }, }); this.maps.set(value.metadata.name, next); return next; } async delete(value: ConfigMap, options?: DeleteOptions): Promise { const current = this.maps.get(value.metadata.name); this.deletes.push({ name: value.metadata.name, options }); if (!current) throw { code: 404 }; if ( options?.preconditions?.resourceVersion !== current.metadata.resourceVersion ) throw { code: 409 }; this.maps.delete(value.metadata.name); return undefined; } async list( _apiVersion: string, _kind: string, _namespace?: string, _pretty?: string, _exact?: boolean, _exportValue?: boolean, _fieldSelector?: string, labelSelector?: string, ): Promise<{ items: unknown[] }> { this.selectors.push(labelSelector ?? ""); const labels = Object.fromEntries( (labelSelector ?? "") .split(",") .filter(Boolean) .map((part) => part.split("=")), ); return { items: [...this.maps.values()].filter((item) => Object.entries(labels).every( ([key, value]) => item.metadata.labels?.[key] === value, ), ), }; } } function store(fake: FakeObjects): KubernetesBuildStore { return new KubernetesBuildStore( fake, namespace, "/tmp/kuber-build-store-test", ); } function configMap(record: BuildRecord, imageLabel = true): ConfigMap { return { apiVersion: "v1", kind: "ConfigMap", metadata: { name: name("build", record.metadata.name), namespace, resourceVersion: record.metadata.resourceVersion, labels: { "kuber.astrxl.dev/type": "build", ...(imageLabel && { "kuber.astrxl.dev/image": createHash("sha256") .update(record.spec.imageKey) .digest("hex") .slice(0, 63), }), }, }, data: { payload: JSON.stringify(record) }, }; } describe("KubernetesBuildStore", () => { test("prunes only old terminal records with resourceVersion preconditions", async () => { const fake = new FakeObjects(); for (let i = 0; i < 52; i++) { const record = build( `history-${i}`, new Date(Date.UTC(2026, 0, 1, 0, 0, i)).toISOString(), ); record.status.state = i === 0 ? "queued" : "failed"; if (i === 1) record.status.reconcileLease = { holder: "worker", token: "active-lease", expiresAt: new Date(Date.now() + 60_000).toISOString(), }; fake.maps.set(name("build", record.metadata.name), configMap(record)); } const old = build( "protected-lock", new Date(Date.UTC(2025, 0, 1)).toISOString(), ); old.status.state = "failed"; fake.maps.set(name("build", old.metadata.name), configMap(old)); const lockName = name("build-lock", old.spec.imageKey); fake.maps.set(lockName, { apiVersion: "v1", kind: "ConfigMap", metadata: { name: lockName, namespace, resourceVersion: "1", labels: { "kuber.astrxl.dev/type": "build-lock" }, }, data: { payload: JSON.stringify({ buildId: old.metadata.name }) }, }); await store(fake).pruneBuilds(50); expect(fake.maps.has(name("build", "history-0"))).toBe(true); expect(fake.maps.has(name("build", "history-1"))).toBe(true); expect(fake.maps.has(name("build", "protected-lock"))).toBe(true); expect(fake.maps.has(name("build", "history-2"))).toBe(false); expect( [...fake.maps.values()].filter((value) => value.metadata.labels?.["kuber.astrxl.dev/type"] === "build"), ).toHaveLength(50); expect(fake.deletes.length).toBeGreaterThan(0); expect( fake.deletes.every( ({ options }) => !!options?.preconditions?.resourceVersion, ), ).toBe(true); await store(fake).pruneBuilds(50); expect(fake.maps.has(name("build", "history-1"))).toBe(true); }); test("creates records with valid, deterministic image index labels", async () => { const fake = new FakeObjects(); const record = build("validated", "2026-09-02T00:00:00.000Z"); const result = await store(fake).createBuild(record); const labels = fake.maps.get(name("build", record.metadata.name))?.metadata .labels; expect(result.created).toBe(true); expect(labels).toBeDefined(); expect(Object.values(labels ?? {}).every(validLabelValue)).toBe(true); expect(labels?.["kuber.astrxl.dev/image"]).toBe( createHash("sha256").update(imageKey).digest("hex").slice(0, 63), ); }); test("uses the image label fast path", async () => { const fake = new FakeObjects(); const result = await store(fake).createBuild( build("old", "2026-09-02T00:00:00.000Z"), ); expect(result.created).toBe(true); expect( fake.selectors.some((selector) => selector.includes("kuber.astrxl.dev/image="), ), ).toBe(true); }); test("finds legacy records through the bounded project/service fallback", async () => { const fake = new FakeObjects(); const legacy = build( "5ef44ce9-f087-4b65-8ff1-2a60dab9112e", "2026-09-02T00:00:00.000Z", ); fake.maps.set( name("build", legacy.metadata.name), configMap(legacy, false), ); expect( fake.maps.get(name("build", legacy.metadata.name))?.metadata.labels, ).toEqual({ "kuber.astrxl.dev/type": "build", }); const result = await store(fake).createBuild( build("replacement", "2026-09-02T00:00:01.000Z"), ); expect( (await store(fake).getBuild("5ef44ce9-f087-4b65-8ff1-2a60dab9112e")) ?.status.state, ).toBe("failed"); expect(fake.selectors).toContain("kuber.astrxl.dev/type=build"); }); test("returns every older record superseded during reconciliation", async () => { const fake = new FakeObjects(); const kuber = store(fake); const oldOne = build("old-one", "2026-09-02T00:00:00.000Z"); const oldTwo = build("old-two", "2026-09-02T00:00:01.000Z"); fake.maps.set(name("build", oldOne.metadata.name), configMap(oldOne)); fake.maps.set(name("build", oldTwo.metadata.name), configMap(oldTwo)); const result = await kuber.createBuild( build("replacement", "2026-09-02T00:00:02.000Z"), ); expect(result.created).toBe(true); expect(result.superseded?.map((record) => record.metadata.name)).toEqual([ "old-one", "old-two", ]); }); test("returns created false when an older candidate is already superseded", async () => { const fake = new FakeObjects(); const kuber = store(fake); await kuber.createBuild(build("newer", "2026-09-02T00:00:01.000Z")); const result = await kuber.createBuild( build("older", "2026-09-02T00:00:00.000Z"), ); expect(result.created).toBe(false); expect(result.record.status.error).toBe("Superseded by newer build"); }); test("orders and reconciles legacy records with invalid metadata timestamps", async () => { const fake = new FakeObjects(); const kuber = store(fake); const legacy = build("legacy", "2026-09-02T00:00:00.000Z"); legacy.metadata.creationTimestamp = undefined as unknown as string; fake.maps.set(name("build", legacy.metadata.name), configMap(legacy)); const replacement = await kuber.createBuild( build("replacement", "2026-09-02T00:00:01.000Z"), ); expect(replacement.created).toBe(true); expect((await kuber.getBuild("legacy"))?.status.state).toBe("failed"); const malformed = build("a", "2026-09-02T00:00:00.000Z"); malformed.metadata.creationTimestamp = 0 as unknown as string; fake.maps.set(name("build", malformed.metadata.name), configMap(malformed)); expect( (await kuber.listBuilds()).map((record) => record.metadata.name), ).toEqual(["a", "legacy", "replacement"]); }); test("recovers an orphan record and takes over its missing lock", async () => { const fake = new FakeObjects(); const kuber = store(fake); const orphan = build("orphan", "2026-09-02T00:00:00.000Z"); fake.maps.set( name("build", orphan.metadata.name), configMap(orphan, false), ); fake.maps.delete(name("build-lock", imageKey)); const result = await kuber.createBuild( build("replacement", "2026-09-02T00:00:01.000Z"), ); expect((await kuber.getBuild("orphan"))?.status.state).toBe("failed"); expect(await kuber.ownsBuild(imageKey, "replacement")).toBe(true); }); test("does not delete a successor lock after a read/delete race", async () => { const fake = new FakeObjects(); const kuber = store(fake); await kuber.createBuild(build("old", "2026-09-02T00:00:00.000Z")); fake.onLockRead = () => { const lock = fake.maps.get(name("build-lock", imageKey))!; fake.maps.set(lock.metadata.name, { ...lock, data: { payload: JSON.stringify({ buildId: "successor" }) }, metadata: { ...lock.metadata, resourceVersion: "2" }, }); }; const old = (await kuber.getBuild("old"))!; old.status.state = "succeeded"; await kuber.replaceBuild(old, "1"); expect(await kuber.ownsBuild(imageKey, "successor")).toBe(true); expect(fake.maps.has(name("build-lock", imageKey))).toBe(true); expect(fake.deletes.at(-1)?.options?.preconditions?.resourceVersion).toBe( "1", ); }); test("fences a stale replace when an expired lease is taken over", async () => { const fake = new FakeObjects(); const kuber = store(fake); const record = build("fenced", "2026-09-02T00:00:00.000Z"); record.status.reconcileLease = { holder: "stale-replica", token: "stale-token", expiresAt: new Date(0).toISOString(), }; fake.maps.set(name("build", record.metadata.name), configMap(record)); const stale = (await kuber.getBuild(record.metadata.name))!; stale.metadata.resourceVersion = "2"; stale.status.logBytes = 99; stale.status.reconcileLease!.expiresAt = "2100-01-01T00:00:00.000Z"; let successorToken = ""; fake.onReplace = async () => { const successor = await kuber.acquireBuildReconciliationLease( record.metadata.name, "current-replica", 60_000, ); successorToken = successor!.token; }; await expect(kuber.replaceBuild(stale, "1", "stale-token")).rejects.toThrow( "concurrently modified", ); fake.onReplace = undefined; const takenOver = (await kuber.getBuild(record.metadata.name))!; expect(takenOver.metadata.resourceVersion).toBe("2"); expect(takenOver.status.logBytes).toBe(0); expect(takenOver.status.reconcileLease).toMatchObject({ holder: "current-replica", token: successorToken, }); expect(takenOver.status.reconcileLease?.expiresAt).not.toBe( "2100-01-01T00:00:00.000Z", ); const current = structuredClone(takenOver); current.metadata.resourceVersion = "3"; current.status.logBytes = 7; await kuber.replaceBuild( current, takenOver.metadata.resourceVersion, successorToken, ); expect((await kuber.getBuild(record.metadata.name))?.status).toMatchObject({ logBytes: 7, reconcileLease: { token: successorToken }, }); }); });