feat: harden self-managed reconciliation
This commit is contained in:
@@ -0,0 +1,349 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import type { V1DeleteOptions } from "@kubernetes/client-node";
|
||||
import {
|
||||
BUILD_RECORD_API_VERSION,
|
||||
type BuildRecord,
|
||||
} from "../../server/build-store";
|
||||
import {
|
||||
KubernetesBuildStore,
|
||||
type BuildObjectApi,
|
||||
} from "../../server/build-kubernetes";
|
||||
import {
|
||||
BUILD_PROTOCOL_VERSION,
|
||||
type BuildRequest,
|
||||
type Sha256Digest,
|
||||
} from "../../shared/build-protocol";
|
||||
|
||||
const namespace = "kuber-test";
|
||||
const imageKey = "project\0service\0registry.test/app:latest";
|
||||
const workspace = `sha256:${"a".repeat(64)}` as Sha256Digest;
|
||||
|
||||
function validLabelValue(value: string): boolean {
|
||||
return (
|
||||
value.length <= 63 &&
|
||||
/^[A-Za-z0-9](?:[-_.A-Za-z0-9]*[A-Za-z0-9])?$/.test(value)
|
||||
);
|
||||
}
|
||||
|
||||
function name(prefix: string, value: string): string {
|
||||
return `${prefix}-${createHash("sha256").update(value).digest("hex").slice(0, 48)}`;
|
||||
}
|
||||
|
||||
function build(id: string, createdAt: string): BuildRecord {
|
||||
const request: BuildRequest = {
|
||||
version: BUILD_PROTOCOL_VERSION,
|
||||
id,
|
||||
project: "project",
|
||||
service: "service",
|
||||
spec: {
|
||||
architecture: "amd64",
|
||||
image: "registry.test/app:latest",
|
||||
context: ".",
|
||||
buildArgs: [],
|
||||
workspace,
|
||||
},
|
||||
};
|
||||
const status = {
|
||||
version: BUILD_PROTOCOL_VERSION,
|
||||
id,
|
||||
state: "queued" as const,
|
||||
createdAt,
|
||||
};
|
||||
return {
|
||||
apiVersion: BUILD_RECORD_API_VERSION,
|
||||
kind: "BuildRecord",
|
||||
metadata: {
|
||||
name: id,
|
||||
resourceVersion: "1",
|
||||
creationTimestamp: createdAt,
|
||||
labels: { project: "project", service: "service" },
|
||||
},
|
||||
spec: { request, imageKey, jobName: `job-${id}`, workspaceSubPath: id },
|
||||
status: {
|
||||
...status,
|
||||
logBytes: 0,
|
||||
logOffset: 0,
|
||||
nextSequence: 1,
|
||||
events: [{ type: "status", status }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
type ConfigMap = Parameters<BuildObjectApi["create"]>[0];
|
||||
type DeleteOptions = Pick<V1DeleteOptions, "preconditions">;
|
||||
|
||||
class FakeObjects implements BuildObjectApi {
|
||||
readonly maps = new Map<string, ConfigMap>();
|
||||
readonly selectors: string[] = [];
|
||||
readonly deletes: Array<{ name: string; options?: DeleteOptions }> = [];
|
||||
onLockRead?: () => void;
|
||||
private lockRead = false;
|
||||
|
||||
async create(value: ConfigMap): Promise<unknown> {
|
||||
if (
|
||||
Object.values(value.metadata.labels ?? {}).some(
|
||||
(label) => !validLabelValue(label),
|
||||
)
|
||||
)
|
||||
throw { code: 422 };
|
||||
if (this.maps.has(value.metadata.name)) throw { code: 409 };
|
||||
this.maps.set(
|
||||
value.metadata.name,
|
||||
structuredClone({
|
||||
...value,
|
||||
metadata: { ...value.metadata, resourceVersion: "1" },
|
||||
}),
|
||||
);
|
||||
return value;
|
||||
}
|
||||
|
||||
async read(value: ConfigMap): Promise<unknown> {
|
||||
const found = this.maps.get(value.metadata.name);
|
||||
if (!found) throw { code: 404 };
|
||||
if (value.metadata.name.startsWith("build-lock-") && !this.lockRead) {
|
||||
this.lockRead = true;
|
||||
this.onLockRead?.();
|
||||
}
|
||||
return structuredClone(found);
|
||||
}
|
||||
|
||||
async replace(value: ConfigMap): Promise<unknown> {
|
||||
if (
|
||||
Object.values(value.metadata.labels ?? {}).some(
|
||||
(label) => !validLabelValue(label),
|
||||
)
|
||||
)
|
||||
throw { code: 422 };
|
||||
const current = this.maps.get(value.metadata.name);
|
||||
if (
|
||||
!current ||
|
||||
current.metadata.resourceVersion !== value.metadata.resourceVersion
|
||||
)
|
||||
throw { code: 409 };
|
||||
const next = structuredClone({
|
||||
...value,
|
||||
metadata: {
|
||||
...value.metadata,
|
||||
resourceVersion: String(Number(current.metadata.resourceVersion) + 1),
|
||||
},
|
||||
});
|
||||
this.maps.set(value.metadata.name, next);
|
||||
return next;
|
||||
}
|
||||
|
||||
async delete(value: ConfigMap, options?: DeleteOptions): Promise<unknown> {
|
||||
const current = this.maps.get(value.metadata.name);
|
||||
this.deletes.push({ name: value.metadata.name, options });
|
||||
if (!current) throw { code: 404 };
|
||||
if (
|
||||
options?.preconditions?.resourceVersion !==
|
||||
current.metadata.resourceVersion
|
||||
)
|
||||
throw { code: 409 };
|
||||
this.maps.delete(value.metadata.name);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async list(
|
||||
_apiVersion: string,
|
||||
_kind: string,
|
||||
_namespace?: string,
|
||||
_pretty?: string,
|
||||
_exact?: boolean,
|
||||
_exportValue?: boolean,
|
||||
_fieldSelector?: string,
|
||||
labelSelector?: string,
|
||||
): Promise<{ items: unknown[] }> {
|
||||
this.selectors.push(labelSelector ?? "");
|
||||
const labels = Object.fromEntries(
|
||||
(labelSelector ?? "")
|
||||
.split(",")
|
||||
.filter(Boolean)
|
||||
.map((part) => part.split("=")),
|
||||
);
|
||||
return {
|
||||
items: [...this.maps.values()].filter((item) =>
|
||||
Object.entries(labels).every(
|
||||
([key, value]) => item.metadata.labels?.[key] === value,
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function store(fake: FakeObjects): KubernetesBuildStore {
|
||||
return new KubernetesBuildStore(
|
||||
fake,
|
||||
namespace,
|
||||
"/tmp/kuber-build-store-test",
|
||||
);
|
||||
}
|
||||
|
||||
function configMap(record: BuildRecord, imageLabel = true): ConfigMap {
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "ConfigMap",
|
||||
metadata: {
|
||||
name: name("build", record.metadata.name),
|
||||
namespace,
|
||||
resourceVersion: record.metadata.resourceVersion,
|
||||
labels: {
|
||||
"kuber.astrxl.dev/type": "build",
|
||||
...(imageLabel && {
|
||||
"kuber.astrxl.dev/image": createHash("sha256")
|
||||
.update(record.spec.imageKey)
|
||||
.digest("hex")
|
||||
.slice(0, 63),
|
||||
}),
|
||||
},
|
||||
},
|
||||
data: { payload: JSON.stringify(record) },
|
||||
};
|
||||
}
|
||||
|
||||
describe("KubernetesBuildStore", () => {
|
||||
test("creates records with valid, deterministic image index labels", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const record = build("validated", "2026-09-02T00:00:00.000Z");
|
||||
const result = await store(fake).createBuild(record);
|
||||
const labels = fake.maps.get(name("build", record.metadata.name))?.metadata
|
||||
.labels;
|
||||
|
||||
expect(result.created).toBe(true);
|
||||
expect(labels).toBeDefined();
|
||||
expect(Object.values(labels ?? {}).every(validLabelValue)).toBe(true);
|
||||
expect(labels?.["kuber.astrxl.dev/image"]).toBe(
|
||||
createHash("sha256").update(imageKey).digest("hex").slice(0, 63),
|
||||
);
|
||||
});
|
||||
|
||||
test("uses the image label fast path", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const result = await store(fake).createBuild(
|
||||
build("old", "2026-09-02T00:00:00.000Z"),
|
||||
);
|
||||
expect(result.created).toBe(true);
|
||||
expect(
|
||||
fake.selectors.some((selector) =>
|
||||
selector.includes("kuber.astrxl.dev/image="),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("finds legacy records through the bounded project/service fallback", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const legacy = build(
|
||||
"5ef44ce9-f087-4b65-8ff1-2a60dab9112e",
|
||||
"2026-09-02T00:00:00.000Z",
|
||||
);
|
||||
fake.maps.set(
|
||||
name("build", legacy.metadata.name),
|
||||
configMap(legacy, false),
|
||||
);
|
||||
expect(
|
||||
fake.maps.get(name("build", legacy.metadata.name))?.metadata.labels,
|
||||
).toEqual({
|
||||
"kuber.astrxl.dev/type": "build",
|
||||
});
|
||||
const result = await store(fake).createBuild(
|
||||
build("replacement", "2026-09-02T00:00:01.000Z"),
|
||||
);
|
||||
expect(
|
||||
(await store(fake).getBuild("5ef44ce9-f087-4b65-8ff1-2a60dab9112e"))
|
||||
?.status.state,
|
||||
).toBe("failed");
|
||||
expect(fake.selectors).toContain("kuber.astrxl.dev/type=build");
|
||||
});
|
||||
|
||||
test("returns every older record superseded during reconciliation", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const kuber = store(fake);
|
||||
const oldOne = build("old-one", "2026-09-02T00:00:00.000Z");
|
||||
const oldTwo = build("old-two", "2026-09-02T00:00:01.000Z");
|
||||
fake.maps.set(name("build", oldOne.metadata.name), configMap(oldOne));
|
||||
fake.maps.set(name("build", oldTwo.metadata.name), configMap(oldTwo));
|
||||
|
||||
const result = await kuber.createBuild(
|
||||
build("replacement", "2026-09-02T00:00:02.000Z"),
|
||||
);
|
||||
|
||||
expect(result.created).toBe(true);
|
||||
expect(result.superseded?.map((record) => record.metadata.name)).toEqual([
|
||||
"old-one",
|
||||
"old-two",
|
||||
]);
|
||||
});
|
||||
|
||||
test("returns created false when an older candidate is already superseded", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const kuber = store(fake);
|
||||
await kuber.createBuild(build("newer", "2026-09-02T00:00:01.000Z"));
|
||||
|
||||
const result = await kuber.createBuild(
|
||||
build("older", "2026-09-02T00:00:00.000Z"),
|
||||
);
|
||||
|
||||
expect(result.created).toBe(false);
|
||||
expect(result.record.status.error).toBe("Superseded by newer build");
|
||||
});
|
||||
|
||||
test("orders and reconciles legacy records with invalid metadata timestamps", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const kuber = store(fake);
|
||||
const legacy = build("legacy", "2026-09-02T00:00:00.000Z");
|
||||
legacy.metadata.creationTimestamp = undefined as unknown as string;
|
||||
fake.maps.set(name("build", legacy.metadata.name), configMap(legacy));
|
||||
|
||||
const replacement = await kuber.createBuild(
|
||||
build("replacement", "2026-09-02T00:00:01.000Z"),
|
||||
);
|
||||
expect(replacement.created).toBe(true);
|
||||
expect((await kuber.getBuild("legacy"))?.status.state).toBe("failed");
|
||||
|
||||
const malformed = build("a", "2026-09-02T00:00:00.000Z");
|
||||
malformed.metadata.creationTimestamp = 0 as unknown as string;
|
||||
fake.maps.set(name("build", malformed.metadata.name), configMap(malformed));
|
||||
expect(
|
||||
(await kuber.listBuilds()).map((record) => record.metadata.name),
|
||||
).toEqual(["a", "legacy", "replacement"]);
|
||||
});
|
||||
|
||||
test("recovers an orphan record and takes over its missing lock", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const kuber = store(fake);
|
||||
const orphan = build("orphan", "2026-09-02T00:00:00.000Z");
|
||||
fake.maps.set(
|
||||
name("build", orphan.metadata.name),
|
||||
configMap(orphan, false),
|
||||
);
|
||||
fake.maps.delete(name("build-lock", imageKey));
|
||||
const result = await kuber.createBuild(
|
||||
build("replacement", "2026-09-02T00:00:01.000Z"),
|
||||
);
|
||||
expect((await kuber.getBuild("orphan"))?.status.state).toBe("failed");
|
||||
expect(await kuber.ownsBuild(imageKey, "replacement")).toBe(true);
|
||||
});
|
||||
|
||||
test("does not delete a successor lock after a read/delete race", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const kuber = store(fake);
|
||||
await kuber.createBuild(build("old", "2026-09-02T00:00:00.000Z"));
|
||||
fake.onLockRead = () => {
|
||||
const lock = fake.maps.get(name("build-lock", imageKey))!;
|
||||
fake.maps.set(lock.metadata.name, {
|
||||
...lock,
|
||||
data: { payload: JSON.stringify({ buildId: "successor" }) },
|
||||
metadata: { ...lock.metadata, resourceVersion: "2" },
|
||||
});
|
||||
};
|
||||
const old = (await kuber.getBuild("old"))!;
|
||||
old.status.state = "succeeded";
|
||||
await kuber.replaceBuild(old, "1");
|
||||
expect(await kuber.ownsBuild(imageKey, "successor")).toBe(true);
|
||||
expect(fake.maps.has(name("build-lock", imageKey))).toBe(true);
|
||||
expect(fake.deletes.at(-1)?.options?.preconditions?.resourceVersion).toBe(
|
||||
"1",
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user