feat: harden self-managed reconciliation

This commit is contained in:
2026-09-05 10:17:55 +00:00 Unverified
parent 8e9d207915
commit 321f4e807a
40 changed files with 4977 additions and 404 deletions
+498 -5
View File
@@ -2,12 +2,15 @@ import { afterEach, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { ApiRequestInit } from "../../lib/api";
import type { ApiRequestInit, ApiRequestOptions } from "../../lib/api";
import {
buildServices,
resolveBuildImages,
uploadWorkspaceSnapshot,
type ApiRequester,
TaskScheduler,
MAX_CONCURRENT_REQUESTS,
MAX_REQUESTS_PER_SECOND,
} from "../../lib/build";
import {
workspaceManifestDigest,
@@ -22,6 +25,42 @@ function emptySnapshot(): WorkspaceSnapshot {
return { manifest, digest: workspaceManifestDigest(manifest), blobs: [] };
}
function createSnapshotWithBlobs(
count: number,
blobSize: number,
): WorkspaceSnapshot {
const blobs = Array.from({ length: count }, (_, i) => {
const data = new Uint8Array(blobSize);
data.fill(i + 1);
const digest = `sha256:${"0".repeat(62)}${String(i + 1).padStart(2, "0")}`;
return {
digest: digest as import("../../shared/build-protocol").Sha256Digest,
data,
};
});
const manifest = {
version: 1 as const,
files: blobs
.map((blob, i) => ({
path: `file${i}.txt`,
type: "file" as const,
digest: blob.digest,
size: blob.data.byteLength,
mode: 0o644 as const,
}))
.sort((left, right) =>
Buffer.from(left.path).compare(Buffer.from(right.path)),
),
};
return {
manifest,
digest: workspaceManifestDigest(manifest),
blobs,
};
}
afterEach(async () => {
await Promise.all(
directories
@@ -33,13 +72,18 @@ afterEach(async () => {
describe("authenticated build API pipeline", () => {
test("negotiates and uploads the manifest through resumable blob routes", async () => {
const snapshot = emptySnapshot();
const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
const calls: Array<{
path: string;
init?: ApiRequestInit;
options?: ApiRequestOptions;
}> = [];
let negotiations = 0;
const request: ApiRequester = async <T>(
path: string,
init?: ApiRequestInit,
options?: ApiRequestOptions,
) => {
calls.push({ path, init });
calls.push({ path, init, options });
if (path === "/snapshots/negotiate") {
negotiations += 1;
return (
@@ -76,20 +120,385 @@ describe("authenticated build API pipeline", () => {
expect(new Headers(calls[2]!.init?.headers).get("upload-offset")).toBe("0");
});
test("uses project-scoped URLs for every resumable blob upload endpoint", async () => {
const snapshot = emptySnapshot();
const project = "shop & staging";
const uploadPath = `/blobs/${encodeURIComponent(snapshot.digest)}/uploads`;
const calls: Array<{ method: string; path: string }> = [];
let negotiations = 0;
const request: ApiRequester = async <T>(
path: string,
init?: ApiRequestInit,
) => {
const url = new URL(path, "https://kuber.test");
const method = init?.method ?? "GET";
calls.push({ method, path });
if (url.pathname === "/snapshots/negotiate") {
negotiations += 1;
return (
negotiations === 1
? {
workspace: snapshot.digest,
missing: [snapshot.digest],
ready: false,
}
: { workspace: snapshot.digest, missing: [], ready: true }
) as T;
}
if (url.pathname === uploadPath && method === "POST")
return { offset: 0, complete: false } as T;
if (url.pathname === uploadPath && method === "PATCH")
return { offset: (init!.body as Uint8Array).byteLength } as T;
if (url.pathname === `${uploadPath}/complete` && method === "POST")
return { complete: true } as T;
throw new Error(`Unexpected request ${method} ${path}`);
};
await uploadWorkspaceSnapshot(
snapshot,
request,
undefined,
undefined,
project,
);
const projectQuery = `?project=${encodeURIComponent(project)}`;
expect(calls).toEqual([
{ method: "POST", path: "/snapshots/negotiate" },
{ method: "POST", path: `${uploadPath}${projectQuery}` },
{ method: "PATCH", path: `${uploadPath}${projectQuery}` },
{ method: "POST", path: `${uploadPath}/complete${projectQuery}` },
{ method: "POST", path: "/snapshots/negotiate" },
]);
});
describe("TaskScheduler", () => {
test("uses the production request limits", () => {
expect(MAX_CONCURRENT_REQUESTS).toBe(20);
expect(MAX_REQUESTS_PER_SECOND).toBe(40);
});
test("limits concurrent in-flight operations", async () => {
let maxInflight = 0;
let currentInflight = 0;
const scheduler = new TaskScheduler({
maxInflight: 5,
maxPerSecond: 100,
});
const tasks = Array.from({ length: 10 }, () =>
scheduler.run(async () => {
currentInflight++;
maxInflight = Math.max(maxInflight, currentInflight);
await Bun.sleep(10);
currentInflight--;
}),
);
await Promise.all(tasks);
expect(maxInflight).toBeLessThanOrEqual(5);
});
test("rate limits request starts to maxPerSecond", async () => {
const requestStarts: number[] = [];
let currentTime = 0;
const clock = { now: () => currentTime };
const sleep = async (ms: number) => {
currentTime += ms;
};
const scheduler = new TaskScheduler({
maxInflight: 100,
maxPerSecond: 4,
clock,
sleep,
});
const tasks = Array.from({ length: 8 }, () =>
scheduler.run(async () => {
requestStarts.push(currentTime);
}),
);
await Promise.all(tasks);
for (const start of requestStarts) {
expect(
requestStarts.filter(
(candidate) => candidate >= start && candidate < start + 1000,
).length,
).toBeLessThanOrEqual(4);
}
});
});
describe("concurrent blob uploads", () => {
test("uploads multiple blobs concurrently", async () => {
const snapshot = createSnapshotWithBlobs(5, 1024);
let negotiations = 0;
let inflight = 0;
let maxInflight = 0;
const request: ApiRequester = async <T>(
path: string,
init?: ApiRequestInit,
) => {
if (path === "/snapshots/negotiate") {
negotiations += 1;
return (
negotiations === 1
? {
workspace: snapshot.digest,
missing: snapshot.blobs.map((blob) => blob.digest),
ready: false,
}
: { workspace: snapshot.digest, missing: [], ready: true }
) as T;
}
inflight += 1;
maxInflight = Math.max(maxInflight, inflight);
await Bun.sleep(5);
inflight -= 1;
if (init?.method === "POST" && !path.endsWith("/complete"))
return { offset: 0, complete: false } as T;
if (init?.method === "PATCH") {
return { offset: (init.body as Uint8Array).byteLength } as T;
}
return { complete: true } as T;
};
await uploadWorkspaceSnapshot(
snapshot,
request,
undefined,
new TaskScheduler({ maxInflight: 5, maxPerSecond: 100 }),
);
expect(maxInflight).toBeGreaterThan(1);
expect(negotiations).toBe(2);
});
test("completes active blobs before initializing the full backlog", async () => {
const snapshot = createSnapshotWithBlobs(12, 1024);
const methods: string[] = [];
let negotiations = 0;
const request: ApiRequester = async <T>(
path: string,
init?: ApiRequestInit,
) => {
if (path === "/snapshots/negotiate") {
negotiations += 1;
return (
negotiations === 1
? {
workspace: snapshot.digest,
missing: snapshot.blobs.map((blob) => blob.digest),
ready: false,
}
: { workspace: snapshot.digest, missing: [], ready: true }
) as T;
}
methods.push(
path.endsWith("/complete") ? "complete" : (init?.method ?? "GET"),
);
if (init?.method === "POST" && !path.endsWith("/complete"))
return { offset: 0, complete: false } as T;
if (init?.method === "PATCH")
return { offset: (init.body as Uint8Array).byteLength } as T;
return { complete: true } as T;
};
await uploadWorkspaceSnapshot(
snapshot,
request,
undefined,
new TaskScheduler({ maxInflight: 4, maxPerSecond: 100 }),
);
expect(methods.indexOf("complete")).toBeLessThan(
methods.lastIndexOf("POST"),
);
});
test("inflight never exceeds configured maximum", async () => {
const snapshot = createSnapshotWithBlobs(30, 1024);
let currentInflight = 0;
let maxObservedInflight = 0;
const maxInflight = 10;
let negotiations = 0;
const request: ApiRequester = async <T>(
path: string,
init?: ApiRequestInit,
) => {
if (path === "/snapshots/negotiate") {
negotiations += 1;
return (
negotiations === 1
? {
workspace: snapshot.digest,
missing: snapshot.blobs.map((blob) => blob.digest),
ready: false,
}
: { workspace: snapshot.digest, missing: [], ready: true }
) as T;
}
currentInflight += 1;
maxObservedInflight = Math.max(maxObservedInflight, currentInflight);
await Bun.sleep(2);
currentInflight -= 1;
if (init?.method === "POST" && !path.endsWith("/complete"))
return { offset: 0, complete: false } as T;
if (init?.method === "PATCH") {
return { offset: (init.body as Uint8Array).byteLength } as T;
}
return { complete: true } as T;
};
await uploadWorkspaceSnapshot(
snapshot,
request,
undefined,
new TaskScheduler({ maxInflight, maxPerSecond: 1000 }),
);
expect(maxObservedInflight).toBeLessThanOrEqual(maxInflight);
});
test("request starts are rate-limited to maxPerSecond", async () => {
const snapshot = createSnapshotWithBlobs(8, 1024);
const requestStarts: Array<{ path: string; time: number }> = [];
let currentTime = 0;
const clock = { now: () => currentTime };
const sleep = async (ms: number) => {
currentTime += ms;
};
let negotiations = 0;
const request: ApiRequester = async <T>(
path: string,
init?: ApiRequestInit,
) => {
requestStarts.push({ path, time: currentTime });
if (path === "/snapshots/negotiate") {
negotiations += 1;
return (
negotiations === 1
? {
workspace: snapshot.digest,
missing: snapshot.blobs.map((blob) => blob.digest),
ready: false,
}
: { workspace: snapshot.digest, missing: [], ready: true }
) as T;
}
if (init?.method === "POST" && !path.endsWith("/complete")) {
return { offset: 0, complete: false } as T;
}
if (init?.method === "PATCH") {
return { offset: (init.body as Uint8Array).byteLength } as T;
}
if (path.endsWith("/complete")) {
return { complete: true } as T;
}
return { complete: true } as T;
};
const maxPerSecond = 4;
await uploadWorkspaceSnapshot(
snapshot,
request,
undefined,
new TaskScheduler({
maxInflight: 100,
maxPerSecond,
clock,
sleep,
}),
);
for (const { time } of requestStarts) {
expect(
requestStarts.filter(
({ time: candidate }) =>
candidate >= time && candidate < time + 1000,
).length,
).toBeLessThanOrEqual(maxPerSecond);
}
});
test("chunks of one blob remain ordered during concurrent uploads", async () => {
const blobSize = 25 * 1024 * 1024;
const snapshot = createSnapshotWithBlobs(3, blobSize);
const chunkOrders = new Map<string, number[]>();
let negotiations = 0;
const request: ApiRequester = async <T>(
path: string,
init?: ApiRequestInit,
) => {
if (path === "/snapshots/negotiate") {
negotiations += 1;
return (
negotiations === 1
? {
workspace: snapshot.digest,
missing: snapshot.blobs.map((blob) => blob.digest),
ready: false,
}
: { workspace: snapshot.digest, missing: [], ready: true }
) as T;
}
if (init?.method === "POST" && !path.endsWith("/complete")) {
return { offset: 0, complete: false } as T;
}
if (init?.method === "PATCH") {
const digest = decodeURIComponent(path.split("/")[2]!);
const offset = Number(
new Headers(init.headers).get("upload-offset") ?? "0",
);
const chunkIndex = Math.floor(offset / (8 * 1024 * 1024));
if (!chunkOrders.has(digest)) chunkOrders.set(digest, []);
chunkOrders.get(digest)!.push(chunkIndex);
return { offset: offset + (init.body as Uint8Array).byteLength } as T;
}
return { complete: true } as T;
};
await uploadWorkspaceSnapshot(
snapshot,
request,
undefined,
new TaskScheduler({ maxInflight: 3, maxPerSecond: 100 }),
);
for (const [, chunks] of chunkOrders) {
for (let i = 1; i < chunks.length; i++) {
expect(chunks[i]!).toBeGreaterThan(chunks[i - 1]!);
}
}
});
});
test("submits, reconciles, reports logs, and returns the server image reference", async () => {
const root = await mkdtemp(join(tmpdir(), "kuber-build-api-"));
directories.push(root);
const git = Bun.spawn(["git", "init", "-q", root]);
expect(await git.exited).toBe(0);
const snapshot = emptySnapshot();
const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
const calls: Array<{
path: string;
init?: ApiRequestInit;
options?: ApiRequestOptions;
}> = [];
let submitted: BuildRequest | undefined;
const output: string[] = [];
const request: ApiRequester = async <T>(
path: string,
init?: ApiRequestInit,
options?: ApiRequestOptions,
) => {
calls.push({ path, init });
calls.push({ path, init, options });
if (path === "/snapshots/negotiate")
return { workspace: snapshot.digest, missing: [], ready: true } as T;
if (path === "/builds") {
@@ -155,9 +564,93 @@ describe("authenticated build API pipeline", () => {
},
});
expect(output).toContain("build log");
expect(calls.find(({ path }) => path === "/builds")?.options).toEqual({
timeoutMs: 300_000,
});
expect(
calls
.filter(
({ path }) => path.includes("/events") || path.endsWith("/reconcile"),
)
.map(({ options }) => options),
).toEqual([
{ timeoutMs: 300_000 },
{ timeoutMs: 300_000 },
{ timeoutMs: 300_000 },
]);
expect(calls.some(({ path }) => path.endsWith("/result"))).toBe(true);
});
test("retries a timed out poll request with the build request timeout", async () => {
const root = await mkdtemp(join(tmpdir(), "kuber-build-api-"));
directories.push(root);
const git = Bun.spawn(["git", "init", "-q", root]);
expect(await git.exited).toBe(0);
const snapshot = emptySnapshot();
const pollCalls: Array<{ path: string; options?: ApiRequestOptions }> = [];
let buildId = "";
let eventRequests = 0;
const request: ApiRequester = async <T>(
path: string,
init?: ApiRequestInit,
options?: ApiRequestOptions,
) => {
if (path === "/snapshots/negotiate")
return { workspace: snapshot.digest, missing: [], ready: true } as T;
if (path === "/builds") {
const buildRequest = init?.json as BuildRequest | undefined;
if (!buildRequest) throw new Error("Expected build request");
buildId = buildRequest.id;
return {
version: 1,
id: buildId,
state: "queued",
createdAt: "2026-01-01T00:00:00Z",
} as T;
}
if (path.includes("/events")) {
pollCalls.push({ path, options });
eventRequests += 1;
if (eventRequests === 1)
throw new DOMException("request timed out", "TimeoutError");
return [] as T;
}
if (path.endsWith("/reconcile")) {
pollCalls.push({ path, options });
return {
version: 1,
id: buildId,
state: "succeeded",
createdAt: "2026-01-01T00:00:00Z",
digest: `sha256:${"a".repeat(64)}`,
} as T;
}
if (path.endsWith("/result"))
return {
image: "registry.server/kuber/shop-web",
digest: `sha256:${"a".repeat(64)}`,
reference: `registry.server/kuber/shop-web@sha256:${"a".repeat(64)}`,
} as T;
throw new Error(`Unexpected request ${path}`);
};
await buildServices(
"shop",
{ services: { web: { build: "." } } },
root,
undefined,
{ request, snapshot, sleep: async () => {}, pollIntervalMs: 0 },
);
expect(eventRequests).toBe(3);
expect(pollCalls).toEqual([
expect.objectContaining({ options: { timeoutMs: 300_000 } }),
expect.objectContaining({ options: { timeoutMs: 300_000 } }),
expect.objectContaining({ options: { timeoutMs: 300_000 } }),
expect.objectContaining({ options: { timeoutMs: 300_000 } }),
]);
});
test("no-build image resolution uses only the resolve route", async () => {
const calls: string[] = [];
const images = await resolveBuildImages(
+36 -1
View File
@@ -94,7 +94,9 @@ describe("kuber config", () => {
{ services: { app: { build: "." } } },
process.cwd(),
{},
{ app: `registry.example.com/team/kuber/project-app:latest@sha256:${"a".repeat(64)}` },
{
app: `registry.example.com/team/kuber/project-app:latest@sha256:${"a".repeat(64)}`,
},
);
const deployment = resources.find(
(resource) => resource.kind === "Deployment",
@@ -103,6 +105,39 @@ describe("kuber config", () => {
`registry.example.com/team/kuber/project-app:latest@sha256:${"a".repeat(64)}`,
);
});
test("scopes the server RBAC self-management rules", async () => {
const config = await loadConfig(process.cwd());
const resources: any[] = [];
await config.postRender?.(resources, {
cwd: process.cwd(),
project: "kuber-system",
composeFile: "compose.yml",
});
const role = resources.find((resource) => resource.kind === "Role");
const manager = resources.find(
(resource) => resource.kind === "ClusterRole",
);
expect(role.rules).toContainEqual({
apiGroups: [""],
resources: ["serviceaccounts"],
resourceNames: ["kuber-server"],
verbs: ["get", "update", "patch"],
});
expect(role.rules).toContainEqual({
apiGroups: ["rbac.authorization.k8s.io"],
resources: ["roles", "rolebindings"],
resourceNames: ["kuber-server-auth"],
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"],
});
expect(manager.rules).toContainEqual({
apiGroups: ["rbac.authorization.k8s.io"],
resources: ["clusterroles", "clusterrolebindings"],
resourceNames: ["kuber-server-manager"],
verbs: ["get", "update", "patch"],
});
});
});
describe("global config argument", () => {
+50
View File
@@ -0,0 +1,50 @@
import { afterEach, describe, expect, test } from "bun:test";
import { realpath, rm, stat } from "node:fs/promises";
import {
getTrustPath,
readTrust,
requireLocalTrust,
resolveTrustIdentity,
updateTrust,
} from "../../lib/trust";
const originalConfig = process.env.XDG_CONFIG_HOME;
afterEach(async () => {
const path = getTrustPath();
if (originalConfig === undefined) delete process.env.XDG_CONFIG_HOME;
else process.env.XDG_CONFIG_HOME = originalConfig;
await rm(path, { force: true });
await rm(path.slice(0, path.lastIndexOf("/")), {
recursive: true,
force: true,
});
});
describe("local namespace trust", () => {
test("grants and revokes a resolved CWD fingerprint in a mode-0600 store", async () => {
process.env.XDG_CONFIG_HOME = `/tmp/kuber-trust-${crypto.randomUUID()}`;
const cwd = await realpath(".");
const identity = await resolveTrustIdentity("demo", cwd);
await updateTrust((records) => [...records, identity]);
expect(await requireLocalTrust(identity)).toEqual(identity);
expect((await stat(getTrustPath())).mode & 0o777).toBe(0o600);
await updateTrust((records) =>
records.filter(
(record) =>
record.project !== identity.project ||
record.fingerprint !== identity.fingerprint,
),
);
await expect(requireLocalTrust(identity)).rejects.toThrow("TRUST_REQUIRED");
});
test("rejects an unregistered directory in the same namespace", async () => {
process.env.XDG_CONFIG_HOME = `/tmp/kuber-trust-${crypto.randomUUID()}`;
const first = { project: "demo", fingerprint: "a".repeat(64) };
const second = { project: "demo", fingerprint: "b".repeat(64) };
await updateTrust(() => [first]);
await expect(requireLocalTrust(second)).rejects.toThrow("TRUST_REQUIRED");
expect(await readTrust()).toEqual([first]);
});
});