feat: harden self-managed reconciliation
This commit is contained in:
+716
-1
@@ -8,6 +8,7 @@ import {
|
||||
MemoryWorkspaceLeaseProvider,
|
||||
} from "../../server/operation-store";
|
||||
import { MemoryWorkspaceStore } from "../../server/workspace-store";
|
||||
import { MemoryTrustStore } from "../../server/trust-store";
|
||||
|
||||
function request(
|
||||
path: string,
|
||||
@@ -137,6 +138,151 @@ describe("kuber API authentication", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("operation response safety", () => {
|
||||
test("redacts database and storage reconciliation results immediately", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "demo",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
});
|
||||
const seen: unknown[] = [];
|
||||
const databaseResult = {
|
||||
credentials: { password: "database-password" },
|
||||
env: [{ name: "DB_PASSWORD", value: "database-password" }],
|
||||
};
|
||||
const storageResult = {
|
||||
credentials: { secretKey: "storage-secret" },
|
||||
env: [{ name: "STORAGE_SECRET", value: "storage-secret" }],
|
||||
};
|
||||
const management = {
|
||||
reconcileDatabases: async (_workspace: unknown, compose: unknown) => {
|
||||
seen.push(compose);
|
||||
return databaseResult;
|
||||
},
|
||||
reconcileStorage: async (_workspace: unknown, compose: unknown) => {
|
||||
seen.push(compose);
|
||||
return storageResult;
|
||||
},
|
||||
} as unknown as ManagementService;
|
||||
const app = createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore: new MemoryOperationStore(undefined, () =>
|
||||
crypto.randomUUID(),
|
||||
),
|
||||
management,
|
||||
});
|
||||
const compose = {
|
||||
services: {},
|
||||
secret: "internal-compose-secret",
|
||||
};
|
||||
for (const [path, key, secret] of [
|
||||
["databases", "db-once", "database-password"],
|
||||
["storage", "storage-once", "storage-secret"],
|
||||
] as const) {
|
||||
const result = await app(
|
||||
request(
|
||||
`/api/v2/workspaces/demo/${path}`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "idempotency-key": key },
|
||||
body: JSON.stringify({ compose }),
|
||||
},
|
||||
"token",
|
||||
),
|
||||
);
|
||||
expect(result.status).toBe(200);
|
||||
const body = JSON.stringify(await result.json());
|
||||
expect(body).not.toContain(secret);
|
||||
expect(body).not.toContain("internal-compose-secret");
|
||||
expect(body).toContain('"redacted":true');
|
||||
}
|
||||
expect(seen).toHaveLength(2);
|
||||
expect(JSON.stringify(seen[0])).toContain("internal-compose-secret");
|
||||
expect(databaseResult.credentials.password).toBe("database-password");
|
||||
expect(storageResult.credentials.secretKey).toBe("storage-secret");
|
||||
});
|
||||
|
||||
test("redacts failed reconciliation errors immediately and when retrieved", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "demo",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
});
|
||||
const operationStore = new MemoryOperationStore(undefined, () =>
|
||||
crypto.randomUUID(),
|
||||
);
|
||||
const app = createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore,
|
||||
management: {
|
||||
reconcileDatabases: async () => {
|
||||
throw new Error(
|
||||
'database provider failed DB_PASSWORD=database-password response={"data":{"token":"kube-secret"}}',
|
||||
);
|
||||
},
|
||||
reconcileStorage: async () => {
|
||||
throw new Error(
|
||||
'storage provider failed STORAGE_SECRET=storage-secret response={"data":{"password":"storage-kube-secret"}}',
|
||||
);
|
||||
},
|
||||
} as unknown as ManagementService,
|
||||
});
|
||||
const path = "/api/v2/workspaces/demo/databases";
|
||||
const init = {
|
||||
method: "POST",
|
||||
headers: { "idempotency-key": "failed-reconcile" },
|
||||
body: JSON.stringify({ compose: { services: {} } }),
|
||||
};
|
||||
const immediate = await app(request(path, init, "token"));
|
||||
const immediateBody = JSON.stringify(await immediate.json());
|
||||
expect(immediate.status).toBe(500);
|
||||
expect(immediateBody).not.toContain("database-password");
|
||||
expect(immediateBody).not.toContain("kube-secret");
|
||||
expect(immediateBody).toContain("OPERATION_FAILED");
|
||||
|
||||
const operationId = (await operationStore.list())[0]!.metadata.name;
|
||||
const retrieved = await app(
|
||||
request(`/api/v2/operations/${operationId}`, {}, "token"),
|
||||
);
|
||||
const retrievedBody = JSON.stringify(await retrieved.json());
|
||||
expect(retrievedBody).not.toContain("database-password");
|
||||
expect(retrievedBody).not.toContain("kube-secret");
|
||||
|
||||
const storageImmediate = await app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/storage",
|
||||
{
|
||||
...init,
|
||||
headers: { "idempotency-key": "failed-storage" },
|
||||
},
|
||||
"token",
|
||||
),
|
||||
);
|
||||
const storageBody = JSON.stringify(await storageImmediate.json());
|
||||
expect(storageImmediate.status).toBe(500);
|
||||
expect(storageBody).not.toContain("storage-secret");
|
||||
expect(storageBody).not.toContain("storage-kube-secret");
|
||||
const storageId = (await operationStore.list())[1]!.metadata.name;
|
||||
const storageRetrieved = await app(
|
||||
request(`/api/v2/operations/${storageId}`, {}, "token"),
|
||||
);
|
||||
expect(JSON.stringify(await storageRetrieved.json())).not.toContain(
|
||||
"storage-kube-secret",
|
||||
);
|
||||
|
||||
const idempotent = await app(request(path, init, "token"));
|
||||
const idempotentBody = JSON.stringify(await idempotent.json());
|
||||
expect(idempotentBody).not.toContain("database-password");
|
||||
expect(idempotentBody).not.toContain("kube-secret");
|
||||
});
|
||||
});
|
||||
|
||||
async function authenticatedStore(role: "viewer" | "operator" | "admin") {
|
||||
const store = new MemoryAuthStore();
|
||||
await store.putUser({ username: role, passwordHash: "hash", roles: [role] });
|
||||
@@ -150,6 +296,101 @@ async function authenticatedStore(role: "viewer" | "operator" | "admin") {
|
||||
}
|
||||
|
||||
describe("kuber v2 HTTP routes", () => {
|
||||
test("grants, lists, revokes, and enforces namespace trust for applies", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "demo",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
});
|
||||
const trustStore = new MemoryTrustStore();
|
||||
const fingerprint = "a".repeat(64);
|
||||
const headers = {
|
||||
"x-kuber-trust-project": "demo",
|
||||
"x-kuber-trust-fingerprint": fingerprint,
|
||||
};
|
||||
const apply = (app: ReturnType<typeof createApp>) =>
|
||||
app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/resources/apply",
|
||||
{
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ resources: [] }),
|
||||
},
|
||||
"token",
|
||||
),
|
||||
);
|
||||
const unavailable = await createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore: new MemoryOperationStore(),
|
||||
management: {
|
||||
applyResources: async () => [],
|
||||
} as unknown as ManagementService,
|
||||
});
|
||||
const unavailableApply = await apply(unavailable);
|
||||
expect(unavailableApply.status).toBe(503);
|
||||
expect(await unavailableApply.json()).toMatchObject({
|
||||
code: "TRUST_STORE_UNAVAILABLE",
|
||||
});
|
||||
|
||||
const app = createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore: new MemoryOperationStore(),
|
||||
trustStore,
|
||||
management: {
|
||||
applyResources: async () => [],
|
||||
} as unknown as ManagementService,
|
||||
});
|
||||
const untrustedApply = (headers?: RequestInit["headers"]) =>
|
||||
app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/resources/apply",
|
||||
{
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify({ resources: [] }),
|
||||
},
|
||||
"token",
|
||||
),
|
||||
);
|
||||
|
||||
expect((await untrustedApply()).status).toBe(428);
|
||||
expect((await apply(app)).status).toBe(403);
|
||||
expect(
|
||||
(
|
||||
await app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/trust",
|
||||
{ method: "POST", body: JSON.stringify({ fingerprint }) },
|
||||
"token",
|
||||
),
|
||||
)
|
||||
).status,
|
||||
).toBe(204);
|
||||
const status = await app(
|
||||
request("/api/v2/workspaces/demo/trust", {}, "token"),
|
||||
);
|
||||
expect(status.status).toBe(200);
|
||||
expect(await status.json()).toEqual({ fingerprints: [fingerprint] });
|
||||
expect((await apply(app)).status).toBe(200);
|
||||
expect(
|
||||
(
|
||||
await app(
|
||||
request(
|
||||
`/api/v2/workspaces/demo/trust?fingerprint=${fingerprint}`,
|
||||
{ method: "DELETE" },
|
||||
"token",
|
||||
),
|
||||
)
|
||||
).status,
|
||||
).toBe(204);
|
||||
expect((await apply(app)).status).toBe(403);
|
||||
});
|
||||
|
||||
test("uses exact origins, request IDs, and problem+json errors", async () => {
|
||||
const app = createApp({
|
||||
store: new MemoryAuthStore(),
|
||||
@@ -405,10 +646,484 @@ describe("kuber v2 HTTP routes", () => {
|
||||
operationId: "operation-blocked",
|
||||
});
|
||||
expect((await operationStore.get("operation-blocked"))?.status.state).toBe(
|
||||
"failed",
|
||||
"pending",
|
||||
);
|
||||
});
|
||||
|
||||
test("keeps a concurrent idempotent operation pending when its lease acquisition is denied", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "demo",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
});
|
||||
const operationStore = new MemoryOperationStore(
|
||||
undefined,
|
||||
() => "same-key",
|
||||
);
|
||||
let acquireCount = 0;
|
||||
let releaseCount = 0;
|
||||
let allowFirstAcquire!: () => void;
|
||||
let signalFirstAcquire!: () => void;
|
||||
const firstAcquireStarted = new Promise<void>((resolve) => {
|
||||
signalFirstAcquire = resolve;
|
||||
});
|
||||
const leases = {
|
||||
acquire: async () => {
|
||||
acquireCount += 1;
|
||||
if (acquireCount === 2) return undefined;
|
||||
signalFirstAcquire();
|
||||
await new Promise<void>((resolve) => {
|
||||
allowFirstAcquire = resolve;
|
||||
});
|
||||
return {
|
||||
workspaceId: "demo",
|
||||
holder: "operation-same-key",
|
||||
expiresAt: new Date().toISOString(),
|
||||
renew: async () => true,
|
||||
release: async () => {
|
||||
releaseCount += 1;
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
const app = createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore,
|
||||
leases,
|
||||
management: { stop: async () => ["api"] } as unknown as ManagementService,
|
||||
});
|
||||
const stop = () =>
|
||||
app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/lifecycle",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "idempotency-key": "same-key" },
|
||||
body: JSON.stringify({ action: "stop" }),
|
||||
},
|
||||
"token",
|
||||
),
|
||||
);
|
||||
|
||||
const owner = stop();
|
||||
await firstAcquireStarted;
|
||||
const duplicate = await stop();
|
||||
expect(duplicate.status).toBe(409);
|
||||
expect((await operationStore.get("operation-same-key"))?.status.state).toBe(
|
||||
"pending",
|
||||
);
|
||||
|
||||
allowFirstAcquire();
|
||||
expect((await owner).status).toBe(200);
|
||||
expect((await operationStore.get("operation-same-key"))?.status.state).toBe(
|
||||
"succeeded",
|
||||
);
|
||||
expect(acquireCount).toBe(2);
|
||||
expect(releaseCount).toBe(1);
|
||||
});
|
||||
|
||||
test("lets the lease winner claim execution even when a duplicate created the operation", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "demo",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
});
|
||||
const operationStore = new MemoryOperationStore(
|
||||
undefined,
|
||||
() => "same-key",
|
||||
);
|
||||
let acquireCount = 0;
|
||||
let releaseFirstAcquire!: () => void;
|
||||
let firstAcquireStarted!: () => void;
|
||||
const firstAcquire = new Promise<void>((resolve) => {
|
||||
firstAcquireStarted = resolve;
|
||||
});
|
||||
const leases = {
|
||||
acquire: async () => {
|
||||
acquireCount += 1;
|
||||
if (acquireCount === 1) {
|
||||
firstAcquireStarted();
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseFirstAcquire = resolve;
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
workspaceId: "demo",
|
||||
holder: "operation-same-key",
|
||||
expiresAt: new Date().toISOString(),
|
||||
renew: async () => true,
|
||||
release: async () => {},
|
||||
};
|
||||
},
|
||||
};
|
||||
let executions = 0;
|
||||
const app = createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore,
|
||||
leases,
|
||||
management: {
|
||||
stop: async () => {
|
||||
executions += 1;
|
||||
return ["api"];
|
||||
},
|
||||
} as unknown as ManagementService,
|
||||
});
|
||||
const stop = () =>
|
||||
app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/lifecycle",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "idempotency-key": "same-key" },
|
||||
body: JSON.stringify({ action: "stop" }),
|
||||
},
|
||||
"token",
|
||||
),
|
||||
);
|
||||
|
||||
const creator = stop();
|
||||
await firstAcquire;
|
||||
expect((await stop()).status).toBe(200);
|
||||
releaseFirstAcquire();
|
||||
expect((await creator).status).toBe(409);
|
||||
expect(executions).toBe(1);
|
||||
expect((await operationStore.get("operation-same-key"))?.status.state).toBe(
|
||||
"succeeded",
|
||||
);
|
||||
});
|
||||
|
||||
test("releases a lease when the initial execution claim fails", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "demo",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
});
|
||||
const operationStore = new MemoryOperationStore(
|
||||
undefined,
|
||||
() => "claim-fails",
|
||||
);
|
||||
spyOn(operationStore, "claimExecution").mockRejectedValue(
|
||||
new Error("claim unavailable"),
|
||||
);
|
||||
let releases = 0;
|
||||
const app = createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore,
|
||||
leases: {
|
||||
acquire: async () => ({
|
||||
workspaceId: "demo",
|
||||
holder: "operation-claim-fails",
|
||||
expiresAt: new Date().toISOString(),
|
||||
renew: async () => true,
|
||||
release: async () => {
|
||||
releases += 1;
|
||||
},
|
||||
}),
|
||||
},
|
||||
management: { stop: async () => [] } as unknown as ManagementService,
|
||||
});
|
||||
|
||||
expect(
|
||||
(
|
||||
await app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/lifecycle",
|
||||
{ method: "POST", body: JSON.stringify({ action: "stop" }) },
|
||||
"token",
|
||||
),
|
||||
)
|
||||
).status,
|
||||
).toBe(500);
|
||||
expect(releases).toBe(1);
|
||||
});
|
||||
|
||||
test("fails a claimed operation before execution when lease ownership is lost", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "demo",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
});
|
||||
const operationStore = new MemoryOperationStore(
|
||||
undefined,
|
||||
() => "lost-lease",
|
||||
);
|
||||
let executions = 0;
|
||||
let releases = 0;
|
||||
const app = createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore,
|
||||
leases: {
|
||||
acquire: async () => ({
|
||||
workspaceId: "demo",
|
||||
holder: "operation-lost-lease",
|
||||
expiresAt: new Date().toISOString(),
|
||||
renew: async () => false,
|
||||
release: async () => {
|
||||
releases += 1;
|
||||
},
|
||||
}),
|
||||
},
|
||||
management: {
|
||||
stop: async () => {
|
||||
executions += 1;
|
||||
return [];
|
||||
},
|
||||
} as unknown as ManagementService,
|
||||
});
|
||||
|
||||
const result = await app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/lifecycle",
|
||||
{ method: "POST", body: JSON.stringify({ action: "stop" }) },
|
||||
"token",
|
||||
),
|
||||
);
|
||||
expect(result.status).toBe(409);
|
||||
expect(executions).toBe(0);
|
||||
expect(releases).toBe(1);
|
||||
expect(await result.json()).toMatchObject({
|
||||
code: "WORKSPACE_LEASE_LOST",
|
||||
});
|
||||
expect(
|
||||
(await operationStore.get("operation-lost-lease"))?.status,
|
||||
).toMatchObject({
|
||||
state: "failed",
|
||||
error: { code: "WORKSPACE_LEASE_LOST" },
|
||||
});
|
||||
});
|
||||
|
||||
test("aborts blocked execution and fails the operation when renewal loses the lease", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "demo",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
});
|
||||
const operationStore = new MemoryOperationStore(
|
||||
undefined,
|
||||
() => "blocked-loss",
|
||||
);
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
let scheduledRenewal!: () => void;
|
||||
globalThis.setTimeout = ((callback: Parameters<typeof setTimeout>[0]) => {
|
||||
scheduledRenewal = callback as () => void;
|
||||
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||
}) as typeof setTimeout;
|
||||
try {
|
||||
let executionStarted!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
executionStarted = resolve;
|
||||
});
|
||||
let observedAbort = false;
|
||||
let renewals = 0;
|
||||
const app = createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore,
|
||||
leases: {
|
||||
acquire: async () => ({
|
||||
workspaceId: "demo",
|
||||
holder: "operation-blocked-loss",
|
||||
expiresAt: new Date().toISOString(),
|
||||
renew: async () => ++renewals === 1,
|
||||
release: async () => {},
|
||||
}),
|
||||
},
|
||||
management: {
|
||||
stop: async (
|
||||
_workspace: { project: string; uid: string },
|
||||
_names?: string[],
|
||||
execution?: { signal?: AbortSignal },
|
||||
) => {
|
||||
executionStarted();
|
||||
await new Promise<void>((resolve) => {
|
||||
execution?.signal?.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
observedAbort = true;
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
return [];
|
||||
},
|
||||
} as unknown as ManagementService,
|
||||
});
|
||||
const pending = app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/lifecycle",
|
||||
{ method: "POST", body: JSON.stringify({ action: "stop" }) },
|
||||
"token",
|
||||
),
|
||||
);
|
||||
await started;
|
||||
scheduledRenewal();
|
||||
const result = await pending;
|
||||
|
||||
expect(observedAbort).toBe(true);
|
||||
expect(result.status).toBe(409);
|
||||
expect(await result.json()).toMatchObject({
|
||||
code: "WORKSPACE_LEASE_LOST",
|
||||
});
|
||||
expect(
|
||||
(await operationStore.get("operation-blocked-loss"))?.status,
|
||||
).toMatchObject({
|
||||
state: "failed",
|
||||
error: { code: "WORKSPACE_LEASE_LOST" },
|
||||
});
|
||||
} finally {
|
||||
globalThis.setTimeout = originalSetTimeout;
|
||||
}
|
||||
});
|
||||
|
||||
test("renews the workspace lease while an operation remains in flight", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "demo",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
});
|
||||
let scheduledRenewal: (() => void) | undefined;
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
const originalClearTimeout = globalThis.clearTimeout;
|
||||
const clearedTimers: unknown[] = [];
|
||||
globalThis.setTimeout = ((callback: Parameters<typeof setTimeout>[0]) => {
|
||||
scheduledRenewal = callback as () => void;
|
||||
return 0 as unknown as ReturnType<typeof setTimeout>;
|
||||
}) as typeof setTimeout;
|
||||
globalThis.clearTimeout = ((timer: ReturnType<typeof setTimeout>) => {
|
||||
clearedTimers.push(timer);
|
||||
}) as typeof clearTimeout;
|
||||
try {
|
||||
let releaseOperation!: () => void;
|
||||
let operationStarted!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
operationStarted = resolve;
|
||||
});
|
||||
const completed = new Promise<string[]>((resolve) => {
|
||||
releaseOperation = () => resolve(["api"]);
|
||||
});
|
||||
let renewals = 0;
|
||||
const app = createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore: new MemoryOperationStore(undefined, () => "renewing"),
|
||||
leases: {
|
||||
acquire: async () => ({
|
||||
workspaceId: "demo",
|
||||
holder: "operation-renewing",
|
||||
expiresAt: new Date().toISOString(),
|
||||
renew: async () => {
|
||||
renewals += 1;
|
||||
return true;
|
||||
},
|
||||
release: async () => {},
|
||||
}),
|
||||
},
|
||||
management: {
|
||||
stop: async () => {
|
||||
operationStarted();
|
||||
return completed;
|
||||
},
|
||||
} as unknown as ManagementService,
|
||||
});
|
||||
const response = app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/lifecycle",
|
||||
{ method: "POST", body: JSON.stringify({ action: "stop" }) },
|
||||
"token",
|
||||
),
|
||||
);
|
||||
await started;
|
||||
expect(renewals).toBe(1);
|
||||
scheduledRenewal?.();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
expect(renewals).toBe(2);
|
||||
releaseOperation();
|
||||
expect((await response).status).toBe(200);
|
||||
expect(renewals).toBe(3);
|
||||
expect(clearedTimers).toEqual([0]);
|
||||
} finally {
|
||||
globalThis.setTimeout = originalSetTimeout;
|
||||
globalThis.clearTimeout = originalClearTimeout;
|
||||
}
|
||||
});
|
||||
|
||||
test("reuses a failed reconciliation without attempting a second failure transition", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "demo",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
});
|
||||
const operationStore = new MemoryOperationStore(
|
||||
undefined,
|
||||
() => "reconcile",
|
||||
);
|
||||
let reconciliations = 0;
|
||||
const management = {
|
||||
reconcileDatabases: async () => {
|
||||
reconciliations += 1;
|
||||
const [operation] = await operationStore.list("demo");
|
||||
await operationStore.transition(operation!.metadata.name, "failed", {
|
||||
error: {
|
||||
code: "RECONCILE_FAILED",
|
||||
message: "Database reconciliation failed",
|
||||
},
|
||||
});
|
||||
return {};
|
||||
},
|
||||
} as unknown as ManagementService;
|
||||
const app = createApp({
|
||||
store: await authenticatedStore("operator"),
|
||||
workspaceStore,
|
||||
operationStore,
|
||||
management,
|
||||
});
|
||||
const reconcile = () =>
|
||||
app(
|
||||
request(
|
||||
"/api/v2/workspaces/demo/databases",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "idempotency-key": "reconcile-once" },
|
||||
body: JSON.stringify({ compose: {} }),
|
||||
},
|
||||
"token",
|
||||
),
|
||||
);
|
||||
|
||||
for (const result of [await reconcile(), await reconcile()]) {
|
||||
expect(result.status).toBe(500);
|
||||
expect(await result.json()).toMatchObject({
|
||||
code: "RECONCILE_FAILED",
|
||||
detail: "Database reconciliation failed",
|
||||
});
|
||||
}
|
||||
expect(reconciliations).toBe(1);
|
||||
expect(
|
||||
(await operationStore.get("operation-reconcile"))?.status.state,
|
||||
).toBe("failed");
|
||||
});
|
||||
|
||||
test("routes workspace adoption and keeps platform adoption admin-only", async () => {
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-uid",
|
||||
|
||||
@@ -10,7 +10,9 @@ import {
|
||||
type BuildJobObservation,
|
||||
type BuildKubernetesOperations,
|
||||
} from "../../server/build-controller";
|
||||
import { MemoryBuildStore } from "../../server/build-store";
|
||||
import { createApp } from "../../server/app";
|
||||
import { hashToken, MemoryAuthStore } from "../../server/auth";
|
||||
import { MemoryBuildStore, type BuildStore } from "../../server/build-store";
|
||||
import { FilesystemCas } from "../../server/cas";
|
||||
import type { KubernetesJob } from "../../server/build-job";
|
||||
import {
|
||||
@@ -36,7 +38,10 @@ class FakeKubernetes implements BuildKubernetesOperations {
|
||||
deleted: string[] = [];
|
||||
observation: BuildJobObservation | undefined = { phase: "queued" };
|
||||
logs = "";
|
||||
createError?: Error;
|
||||
preserveAfterDelete = false;
|
||||
async createJob(job: KubernetesJob) {
|
||||
if (this.createError) throw this.createError;
|
||||
this.jobs.push(job);
|
||||
}
|
||||
async getJob() {
|
||||
@@ -47,14 +52,39 @@ class FakeKubernetes implements BuildKubernetesOperations {
|
||||
}
|
||||
async deleteJob(_namespace: string, name: string) {
|
||||
this.deleted.push(name);
|
||||
if (!this.preserveAfterDelete) this.observation = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function fixture(maxLogBytes = 1024) {
|
||||
class SupersededBeforeJobStore extends MemoryBuildStore {
|
||||
private superseded = false;
|
||||
|
||||
override async ownsBuild(
|
||||
imageKey: string,
|
||||
buildId: string,
|
||||
): Promise<boolean> {
|
||||
if (!this.superseded) {
|
||||
this.superseded = true;
|
||||
const current = await this.getBuild(buildId);
|
||||
if (current) {
|
||||
const newer = structuredClone(current);
|
||||
newer.metadata.name = "newer-build";
|
||||
newer.metadata.creationTimestamp = "2026-09-02T00:00:01.000Z";
|
||||
newer.spec.request.id = "newer-build";
|
||||
await super.createBuild(newer);
|
||||
}
|
||||
}
|
||||
return super.ownsBuild(imageKey, buildId);
|
||||
}
|
||||
}
|
||||
|
||||
async function fixture(
|
||||
maxLogBytes = 1024,
|
||||
store: BuildStore = new MemoryBuildStore(),
|
||||
) {
|
||||
const root = await mkdtemp(join(tmpdir(), "kuber-controller-"));
|
||||
roots.push(root);
|
||||
const cas = new FilesystemCas(join(root, "cas"));
|
||||
const store = new MemoryBuildStore();
|
||||
const kubernetes = new FakeKubernetes();
|
||||
const source = Buffer.from("FROM scratch\n");
|
||||
const sourceDigest = await cas.put(source);
|
||||
@@ -237,7 +267,7 @@ describe("build controller", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("submits once, blocks competing image builds, captures bounded logs, and resolves immutable results", async () => {
|
||||
test("submits once, supersedes competing image builds, captures bounded logs, and resolves immutable results", async () => {
|
||||
const { controller, kubernetes, request } = await fixture(8);
|
||||
expect(await controller.submitBuild(request)).toMatchObject({
|
||||
state: "queued",
|
||||
@@ -247,18 +277,18 @@ describe("build controller", () => {
|
||||
).toMatchObject({ state: "queued" });
|
||||
expect(kubernetes.jobs).toHaveLength(1);
|
||||
const jobSpec = kubernetes.jobs[0]!.spec as any;
|
||||
expect(
|
||||
jobSpec.template.spec.containers[0].volumeMounts,
|
||||
).toContainEqual(
|
||||
expect(jobSpec.template.spec.containers[0].volumeMounts).toContainEqual(
|
||||
expect.objectContaining({
|
||||
name: "workspace",
|
||||
mountPath: "/workspace",
|
||||
subPath: `workspaces/${kubernetes.jobs[0]!.metadata.name}`,
|
||||
}),
|
||||
);
|
||||
await expect(
|
||||
controller.submitBuild({ ...request, id: "request-two" }),
|
||||
).rejects.toBeInstanceOf(BuildConflictError);
|
||||
const replacement = { ...request, id: "request-two" };
|
||||
expect(await controller.submitBuild(replacement)).toMatchObject({
|
||||
state: "queued",
|
||||
});
|
||||
expect(kubernetes.jobs).toHaveLength(2);
|
||||
await expect(
|
||||
controller.submitBuild({ ...request, project: "changed" }),
|
||||
).rejects.toBeInstanceOf(BuildConflictError);
|
||||
@@ -268,10 +298,10 @@ describe("build controller", () => {
|
||||
phase: "running",
|
||||
startedAt: "2026-09-02T00:00:03.000Z",
|
||||
};
|
||||
expect(await controller.reconcileBuild(request.id)).toMatchObject({
|
||||
expect(await controller.reconcileBuild(replacement.id)).toMatchObject({
|
||||
state: "running",
|
||||
});
|
||||
const logs = (await controller.getBuildEvents(request.id)).filter(
|
||||
const logs = (await controller.getBuildEvents(replacement.id)).filter(
|
||||
(event) => event.type === "log",
|
||||
);
|
||||
expect(
|
||||
@@ -286,17 +316,137 @@ describe("build controller", () => {
|
||||
phase: "succeeded",
|
||||
finishedAt: "2026-09-02T00:00:04.000Z",
|
||||
};
|
||||
const status = await controller.reconcileBuild(request.id);
|
||||
const status = await controller.reconcileBuild(replacement.id);
|
||||
expect(status).toMatchObject({
|
||||
state: "succeeded",
|
||||
digest: `sha256:${"f".repeat(64)}`,
|
||||
});
|
||||
expect(await controller.getBuildResult(request.id)).toEqual({
|
||||
expect(await controller.getBuildResult(replacement.id)).toEqual({
|
||||
image: "registry.test/demo/web",
|
||||
digest: `sha256:${"f".repeat(64)}`,
|
||||
reference: `registry.test/demo/web@sha256:${"f".repeat(64)}`,
|
||||
});
|
||||
expect(await controller.reconcileBuild(request.id)).toEqual(status);
|
||||
expect(await controller.reconcileBuild(replacement.id)).toEqual(status);
|
||||
});
|
||||
|
||||
test("concurrent controllers converge on one same-ID record and Job", async () => {
|
||||
const first = await fixture();
|
||||
const second = new BuildController({
|
||||
cas: first.cas,
|
||||
store: first.store,
|
||||
kubernetes: first.kubernetes,
|
||||
namespace: "builds",
|
||||
workspaceRoot: join(first.root, "workspaces"),
|
||||
workspaceClaimName: "workspaces",
|
||||
cacheImage: "registry.test/cache/app",
|
||||
});
|
||||
await Promise.all([
|
||||
first.controller.submitBuild(first.request),
|
||||
second.submitBuild(structuredClone(first.request)),
|
||||
]);
|
||||
expect(first.kubernetes.jobs).toHaveLength(1);
|
||||
expect(await first.store.getBuild(first.request.id)).toMatchObject({
|
||||
spec: { request: { id: first.request.id } },
|
||||
});
|
||||
});
|
||||
|
||||
test("starts a replacement without waiting for superseded Job deletion", async () => {
|
||||
const { controller, kubernetes, request, root, store } = await fixture();
|
||||
await controller.submitBuild(request);
|
||||
const oldJobName = kubernetes.jobs[0]!.metadata.name;
|
||||
kubernetes.preserveAfterDelete = true;
|
||||
await expect(
|
||||
controller.submitBuild({ ...request, id: "replacement" }),
|
||||
).resolves.toMatchObject({
|
||||
state: "queued",
|
||||
});
|
||||
expect(kubernetes.deleted).toEqual([oldJobName]);
|
||||
expect(kubernetes.jobs).toHaveLength(2);
|
||||
await expect(
|
||||
lstat(join(root, "workspaces", oldJobName)),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect((await store.getBuild(request.id))?.status).toMatchObject({
|
||||
state: "failed",
|
||||
error: "Superseded by newer build",
|
||||
cancelled: true,
|
||||
});
|
||||
});
|
||||
|
||||
test("deletes an ambiguous superseded Job even when status was not persisted", async () => {
|
||||
const { controller, kubernetes, request, store } = await fixture();
|
||||
await controller.submitBuild(request);
|
||||
const oldJobName = kubernetes.jobs[0]!.metadata.name;
|
||||
const old = (await store.getBuild(request.id))!;
|
||||
old.status.jobCreated = false;
|
||||
await store.replaceBuild(old, old.metadata.resourceVersion);
|
||||
|
||||
await expect(
|
||||
controller.submitBuild({ ...request, id: "replacement" }),
|
||||
).resolves.toMatchObject({ state: "queued" });
|
||||
expect(kubernetes.deleted).toEqual([oldJobName]);
|
||||
});
|
||||
|
||||
test("does not create a Job when the candidate is superseded before Job creation", async () => {
|
||||
const store = new SupersededBeforeJobStore();
|
||||
const { controller, kubernetes, request } = await fixture(1024, store);
|
||||
|
||||
await expect(controller.submitBuild(request)).rejects.toBeInstanceOf(
|
||||
BuildConflictError,
|
||||
);
|
||||
expect(kubernetes.jobs).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("maps supersession before Job creation to HTTP 409", async () => {
|
||||
const auth = new MemoryAuthStore();
|
||||
await auth.putUser({
|
||||
username: "operator",
|
||||
passwordHash: "hash",
|
||||
roles: ["operator"],
|
||||
});
|
||||
await auth.putSession({
|
||||
tokenHash: hashToken("token"),
|
||||
username: "operator",
|
||||
authVersion: 1,
|
||||
expiresAt: "2030-01-01T00:00:00.000Z",
|
||||
});
|
||||
const store = new SupersededBeforeJobStore();
|
||||
const {
|
||||
controller,
|
||||
kubernetes,
|
||||
request: buildRequest,
|
||||
} = await fixture(1024, store);
|
||||
const app = createApp({ store: auth, builds: controller });
|
||||
|
||||
const response = await app(
|
||||
new Request("https://kuber.test/api/v2/builds", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: "Bearer token",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(buildRequest),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(409);
|
||||
expect(await response.json()).toMatchObject({ code: "BUILD_CONFLICT" });
|
||||
expect(kubernetes.jobs).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("releases the image lock after an ambiguous Job creation failure", async () => {
|
||||
const { controller, kubernetes, request, store } = await fixture();
|
||||
kubernetes.createError = new Error("create response lost");
|
||||
kubernetes.preserveAfterDelete = true;
|
||||
await expect(controller.submitBuild(request)).rejects.toThrow(
|
||||
"create response lost",
|
||||
);
|
||||
expect((await store.getBuild(request.id))?.status).toMatchObject({
|
||||
state: "failed",
|
||||
});
|
||||
kubernetes.createError = undefined;
|
||||
await expect(
|
||||
controller.submitBuild({ ...request, id: "replacement" }),
|
||||
).resolves.toMatchObject({ state: "queued" });
|
||||
});
|
||||
|
||||
test("cancels idempotently and cleans up only terminal build resources", async () => {
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import type { V1DeleteOptions } from "@kubernetes/client-node";
|
||||
import {
|
||||
BUILD_RECORD_API_VERSION,
|
||||
type BuildRecord,
|
||||
} from "../../server/build-store";
|
||||
import {
|
||||
KubernetesBuildStore,
|
||||
type BuildObjectApi,
|
||||
} from "../../server/build-kubernetes";
|
||||
import {
|
||||
BUILD_PROTOCOL_VERSION,
|
||||
type BuildRequest,
|
||||
type Sha256Digest,
|
||||
} from "../../shared/build-protocol";
|
||||
|
||||
const namespace = "kuber-test";
|
||||
const imageKey = "project\0service\0registry.test/app:latest";
|
||||
const workspace = `sha256:${"a".repeat(64)}` as Sha256Digest;
|
||||
|
||||
function validLabelValue(value: string): boolean {
|
||||
return (
|
||||
value.length <= 63 &&
|
||||
/^[A-Za-z0-9](?:[-_.A-Za-z0-9]*[A-Za-z0-9])?$/.test(value)
|
||||
);
|
||||
}
|
||||
|
||||
function name(prefix: string, value: string): string {
|
||||
return `${prefix}-${createHash("sha256").update(value).digest("hex").slice(0, 48)}`;
|
||||
}
|
||||
|
||||
function build(id: string, createdAt: string): BuildRecord {
|
||||
const request: BuildRequest = {
|
||||
version: BUILD_PROTOCOL_VERSION,
|
||||
id,
|
||||
project: "project",
|
||||
service: "service",
|
||||
spec: {
|
||||
architecture: "amd64",
|
||||
image: "registry.test/app:latest",
|
||||
context: ".",
|
||||
buildArgs: [],
|
||||
workspace,
|
||||
},
|
||||
};
|
||||
const status = {
|
||||
version: BUILD_PROTOCOL_VERSION,
|
||||
id,
|
||||
state: "queued" as const,
|
||||
createdAt,
|
||||
};
|
||||
return {
|
||||
apiVersion: BUILD_RECORD_API_VERSION,
|
||||
kind: "BuildRecord",
|
||||
metadata: {
|
||||
name: id,
|
||||
resourceVersion: "1",
|
||||
creationTimestamp: createdAt,
|
||||
labels: { project: "project", service: "service" },
|
||||
},
|
||||
spec: { request, imageKey, jobName: `job-${id}`, workspaceSubPath: id },
|
||||
status: {
|
||||
...status,
|
||||
logBytes: 0,
|
||||
logOffset: 0,
|
||||
nextSequence: 1,
|
||||
events: [{ type: "status", status }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
type ConfigMap = Parameters<BuildObjectApi["create"]>[0];
|
||||
type DeleteOptions = Pick<V1DeleteOptions, "preconditions">;
|
||||
|
||||
class FakeObjects implements BuildObjectApi {
|
||||
readonly maps = new Map<string, ConfigMap>();
|
||||
readonly selectors: string[] = [];
|
||||
readonly deletes: Array<{ name: string; options?: DeleteOptions }> = [];
|
||||
onLockRead?: () => void;
|
||||
private lockRead = false;
|
||||
|
||||
async create(value: ConfigMap): Promise<unknown> {
|
||||
if (
|
||||
Object.values(value.metadata.labels ?? {}).some(
|
||||
(label) => !validLabelValue(label),
|
||||
)
|
||||
)
|
||||
throw { code: 422 };
|
||||
if (this.maps.has(value.metadata.name)) throw { code: 409 };
|
||||
this.maps.set(
|
||||
value.metadata.name,
|
||||
structuredClone({
|
||||
...value,
|
||||
metadata: { ...value.metadata, resourceVersion: "1" },
|
||||
}),
|
||||
);
|
||||
return value;
|
||||
}
|
||||
|
||||
async read(value: ConfigMap): Promise<unknown> {
|
||||
const found = this.maps.get(value.metadata.name);
|
||||
if (!found) throw { code: 404 };
|
||||
if (value.metadata.name.startsWith("build-lock-") && !this.lockRead) {
|
||||
this.lockRead = true;
|
||||
this.onLockRead?.();
|
||||
}
|
||||
return structuredClone(found);
|
||||
}
|
||||
|
||||
async replace(value: ConfigMap): Promise<unknown> {
|
||||
if (
|
||||
Object.values(value.metadata.labels ?? {}).some(
|
||||
(label) => !validLabelValue(label),
|
||||
)
|
||||
)
|
||||
throw { code: 422 };
|
||||
const current = this.maps.get(value.metadata.name);
|
||||
if (
|
||||
!current ||
|
||||
current.metadata.resourceVersion !== value.metadata.resourceVersion
|
||||
)
|
||||
throw { code: 409 };
|
||||
const next = structuredClone({
|
||||
...value,
|
||||
metadata: {
|
||||
...value.metadata,
|
||||
resourceVersion: String(Number(current.metadata.resourceVersion) + 1),
|
||||
},
|
||||
});
|
||||
this.maps.set(value.metadata.name, next);
|
||||
return next;
|
||||
}
|
||||
|
||||
async delete(value: ConfigMap, options?: DeleteOptions): Promise<unknown> {
|
||||
const current = this.maps.get(value.metadata.name);
|
||||
this.deletes.push({ name: value.metadata.name, options });
|
||||
if (!current) throw { code: 404 };
|
||||
if (
|
||||
options?.preconditions?.resourceVersion !==
|
||||
current.metadata.resourceVersion
|
||||
)
|
||||
throw { code: 409 };
|
||||
this.maps.delete(value.metadata.name);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async list(
|
||||
_apiVersion: string,
|
||||
_kind: string,
|
||||
_namespace?: string,
|
||||
_pretty?: string,
|
||||
_exact?: boolean,
|
||||
_exportValue?: boolean,
|
||||
_fieldSelector?: string,
|
||||
labelSelector?: string,
|
||||
): Promise<{ items: unknown[] }> {
|
||||
this.selectors.push(labelSelector ?? "");
|
||||
const labels = Object.fromEntries(
|
||||
(labelSelector ?? "")
|
||||
.split(",")
|
||||
.filter(Boolean)
|
||||
.map((part) => part.split("=")),
|
||||
);
|
||||
return {
|
||||
items: [...this.maps.values()].filter((item) =>
|
||||
Object.entries(labels).every(
|
||||
([key, value]) => item.metadata.labels?.[key] === value,
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function store(fake: FakeObjects): KubernetesBuildStore {
|
||||
return new KubernetesBuildStore(
|
||||
fake,
|
||||
namespace,
|
||||
"/tmp/kuber-build-store-test",
|
||||
);
|
||||
}
|
||||
|
||||
function configMap(record: BuildRecord, imageLabel = true): ConfigMap {
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "ConfigMap",
|
||||
metadata: {
|
||||
name: name("build", record.metadata.name),
|
||||
namespace,
|
||||
resourceVersion: record.metadata.resourceVersion,
|
||||
labels: {
|
||||
"kuber.astrxl.dev/type": "build",
|
||||
...(imageLabel && {
|
||||
"kuber.astrxl.dev/image": createHash("sha256")
|
||||
.update(record.spec.imageKey)
|
||||
.digest("hex")
|
||||
.slice(0, 63),
|
||||
}),
|
||||
},
|
||||
},
|
||||
data: { payload: JSON.stringify(record) },
|
||||
};
|
||||
}
|
||||
|
||||
describe("KubernetesBuildStore", () => {
|
||||
test("creates records with valid, deterministic image index labels", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const record = build("validated", "2026-09-02T00:00:00.000Z");
|
||||
const result = await store(fake).createBuild(record);
|
||||
const labels = fake.maps.get(name("build", record.metadata.name))?.metadata
|
||||
.labels;
|
||||
|
||||
expect(result.created).toBe(true);
|
||||
expect(labels).toBeDefined();
|
||||
expect(Object.values(labels ?? {}).every(validLabelValue)).toBe(true);
|
||||
expect(labels?.["kuber.astrxl.dev/image"]).toBe(
|
||||
createHash("sha256").update(imageKey).digest("hex").slice(0, 63),
|
||||
);
|
||||
});
|
||||
|
||||
test("uses the image label fast path", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const result = await store(fake).createBuild(
|
||||
build("old", "2026-09-02T00:00:00.000Z"),
|
||||
);
|
||||
expect(result.created).toBe(true);
|
||||
expect(
|
||||
fake.selectors.some((selector) =>
|
||||
selector.includes("kuber.astrxl.dev/image="),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("finds legacy records through the bounded project/service fallback", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const legacy = build(
|
||||
"5ef44ce9-f087-4b65-8ff1-2a60dab9112e",
|
||||
"2026-09-02T00:00:00.000Z",
|
||||
);
|
||||
fake.maps.set(
|
||||
name("build", legacy.metadata.name),
|
||||
configMap(legacy, false),
|
||||
);
|
||||
expect(
|
||||
fake.maps.get(name("build", legacy.metadata.name))?.metadata.labels,
|
||||
).toEqual({
|
||||
"kuber.astrxl.dev/type": "build",
|
||||
});
|
||||
const result = await store(fake).createBuild(
|
||||
build("replacement", "2026-09-02T00:00:01.000Z"),
|
||||
);
|
||||
expect(
|
||||
(await store(fake).getBuild("5ef44ce9-f087-4b65-8ff1-2a60dab9112e"))
|
||||
?.status.state,
|
||||
).toBe("failed");
|
||||
expect(fake.selectors).toContain("kuber.astrxl.dev/type=build");
|
||||
});
|
||||
|
||||
test("returns every older record superseded during reconciliation", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const kuber = store(fake);
|
||||
const oldOne = build("old-one", "2026-09-02T00:00:00.000Z");
|
||||
const oldTwo = build("old-two", "2026-09-02T00:00:01.000Z");
|
||||
fake.maps.set(name("build", oldOne.metadata.name), configMap(oldOne));
|
||||
fake.maps.set(name("build", oldTwo.metadata.name), configMap(oldTwo));
|
||||
|
||||
const result = await kuber.createBuild(
|
||||
build("replacement", "2026-09-02T00:00:02.000Z"),
|
||||
);
|
||||
|
||||
expect(result.created).toBe(true);
|
||||
expect(result.superseded?.map((record) => record.metadata.name)).toEqual([
|
||||
"old-one",
|
||||
"old-two",
|
||||
]);
|
||||
});
|
||||
|
||||
test("returns created false when an older candidate is already superseded", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const kuber = store(fake);
|
||||
await kuber.createBuild(build("newer", "2026-09-02T00:00:01.000Z"));
|
||||
|
||||
const result = await kuber.createBuild(
|
||||
build("older", "2026-09-02T00:00:00.000Z"),
|
||||
);
|
||||
|
||||
expect(result.created).toBe(false);
|
||||
expect(result.record.status.error).toBe("Superseded by newer build");
|
||||
});
|
||||
|
||||
test("orders and reconciles legacy records with invalid metadata timestamps", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const kuber = store(fake);
|
||||
const legacy = build("legacy", "2026-09-02T00:00:00.000Z");
|
||||
legacy.metadata.creationTimestamp = undefined as unknown as string;
|
||||
fake.maps.set(name("build", legacy.metadata.name), configMap(legacy));
|
||||
|
||||
const replacement = await kuber.createBuild(
|
||||
build("replacement", "2026-09-02T00:00:01.000Z"),
|
||||
);
|
||||
expect(replacement.created).toBe(true);
|
||||
expect((await kuber.getBuild("legacy"))?.status.state).toBe("failed");
|
||||
|
||||
const malformed = build("a", "2026-09-02T00:00:00.000Z");
|
||||
malformed.metadata.creationTimestamp = 0 as unknown as string;
|
||||
fake.maps.set(name("build", malformed.metadata.name), configMap(malformed));
|
||||
expect(
|
||||
(await kuber.listBuilds()).map((record) => record.metadata.name),
|
||||
).toEqual(["a", "legacy", "replacement"]);
|
||||
});
|
||||
|
||||
test("recovers an orphan record and takes over its missing lock", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const kuber = store(fake);
|
||||
const orphan = build("orphan", "2026-09-02T00:00:00.000Z");
|
||||
fake.maps.set(
|
||||
name("build", orphan.metadata.name),
|
||||
configMap(orphan, false),
|
||||
);
|
||||
fake.maps.delete(name("build-lock", imageKey));
|
||||
const result = await kuber.createBuild(
|
||||
build("replacement", "2026-09-02T00:00:01.000Z"),
|
||||
);
|
||||
expect((await kuber.getBuild("orphan"))?.status.state).toBe("failed");
|
||||
expect(await kuber.ownsBuild(imageKey, "replacement")).toBe(true);
|
||||
});
|
||||
|
||||
test("does not delete a successor lock after a read/delete race", async () => {
|
||||
const fake = new FakeObjects();
|
||||
const kuber = store(fake);
|
||||
await kuber.createBuild(build("old", "2026-09-02T00:00:00.000Z"));
|
||||
fake.onLockRead = () => {
|
||||
const lock = fake.maps.get(name("build-lock", imageKey))!;
|
||||
fake.maps.set(lock.metadata.name, {
|
||||
...lock,
|
||||
data: { payload: JSON.stringify({ buildId: "successor" }) },
|
||||
metadata: { ...lock.metadata, resourceVersion: "2" },
|
||||
});
|
||||
};
|
||||
const old = (await kuber.getBuild("old"))!;
|
||||
old.status.state = "succeeded";
|
||||
await kuber.replaceBuild(old, "1");
|
||||
expect(await kuber.ownsBuild(imageKey, "successor")).toBe(true);
|
||||
expect(fake.maps.has(name("build-lock", imageKey))).toBe(true);
|
||||
expect(fake.deletes.at(-1)?.options?.preconditions?.resourceVersion).toBe(
|
||||
"1",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -13,7 +13,11 @@ import {
|
||||
|
||||
const workspace = `sha256:${"a".repeat(64)}` as Sha256Digest;
|
||||
|
||||
function build(id: string, imageKey = "project\0service\0image"): BuildRecord {
|
||||
function build(
|
||||
id: string,
|
||||
imageKey = "project\0service\0image",
|
||||
createdAt = "2026-09-02T00:00:00.000Z",
|
||||
): BuildRecord {
|
||||
const request: BuildRequest = {
|
||||
version: BUILD_PROTOCOL_VERSION,
|
||||
id,
|
||||
@@ -31,7 +35,7 @@ function build(id: string, imageKey = "project\0service\0image"): BuildRecord {
|
||||
version: BUILD_PROTOCOL_VERSION,
|
||||
id,
|
||||
state: "queued" as const,
|
||||
createdAt: "2026-09-02T00:00:00.000Z",
|
||||
createdAt,
|
||||
};
|
||||
return {
|
||||
apiVersion: BUILD_RECORD_API_VERSION,
|
||||
@@ -61,25 +65,28 @@ describe("build store", () => {
|
||||
await expect(
|
||||
store.createBuild(build("one", "different-key")),
|
||||
).rejects.toBeInstanceOf(BuildStoreConflictError);
|
||||
await expect(store.createBuild(build("two"))).rejects.toBeInstanceOf(
|
||||
BuildStoreConflictError,
|
||||
);
|
||||
const replacement = await store.createBuild(build("two"));
|
||||
expect(replacement.created).toBe(true);
|
||||
expect(replacement.superseded?.[0]?.metadata.name).toBe("one");
|
||||
|
||||
const first = (await store.getBuild("one"))!;
|
||||
const digest = `sha256:${"b".repeat(64)}` as Sha256Digest;
|
||||
first.metadata.resourceVersion = "2";
|
||||
first.metadata.resourceVersion = "3";
|
||||
Object.assign(first.status, {
|
||||
state: "succeeded",
|
||||
digest,
|
||||
finishedAt: first.status.createdAt,
|
||||
});
|
||||
await store.replaceBuild(first, "1");
|
||||
expect((await store.createBuild(build("two"))).created).toBe(true);
|
||||
await store.replaceBuild(first, "2");
|
||||
expect(
|
||||
(await store.createBuild(build("three", "2026-09-02T00:00:02.000Z")))
|
||||
.created,
|
||||
).toBe(true);
|
||||
|
||||
const changed = (await store.getBuild("one"))!;
|
||||
changed.metadata.resourceVersion = "3";
|
||||
changed.status.digest = `sha256:${"c".repeat(64)}`;
|
||||
await expect(store.replaceBuild(changed, "2")).rejects.toThrow("immutable");
|
||||
await expect(store.replaceBuild(changed, "3")).rejects.toThrow("immutable");
|
||||
});
|
||||
|
||||
test("optimistically updates resumable upload records without leaking mutable data", async () => {
|
||||
@@ -107,4 +114,117 @@ describe("build store", () => {
|
||||
BuildStoreConflictError,
|
||||
);
|
||||
});
|
||||
|
||||
test("supersedes only the current image and leaves terminal history", async () => {
|
||||
const store = new MemoryBuildStore();
|
||||
await store.createBuild(
|
||||
build("old", "project\0service\0image", "2026-09-02T00:00:00.000Z"),
|
||||
);
|
||||
const result = await store.createBuild(
|
||||
build("new", "project\0service\0image", "2026-09-02T00:00:01.000Z"),
|
||||
);
|
||||
expect(result.superseded?.[0]?.status.error).toBe(
|
||||
"Superseded by newer build",
|
||||
);
|
||||
expect((await store.getBuild("old"))?.status.state).toBe("failed");
|
||||
expect((await store.createBuild(build("new"))).created).toBe(false);
|
||||
expect(
|
||||
(await store.createBuild(build("other", "project\0service\0other")))
|
||||
.created,
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("concurrent same-image submissions have one active record", async () => {
|
||||
const store = new MemoryBuildStore();
|
||||
const results = await Promise.all([
|
||||
store.createBuild(build("first")),
|
||||
store.createBuild(build("second")),
|
||||
store.createBuild(build("third")),
|
||||
]);
|
||||
expect(results.filter((result) => result.created)).toHaveLength(3);
|
||||
const active = (await store.listBuilds()).filter(
|
||||
(record) =>
|
||||
record.status.state !== "succeeded" && record.status.state !== "failed",
|
||||
);
|
||||
expect(active).toHaveLength(1);
|
||||
expect(active[0]?.metadata.name).toBe("third");
|
||||
});
|
||||
|
||||
test("recovers an active record whose in-memory lock was lost", async () => {
|
||||
const store = new MemoryBuildStore();
|
||||
await store.createBuild(build("orphan"));
|
||||
(store as unknown as { active: Map<string, string> }).active.clear();
|
||||
|
||||
const replacement = await store.createBuild(build("replacement"));
|
||||
expect(replacement.superseded?.[0]?.metadata.name).toBe("orphan");
|
||||
expect((await store.getBuild("orphan"))?.status.state).toBe("failed");
|
||||
expect(
|
||||
await store.ownsBuild("project\0service\0image", "replacement"),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("a delayed older retry cannot reclaim a newer same-image lock", async () => {
|
||||
const store = new MemoryBuildStore();
|
||||
await store.createBuild(
|
||||
build("a", "project\0service\0image", "2026-09-02T00:00:00.000Z"),
|
||||
);
|
||||
(store as unknown as { active: Map<string, string> }).active.clear();
|
||||
await store.createBuild(
|
||||
build("b", "project\0service\0image", "2026-09-02T00:00:01.000Z"),
|
||||
);
|
||||
|
||||
const delayed = await store.createBuild(
|
||||
build("a", "project\0service\0image", "2026-09-02T00:00:00.000Z"),
|
||||
);
|
||||
expect(delayed.created).toBe(false);
|
||||
expect(delayed.record.status.error).toBe("Superseded by newer build");
|
||||
expect(await store.ownsBuild("project\0service\0image", "b")).toBe(true);
|
||||
});
|
||||
|
||||
test("returns cleanup metadata when a delayed candidate loses to a newer record", async () => {
|
||||
const store = new MemoryBuildStore();
|
||||
await store.createBuild(
|
||||
build("newer", "project\0service\0image", "2026-09-02T00:00:01.000Z"),
|
||||
);
|
||||
|
||||
const result = await store.createBuild(
|
||||
build("delayed", "project\0service\0image", "2026-09-02T00:00:00.000Z"),
|
||||
);
|
||||
|
||||
expect(result.created).toBe(false);
|
||||
expect(result.superseded?.map((record) => record.metadata.name)).toEqual([
|
||||
"delayed",
|
||||
]);
|
||||
expect(
|
||||
(await store.createBuild(build("delayed"))).superseded,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
test("orders and reconciles legacy records with invalid metadata timestamps", async () => {
|
||||
const store = new MemoryBuildStore();
|
||||
const legacy = build(
|
||||
"legacy",
|
||||
"project\0service\0image",
|
||||
"2026-09-02T00:00:00.000Z",
|
||||
);
|
||||
legacy.metadata.creationTimestamp = undefined as unknown as string;
|
||||
await store.createBuild(legacy);
|
||||
|
||||
const replacement = await store.createBuild(
|
||||
build(
|
||||
"replacement",
|
||||
"project\0service\0image",
|
||||
"2026-09-02T00:00:01.000Z",
|
||||
),
|
||||
);
|
||||
expect(replacement.created).toBe(true);
|
||||
expect((await store.getBuild("legacy"))?.status.state).toBe("failed");
|
||||
|
||||
const malformed = build("a", "project\0service\0other");
|
||||
malformed.metadata.creationTimestamp = 0 as unknown as string;
|
||||
await store.createBuild(malformed);
|
||||
expect(
|
||||
(await store.listBuilds()).map((record) => record.metadata.name),
|
||||
).toEqual(["a", "legacy", "replacement"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -155,7 +155,9 @@ function connection(role: "viewer" | "operator" | "admin" = "operator") {
|
||||
} as ExecConnection;
|
||||
}
|
||||
|
||||
async function authenticatedStore(role: "viewer" | "operator" | "admin" = "operator") {
|
||||
async function authenticatedStore(
|
||||
role: "viewer" | "operator" | "admin" = "operator",
|
||||
) {
|
||||
const store = new MemoryAuthStore();
|
||||
await store.putUser({ username: role, passwordHash: "hash", roles: [role] });
|
||||
await store.putSession({
|
||||
@@ -176,7 +178,9 @@ function request(path = "/api/v2/workspaces/shop/exec") {
|
||||
describe("authorizeExecConnection", () => {
|
||||
test("resolves the workspace UID server-side for authorized operators", async () => {
|
||||
const store = await authenticatedStore("operator");
|
||||
const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-1" });
|
||||
const workspaceStore = new MemoryWorkspaceStore({
|
||||
uid: () => "workspace-1",
|
||||
});
|
||||
await workspaceStore.create({
|
||||
id: "shop",
|
||||
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||
@@ -285,7 +289,9 @@ describe("WireExecSession", () => {
|
||||
createExecService(new FakeBackend()),
|
||||
connection(),
|
||||
);
|
||||
await session.receive(JSON.stringify({ type: "resize", columns: 1, rows: 1 }));
|
||||
await session.receive(
|
||||
JSON.stringify({ type: "resize", columns: 1, rows: 1 }),
|
||||
);
|
||||
expect(JSON.parse(socket.sent[0]!)).toMatchObject({
|
||||
type: "error",
|
||||
code: "EXEC_INVALID",
|
||||
|
||||
@@ -172,9 +172,10 @@ describe("Kubernetes state persistence", () => {
|
||||
);
|
||||
const first = await persistence.createIdempotent(operation("first"));
|
||||
const second = await persistence.createIdempotent(operation("second"));
|
||||
expect(second).toEqual(first);
|
||||
expect(first.created).toBe(true);
|
||||
expect(second).toEqual({ operation: first.operation, created: false });
|
||||
expect(fake.objects.size).toBe(1);
|
||||
expect(first.metadata.name).toBe(
|
||||
expect(first.operation.metadata.name).toBe(
|
||||
`operation-${createHash("sha256")
|
||||
.update("demo\0same-key")
|
||||
.digest("hex")
|
||||
@@ -239,7 +240,7 @@ describe("Kubernetes state persistence", () => {
|
||||
item.metadata?.labels?.[WORKSPACE_UID_LABEL] === "workspace-uid",
|
||||
),
|
||||
).toBe(true);
|
||||
await expect(adoption.adopt("kuber-system", "uid")).rejects.toThrow(
|
||||
await expect(adoption.adopt("kube-system", "uid")).rejects.toThrow(
|
||||
"reserved",
|
||||
);
|
||||
});
|
||||
@@ -302,7 +303,10 @@ describe("Kubernetes state persistence", () => {
|
||||
});
|
||||
expect(fake.patches).toHaveLength(2);
|
||||
const [namespacePatch, resourcePatch] = fake.patches;
|
||||
expect(namespacePatch).toMatchObject({ apiVersion: "v1", kind: "Namespace" });
|
||||
expect(namespacePatch).toMatchObject({
|
||||
apiVersion: "v1",
|
||||
kind: "Namespace",
|
||||
});
|
||||
expect(resourcePatch).toMatchObject({
|
||||
apiVersion: "apps/v1",
|
||||
kind: "Deployment",
|
||||
@@ -314,6 +318,7 @@ describe("Kubernetes state persistence", () => {
|
||||
class FakeLeaseStore implements LeaseObjects {
|
||||
readonly leases = new Map<string, V1Lease>();
|
||||
private rv = 0;
|
||||
beforeDelete?: () => void;
|
||||
|
||||
private key(name: string, namespace: string) {
|
||||
return `${namespace}/${name}`;
|
||||
@@ -360,8 +365,18 @@ class FakeLeaseStore implements LeaseObjects {
|
||||
return structuredClone(stored);
|
||||
}
|
||||
|
||||
async delete(name: string, namespace: string) {
|
||||
this.leases.delete(this.key(name, namespace));
|
||||
async delete(name: string, namespace: string, expectedResourceVersion?: string) {
|
||||
this.beforeDelete?.();
|
||||
this.beforeDelete = undefined;
|
||||
const key = this.key(name, namespace);
|
||||
const current = this.leases.get(key);
|
||||
if (!current) throw { code: 404 };
|
||||
if (
|
||||
expectedResourceVersion &&
|
||||
current.metadata?.resourceVersion !== expectedResourceVersion
|
||||
)
|
||||
throw { code: 409 };
|
||||
this.leases.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -374,9 +389,7 @@ describe("Kubernetes workspace lease provider", () => {
|
||||
expect(lease!.workspaceId).toBe("demo");
|
||||
expect(lease!.holder).toBe("worker-a");
|
||||
expect(fake.leases.size).toBe(1);
|
||||
expect(
|
||||
await provider.acquire("demo", "worker-b", 1000),
|
||||
).toBeUndefined();
|
||||
expect(await provider.acquire("demo", "worker-b", 1000)).toBeUndefined();
|
||||
});
|
||||
|
||||
test("renews optimistically and refuses after expiry or holder change", async () => {
|
||||
@@ -428,6 +441,36 @@ describe("Kubernetes workspace lease provider", () => {
|
||||
expect(await provider.acquire("demo", "worker-b", 1000)).toBeDefined();
|
||||
});
|
||||
|
||||
test("stale release cannot delete a successor that takes over after expiry", async () => {
|
||||
const fake = new FakeLeaseStore();
|
||||
let now = 0;
|
||||
const provider = new KubernetesWorkspaceLeaseProvider(
|
||||
fake,
|
||||
"kuber-system",
|
||||
() => now,
|
||||
);
|
||||
const stale = await provider.acquire("demo", "worker-a", 1000);
|
||||
now = 1500;
|
||||
fake.beforeDelete = () => {
|
||||
const current = [...fake.leases.values()][0]!;
|
||||
fake.leases.set(
|
||||
`${current.metadata!.namespace}/${current.metadata!.name}`,
|
||||
{
|
||||
...current,
|
||||
metadata: {
|
||||
...current.metadata,
|
||||
resourceVersion: "successor-version",
|
||||
},
|
||||
spec: { ...current.spec, holderIdentity: "worker-b" },
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
await stale!.release();
|
||||
|
||||
expect([...fake.leases.values()][0]?.spec?.holderIdentity).toBe("worker-b");
|
||||
});
|
||||
|
||||
test("writes acquireTime and renewTime as microsecond MicroTime strings", async () => {
|
||||
const fake = new FakeLeaseStore();
|
||||
const provider = new KubernetesWorkspaceLeaseProvider(
|
||||
@@ -436,8 +479,7 @@ describe("Kubernetes workspace lease provider", () => {
|
||||
() => Date.parse("2026-09-03T00:23:00.205Z"),
|
||||
);
|
||||
const lease = await provider.acquire("demo", "worker-a", 1000);
|
||||
const microRegex =
|
||||
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/;
|
||||
const microRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/;
|
||||
for (const stored of fake.leases.values()) {
|
||||
expect(String(stored.spec?.acquireTime)).toMatch(microRegex);
|
||||
expect(String(stored.spec?.renewTime)).toMatch(microRegex);
|
||||
@@ -505,7 +547,9 @@ class TakeoverContentionStore implements LeaseObjects {
|
||||
spec: {
|
||||
holderIdentity: "contender",
|
||||
leaseDurationSeconds: 1,
|
||||
renewTime: new Date(this.now - 5000).toISOString() as unknown as V1LeaseSpec["renewTime"],
|
||||
renewTime: new Date(
|
||||
this.now - 5000,
|
||||
).toISOString() as unknown as V1LeaseSpec["renewTime"],
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -557,7 +601,9 @@ class SustainedContentionStore implements LeaseObjects {
|
||||
spec: {
|
||||
holderIdentity: "contender",
|
||||
leaseDurationSeconds: 1,
|
||||
renewTime: new Date(this.now - 5000).toISOString() as unknown as V1LeaseSpec["renewTime"],
|
||||
renewTime: new Date(
|
||||
this.now - 5000,
|
||||
).toISOString() as unknown as V1LeaseSpec["renewTime"],
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -671,7 +717,9 @@ function replicaSet(
|
||||
metadata: {
|
||||
name,
|
||||
namespace,
|
||||
...(hasManagedBy && { labels: { ...labels, "app.kubernetes.io/managed-by": "kuber" } }),
|
||||
...(hasManagedBy && {
|
||||
labels: { ...labels, "app.kubernetes.io/managed-by": "kuber" },
|
||||
}),
|
||||
...(!hasManagedBy && Object.keys(labels).length && { labels }),
|
||||
annotations: {
|
||||
"deployment.kubernetes.io/revision": String(revisionNumber),
|
||||
@@ -682,7 +730,9 @@ function replicaSet(
|
||||
},
|
||||
spec: {
|
||||
template: {
|
||||
metadata: { labels: { app: name, "pod-template-hash": `hash${revisionNumber}` } },
|
||||
metadata: {
|
||||
labels: { app: name, "pod-template-hash": `hash${revisionNumber}` },
|
||||
},
|
||||
spec: { containers: [{ name: "app", image }] },
|
||||
},
|
||||
},
|
||||
@@ -721,13 +771,17 @@ class FakeApps {
|
||||
|
||||
async listNamespacedDeployment({ namespace }: { namespace: string }) {
|
||||
return {
|
||||
items: this.deployments.filter((d) => d.metadata?.namespace === namespace),
|
||||
items: this.deployments.filter(
|
||||
(d) => d.metadata?.namespace === namespace,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
async listNamespacedReplicaSet({ namespace }: { namespace: string }) {
|
||||
return {
|
||||
items: this.replicaSets.filter((rs) => rs.metadata?.namespace === namespace),
|
||||
items: this.replicaSets.filter(
|
||||
(rs) => rs.metadata?.namespace === namespace,
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -819,7 +873,15 @@ describe("rollback ReplicaSet discovery", () => {
|
||||
replicaSet("web-2", "demo", depUid, 2, "img:v2", { app: "web" }),
|
||||
// An unrelated RS that happens to carry the managed-by label (e.g. a
|
||||
// standalone non-Deployment object) must still be ignored.
|
||||
replicaSet("standalone", "demo", "other-uid", 1, "img:standalone", { app: "standalone" }, true),
|
||||
replicaSet(
|
||||
"standalone",
|
||||
"demo",
|
||||
"other-uid",
|
||||
1,
|
||||
"img:standalone",
|
||||
{ app: "standalone" },
|
||||
true,
|
||||
),
|
||||
];
|
||||
const deps = createKubernetesManagementDependencies(
|
||||
managementClients(deployments, replicaSets),
|
||||
|
||||
@@ -233,13 +233,22 @@ describe("server management service", () => {
|
||||
dependencies({
|
||||
listDeployments: async () =>
|
||||
["api", "worker", "batch"].map(
|
||||
(name) =>
|
||||
object("Deployment", name, `${name}-uid`) as V1Deployment,
|
||||
(name) => object("Deployment", name, `${name}-uid`) as V1Deployment,
|
||||
),
|
||||
listProjectResources: async () => [
|
||||
object("HorizontalPodAutoscaler", "api", "api-hpa-uid", undefined),
|
||||
object("HorizontalPodAutoscaler", "worker", "worker-hpa-uid", undefined),
|
||||
object("HorizontalPodAutoscaler", "batch", "batch-hpa-uid", undefined),
|
||||
object(
|
||||
"HorizontalPodAutoscaler",
|
||||
"worker",
|
||||
"worker-hpa-uid",
|
||||
undefined,
|
||||
),
|
||||
object(
|
||||
"HorizontalPodAutoscaler",
|
||||
"batch",
|
||||
"batch-hpa-uid",
|
||||
undefined,
|
||||
),
|
||||
],
|
||||
scaleDeployment: async (_project, name, replicas) => {
|
||||
calls.push(`scale:${name}:${replicas}`);
|
||||
@@ -268,13 +277,22 @@ describe("server management service", () => {
|
||||
dependencies({
|
||||
listDeployments: async () =>
|
||||
["api", "worker"].map(
|
||||
(name) =>
|
||||
object("Deployment", name, `${name}-uid`) as V1Deployment,
|
||||
(name) => object("Deployment", name, `${name}-uid`) as V1Deployment,
|
||||
),
|
||||
listProjectResources: async () => [
|
||||
object("HorizontalPodAutoscaler", "api", "api-hpa-uid", undefined),
|
||||
object("HorizontalPodAutoscaler", "worker", "worker-hpa-uid", undefined),
|
||||
object("HorizontalPodAutoscaler", "legacy", "legacy-hpa-uid", undefined),
|
||||
object(
|
||||
"HorizontalPodAutoscaler",
|
||||
"worker",
|
||||
"worker-hpa-uid",
|
||||
undefined,
|
||||
),
|
||||
object(
|
||||
"HorizontalPodAutoscaler",
|
||||
"legacy",
|
||||
"legacy-hpa-uid",
|
||||
undefined,
|
||||
),
|
||||
],
|
||||
scaleDeployment: async (_project, name, replicas) => {
|
||||
calls.push(`scale:${name}:${replicas}`);
|
||||
@@ -297,8 +315,9 @@ describe("server management service", () => {
|
||||
test("refuses to delete an HPA not owned by the workspace", async () => {
|
||||
const service = createManagementService(
|
||||
dependencies({
|
||||
listDeployments: async () =>
|
||||
[object("Deployment", "api", "api-uid") as V1Deployment],
|
||||
listDeployments: async () => [
|
||||
object("Deployment", "api", "api-uid") as V1Deployment,
|
||||
],
|
||||
listProjectResources: async () => [
|
||||
object("HorizontalPodAutoscaler", "api", "api-hpa-uid", {
|
||||
"app.kubernetes.io/managed-by": "kuber",
|
||||
@@ -319,8 +338,7 @@ describe("server management service", () => {
|
||||
dependencies({
|
||||
listDeployments: async () =>
|
||||
["api", "worker"].map(
|
||||
(name) =>
|
||||
object("Deployment", name, `${name}-uid`) as V1Deployment,
|
||||
(name) => object("Deployment", name, `${name}-uid`) as V1Deployment,
|
||||
),
|
||||
listProjectResources: async () => [
|
||||
object("ConfigMap", "env", "env-uid"),
|
||||
@@ -351,4 +369,16 @@ describe("server management service", () => {
|
||||
).rejects.toThrow("reserved");
|
||||
expect(read).toBe(false);
|
||||
});
|
||||
|
||||
test("allows the kuber-system workspace to manage itself", async () => {
|
||||
const service = createManagementService(
|
||||
dependencies({
|
||||
readNamespace: async () => ({ uid: "namespace-uid" }),
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
service.namespaceSafety({ project: "kuber-system", uid: "workspace-1" }),
|
||||
).resolves.toMatchObject({ project: "kuber-system", status: "external" });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -26,6 +26,44 @@ function digest(data: Uint8Array | string): Sha256Digest {
|
||||
}
|
||||
|
||||
describe("source materialization", () => {
|
||||
test("bounds concurrent CAS reads while materializing many files", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "kuber-materialize-"));
|
||||
roots.push(root);
|
||||
const data = Buffer.from("x");
|
||||
const blobDigest = digest(data);
|
||||
const manifest: WorkspaceManifest = {
|
||||
version: BUILD_PROTOCOL_VERSION,
|
||||
files: Array.from({ length: 40 }, (_, index) => ({
|
||||
path: `file-${String(index).padStart(2, "0")}`,
|
||||
type: "file" as const,
|
||||
digest: blobDigest,
|
||||
size: data.byteLength,
|
||||
mode: 0o644 as const,
|
||||
})),
|
||||
};
|
||||
const manifestBytes = Buffer.from(JSON.stringify(manifest));
|
||||
const workspace = digest(manifestBytes);
|
||||
let inflight = 0;
|
||||
let maxInflight = 0;
|
||||
const read = async <T>(result: T): Promise<T> => {
|
||||
inflight += 1;
|
||||
maxInflight = Math.max(maxInflight, inflight);
|
||||
await Bun.sleep(2);
|
||||
inflight -= 1;
|
||||
return result;
|
||||
};
|
||||
const cas = {
|
||||
has: async () => read(true),
|
||||
get: async (requested: Sha256Digest) =>
|
||||
requested === workspace ? manifestBytes : read(data),
|
||||
};
|
||||
|
||||
await materializeWorkspace(cas, workspace, join(root, "workspace"));
|
||||
|
||||
expect(maxInflight).toBeGreaterThan(1);
|
||||
expect(maxInflight).toBeLessThanOrEqual(20);
|
||||
});
|
||||
|
||||
test("materializes files and safe symlinks with declared modes", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "kuber-materialize-"));
|
||||
roots.push(root);
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
OperationValidationError,
|
||||
recoverStaleOperations,
|
||||
sanitizeOperationResult,
|
||||
sanitizeOperationError,
|
||||
} from "../../server/operation-store";
|
||||
|
||||
describe("operation store", () => {
|
||||
@@ -92,10 +93,33 @@ describe("operation store", () => {
|
||||
"checkout https://[email protected]/org/repo.git",
|
||||
"AKIAIOSFODNN7EXAMPLE",
|
||||
]),
|
||||
).toEqual([
|
||||
"checkout https://[email protected]/org/repo.git",
|
||||
"[REDACTED]",
|
||||
]);
|
||||
).toEqual(["checkout https://[email protected]/org/repo.git", "[REDACTED]"]);
|
||||
});
|
||||
|
||||
test("sanitizes operation failures while preserving their codes", () => {
|
||||
expect(
|
||||
sanitizeOperationError(
|
||||
{
|
||||
code: "PROVIDER_FAILED",
|
||||
message:
|
||||
'database failed: DB_PASSWORD=database-password response={"data":{"token":"kube-secret"}}',
|
||||
},
|
||||
"deploy",
|
||||
),
|
||||
).toEqual({
|
||||
code: "PROVIDER_FAILED",
|
||||
message:
|
||||
'database failed: DB_PASSWORD=[REDACTED] response={"data":{"token":"[REDACTED]"}}',
|
||||
});
|
||||
expect(
|
||||
sanitizeOperationError(
|
||||
{ code: "RECONCILE_FAILED", message: "secret: db-password" },
|
||||
"databases.reconcile",
|
||||
),
|
||||
).toEqual({
|
||||
code: "RECONCILE_FAILED",
|
||||
message: "Database reconciliation failed",
|
||||
});
|
||||
});
|
||||
|
||||
test("enforces the operation state machine", async () => {
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import type { KubernetesObjectApi } from "@kubernetes/client-node";
|
||||
import { KubernetesTrustStore } from "../../server/kubernetes-state";
|
||||
import { MemoryTrustStore } from "../../server/trust-store";
|
||||
|
||||
type ConfigMap = {
|
||||
apiVersion: string;
|
||||
kind: string;
|
||||
metadata: {
|
||||
name: string;
|
||||
namespace?: string;
|
||||
labels?: Record<string, string>;
|
||||
};
|
||||
data?: Record<string, string>;
|
||||
};
|
||||
|
||||
class FakeObjects {
|
||||
readonly maps = new Map<string, ConfigMap>();
|
||||
readonly selectors: string[] = [];
|
||||
|
||||
async create(value: ConfigMap): Promise<void> {
|
||||
this.maps.set(value.metadata.name, structuredClone(value));
|
||||
}
|
||||
|
||||
async read(value: ConfigMap): Promise<ConfigMap> {
|
||||
const found = this.maps.get(value.metadata.name);
|
||||
if (!found) throw { code: 404 };
|
||||
return structuredClone(found);
|
||||
}
|
||||
|
||||
async delete(value: ConfigMap): Promise<void> {
|
||||
if (!this.maps.delete(value.metadata.name)) throw { code: 404 };
|
||||
}
|
||||
|
||||
async list(
|
||||
_apiVersion: string,
|
||||
_kind: string,
|
||||
_namespace?: string,
|
||||
_pretty?: string,
|
||||
_exact?: boolean,
|
||||
_exportValue?: boolean,
|
||||
_fieldSelector?: string,
|
||||
labelSelector?: string,
|
||||
): Promise<{ items: ConfigMap[] }> {
|
||||
this.selectors.push(labelSelector ?? "");
|
||||
const labels = Object.fromEntries(
|
||||
(labelSelector ?? "")
|
||||
.split(",")
|
||||
.filter(Boolean)
|
||||
.map((part) => part.split("=")),
|
||||
);
|
||||
return {
|
||||
items: [...this.maps.values()].filter((item) =>
|
||||
Object.entries(labels).every(
|
||||
([key, value]) => item.metadata.labels?.[key] === value,
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
describe("namespace trust store", () => {
|
||||
test("rejects invalid namespace and fingerprint", async () => {
|
||||
const store = new MemoryTrustStore();
|
||||
await expect(store.grant("Demo", "a".repeat(64))).rejects.toThrow();
|
||||
await expect(store.grant("a".repeat(64), "a".repeat(64))).rejects.toThrow();
|
||||
await expect(store.grant("demo", "bad")).rejects.toThrow();
|
||||
});
|
||||
|
||||
test("requires explicit registration for each CWD fingerprint", async () => {
|
||||
const store = new MemoryTrustStore();
|
||||
const first = "a".repeat(64);
|
||||
const second = "b".repeat(64);
|
||||
await store.grant("demo", first);
|
||||
expect(await store.has("demo", first)).toBe(true);
|
||||
expect(await store.has("demo", second)).toBe(false);
|
||||
await store.grant("demo", second);
|
||||
expect(await store.list("demo")).toEqual([first, second]);
|
||||
expect(await store.revoke("demo", first)).toBe(true);
|
||||
expect(await store.has("demo", first)).toBe(false);
|
||||
});
|
||||
|
||||
test("persists valid ConfigMap records and ignores malformed payloads", async () => {
|
||||
const objects = new FakeObjects();
|
||||
const store = new KubernetesTrustStore(
|
||||
objects as unknown as KubernetesObjectApi,
|
||||
);
|
||||
const fingerprint = "a".repeat(64);
|
||||
|
||||
await store.grant("demo", fingerprint);
|
||||
const [record] = [...objects.maps.values()];
|
||||
expect(record).toMatchObject({
|
||||
apiVersion: "v1",
|
||||
kind: "ConfigMap",
|
||||
metadata: {
|
||||
namespace: "kuber-system",
|
||||
labels: {
|
||||
"kuber.astrxl.dev/type": "trust",
|
||||
"kuber.astrxl.dev/workspace": "demo",
|
||||
},
|
||||
},
|
||||
data: { payload: JSON.stringify({ project: "demo", fingerprint }) },
|
||||
});
|
||||
expect(record?.metadata.name).toMatch(/^trust-[a-f0-9]{48}$/);
|
||||
expect(await store.list("demo")).toEqual([fingerprint]);
|
||||
expect(objects.selectors).toContain(
|
||||
"kuber.astrxl.dev/type=trust,kuber.astrxl.dev/workspace=demo",
|
||||
);
|
||||
|
||||
objects.maps.set("trust-malformed", {
|
||||
apiVersion: "v1",
|
||||
kind: "ConfigMap",
|
||||
metadata: {
|
||||
name: "trust-malformed",
|
||||
labels: {
|
||||
"kuber.astrxl.dev/type": "trust",
|
||||
"kuber.astrxl.dev/workspace": "demo",
|
||||
},
|
||||
},
|
||||
data: { payload: "not-json" },
|
||||
});
|
||||
objects.maps.set("trust-wrong-project", {
|
||||
apiVersion: "v1",
|
||||
kind: "ConfigMap",
|
||||
metadata: {
|
||||
name: "trust-wrong-project",
|
||||
labels: {
|
||||
"kuber.astrxl.dev/type": "trust",
|
||||
"kuber.astrxl.dev/workspace": "demo",
|
||||
},
|
||||
},
|
||||
data: {
|
||||
payload: JSON.stringify({
|
||||
project: "other",
|
||||
fingerprint: "b".repeat(64),
|
||||
}),
|
||||
},
|
||||
});
|
||||
expect(await store.list("demo")).toEqual([fingerprint]);
|
||||
|
||||
expect(await store.revoke("demo", fingerprint)).toBe(true);
|
||||
expect(await store.has("demo", fingerprint)).toBe(false);
|
||||
expect(await store.revoke("demo", fingerprint)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -16,7 +16,6 @@ describe("workspace store", () => {
|
||||
"Upper",
|
||||
"has_dot",
|
||||
"kube-public",
|
||||
"kuber-system",
|
||||
"database",
|
||||
"garage-system",
|
||||
"routing",
|
||||
@@ -25,6 +24,12 @@ describe("workspace store", () => {
|
||||
WorkspaceValidationError,
|
||||
);
|
||||
}
|
||||
|
||||
await expect(
|
||||
store.create({ id: "kuber-system", source }),
|
||||
).resolves.toMatchObject({
|
||||
metadata: { name: "kuber-system" },
|
||||
});
|
||||
});
|
||||
|
||||
test("creates immutable revisions and uses ETags for replacement", async () => {
|
||||
|
||||
Reference in New Issue
Block a user