1834 lines
56 KiB
TypeScript
1834 lines
56 KiB
TypeScript
import { describe, expect, spyOn, test } from "bun:test";
|
|
import { cleanupExpiredSessions, createApp } from "../../server/app";
|
|
import { hashToken, MemoryAuthStore } from "../../server/auth";
|
|
import { MemoryAuditStore } from "../../server/audit-store";
|
|
import type { ManagementService } from "../../server/management";
|
|
import {
|
|
MemoryOperationStore,
|
|
MemoryWorkspaceLeaseProvider,
|
|
} from "../../server/operation-store";
|
|
import { MemoryWorkspaceStore } from "../../server/workspace-store";
|
|
import { MemoryTrustStore } from "../../server/trust-store";
|
|
|
|
function request(
|
|
path: string,
|
|
init: RequestInit = {},
|
|
token?: string,
|
|
): Request {
|
|
const headers = new Headers(init.headers);
|
|
if (token) headers.set("authorization", `Bearer ${token}`);
|
|
return new Request(`https://kuber.astrxl.dev${path}`, { ...init, headers });
|
|
}
|
|
|
|
describe("kuber API authentication", () => {
|
|
test("logs every request start and completion without changing responses", async () => {
|
|
const logs: Record<string, unknown>[] = [];
|
|
const app = createApp({
|
|
store: new MemoryAuthStore(),
|
|
requestId: () => "request-log-id",
|
|
logger: {
|
|
error: () => {},
|
|
log: (entry) => logs.push(entry),
|
|
},
|
|
});
|
|
|
|
const response = await app(request("/api/v2/health"));
|
|
|
|
expect(response.status).toBe(200);
|
|
expect(logs).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
event: "kuber.server.request.start",
|
|
requestId: "request-log-id",
|
|
method: "GET",
|
|
pathname: "/api/v2/health",
|
|
}),
|
|
expect.objectContaining({
|
|
event: "kuber.server.request.end",
|
|
requestId: "request-log-id",
|
|
status: 200,
|
|
durationMs: expect.any(Number),
|
|
}),
|
|
]),
|
|
);
|
|
});
|
|
|
|
test("ignores request logging failures", async () => {
|
|
const app = createApp({
|
|
store: new MemoryAuthStore(),
|
|
logger: {
|
|
error: () => {},
|
|
log: () => {
|
|
throw new Error("logger unavailable");
|
|
},
|
|
},
|
|
});
|
|
|
|
expect((await app(request("/api/v2/health"))).status).toBe(200);
|
|
});
|
|
|
|
test("logs in, resolves identity, and revokes the session", async () => {
|
|
const store = new MemoryAuthStore();
|
|
await store.putUser({
|
|
username: "dmgnr",
|
|
passwordHash: "stored-hash",
|
|
roles: ["admin"],
|
|
});
|
|
const app = createApp({
|
|
store,
|
|
now: () => Date.parse("2026-09-02T00:00:00.000Z"),
|
|
verifyPassword: async (password, hash) =>
|
|
password === "correct" && hash === "stored-hash",
|
|
});
|
|
|
|
const login = await app(
|
|
request("/api/v2/login", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({
|
|
username: "dmgnr",
|
|
password: "correct",
|
|
persistent: false,
|
|
}),
|
|
}),
|
|
);
|
|
expect(login.status).toBe(200);
|
|
const session = (await login.json()) as {
|
|
token: string;
|
|
expiresAt: string;
|
|
};
|
|
expect(session.expiresAt).toBe("2026-09-03T00:00:00.000Z");
|
|
|
|
const me = await app(request("/api/v2/me", {}, session.token));
|
|
expect(await me.json()).toEqual({ username: "dmgnr", roles: ["admin"] });
|
|
|
|
const logout = await app(
|
|
request("/api/v2/logout", { method: "POST" }, session.token),
|
|
);
|
|
expect(logout.status).toBe(204);
|
|
expect((await app(request("/api/v2/me", {}, session.token))).status).toBe(
|
|
401,
|
|
);
|
|
});
|
|
|
|
test("rejects invalid credentials and unauthenticated requests", async () => {
|
|
const store = new MemoryAuthStore();
|
|
await store.putUser({
|
|
username: "dmgnr",
|
|
passwordHash: "stored-hash",
|
|
roles: ["admin"],
|
|
});
|
|
const app = createApp({
|
|
store,
|
|
verifyPassword: async () => false,
|
|
});
|
|
|
|
const login = await app(
|
|
request("/api/v2/login", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ username: "dmgnr", password: "wrong" }),
|
|
}),
|
|
);
|
|
expect(login.status).toBe(401);
|
|
expect((await app(request("/api/v2/me"))).status).toBe(401);
|
|
});
|
|
|
|
test("provides an unauthenticated health endpoint", async () => {
|
|
const app = createApp({ store: new MemoryAuthStore() });
|
|
const response = await app(request("/api/v2/health"));
|
|
expect(response.status).toBe(200);
|
|
expect(await response.json()).toEqual({ status: "ok" });
|
|
});
|
|
|
|
test("provides an explicit expired-session startup cleanup helper", async () => {
|
|
const store = new MemoryAuthStore();
|
|
await store.putUser({
|
|
username: "user",
|
|
passwordHash: "hash",
|
|
roles: ["viewer"],
|
|
});
|
|
await store.putSession({
|
|
tokenHash: hashToken("expired"),
|
|
username: "user",
|
|
authVersion: 1,
|
|
expiresAt: "2026-09-01T00:00:00.000Z",
|
|
});
|
|
expect(
|
|
await cleanupExpiredSessions(
|
|
store,
|
|
Date.parse("2026-09-02T00:00:00.000Z"),
|
|
),
|
|
).toBe(1);
|
|
});
|
|
|
|
test("rate limits repeated failed logins", async () => {
|
|
const app = createApp({
|
|
store: new MemoryAuthStore(),
|
|
now: () => 0,
|
|
});
|
|
const login = () =>
|
|
app(
|
|
request("/api/v2/login", {
|
|
method: "POST",
|
|
body: JSON.stringify({ username: "missing", password: "wrong" }),
|
|
}),
|
|
);
|
|
|
|
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
expect((await login()).status).toBe(401);
|
|
}
|
|
const limited = await login();
|
|
expect(limited.status).toBe(429);
|
|
expect(limited.headers.get("retry-after")).toBe("300");
|
|
});
|
|
});
|
|
|
|
describe("operation response safety", () => {
|
|
test("database and storage reconcile concurrently while conflicting scopes fail fast", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-uid" });
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore();
|
|
const leases = new MemoryWorkspaceLeaseProvider();
|
|
const entered = new Set<string>();
|
|
let release!: () => void;
|
|
const blocked = new Promise<void>((resolve) => (release = resolve));
|
|
let bothEntered!: () => void;
|
|
const bothStarted = new Promise<void>((resolve) => (bothEntered = resolve));
|
|
const reconcile = (name: string) => async () => {
|
|
entered.add(name);
|
|
if (entered.size === 2) bothEntered();
|
|
await blocked;
|
|
return {};
|
|
};
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
leases,
|
|
management: {
|
|
reconcileDatabases: reconcile("database"),
|
|
reconcileStorage: reconcile("storage"),
|
|
stop: async () => [],
|
|
} as unknown as ManagementService,
|
|
});
|
|
const post = (path: string, key: string, body: unknown = { compose: { services: {} } }) =>
|
|
app(request(`/api/v2/workspaces/demo/${path}`, {
|
|
method: "POST",
|
|
headers: { "idempotency-key": key },
|
|
body: JSON.stringify(body),
|
|
}, "token"));
|
|
|
|
const database = post("databases", "database-one");
|
|
const storage = post("storage", "storage-one");
|
|
await bothStarted;
|
|
expect(entered).toEqual(new Set(["database", "storage"]));
|
|
|
|
const sameScope = await post("databases", "database-two");
|
|
expect(sameScope.status).toBe(409);
|
|
const broad = await post("lifecycle", "lifecycle", { action: "stop" });
|
|
expect(broad.status).toBe(409);
|
|
|
|
release();
|
|
expect((await database).status).toBe(200);
|
|
expect((await storage).status).toBe(200);
|
|
});
|
|
|
|
test("redacts database and storage reconciliation results immediately", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const seen: unknown[] = [];
|
|
const databaseResult = {
|
|
credentials: { password: "database-password" },
|
|
env: [{ name: "DB_PASSWORD", value: "database-password" }],
|
|
};
|
|
const storageResult = {
|
|
credentials: { secretKey: "storage-secret" },
|
|
env: [{ name: "STORAGE_SECRET", value: "storage-secret" }],
|
|
};
|
|
const management = {
|
|
reconcileDatabases: async (_workspace: unknown, compose: unknown) => {
|
|
seen.push(compose);
|
|
return databaseResult;
|
|
},
|
|
reconcileStorage: async (_workspace: unknown, compose: unknown) => {
|
|
seen.push(compose);
|
|
return storageResult;
|
|
},
|
|
} as unknown as ManagementService;
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore: new MemoryOperationStore(undefined, () =>
|
|
crypto.randomUUID(),
|
|
),
|
|
management,
|
|
});
|
|
const compose = {
|
|
services: {},
|
|
secret: "internal-compose-secret",
|
|
};
|
|
for (const [path, key, secret] of [
|
|
["databases", "db-once", "database-password"],
|
|
["storage", "storage-once", "storage-secret"],
|
|
] as const) {
|
|
const result = await app(
|
|
request(
|
|
`/api/v2/workspaces/demo/${path}`,
|
|
{
|
|
method: "POST",
|
|
headers: { "idempotency-key": key },
|
|
body: JSON.stringify({ compose }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
expect(result.status).toBe(200);
|
|
const body = JSON.stringify(await result.json());
|
|
expect(body).not.toContain(secret);
|
|
expect(body).not.toContain("internal-compose-secret");
|
|
expect(body).toContain('"redacted":true');
|
|
}
|
|
expect(seen).toHaveLength(2);
|
|
expect(JSON.stringify(seen[0])).toContain("internal-compose-secret");
|
|
expect(databaseResult.credentials.password).toBe("database-password");
|
|
expect(storageResult.credentials.secretKey).toBe("storage-secret");
|
|
});
|
|
|
|
test("redacts failed reconciliation errors immediately and when retrieved", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore(undefined, () =>
|
|
crypto.randomUUID(),
|
|
);
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
management: {
|
|
reconcileDatabases: async () => {
|
|
throw new Error(
|
|
'database provider failed DB_PASSWORD=database-password response={"data":{"token":"kube-secret"}}',
|
|
);
|
|
},
|
|
reconcileStorage: async () => {
|
|
throw new Error(
|
|
'storage provider failed STORAGE_SECRET=storage-secret response={"data":{"password":"storage-kube-secret"}}',
|
|
);
|
|
},
|
|
} as unknown as ManagementService,
|
|
});
|
|
const path = "/api/v2/workspaces/demo/databases";
|
|
const init = {
|
|
method: "POST",
|
|
headers: { "idempotency-key": "failed-reconcile" },
|
|
body: JSON.stringify({ compose: { services: {} } }),
|
|
};
|
|
const immediate = await app(request(path, init, "token"));
|
|
const immediateBody = JSON.stringify(await immediate.json());
|
|
expect(immediate.status).toBe(500);
|
|
expect(immediateBody).not.toContain("database-password");
|
|
expect(immediateBody).not.toContain("kube-secret");
|
|
expect(immediateBody).toContain("OPERATION_FAILED");
|
|
|
|
const operationId = (await operationStore.list())[0]!.metadata.name;
|
|
const retrieved = await app(
|
|
request(`/api/v2/operations/${operationId}`, {}, "token"),
|
|
);
|
|
const retrievedBody = JSON.stringify(await retrieved.json());
|
|
expect(retrievedBody).not.toContain("database-password");
|
|
expect(retrievedBody).not.toContain("kube-secret");
|
|
|
|
const storageImmediate = await app(
|
|
request(
|
|
"/api/v2/workspaces/demo/storage",
|
|
{
|
|
...init,
|
|
headers: { "idempotency-key": "failed-storage" },
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
const storageBody = JSON.stringify(await storageImmediate.json());
|
|
expect(storageImmediate.status).toBe(500);
|
|
expect(storageBody).not.toContain("storage-secret");
|
|
expect(storageBody).not.toContain("storage-kube-secret");
|
|
const storageId = (await operationStore.list())[1]!.metadata.name;
|
|
const storageRetrieved = await app(
|
|
request(`/api/v2/operations/${storageId}`, {}, "token"),
|
|
);
|
|
expect(JSON.stringify(await storageRetrieved.json())).not.toContain(
|
|
"storage-kube-secret",
|
|
);
|
|
|
|
const idempotent = await app(request(path, init, "token"));
|
|
const idempotentBody = JSON.stringify(await idempotent.json());
|
|
expect(idempotentBody).not.toContain("database-password");
|
|
expect(idempotentBody).not.toContain("kube-secret");
|
|
});
|
|
});
|
|
|
|
async function authenticatedStore(role: "viewer" | "operator" | "admin") {
|
|
const store = new MemoryAuthStore();
|
|
await store.putUser({ username: role, passwordHash: "hash", roles: [role] });
|
|
await store.putSession({
|
|
tokenHash: hashToken("token"),
|
|
username: role,
|
|
authVersion: 1,
|
|
expiresAt: "2030-01-01T00:00:00.000Z",
|
|
});
|
|
return store;
|
|
}
|
|
|
|
describe("kuber v2 HTTP routes", () => {
|
|
test("grants, lists, revokes, and enforces namespace trust for applies", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const trustStore = new MemoryTrustStore();
|
|
const fingerprint = "a".repeat(64);
|
|
const headers = {
|
|
"x-kuber-trust-project": "demo",
|
|
"x-kuber-trust-fingerprint": fingerprint,
|
|
};
|
|
const apply = (app: ReturnType<typeof createApp>) =>
|
|
app(
|
|
request(
|
|
"/api/v2/workspaces/demo/resources/apply",
|
|
{
|
|
method: "POST",
|
|
headers,
|
|
body: JSON.stringify({ resources: [] }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
const unavailable = await createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore: new MemoryOperationStore(),
|
|
management: {
|
|
applyResources: async () => [],
|
|
} as unknown as ManagementService,
|
|
});
|
|
const unavailableApply = await apply(unavailable);
|
|
expect(unavailableApply.status).toBe(503);
|
|
expect(await unavailableApply.json()).toMatchObject({
|
|
code: "TRUST_STORE_UNAVAILABLE",
|
|
});
|
|
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore: new MemoryOperationStore(),
|
|
trustStore,
|
|
management: {
|
|
applyResources: async () => [],
|
|
} as unknown as ManagementService,
|
|
});
|
|
const untrustedApply = (headers?: RequestInit["headers"]) =>
|
|
app(
|
|
request(
|
|
"/api/v2/workspaces/demo/resources/apply",
|
|
{
|
|
method: "POST",
|
|
headers,
|
|
body: JSON.stringify({ resources: [] }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
|
|
expect((await untrustedApply()).status).toBe(428);
|
|
expect((await apply(app)).status).toBe(403);
|
|
expect(
|
|
(
|
|
await app(
|
|
request(
|
|
"/api/v2/workspaces/demo/trust",
|
|
{ method: "POST", body: JSON.stringify({ fingerprint }) },
|
|
"token",
|
|
),
|
|
)
|
|
).status,
|
|
).toBe(204);
|
|
const status = await app(
|
|
request("/api/v2/workspaces/demo/trust", {}, "token"),
|
|
);
|
|
expect(status.status).toBe(200);
|
|
expect(await status.json()).toEqual({ fingerprints: [fingerprint] });
|
|
expect((await apply(app)).status).toBe(200);
|
|
expect(
|
|
(
|
|
await app(
|
|
request(
|
|
`/api/v2/workspaces/demo/trust?fingerprint=${fingerprint}`,
|
|
{ method: "DELETE" },
|
|
"token",
|
|
),
|
|
)
|
|
).status,
|
|
).toBe(204);
|
|
expect((await apply(app)).status).toBe(403);
|
|
});
|
|
|
|
test("returns OPERATION_CONFLICT when an idempotency key is reused for another body", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const trustStore = new MemoryTrustStore();
|
|
const fingerprint = "c".repeat(64);
|
|
await trustStore.grant("demo", fingerprint);
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
trustStore,
|
|
operationStore: new MemoryOperationStore(),
|
|
management: {
|
|
applyResources: async () => [],
|
|
} as unknown as ManagementService,
|
|
});
|
|
const apply = (resources: unknown[]) =>
|
|
app(
|
|
request(
|
|
"/api/v2/workspaces/demo/resources/apply",
|
|
{
|
|
method: "POST",
|
|
headers: {
|
|
"idempotency-key": "same-apply",
|
|
"x-kuber-trust-project": "demo",
|
|
"x-kuber-trust-fingerprint": fingerprint,
|
|
},
|
|
body: JSON.stringify({ resources }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
|
|
expect((await apply([])).status).toBe(200);
|
|
const conflict = await apply([
|
|
{ apiVersion: "v1", kind: "Service", metadata: { name: "web" } },
|
|
]);
|
|
expect(conflict.status).toBe(409);
|
|
expect(await conflict.json()).toMatchObject({ code: "OPERATION_CONFLICT" });
|
|
});
|
|
|
|
test("starts preferred resource operations before returning so progress can be polled", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const trustStore = new MemoryTrustStore();
|
|
const fingerprint = "b".repeat(64);
|
|
let start!: () => void;
|
|
let finish!: () => void;
|
|
let complete!: () => void;
|
|
const started = new Promise<void>((resolve) => (start = resolve));
|
|
const unblock = new Promise<void>((resolve) => (finish = resolve));
|
|
const completed = new Promise<void>((resolve) => (complete = resolve));
|
|
const management = {
|
|
applyResources: async (
|
|
_workspace: unknown,
|
|
_resources: unknown,
|
|
execution: {
|
|
emit?: (event: {
|
|
resource: { apiVersion: string; kind: string; name: string };
|
|
phase: "apply";
|
|
state: "started" | "succeeded";
|
|
}) => Promise<void>;
|
|
},
|
|
) => {
|
|
await execution.emit?.({
|
|
resource: { apiVersion: "v1", kind: "Service", name: "web" },
|
|
phase: "apply",
|
|
state: "started",
|
|
});
|
|
start();
|
|
await unblock;
|
|
await execution.emit?.({
|
|
resource: { apiVersion: "v1", kind: "Service", name: "web" },
|
|
phase: "apply",
|
|
state: "succeeded",
|
|
});
|
|
complete();
|
|
return [];
|
|
},
|
|
} as unknown as ManagementService;
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore: new MemoryOperationStore(),
|
|
trustStore,
|
|
management,
|
|
});
|
|
await app(
|
|
request(
|
|
"/api/v2/workspaces/demo/trust",
|
|
{ method: "POST", body: JSON.stringify({ fingerprint }) },
|
|
"token",
|
|
),
|
|
);
|
|
|
|
const submitted = await app(
|
|
request(
|
|
"/api/v2/workspaces/demo/resources/apply",
|
|
{
|
|
method: "POST",
|
|
headers: {
|
|
"idempotency-key": "progress-once",
|
|
prefer: "respond-async",
|
|
"x-kuber-trust-project": "demo",
|
|
"x-kuber-trust-fingerprint": fingerprint,
|
|
},
|
|
body: JSON.stringify({ resources: [] }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
expect(submitted.status).toBe(202);
|
|
const { operationId } = (await submitted.json()) as { operationId: string };
|
|
await started;
|
|
const events = await app(
|
|
request(`/api/v2/operations/${operationId}/events?after=0`, {}, "token"),
|
|
);
|
|
expect(await events.json()).toMatchObject({
|
|
items: [{ sequence: 1, data: { state: "started" } }],
|
|
});
|
|
finish();
|
|
await completed;
|
|
});
|
|
|
|
test("uses exact origins, request IDs, and problem+json errors", async () => {
|
|
const app = createApp({
|
|
store: new MemoryAuthStore(),
|
|
allowedOrigins: ["https://console.example"],
|
|
requestId: () => "generated-id",
|
|
});
|
|
const denied = await app(
|
|
request("/api/v2/health", {
|
|
headers: { origin: "https://console.example.evil" },
|
|
}),
|
|
);
|
|
expect(denied.status).toBe(403);
|
|
expect(denied.headers.get("content-type")).toContain(
|
|
"application/problem+json",
|
|
);
|
|
expect(denied.headers.get("x-request-id")).toBe("generated-id");
|
|
expect(await denied.json()).toMatchObject({
|
|
code: "ORIGIN_NOT_ALLOWED",
|
|
requestId: "generated-id",
|
|
});
|
|
|
|
const allowed = await app(
|
|
request("/api/v2/health", {
|
|
headers: {
|
|
origin: "https://console.example",
|
|
"x-request-id": "caller-id",
|
|
},
|
|
}),
|
|
);
|
|
expect(allowed.headers.get("access-control-allow-origin")).toBe(
|
|
"https://console.example",
|
|
);
|
|
expect(allowed.headers.get("x-request-id")).toBe("caller-id");
|
|
});
|
|
|
|
test("enforces capabilities and supports user CRUD with revocation", async () => {
|
|
const viewerStore = await authenticatedStore("viewer");
|
|
const viewerApp = createApp({ store: viewerStore });
|
|
expect(
|
|
(await viewerApp(request("/api/v2/users", {}, "token"))).status,
|
|
).toBe(403);
|
|
|
|
const store = await authenticatedStore("admin");
|
|
const auditStore = new MemoryAuditStore();
|
|
const app = createApp({
|
|
store,
|
|
auditStore,
|
|
hashPassword: async (password) => `hashed:${password}`,
|
|
});
|
|
const created = await app(
|
|
request(
|
|
"/api/v2/users",
|
|
{
|
|
method: "POST",
|
|
body: JSON.stringify({
|
|
username: "alice",
|
|
password: "secret",
|
|
roles: ["operator"],
|
|
}),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
expect(created.status).toBe(201);
|
|
expect(await created.json()).toEqual({
|
|
username: "alice",
|
|
roles: ["operator"],
|
|
disabled: false,
|
|
});
|
|
expect((await store.getUser("alice"))?.passwordHash).toBe("hashed:secret");
|
|
|
|
const revoke = await app(
|
|
request(
|
|
"/api/v2/users/alice/sessions/revoke",
|
|
{ method: "POST" },
|
|
"token",
|
|
),
|
|
);
|
|
expect(revoke.status).toBe(200);
|
|
expect(await revoke.json()).toEqual({ username: "alice", revoked: 0 });
|
|
expect((await auditStore.list()).map((event) => event.spec.action)).toEqual(
|
|
["user.create", "sessions.revoke"],
|
|
);
|
|
});
|
|
|
|
test("provides workspace ETags and idempotent synchronous operations", async () => {
|
|
const store = await authenticatedStore("operator");
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
now: () => new Date("2026-09-02T00:00:00.000Z"),
|
|
});
|
|
const operationStore = new MemoryOperationStore(
|
|
() => new Date("2026-09-02T00:00:00.000Z"),
|
|
() => "operation-uid",
|
|
);
|
|
let stops = 0;
|
|
const management = {
|
|
stop: async () => {
|
|
stops += 1;
|
|
return ["api"];
|
|
},
|
|
} as unknown as ManagementService;
|
|
const app = createApp({
|
|
store,
|
|
workspaceStore,
|
|
operationStore,
|
|
management,
|
|
});
|
|
|
|
const created = await app(
|
|
request(
|
|
"/api/v2/workspaces",
|
|
{
|
|
method: "POST",
|
|
body: JSON.stringify({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
}),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
expect(created.status).toBe(201);
|
|
expect(created.headers.get("etag")).toBe('"1"');
|
|
|
|
const missingPrecondition = await app(
|
|
request(
|
|
"/api/v2/workspaces/demo",
|
|
{
|
|
method: "PUT",
|
|
body: JSON.stringify({
|
|
source: { uri: "oci://example/demo", digest: "sha256:def" },
|
|
}),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
expect(missingPrecondition.status).toBe(428);
|
|
|
|
const stop = () =>
|
|
app(
|
|
request(
|
|
"/api/v2/workspaces/demo/lifecycle",
|
|
{
|
|
method: "POST",
|
|
headers: { "idempotency-key": "stop-once" },
|
|
body: JSON.stringify({ action: "stop", services: ["api"] }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
expect((await stop()).status).toBe(200);
|
|
expect((await stop()).status).toBe(200);
|
|
expect(stops).toBe(1);
|
|
expect(await operationStore.list("demo")).toHaveLength(1);
|
|
});
|
|
|
|
test("logs raw diagnostics for correlated generic failures", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const logs: unknown[] = [];
|
|
const providerError = Object.assign(
|
|
new Error(
|
|
'provider failed password=top-secret config={"compose":"private"}',
|
|
),
|
|
{
|
|
name: "KubernetesError",
|
|
stack:
|
|
"KubernetesError: provider failed\n at provider (test.ts:1:1)",
|
|
body: {
|
|
status: "Failure",
|
|
reason: "InternalError",
|
|
code: 500,
|
|
message: "contains top-secret",
|
|
},
|
|
},
|
|
);
|
|
spyOn(workspaceStore, "update").mockRejectedValue(providerError);
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
requestId: () => "request-123",
|
|
logger: { error: (entry) => logs.push(entry) },
|
|
});
|
|
|
|
const result = await app(
|
|
request(
|
|
"/api/v2/workspaces/demo",
|
|
{
|
|
method: "PUT",
|
|
headers: { "if-match": '"1"' },
|
|
body: JSON.stringify({
|
|
source: { uri: "cas://secret-source", digest: "secret-digest" },
|
|
config: { compose: "private-config", password: "top-secret" },
|
|
}),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
|
|
expect(result.status).toBe(500);
|
|
expect(await result.json()).toMatchObject({
|
|
code: "INTERNAL_ERROR",
|
|
requestId: "request-123",
|
|
detail: "The request could not be completed",
|
|
});
|
|
expect(logs).toEqual([
|
|
{
|
|
event: "request.failed",
|
|
requestId: "request-123",
|
|
method: "PUT",
|
|
pathname: "/api/v2/workspaces/demo",
|
|
workspaceId: "demo",
|
|
status: 500,
|
|
code: "INTERNAL_ERROR",
|
|
errorName: "KubernetesError",
|
|
message:
|
|
'provider failed password=top-secret config={"compose":"private"}',
|
|
stack:
|
|
"KubernetesError: provider failed\n at provider (test.ts:1:1)",
|
|
kubernetesStatus: {
|
|
status: "Failure",
|
|
reason: "InternalError",
|
|
code: 500,
|
|
},
|
|
},
|
|
]);
|
|
});
|
|
|
|
test("lists persisted operation events with capability and workspace scope checks", async () => {
|
|
const store = await authenticatedStore("operator");
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore(
|
|
undefined,
|
|
() => "event-id",
|
|
);
|
|
const operation = await operationStore.create({
|
|
workspaceId: "demo",
|
|
action: "resources.apply",
|
|
idempotencyKey: "event-key",
|
|
});
|
|
await operationStore.emit(operation.metadata.name, {
|
|
resource: { apiVersion: "v1", kind: "Service", name: "web" },
|
|
phase: "apply",
|
|
state: "succeeded",
|
|
});
|
|
await store.createApiKey({
|
|
id: "demo-reader-key-01",
|
|
tokenHash: hashToken("demo-reader-token"),
|
|
username: "operator",
|
|
capabilities: ["kubernetes:read"],
|
|
workspace: "demo",
|
|
expiresAt: "2030-01-01T00:00:00.000Z",
|
|
});
|
|
await store.createApiKey({
|
|
id: "other-reader-key-1",
|
|
tokenHash: hashToken("other-reader-token"),
|
|
username: "operator",
|
|
capabilities: ["kubernetes:read"],
|
|
workspace: "other",
|
|
expiresAt: "2030-01-01T00:00:00.000Z",
|
|
});
|
|
const app = createApp({ store, workspaceStore, operationStore });
|
|
const path = `/api/v2/operations/${operation.metadata.name}/events?after=0`;
|
|
const allowed = await app(request(path, {}, "demo-reader-token"));
|
|
expect(allowed.status).toBe(200);
|
|
expect(await allowed.json()).toMatchObject({
|
|
items: [
|
|
{
|
|
sequence: 1,
|
|
data: { resource: { kind: "Service", name: "web" }, phase: "apply" },
|
|
},
|
|
],
|
|
nextCursor: 1,
|
|
retainedFirstSequence: 1,
|
|
cursorGap: false,
|
|
});
|
|
expect((await app(request(path, {}, "other-reader-token"))).status).toBe(
|
|
403,
|
|
);
|
|
expect(
|
|
(await app(request(`${path}x`, {}, "demo-reader-token"))).status,
|
|
).toBe(400);
|
|
});
|
|
|
|
test("reports operation event cursor gaps after retained history is truncated", async () => {
|
|
const operationStore = new MemoryOperationStore(
|
|
undefined,
|
|
() => "retained",
|
|
);
|
|
const operation = await operationStore.create({
|
|
workspaceId: "demo",
|
|
action: "resources.apply",
|
|
idempotencyKey: "retained-events",
|
|
});
|
|
for (let sequence = 0; sequence < 257; sequence += 1) {
|
|
await operationStore.emit(operation.metadata.name, {
|
|
resource: { apiVersion: "v1", kind: "ConfigMap", name: "config" },
|
|
phase: "apply",
|
|
state: "succeeded",
|
|
});
|
|
}
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
operationStore,
|
|
});
|
|
|
|
const response = await app(
|
|
request(
|
|
`/api/v2/operations/${operation.metadata.name}/events?after=0`,
|
|
{},
|
|
"token",
|
|
),
|
|
);
|
|
expect(response.status).toBe(200);
|
|
const events = (await response.json()) as {
|
|
retainedFirstSequence: number;
|
|
cursorGap: boolean;
|
|
items: { sequence: number }[];
|
|
};
|
|
expect(events.retainedFirstSequence).toBe(2);
|
|
expect(events.cursorGap).toBe(true);
|
|
expect(events.items[0]?.sequence).toBe(2);
|
|
});
|
|
|
|
test("rejects JSON bodies over the configured limit", async () => {
|
|
const app = createApp({
|
|
store: await authenticatedStore("admin"),
|
|
workspaceStore: new MemoryWorkspaceStore(),
|
|
jsonBodyLimit: 32,
|
|
});
|
|
const result = await app(
|
|
request(
|
|
"/api/v2/workspaces",
|
|
{ method: "POST", body: JSON.stringify({ value: "x".repeat(64) }) },
|
|
"token",
|
|
),
|
|
);
|
|
expect(result.status).toBe(413);
|
|
expect(await result.json()).toMatchObject({ code: "BODY_TOO_LARGE" });
|
|
});
|
|
|
|
test("does not expose request internals or corrupt success when auditing fails", async () => {
|
|
const logs: unknown[] = [];
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore(
|
|
undefined,
|
|
() => "operation-uid",
|
|
);
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
management: { stop: async () => ["api"] } as unknown as ManagementService,
|
|
auditStore: {
|
|
append: async () => {
|
|
throw new Error("audit unavailable");
|
|
},
|
|
list: async () => [],
|
|
},
|
|
requestId: () => "audit-request-123",
|
|
logger: { error: (entry) => logs.push(entry) },
|
|
});
|
|
const result = await app(
|
|
request(
|
|
"/api/v2/workspaces/demo/lifecycle",
|
|
{
|
|
method: "POST",
|
|
headers: { "idempotency-key": "private-request" },
|
|
body: JSON.stringify({ action: "stop", password: "do-not-store" }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
expect(result.status).toBe(200);
|
|
const body = (await result.json()) as Record<string, any>;
|
|
expect(body.operation.spec).toEqual({
|
|
workspaceId: "demo",
|
|
action: "workspace.stop",
|
|
});
|
|
expect(body.operation.status.state).toBe("succeeded");
|
|
expect(logs).toEqual([
|
|
{
|
|
event: "operation.audit.failed",
|
|
requestId: "audit-request-123",
|
|
method: "POST",
|
|
pathname: "/api/v2/workspaces/demo/lifecycle",
|
|
workspaceId: "demo",
|
|
operationId: "operation-operation-uid",
|
|
status: 500,
|
|
code: "AUDIT_APPEND_FAILED",
|
|
errorName: "Error",
|
|
message: "Successful operation audit event could not be appended",
|
|
},
|
|
]);
|
|
expect(JSON.stringify(logs)).not.toContain("audit unavailable");
|
|
expect(JSON.stringify(logs)).not.toContain("private-request");
|
|
expect(JSON.stringify(logs)).not.toContain("do-not-store");
|
|
expect(
|
|
await operationStore.get("operation-operation-uid"),
|
|
).not.toHaveProperty("spec.request");
|
|
});
|
|
|
|
test("fails and identifies operations that cannot acquire the workspace lease", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore(undefined, () => "blocked");
|
|
const leases = new MemoryWorkspaceLeaseProvider();
|
|
await leases.acquire("demo", "other");
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
leases,
|
|
management: { stop: async () => [] } as unknown as ManagementService,
|
|
});
|
|
const result = await app(
|
|
request(
|
|
"/api/v2/workspaces/demo/lifecycle",
|
|
{ method: "POST", body: JSON.stringify({ action: "stop" }) },
|
|
"token",
|
|
),
|
|
);
|
|
expect(result.status).toBe(409);
|
|
expect(await result.json()).toMatchObject({
|
|
code: "WORKSPACE_BUSY",
|
|
operationId: "operation-blocked",
|
|
});
|
|
expect((await operationStore.get("operation-blocked"))?.status.state).toBe(
|
|
"pending",
|
|
);
|
|
});
|
|
|
|
test("a stalled observation-only wait does not block mutation and remains idempotent", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-uid" });
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore();
|
|
const leases = new MemoryWorkspaceLeaseProvider();
|
|
let waitStarted!: () => void;
|
|
let finishWait!: () => void;
|
|
const started = new Promise<void>((resolve) => (waitStarted = resolve));
|
|
const blocked = new Promise<void>((resolve) => (finishWait = resolve));
|
|
let waits = 0;
|
|
let stops = 0;
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
leases,
|
|
management: {
|
|
waitForResources: async () => {
|
|
waits += 1;
|
|
waitStarted();
|
|
await blocked;
|
|
},
|
|
stop: async () => {
|
|
stops += 1;
|
|
return ["api"];
|
|
},
|
|
} as unknown as ManagementService,
|
|
});
|
|
const waitRequest = () =>
|
|
app(
|
|
request(
|
|
"/api/v2/workspaces/demo/resources/wait",
|
|
{
|
|
method: "POST",
|
|
headers: { "idempotency-key": "wait-once", prefer: "respond-async" },
|
|
body: JSON.stringify({ deployments: ["api"] }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
|
|
const submitted = await waitRequest();
|
|
expect(submitted.status).toBe(202);
|
|
await started;
|
|
const duplicate = await waitRequest();
|
|
expect(duplicate.status).toBe(202);
|
|
expect(waits).toBe(1);
|
|
|
|
const stop = await app(
|
|
request(
|
|
"/api/v2/workspaces/demo/lifecycle",
|
|
{
|
|
method: "POST",
|
|
headers: { "idempotency-key": "stop-during-wait" },
|
|
body: JSON.stringify({ action: "stop" }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
expect(stop.status).toBe(200);
|
|
expect(stops).toBe(1);
|
|
finishWait();
|
|
});
|
|
|
|
test("keeps a concurrent idempotent operation pending when its lease acquisition is denied", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore(
|
|
undefined,
|
|
() => "same-key",
|
|
);
|
|
let acquireCount = 0;
|
|
let releaseCount = 0;
|
|
let allowFirstAcquire!: () => void;
|
|
let signalFirstAcquire!: () => void;
|
|
const firstAcquireStarted = new Promise<void>((resolve) => {
|
|
signalFirstAcquire = resolve;
|
|
});
|
|
const leases = {
|
|
acquire: async () => {
|
|
acquireCount += 1;
|
|
if (acquireCount === 2) return undefined;
|
|
if (acquireCount === 1) {
|
|
signalFirstAcquire();
|
|
await new Promise<void>((resolve) => {
|
|
allowFirstAcquire = resolve;
|
|
});
|
|
}
|
|
return {
|
|
workspaceId: "demo",
|
|
holder: "operation-same-key",
|
|
expiresAt: new Date().toISOString(),
|
|
renew: async () => true,
|
|
release: async () => {
|
|
releaseCount += 1;
|
|
},
|
|
};
|
|
},
|
|
};
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
leases,
|
|
management: { stop: async () => ["api"] } as unknown as ManagementService,
|
|
});
|
|
const stop = () =>
|
|
app(
|
|
request(
|
|
"/api/v2/workspaces/demo/lifecycle",
|
|
{
|
|
method: "POST",
|
|
headers: { "idempotency-key": "same-key" },
|
|
body: JSON.stringify({ action: "stop" }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
|
|
const owner = stop();
|
|
await firstAcquireStarted;
|
|
const duplicate = await stop();
|
|
expect(duplicate.status).toBe(409);
|
|
expect((await operationStore.get("operation-same-key"))?.status.state).toBe(
|
|
"pending",
|
|
);
|
|
|
|
allowFirstAcquire();
|
|
expect((await owner).status).toBe(200);
|
|
expect((await operationStore.get("operation-same-key"))?.status.state).toBe(
|
|
"succeeded",
|
|
);
|
|
expect(acquireCount).toBe(4);
|
|
expect(releaseCount).toBe(3);
|
|
});
|
|
|
|
test("lets the lease winner claim execution even when a duplicate created the operation", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore(
|
|
undefined,
|
|
() => "same-key",
|
|
);
|
|
let acquireCount = 0;
|
|
let releaseFirstAcquire!: () => void;
|
|
let firstAcquireStarted!: () => void;
|
|
const firstAcquire = new Promise<void>((resolve) => {
|
|
firstAcquireStarted = resolve;
|
|
});
|
|
const leases = {
|
|
acquire: async () => {
|
|
acquireCount += 1;
|
|
if (acquireCount === 1) {
|
|
firstAcquireStarted();
|
|
await new Promise<void>((resolve) => {
|
|
releaseFirstAcquire = resolve;
|
|
});
|
|
return undefined;
|
|
}
|
|
return {
|
|
workspaceId: "demo",
|
|
holder: "operation-same-key",
|
|
expiresAt: new Date().toISOString(),
|
|
renew: async () => true,
|
|
release: async () => {},
|
|
};
|
|
},
|
|
};
|
|
let executions = 0;
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
leases,
|
|
management: {
|
|
stop: async () => {
|
|
executions += 1;
|
|
return ["api"];
|
|
},
|
|
} as unknown as ManagementService,
|
|
});
|
|
const stop = () =>
|
|
app(
|
|
request(
|
|
"/api/v2/workspaces/demo/lifecycle",
|
|
{
|
|
method: "POST",
|
|
headers: { "idempotency-key": "same-key" },
|
|
body: JSON.stringify({ action: "stop" }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
|
|
const creator = stop();
|
|
await firstAcquire;
|
|
expect((await stop()).status).toBe(200);
|
|
releaseFirstAcquire();
|
|
expect((await creator).status).toBe(409);
|
|
expect(executions).toBe(1);
|
|
expect((await operationStore.get("operation-same-key"))?.status.state).toBe(
|
|
"succeeded",
|
|
);
|
|
});
|
|
|
|
test("releases a lease when the initial execution claim fails", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore(
|
|
undefined,
|
|
() => "claim-fails",
|
|
);
|
|
spyOn(operationStore, "claimExecution").mockRejectedValue(
|
|
new Error("claim unavailable"),
|
|
);
|
|
let releases = 0;
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
leases: {
|
|
acquire: async () => ({
|
|
workspaceId: "demo",
|
|
holder: "operation-claim-fails",
|
|
expiresAt: new Date().toISOString(),
|
|
renew: async () => true,
|
|
release: async () => {
|
|
releases += 1;
|
|
},
|
|
}),
|
|
},
|
|
management: { stop: async () => [] } as unknown as ManagementService,
|
|
});
|
|
|
|
expect(
|
|
(
|
|
await app(
|
|
request(
|
|
"/api/v2/workspaces/demo/lifecycle",
|
|
{ method: "POST", body: JSON.stringify({ action: "stop" }) },
|
|
"token",
|
|
),
|
|
)
|
|
).status,
|
|
).toBe(500);
|
|
expect(releases).toBe(3);
|
|
});
|
|
|
|
test("fails a claimed operation before execution when lease ownership is lost", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore(
|
|
undefined,
|
|
() => "lost-lease",
|
|
);
|
|
let executions = 0;
|
|
let releases = 0;
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
leases: {
|
|
acquire: async () => ({
|
|
workspaceId: "demo",
|
|
holder: "operation-lost-lease",
|
|
expiresAt: new Date().toISOString(),
|
|
renew: async () => false,
|
|
release: async () => {
|
|
releases += 1;
|
|
},
|
|
}),
|
|
},
|
|
management: {
|
|
stop: async () => {
|
|
executions += 1;
|
|
return [];
|
|
},
|
|
} as unknown as ManagementService,
|
|
});
|
|
|
|
const result = await app(
|
|
request(
|
|
"/api/v2/workspaces/demo/lifecycle",
|
|
{ method: "POST", body: JSON.stringify({ action: "stop" }) },
|
|
"token",
|
|
),
|
|
);
|
|
expect(result.status).toBe(409);
|
|
expect(executions).toBe(0);
|
|
expect(releases).toBe(3);
|
|
expect(await result.json()).toMatchObject({
|
|
code: "WORKSPACE_LEASE_LOST",
|
|
});
|
|
expect(
|
|
(await operationStore.get("operation-lost-lease"))?.status,
|
|
).toMatchObject({
|
|
state: "failed",
|
|
error: { code: "WORKSPACE_LEASE_LOST" },
|
|
});
|
|
});
|
|
|
|
test("aborts blocked execution and fails the operation when renewal loses the lease", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore(
|
|
undefined,
|
|
() => "blocked-loss",
|
|
);
|
|
const originalSetTimeout = globalThis.setTimeout;
|
|
let scheduledRenewal!: () => void;
|
|
globalThis.setTimeout = ((callback: Parameters<typeof setTimeout>[0]) => {
|
|
scheduledRenewal = callback as () => void;
|
|
return 0 as unknown as ReturnType<typeof setTimeout>;
|
|
}) as typeof setTimeout;
|
|
try {
|
|
let executionStarted!: () => void;
|
|
const started = new Promise<void>((resolve) => {
|
|
executionStarted = resolve;
|
|
});
|
|
let observedAbort = false;
|
|
let renewals = 0;
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
leases: {
|
|
acquire: async () => ({
|
|
workspaceId: "demo",
|
|
holder: "operation-blocked-loss",
|
|
expiresAt: new Date().toISOString(),
|
|
renew: async () => ++renewals <= 3,
|
|
release: async () => {},
|
|
}),
|
|
},
|
|
management: {
|
|
stop: async (
|
|
_workspace: { project: string; uid: string },
|
|
_names?: string[],
|
|
execution?: { signal?: AbortSignal },
|
|
) => {
|
|
executionStarted();
|
|
await new Promise<void>((resolve) => {
|
|
execution?.signal?.addEventListener(
|
|
"abort",
|
|
() => {
|
|
observedAbort = true;
|
|
resolve();
|
|
},
|
|
{ once: true },
|
|
);
|
|
});
|
|
return [];
|
|
},
|
|
} as unknown as ManagementService,
|
|
});
|
|
const pending = app(
|
|
request(
|
|
"/api/v2/workspaces/demo/lifecycle",
|
|
{ method: "POST", body: JSON.stringify({ action: "stop" }) },
|
|
"token",
|
|
),
|
|
);
|
|
await started;
|
|
scheduledRenewal();
|
|
const result = await pending;
|
|
|
|
expect(observedAbort).toBe(true);
|
|
expect(result.status).toBe(409);
|
|
expect(await result.json()).toMatchObject({
|
|
code: "WORKSPACE_LEASE_LOST",
|
|
});
|
|
expect(
|
|
(await operationStore.get("operation-blocked-loss"))?.status,
|
|
).toMatchObject({
|
|
state: "failed",
|
|
error: { code: "WORKSPACE_LEASE_LOST" },
|
|
});
|
|
} finally {
|
|
globalThis.setTimeout = originalSetTimeout;
|
|
}
|
|
});
|
|
|
|
test("renews the workspace lease while an operation remains in flight", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
let scheduledRenewal: (() => void) | undefined;
|
|
const originalSetTimeout = globalThis.setTimeout;
|
|
const originalClearTimeout = globalThis.clearTimeout;
|
|
const clearedTimers: unknown[] = [];
|
|
globalThis.setTimeout = ((callback: Parameters<typeof setTimeout>[0]) => {
|
|
scheduledRenewal = callback as () => void;
|
|
return 0 as unknown as ReturnType<typeof setTimeout>;
|
|
}) as typeof setTimeout;
|
|
globalThis.clearTimeout = ((timer: ReturnType<typeof setTimeout>) => {
|
|
clearedTimers.push(timer);
|
|
}) as typeof clearTimeout;
|
|
try {
|
|
let releaseOperation!: () => void;
|
|
let operationStarted!: () => void;
|
|
const started = new Promise<void>((resolve) => {
|
|
operationStarted = resolve;
|
|
});
|
|
const completed = new Promise<string[]>((resolve) => {
|
|
releaseOperation = () => resolve(["api"]);
|
|
});
|
|
let renewals = 0;
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore: new MemoryOperationStore(undefined, () => "renewing"),
|
|
leases: {
|
|
acquire: async () => ({
|
|
workspaceId: "demo",
|
|
holder: "operation-renewing",
|
|
expiresAt: new Date().toISOString(),
|
|
renew: async () => {
|
|
renewals += 1;
|
|
return true;
|
|
},
|
|
release: async () => {},
|
|
}),
|
|
},
|
|
management: {
|
|
stop: async () => {
|
|
operationStarted();
|
|
return completed;
|
|
},
|
|
} as unknown as ManagementService,
|
|
});
|
|
const response = app(
|
|
request(
|
|
"/api/v2/workspaces/demo/lifecycle",
|
|
{ method: "POST", body: JSON.stringify({ action: "stop" }) },
|
|
"token",
|
|
),
|
|
);
|
|
await started;
|
|
expect(renewals).toBe(3);
|
|
scheduledRenewal?.();
|
|
await Promise.resolve();
|
|
await Promise.resolve();
|
|
expect(renewals).toBe(6);
|
|
releaseOperation();
|
|
expect((await response).status).toBe(200);
|
|
expect(renewals).toBe(9);
|
|
expect(clearedTimers).toEqual([0]);
|
|
} finally {
|
|
globalThis.setTimeout = originalSetTimeout;
|
|
globalThis.clearTimeout = originalClearTimeout;
|
|
}
|
|
});
|
|
|
|
test("reuses a failed reconciliation without attempting a second failure transition", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const operationStore = new MemoryOperationStore(
|
|
undefined,
|
|
() => "reconcile",
|
|
);
|
|
let reconciliations = 0;
|
|
const management = {
|
|
reconcileDatabases: async () => {
|
|
reconciliations += 1;
|
|
const [operation] = await operationStore.list("demo");
|
|
await operationStore.transition(operation!.metadata.name, "failed", {
|
|
error: {
|
|
code: "RECONCILE_FAILED",
|
|
message: "Database reconciliation failed",
|
|
},
|
|
});
|
|
return {};
|
|
},
|
|
} as unknown as ManagementService;
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
operationStore,
|
|
management,
|
|
});
|
|
const reconcile = () =>
|
|
app(
|
|
request(
|
|
"/api/v2/workspaces/demo/databases",
|
|
{
|
|
method: "POST",
|
|
headers: { "idempotency-key": "reconcile-once" },
|
|
body: JSON.stringify({ compose: {} }),
|
|
},
|
|
"token",
|
|
),
|
|
);
|
|
|
|
for (const result of [await reconcile(), await reconcile()]) {
|
|
expect(result.status).toBe(500);
|
|
expect(await result.json()).toMatchObject({
|
|
code: "RECONCILE_FAILED",
|
|
detail: "Database reconciliation failed",
|
|
});
|
|
}
|
|
expect(reconciliations).toBe(1);
|
|
expect(
|
|
(await operationStore.get("operation-reconcile"))?.status.state,
|
|
).toBe("failed");
|
|
});
|
|
|
|
test("routes workspace adoption and keeps platform adoption admin-only", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const adopted: string[] = [];
|
|
const adoption = {
|
|
adopt: async (workspaceId: string, workspaceUid: string) => {
|
|
adopted.push(`${workspaceId}:${workspaceUid}`);
|
|
return { workspaceId, workspaceUid, resourcesAdopted: 2 };
|
|
},
|
|
adoptPlatform: async (workspaceUid: string) => ({
|
|
workspaceId: "kuber-system",
|
|
workspaceUid,
|
|
resourcesAdopted: 1,
|
|
}),
|
|
};
|
|
const operatorApp = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
adoption,
|
|
});
|
|
const regular = await operatorApp(
|
|
request("/api/v2/workspaces/demo/adopt", { method: "POST" }, "token"),
|
|
);
|
|
expect(regular.status).toBe(200);
|
|
expect(adopted).toEqual(["demo:workspace-uid"]);
|
|
expect(
|
|
(
|
|
await operatorApp(
|
|
request(
|
|
"/api/v2/platform/kuber-system/adopt",
|
|
{
|
|
method: "POST",
|
|
body: JSON.stringify({ workspaceUid: "platform" }),
|
|
},
|
|
"token",
|
|
),
|
|
)
|
|
).status,
|
|
).toBe(403);
|
|
|
|
const adminApp = createApp({
|
|
store: await authenticatedStore("admin"),
|
|
workspaceStore,
|
|
adoption,
|
|
});
|
|
const platform = await adminApp(
|
|
request(
|
|
"/api/v2/platform/kuber-system/adopt",
|
|
{ method: "POST", body: JSON.stringify({ workspaceUid: "platform" }) },
|
|
"token",
|
|
),
|
|
);
|
|
expect(platform.status).toBe(200);
|
|
});
|
|
|
|
test("audits failed workspace adoption with request correlation", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const auditStore = new MemoryAuditStore();
|
|
const logs: unknown[] = [];
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
auditStore,
|
|
requestId: () => "adopt-request-123",
|
|
logger: { error: (entry) => logs.push(entry) },
|
|
adoption: {
|
|
adopt: async () => {
|
|
throw new Error("provider adoption failed");
|
|
},
|
|
adoptPlatform: async () => ({
|
|
workspaceId: "kuber-system",
|
|
workspaceUid: "platform",
|
|
resourcesAdopted: 0,
|
|
}),
|
|
},
|
|
});
|
|
|
|
const result = await app(
|
|
request("/api/v2/workspaces/demo/adopt", { method: "POST" }, "token"),
|
|
);
|
|
|
|
expect(result.status).toBe(500);
|
|
expect(await result.json()).toMatchObject({
|
|
code: "INTERNAL_ERROR",
|
|
detail: "The request could not be completed",
|
|
requestId: "adopt-request-123",
|
|
});
|
|
expect(await auditStore.list("demo")).toMatchObject([
|
|
{
|
|
spec: {
|
|
action: "workspace.adopt",
|
|
outcome: "failure",
|
|
workspaceId: "demo",
|
|
details: {
|
|
route: "/api/v2/workspaces/demo/adopt",
|
|
requestId: "adopt-request-123",
|
|
},
|
|
},
|
|
},
|
|
]);
|
|
expect(logs).toMatchObject([
|
|
{
|
|
event: "request.failed",
|
|
requestId: "adopt-request-123",
|
|
method: "POST",
|
|
pathname: "/api/v2/workspaces/demo/adopt",
|
|
workspaceId: "demo",
|
|
status: 500,
|
|
code: "INTERNAL_ERROR",
|
|
errorName: "Error",
|
|
message: "provider adoption failed",
|
|
},
|
|
]);
|
|
});
|
|
|
|
test("forwards Kubernetes adoption errors with request correlation", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
requestId: () => "adopt-kubernetes-request-123",
|
|
adoption: {
|
|
adopt: async () => {
|
|
throw {
|
|
statusCode: 422,
|
|
body: {
|
|
apiVersion: "v1",
|
|
kind: "Status",
|
|
status: "Failure",
|
|
reason: "Invalid",
|
|
message:
|
|
'Ingress.networking.k8s.io "web" is invalid: spec: Required value',
|
|
code: 422,
|
|
},
|
|
};
|
|
},
|
|
adoptPlatform: async () => ({
|
|
workspaceId: "kuber-system",
|
|
workspaceUid: "platform",
|
|
resourcesAdopted: 0,
|
|
}),
|
|
},
|
|
});
|
|
|
|
const result = await app(
|
|
request("/api/v2/workspaces/demo/adopt", { method: "POST" }, "token"),
|
|
);
|
|
|
|
expect(result.status).toBe(422);
|
|
expect(await result.json()).toMatchObject({
|
|
code: "KUBERNETES_ERROR",
|
|
detail:
|
|
'Ingress.networking.k8s.io "web" is invalid: spec: Required value',
|
|
requestId: "adopt-kubernetes-request-123",
|
|
});
|
|
});
|
|
|
|
test("does not forward arbitrary adoption errors as Kubernetes errors", async () => {
|
|
const workspaceStore = new MemoryWorkspaceStore({
|
|
uid: () => "workspace-uid",
|
|
});
|
|
await workspaceStore.create({
|
|
id: "demo",
|
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
|
});
|
|
const app = createApp({
|
|
store: await authenticatedStore("operator"),
|
|
workspaceStore,
|
|
adoption: {
|
|
adopt: async () => {
|
|
throw {
|
|
statusCode: 422,
|
|
body: { message: "arbitrary provider failure" },
|
|
};
|
|
},
|
|
adoptPlatform: async () => ({
|
|
workspaceId: "kuber-system",
|
|
workspaceUid: "platform",
|
|
resourcesAdopted: 0,
|
|
}),
|
|
},
|
|
});
|
|
|
|
const result = await app(
|
|
request("/api/v2/workspaces/demo/adopt", { method: "POST" }, "token"),
|
|
);
|
|
|
|
expect(result.status).toBe(500);
|
|
expect(await result.json()).toMatchObject({
|
|
code: "INTERNAL_ERROR",
|
|
detail: "The request could not be completed",
|
|
});
|
|
});
|
|
});
|