Files
kuber/tests/server/management.test.ts
T

633 lines
22 KiB
TypeScript

import { describe, expect, test } from "bun:test";
import type { KubernetesObject, V1Deployment } from "@kubernetes/client-node";
import { DatabaseReconciliationError } from "../../lib/database";
import {
createManagementService,
type ManagementDependencies,
type ResourceIdentity,
WORKSPACE_PROJECT_LABEL,
WORKSPACE_UID_LABEL,
} from "../../server/management";
const workspace = { project: "shop", uid: "workspace-1" };
function object(
kind: string,
name: string,
uid: string,
labels: Record<string, string> = {
"app.kubernetes.io/managed-by": "kuber",
[WORKSPACE_UID_LABEL]: workspace.uid,
},
): KubernetesObject {
return {
apiVersion:
kind === "Deployment"
? "apps/v1"
: kind === "HorizontalPodAutoscaler"
? "autoscaling/v2"
: "v1",
kind,
metadata: { name, namespace: workspace.project, uid, labels },
};
}
function dependencies(
overrides: Partial<ManagementDependencies> = {},
): ManagementDependencies {
return {
readNamespace: async () => ({
uid: "namespace-uid",
labels: {
"app.kubernetes.io/managed-by": "kuber",
[WORKSPACE_UID_LABEL]: workspace.uid,
},
}),
listDeployments: async () => [],
scaleDeployment: async () => {},
restartDeployment: async () => {},
waitForDeployment: async () => {},
fetchGraphObjects: async () => [],
planRollback: async () => [],
rollbackDeployment: async () => {},
listProjectResources: async () => [],
listDatabaseResources: async () => [],
listStorageResources: async () => [],
findStaleResources: async () => [],
applyResource: async (resource) => resource,
deleteResource: async () => {},
reconcileDatabases: async () => ({}),
getDatabaseCredentials: async () => ({
username: "user",
password: "pass",
}),
reconcileStorage: async () => ({}),
getStorageCredentials: async () => ({}),
...overrides,
};
}
describe("server management service", () => {
test.each([
["namespace precheck", { readNamespace: async () => { throw new Error("token=private"); } }],
["database dependency", { reconcileDatabases: async () => { throw new Error("token=private"); } }],
["database resource listing", { listDatabaseResources: async () => { throw new Error("token=private"); } }],
["database resource ownership", { applyResource: async () => { throw new Error("token=private"); }, listDatabaseResources: async () => [object("Database", "DB_PASSWORD_private123", "db-uid")] }],
] as const)("labels database %s failures without leaking details", async (phase, overrides) => {
const service = createManagementService(dependencies(overrides));
let failure: unknown;
try { await service.reconcileDatabases(workspace, { services: {} }); }
catch (error) { failure = error; }
expect(failure).toBeInstanceOf(DatabaseReconciliationError);
expect((failure as DatabaseReconciliationError).phase).toBe(phase);
expect((failure as Error).message).not.toContain("private");
});
test("preserves an existing database substep rather than masking it", async () => {
const service = createManagementService(dependencies({
reconcileDatabases: async () => { throw new DatabaseReconciliationError("database apply", new Error("private")); },
}));
await expect(service.reconcileDatabases(workspace, { services: {} })).rejects.toMatchObject({ phase: "database apply" });
});
test("waits on deployments together and cancels siblings after a crash", async () => {
const started: string[] = [];
let siblingCancelled = false;
const service = createManagementService(
dependencies({
listDeployments: async () => [
object("Deployment", "healthy", "healthy-uid") as V1Deployment,
object("Deployment", "broken", "broken-uid") as V1Deployment,
],
waitForDeployment: async (_project, name, timeout, execution) => {
expect(timeout).toBeGreaterThan(0);
started.push(name);
if (name === "broken")
throw new Error(
"Deployment broken rollout failed: CrashLoopBackOff",
);
await new Promise<void>((resolve) => {
if (execution?.signal?.aborted) {
siblingCancelled = true;
return resolve();
}
execution?.signal?.addEventListener(
"abort",
() => {
siblingCancelled = true;
resolve();
},
{ once: true },
);
});
},
}),
);
await expect(
service.waitForResources(workspace, ["healthy", "broken"], 1000),
).rejects.toThrow("Deployment broken rollout failed: CrashLoopBackOff");
expect(started).toEqual(["healthy", "broken"]);
expect(siblingCancelled).toBe(true);
});
test("contains no authorization policy and operates on an explicit workspace", async () => {
const calls: string[] = [];
const service = createManagementService(
dependencies({
listDeployments: async (project) => {
calls.push(`list:${project}`);
return [object("Deployment", "api", "api-uid") as V1Deployment];
},
scaleDeployment: async (project, name, replicas) => {
calls.push(`scale:${project}:${name}:${replicas}`);
},
}),
);
expect(await service.stop(workspace)).toEqual(["api"]);
expect(calls).toEqual(["list:shop", "scale:shop:api:0"]);
});
test("targets requested deployments and rejects unmanaged names", async () => {
const restarted: string[] = [];
const service = createManagementService(
dependencies({
listDeployments: async () =>
["api", "worker"].map(
(name) => object("Deployment", name, `${name}-uid`) as V1Deployment,
),
restartDeployment: async (_project, name) => {
restarted.push(name);
},
}),
);
expect(await service.restart(workspace, ["worker", "worker"])).toEqual([
"worker",
]);
expect(restarted).toEqual(["worker"]);
await expect(service.restart(workspace, ["missing"])).rejects.toThrow(
"No managed deployment named missing",
);
});
test("normal down retains ingress and persistent volume claims", async () => {
const resources = [
object("Deployment", "api", "deployment-uid"),
object("Ingress", "api", "ingress-uid"),
object("PersistentVolumeClaim", "data", "pvc-uid"),
object("Service", "api", "service-uid"),
];
const service = createManagementService(
dependencies({ listProjectResources: async () => resources }),
);
const plan = await service.planDown(workspace);
expect(plan.retained.map((item) => item.kind)).toEqual([
"PersistentVolumeClaim",
"Ingress",
]);
expect(plan.delete.map((item) => item.kind)).toEqual([
"Deployment",
"Service",
]);
});
test("uses stale object UIDs as immutable deletion identities", async () => {
const stale = object("Secret", "old-env", "old-secret-uid");
const deleted: ResourceIdentity[] = [];
const service = createManagementService(
dependencies({
findStaleResources: async () => [stale],
deleteResource: async (item) => {
deleted.push(item);
},
}),
);
const plan = await service.planResources(workspace, [
{ apiVersion: "v1", kind: "ConfigMap", metadata: { name: "env" } },
]);
expect(plan.desired[0]?.metadata?.labels).toMatchObject({
[WORKSPACE_UID_LABEL]: workspace.uid,
[WORKSPACE_PROJECT_LABEL]: workspace.project,
});
expect(plan.stale).toEqual([
{
apiVersion: "v1",
kind: "Secret",
name: "old-env",
namespace: "shop",
uid: "old-secret-uid",
workspaceUid: workspace.uid,
},
]);
await service.deleteResources(workspace, plan.stale);
expect(deleted).toEqual(plan.stale);
});
test("emits safe per-resource apply, wait, delete, and failure progress", async () => {
const events: Array<{
phase: string;
state: string;
resource: { name: string };
}> = [];
const service = createManagementService(
dependencies({
listDeployments: async () => [
object("Deployment", "web", "web-uid") as V1Deployment,
],
applyResource: async (resource) => resource,
deleteResource: async () => {},
}),
);
const execution = {
emit: async (event: (typeof events)[number]) => {
events.push(event);
},
};
await service.applyResources(
workspace,
[object("Service", "web", "service-uid")],
execution,
);
await service.waitForResources(workspace, ["web"], undefined, execution);
await service.deleteResources(
workspace,
[
{
apiVersion: "v1",
kind: "Service",
name: "old",
uid: "old-uid",
workspaceUid: workspace.uid,
},
],
execution,
);
expect(
events.map(
({ phase, state, resource }) => `${phase}:${resource.name}:${state}`,
),
).toEqual([
"apply:web:started",
"apply:web:succeeded",
"wait:web:started",
"wait:web:succeeded",
"delete:old:started",
"delete:old:succeeded",
]);
const failing = createManagementService(
dependencies({
applyResource: async () => {
throw new Error("provider failed");
},
}),
);
await expect(
failing.applyResources(
workspace,
[object("Service", "broken", "broken-uid")],
execution,
),
).rejects.toThrow("provider failed");
expect(events.at(-1)).toMatchObject({
phase: "apply",
state: "failed",
resource: { name: "broken" },
});
});
test("emits aborted resource progress when an execution signal aborts", async () => {
const controller = new AbortController();
const events: Array<{ state: string; resource: { name: string } }> = [];
const service = createManagementService(
dependencies({
applyResource: async (_resource, execution) => {
controller.abort();
if (execution?.signal?.aborted)
throw new Error("Workspace operation execution was cancelled");
return _resource;
},
}),
);
await expect(
service.applyResources(
workspace,
[object("Service", "web", "service-uid")],
{
signal: controller.signal,
emit: async (event) => {
events.push(event);
},
},
),
).rejects.toThrow("cancelled");
expect(events.map(({ state }) => state)).toEqual(["started", "aborted"]);
});
test("rejects namespace and resource ownership mismatches", async () => {
const wrongNamespace = createManagementService(
dependencies({
readNamespace: async () => ({
uid: "namespace-uid",
labels: {
"app.kubernetes.io/managed-by": "kuber",
[WORKSPACE_UID_LABEL]: "someone-else",
},
}),
}),
);
await expect(wrongNamespace.stop(workspace)).rejects.toThrow(
"different-workspace",
);
const wrongResource = createManagementService(
dependencies({
findStaleResources: async () => [
object("Secret", "foreign", "foreign-uid", {
"app.kubernetes.io/managed-by": "kuber",
[WORKSPACE_UID_LABEL]: "someone-else",
}),
],
}),
);
await expect(wrongResource.planResources(workspace, [])).rejects.toThrow(
"is not owned by workspace",
);
});
test("full down requires the namespace UID and includes owned external resources", async () => {
const database = {
...object("Database", "orders", "database-uid"),
apiVersion: "postgresql.cnpg.io/v1",
metadata: {
...object("Database", "orders", "database-uid").metadata,
namespace: "database",
},
};
const deleted: ResourceIdentity[] = [];
const service = createManagementService(dependencies({
listDatabaseResources: async () => [database],
deleteResource: async (identity) => { deleted.push(identity); },
}));
const plan = await service.planDown(workspace, true);
expect(plan.delete.map(({ kind, uid }) => [kind, uid])).toEqual([
["Namespace", "namespace-uid"],
]);
expect(plan.retained.map(({ kind, uid }) => [kind, uid])).toEqual([["Database", "database-uid"]]);
await service.down(workspace, true);
expect(deleted.map(({ kind }) => kind)).toEqual(["Namespace"]);
const missingUid = createManagementService(
dependencies({
readNamespace: async () => ({
labels: {
"app.kubernetes.io/managed-by": "kuber",
[WORKSPACE_UID_LABEL]: workspace.uid,
},
}),
}),
);
await expect(missingUid.planDown(workspace, true)).rejects.toThrow(
"no UID deletion precondition",
);
});
test("preserves foreign and owned database CRs on full down while deleting other resources", async () => {
const shared = { ...object("Database", "sastify-store", "shared-uid", {
"app.kubernetes.io/managed-by": "kuber", [WORKSPACE_PROJECT_LABEL]: workspace.project,
[WORKSPACE_UID_LABEL]: "other-workspace",
}), metadata: { ...object("Database", "sastify-store", "shared-uid").metadata,
namespace: "database", labels: {
"app.kubernetes.io/managed-by": "kuber", [WORKSPACE_PROJECT_LABEL]: workspace.project,
[WORKSPACE_UID_LABEL]: "other-workspace",
} } };
const owned = { ...object("Database", "local", "local-uid"), metadata: {
...object("Database", "local", "local-uid").metadata, namespace: "database",
} };
const deleted: string[] = [];
const service = createManagementService(dependencies({
listProjectResources: async () => [object("Deployment", "api", "deployment-uid")],
listDatabaseResources: async () => [shared, owned],
deleteResource: async (identity) => { deleted.push(identity.kind); },
}));
const plan = await service.down(workspace, true);
expect(plan.delete.map(({ kind }) => kind)).toEqual(["Deployment", "Namespace"]);
expect(plan.retained.map(({ name }) => name)).toEqual(["local"]);
expect(deleted).toEqual(["Deployment", "Namespace"]);
});
test("reuses declared foreign database without taking its workspace UID", async () => {
const db = { ...object("Database", "sastify-store", "shared-uid", {
"app.kubernetes.io/managed-by": "kuber", [WORKSPACE_PROJECT_LABEL]: workspace.project,
[WORKSPACE_UID_LABEL]: "other-workspace",
}), metadata: {
...object("Database", "sastify-store", "shared-uid").metadata,
namespace: "database",
labels: { "app.kubernetes.io/managed-by": "kuber",
[WORKSPACE_PROJECT_LABEL]: workspace.project, [WORKSPACE_UID_LABEL]: "other-workspace" },
}, spec: { owner: "sastify", cluster: { name: "postgres" } } };
const before = structuredClone(db);
const applied: KubernetesObject[] = [];
const service = createManagementService(dependencies({
readNamespace: async (project) => ({ uid: "namespace-uid", labels: {
"app.kubernetes.io/managed-by": "kuber",
[WORKSPACE_UID_LABEL]: project === workspace.project ? workspace.uid : "second-uid",
} }),
listDatabaseResources: async () => [db],
applyResource: async (resource) => { applied.push(resource); return resource; },
}));
const compose = { services: { web: { volumes: ["postgresql:sastify/sastify-store"] } } } as never;
for (const project of [workspace, { project: "second-project", uid: "second-uid" }]) {
await service.reconcileDatabases(project, compose);
}
expect(applied).toEqual([]);
expect(db).toEqual(before);
await expect(service.reconcileDatabases(workspace, { services: {} })).rejects.toMatchObject({ phase: "database resource ownership" });
expect(applied).toEqual([]);
});
test("adopts unowned and same-workspace databases and rejects foreign conflicting specs", async () => {
const resources: KubernetesObject[] = [
{ ...object("Database", "new-db", "new-uid"), metadata: {
...object("Database", "new-db", "new-uid").metadata,
namespace: "database", labels: { "app.kubernetes.io/managed-by": "kuber" },
} },
{ ...object("Database", "owned-db", "owned-uid"), metadata: {
...object("Database", "owned-db", "owned-uid").metadata, namespace: "database",
} },
];
const applied: KubernetesObject[] = [];
const service = createManagementService(dependencies({
listDatabaseResources: async () => resources,
applyResource: async (resource) => { applied.push(resource); return resource; },
}));
await service.reconcileDatabases(workspace, { services: {} });
expect(applied.map((resource) => resource.metadata?.labels?.[WORKSPACE_UID_LABEL])).toEqual([workspace.uid, workspace.uid]);
resources[0] = { ...resources[0]!, metadata: { ...resources[0]!.metadata,
labels: { [WORKSPACE_UID_LABEL]: "other-workspace" } }, spec: {
owner: "wrong", cluster: { name: "postgres" },
} } as KubernetesObject;
await expect(service.reconcileDatabases(workspace, { services: { web: {
volumes: ["postgresql:sastify/new-db"],
} } } as never)).rejects.toMatchObject({ phase: "database resource ownership" });
expect(applied).toHaveLength(2);
});
test("stop deletes matching HPAs before scaling selected deployments", async () => {
const calls: string[] = [];
const service = createManagementService(
dependencies({
listDeployments: async () =>
["api", "worker", "batch"].map(
(name) => object("Deployment", name, `${name}-uid`) as V1Deployment,
),
listProjectResources: async () => [
object("HorizontalPodAutoscaler", "api", "api-hpa-uid", undefined),
object(
"HorizontalPodAutoscaler",
"worker",
"worker-hpa-uid",
undefined,
),
object(
"HorizontalPodAutoscaler",
"batch",
"batch-hpa-uid",
undefined,
),
],
scaleDeployment: async (_project, name, replicas) => {
calls.push(`scale:${name}:${replicas}`);
},
deleteResource: async (item) => {
calls.push(`delete:${item.kind}:${item.name}`);
},
}),
);
expect(await service.stop(workspace, ["api", "worker"])).toEqual([
"api",
"worker",
]);
expect(calls).toEqual([
"delete:HorizontalPodAutoscaler:api",
"delete:HorizontalPodAutoscaler:worker",
"scale:api:0",
"scale:worker:0",
]);
});
test("stop deletes HPAs for all deployments and leaves unrelated HPAs alone", async () => {
const calls: string[] = [];
const service = createManagementService(
dependencies({
listDeployments: async () =>
["api", "worker"].map(
(name) => object("Deployment", name, `${name}-uid`) as V1Deployment,
),
listProjectResources: async () => [
object("HorizontalPodAutoscaler", "api", "api-hpa-uid", undefined),
object(
"HorizontalPodAutoscaler",
"worker",
"worker-hpa-uid",
undefined,
),
object(
"HorizontalPodAutoscaler",
"legacy",
"legacy-hpa-uid",
undefined,
),
],
scaleDeployment: async (_project, name, replicas) => {
calls.push(`scale:${name}:${replicas}`);
},
deleteResource: async (item) => {
calls.push(`delete:${item.kind}:${item.name}`);
},
}),
);
expect(await service.stop(workspace)).toEqual(["api", "worker"]);
expect(calls).toEqual([
"delete:HorizontalPodAutoscaler:api",
"delete:HorizontalPodAutoscaler:worker",
"scale:api:0",
"scale:worker:0",
]);
});
test("refuses to delete an HPA not owned by the workspace", async () => {
const service = createManagementService(
dependencies({
listDeployments: async () => [
object("Deployment", "api", "api-uid") as V1Deployment,
],
listProjectResources: async () => [
object("HorizontalPodAutoscaler", "api", "api-hpa-uid", {
"app.kubernetes.io/managed-by": "kuber",
[WORKSPACE_UID_LABEL]: "someone-else",
}),
],
}),
);
await expect(service.stop(workspace, ["api"])).rejects.toThrow(
"is not owned by workspace",
);
});
test("stop scales deployments to zero when there are no HPAs", async () => {
const calls: string[] = [];
const service = createManagementService(
dependencies({
listDeployments: async () =>
["api", "worker"].map(
(name) => object("Deployment", name, `${name}-uid`) as V1Deployment,
),
listProjectResources: async () => [
object("ConfigMap", "env", "env-uid"),
object("Service", "api", "api-svc-uid"),
],
scaleDeployment: async (_project, name, replicas) => {
calls.push(`scale:${name}:${replicas}`);
},
}),
);
expect(await service.stop(workspace)).toEqual(["api", "worker"]);
expect(calls).toEqual(["scale:api:0", "scale:worker:0"]);
});
test("protects reserved namespaces before consulting Kubernetes", async () => {
let read = false;
const service = createManagementService(
dependencies({
readNamespace: async () => {
read = true;
return undefined;
},
}),
);
await expect(
service.namespaceSafety({ project: "kube-system", uid: "workspace-1" }),
).rejects.toThrow("reserved");
expect(read).toBe(false);
});
test("allows the kuber-system workspace to manage itself", async () => {
const service = createManagementService(
dependencies({
readNamespace: async () => ({ uid: "namespace-uid" }),
}),
);
await expect(
service.namespaceSafety({ project: "kuber-system", uid: "workspace-1" }),
).resolves.toMatchObject({ project: "kuber-system", status: "external" });
});
});