Files
kuber/server/management.ts
T

904 lines
27 KiB
TypeScript

import type { KubernetesObject, V1Deployment } from "@kubernetes/client-node";
import type { ComposeSpecification } from "../schema/docker.d";
import { LABELS } from "../const";
import {
applyResource,
getStaleResources,
listManagedResources,
sortResources,
} from "../lib/apply";
import {
DATABASE_CLUSTER,
DATABASE_NAMESPACE,
DatabaseReconciliationError,
getComposePostgresClaims,
getRoleCredentials,
listManagedDatabaseResources,
reconcilePostgresClaims,
type PostgresClaim,
type RoleCredentials,
} from "../lib/database";
import {
buildNamespaceGraphs,
fetchNamespaceObjects,
type NamespaceGraph,
} from "../lib/graph";
import {
planRollback,
rollbackDeployment,
type RollbackCandidate,
} from "../lib/rollback";
import {
listManagedDeployments,
restartDeployment,
scaleDeployment,
waitForDeploymentRollout,
} from "../lib/shared";
import {
getComposeS3Claims,
getS3Credentials,
listManagedStorageResources,
reconcileS3Claims,
STORAGE_NAMESPACE,
type S3Claim,
} from "../lib/storage";
export const WORKSPACE_UID_LABEL = "kuber.dev/workspace-uid";
export const WORKSPACE_PROJECT_LABEL = "kuber.dev/project";
export const RESERVED_NAMESPACES = new Set([
"default",
"kube-system",
"kube-public",
"kube-node-lease",
DATABASE_NAMESPACE,
STORAGE_NAMESPACE,
]);
export type Workspace = {
project: string;
uid: string;
};
export type NamespaceRecord = {
uid?: string;
labels?: Record<string, string>;
};
export type NamespaceSafety = {
project: string;
status: "missing" | "owned" | "external" | "different-workspace";
namespaceUid?: string;
};
export type ResourceIdentity = {
apiVersion: string;
kind: string;
name: string;
namespace?: string;
uid: string;
workspaceUid: string;
};
export type ResourcePlan = {
desired: KubernetesObject[];
stale: ResourceIdentity[];
};
export type DownPlan = {
full: boolean;
retained: ResourceIdentity[];
delete: ResourceIdentity[];
};
export type CredentialMetadata = {
service: string;
provider: "postgres" | "s3";
principal: string;
resource: string;
secretNamespace: string;
secretName?: string;
};
export type OperationProgressEmitter = (event: {
resource: {
apiVersion: string;
kind: string;
name: string;
namespace?: string;
};
phase: "apply" | "wait" | "delete";
state: "started" | "succeeded" | "failed" | "aborted";
}) => Promise<void>;
export type OperationExecution = {
signal?: AbortSignal;
emit?: OperationProgressEmitter;
};
export type ManagementDependencies = {
readNamespace(project: string): Promise<NamespaceRecord | undefined>;
listDeployments(project: string): Promise<V1Deployment[]>;
scaleDeployment(
project: string,
name: string,
replicas: number,
execution?: OperationExecution,
): Promise<unknown>;
restartDeployment(
project: string,
name: string,
execution?: OperationExecution,
): Promise<unknown>;
waitForDeployment(
project: string,
name: string,
timeoutMs?: number,
execution?: OperationExecution,
): Promise<void>;
fetchGraphObjects(project: string): Promise<KubernetesObject[]>;
planRollback(project: string, names?: string[]): Promise<RollbackCandidate[]>;
rollbackDeployment(
project: string,
candidate: RollbackCandidate,
execution?: OperationExecution,
): Promise<unknown>;
listProjectResources(project: string): Promise<KubernetesObject[]>;
listDatabaseResources(project: string): Promise<KubernetesObject[]>;
listStorageResources(project: string): Promise<KubernetesObject[]>;
findStaleResources(
project: string,
desired: KubernetesObject[],
): Promise<KubernetesObject[]>;
applyResource(
resource: KubernetesObject,
execution?: OperationExecution,
): Promise<KubernetesObject>;
deleteResource(
identity: ResourceIdentity,
execution?: OperationExecution,
): Promise<void>;
reconcileDatabases(
project: string,
compose: ComposeSpecification,
execution?: OperationExecution,
): Promise<Record<string, Record<string, string>>>;
getDatabaseCredentials(username: string): Promise<RoleCredentials>;
reconcileStorage(
project: string,
compose: ComposeSpecification,
execution?: OperationExecution,
): Promise<Record<string, Record<string, string>>>;
getStorageCredentials(claim: S3Claim): Promise<Record<string, string>>;
};
export type ManagementService = ReturnType<typeof createManagementService>;
const defaultOperations: Omit<
ManagementDependencies,
"readNamespace" | "deleteResource"
> = {
listDeployments: async () => listManagedDeployments(),
scaleDeployment: async (_project, name, replicas) =>
scaleDeployment(name, replicas),
restartDeployment: async (_project, name) => restartDeployment(name),
waitForDeployment: async (_project, name, timeoutMs) =>
waitForDeploymentRollout(name, timeoutMs),
fetchGraphObjects: fetchNamespaceObjects,
planRollback: async (_project, names) => planRollback(names),
rollbackDeployment: async (_project, candidate) =>
rollbackDeployment(candidate),
listProjectResources: listManagedResources,
listDatabaseResources: listManagedDatabaseResources,
listStorageResources: listManagedStorageResources,
findStaleResources: getStaleResources,
applyResource,
reconcileDatabases: async (project, compose, execution) =>
reconcilePostgresClaims(project, compose, execution?.signal),
getDatabaseCredentials: getRoleCredentials,
reconcileStorage: async (project, compose, execution) =>
reconcileS3Claims(project, compose, execution?.signal),
getStorageCredentials: getS3Credentials,
};
export function managementDependencies(
infrastructure: Pick<
ManagementDependencies,
"readNamespace" | "deleteResource"
>,
overrides: Partial<ManagementDependencies> = {},
): ManagementDependencies {
return { ...defaultOperations, ...infrastructure, ...overrides };
}
function validateWorkspace(workspace: Workspace): void {
if (!workspace.uid.trim()) throw new Error("Workspace UID is required");
if (
!workspace.project ||
workspace.project.length > 63 ||
!/^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/.test(workspace.project)
) {
throw new Error(`Invalid project namespace ${workspace.project}`);
}
if (RESERVED_NAMESPACES.has(workspace.project)) {
throw new Error(`Namespace ${workspace.project} is reserved`);
}
}
function throwIfExecutionAborted(execution?: OperationExecution): void {
if (!execution?.signal?.aborted) return;
throw new Error("Workspace operation execution was cancelled");
}
function resourceName(resource: KubernetesObject): string {
const name = resource.metadata?.name;
if (!resource.apiVersion || !resource.kind || !name) {
throw new Error("Resources require apiVersion, kind, and metadata.name");
}
return name;
}
function resourceProgressIdentity(
resource: Pick<
ResourceIdentity,
"apiVersion" | "kind" | "name" | "namespace"
>,
) {
return {
apiVersion: resource.apiVersion,
kind: resource.kind,
name: resource.name,
...(resource.namespace && { namespace: resource.namespace }),
};
}
function assertResourceOwnership(
workspace: Workspace,
resource: KubernetesObject,
): void {
const owner = resource.metadata?.labels?.[WORKSPACE_UID_LABEL];
if (owner !== workspace.uid) {
throw new Error(
`${resource.kind}/${resourceName(resource)} is not owned by workspace ${workspace.uid}`,
);
}
}
function identity(
workspace: Workspace,
resource: KubernetesObject,
): ResourceIdentity {
assertResourceOwnership(workspace, resource);
const uid = resource.metadata?.uid;
if (!uid) {
throw new Error(
`${resource.kind}/${resourceName(resource)} has no UID deletion precondition`,
);
}
return {
apiVersion: resource.apiVersion!,
kind: resource.kind!,
name: resourceName(resource),
namespace: resource.metadata?.namespace,
uid,
workspaceUid: workspace.uid,
};
}
function labelDesired(
workspace: Workspace,
resource: KubernetesObject,
): KubernetesObject {
const name = resourceName(resource);
const namespace =
resource.kind === "Namespace"
? undefined
: (resource.metadata?.namespace ?? workspace.project);
if (resource.kind === "Namespace" && name !== workspace.project) {
throw new Error(
`Cannot manage namespace ${name} from project ${workspace.project}`,
);
}
if (namespace && namespace !== workspace.project) {
throw new Error(
`Cannot manage ${resource.kind}/${name} in namespace ${namespace}`,
);
}
const existingOwner = resource.metadata?.labels?.[WORKSPACE_UID_LABEL];
if (existingOwner && existingOwner !== workspace.uid) {
throw new Error(`${resource.kind}/${name} belongs to another workspace`);
}
return {
...resource,
metadata: {
...resource.metadata,
name,
namespace,
labels: {
...resource.metadata?.labels,
...LABELS,
[WORKSPACE_PROJECT_LABEL]: workspace.project,
[WORKSPACE_UID_LABEL]: workspace.uid,
},
},
};
}
function labelExternal(
workspace: Workspace,
resource: KubernetesObject,
namespace: string,
): KubernetesObject {
const name = resourceName(resource);
if (resource.metadata?.namespace !== namespace) {
throw new Error(
`${resource.kind}/${name} is outside expected namespace ${namespace}`,
);
}
const existingOwner = resource.metadata?.labels?.[WORKSPACE_UID_LABEL];
if (existingOwner && existingOwner !== workspace.uid) {
throw new Error(`${resource.kind}/${name} belongs to another workspace`);
}
const { status: _status, ...body } = resource as KubernetesObject & {
status?: unknown;
};
return {
...body,
metadata: {
...body.metadata,
labels: {
...body.metadata?.labels,
[WORKSPACE_PROJECT_LABEL]: workspace.project,
[WORKSPACE_UID_LABEL]: workspace.uid,
},
},
};
}
function deploymentNames(deployments: V1Deployment[]): string[] {
return deployments
.map((deployment) => deployment.metadata?.name)
.filter((name): name is string => Boolean(name));
}
function selectTargets(all: string[], requested?: string[]): string[] {
if (!requested) return all;
const unique = [...new Set(requested)];
const available = new Set(all);
for (const name of unique) {
if (!available.has(name))
throw new Error(`No managed deployment named ${name}`);
}
return unique;
}
export function createManagementService(dependencies: ManagementDependencies) {
async function namespaceSafety(
workspace: Workspace,
): Promise<NamespaceSafety> {
validateWorkspace(workspace);
const namespace = await dependencies.readNamespace(workspace.project);
if (!namespace) return { project: workspace.project, status: "missing" };
const managed =
namespace.labels?.["app.kubernetes.io/managed-by"] ===
LABELS["app.kubernetes.io/managed-by"];
if (!managed) {
return {
project: workspace.project,
status: "external",
namespaceUid: namespace.uid,
};
}
if (namespace.labels?.[WORKSPACE_UID_LABEL] !== workspace.uid) {
return {
project: workspace.project,
status: "different-workspace",
namespaceUid: namespace.uid,
};
}
return {
project: workspace.project,
status: "owned",
namespaceUid: namespace.uid,
};
}
async function assertSafe(
workspace: Workspace,
allowMissing = false,
): Promise<NamespaceSafety> {
const safety = await namespaceSafety(workspace);
if (
safety.status === "owned" ||
(allowMissing && safety.status === "missing")
) {
return safety;
}
throw new Error(
`Namespace ${workspace.project} is ${safety.status}; refusing workspace mutation`,
);
}
async function targets(workspace: Workspace, names?: string[]) {
await assertSafe(workspace);
return selectTargets(
deploymentNames(await dependencies.listDeployments(workspace.project)),
names,
);
}
async function ownExternalResources(
workspace: Workspace,
namespace: string,
resources: KubernetesObject[],
execution?: OperationExecution,
): Promise<void> {
for (const resource of resources) {
throwIfExecutionAborted(execution);
await dependencies.applyResource(
labelExternal(workspace, resource, namespace),
execution,
);
}
}
async function deleteResources(
workspace: Workspace,
resources: ResourceIdentity[],
execution?: OperationExecution,
): Promise<void> {
await assertSafe(workspace);
for (const resource of resources) {
if (!resource.uid || resource.workspaceUid !== workspace.uid) {
throw new Error(
`${resource.kind}/${resource.name} has an invalid workspace deletion identity`,
);
}
}
for (const resource of resources) {
const event = {
resource: resourceProgressIdentity(resource),
phase: "delete" as const,
};
try {
throwIfExecutionAborted(execution);
await execution?.emit?.({ ...event, state: "started" });
await dependencies.deleteResource(resource, execution);
await execution?.emit?.({ ...event, state: "succeeded" });
} catch (error) {
await execution?.emit?.({
...event,
state: execution?.signal?.aborted ? "aborted" : "failed",
});
throw error;
}
}
}
async function planDown(
workspace: Workspace,
full = false,
): Promise<DownPlan> {
const safety = await assertSafe(workspace);
const projectResources = sortResources(
await dependencies.listProjectResources(workspace.project),
);
const retainedResources = full
? []
: projectResources.filter(
(resource) =>
resource.kind === "Ingress" ||
resource.kind === "PersistentVolumeClaim",
);
const deleteResources = full
? [...projectResources]
: projectResources.filter(
(resource) => !retainedResources.includes(resource),
);
if (full) {
// Database CRs may have consumers in other projects. Labels are not a
// reference count, including when this workspace created the CR.
const databases = await dependencies.listDatabaseResources(workspace.project);
retainedResources.push(...databases.filter(
(resource) => resource.kind === "Database" &&
resource.metadata?.labels?.[WORKSPACE_UID_LABEL] === workspace.uid,
));
deleteResources.push(
...databases.filter((resource) => resource.kind !== "Database"),
...(await dependencies.listStorageResources(workspace.project)),
);
if (!safety.namespaceUid) {
throw new Error(
`Namespace ${workspace.project} has no UID deletion precondition`,
);
}
deleteResources.push({
apiVersion: "v1",
kind: "Namespace",
metadata: {
name: workspace.project,
uid: safety.namespaceUid,
labels: {
...LABELS,
[WORKSPACE_PROJECT_LABEL]: workspace.project,
[WORKSPACE_UID_LABEL]: workspace.uid,
},
},
});
}
return {
full,
retained: retainedResources.map((item) => identity(workspace, item)),
delete: sortResources(deleteResources)
.reverse()
.map((item) => identity(workspace, item)),
};
}
return {
namespaceSafety,
async graphStatus(
workspace: Workspace,
options: { includeIdle?: boolean } = {},
): Promise<NamespaceGraph[]> {
await assertSafe(workspace);
return buildNamespaceGraphs(
await dependencies.fetchGraphObjects(workspace.project),
options,
);
},
async stop(
workspace: Workspace,
names?: string[],
execution?: OperationExecution,
): Promise<string[]> {
throwIfExecutionAborted(execution);
const selected = await targets(workspace, names);
const selectedSet = new Set(selected);
const hpas = (await dependencies.listProjectResources(workspace.project))
.filter(
(resource) =>
resource.apiVersion === "autoscaling/v2" &&
resource.kind === "HorizontalPodAutoscaler" &&
resource.metadata?.name !== undefined &&
selectedSet.has(resource.metadata.name),
)
.map((resource) => identity(workspace, resource));
if (hpas.length > 0) {
await deleteResources(workspace, hpas, execution);
}
for (const name of selected) {
throwIfExecutionAborted(execution);
await dependencies.scaleDeployment(
workspace.project,
name,
0,
execution,
);
}
return selected;
},
async restart(
workspace: Workspace,
names?: string[],
execution?: OperationExecution,
): Promise<string[]> {
const selected = await targets(workspace, names);
for (const name of selected) {
throwIfExecutionAborted(execution);
await dependencies.restartDeployment(
workspace.project,
name,
execution,
);
}
return selected;
},
async rollback(
workspace: Workspace,
names?: string[],
timeoutMs?: number,
execution?: OperationExecution,
): Promise<RollbackCandidate[]> {
throwIfExecutionAborted(execution);
await assertSafe(workspace);
const candidates = await dependencies.planRollback(
workspace.project,
names,
);
for (const candidate of candidates) {
throwIfExecutionAborted(execution);
await dependencies.rollbackDeployment(
workspace.project,
candidate,
execution,
);
}
for (const candidate of candidates) {
throwIfExecutionAborted(execution);
await dependencies.waitForDeployment(
workspace.project,
candidate.name,
timeoutMs,
execution,
);
}
return candidates;
},
databaseCredentialsMetadata(
compose: ComposeSpecification,
): CredentialMetadata[] {
return getComposePostgresClaims(compose).map((claim: PostgresClaim) => ({
service: claim.service,
provider: "postgres",
principal: claim.username,
resource: claim.database,
secretNamespace: DATABASE_NAMESPACE,
secretName: claim.secretName,
}));
},
async reconcileDatabases(
workspace: Workspace,
compose: ComposeSpecification,
execution?: OperationExecution,
) {
try {
throwIfExecutionAborted(execution);
await assertSafe(workspace, true);
} catch (error) {
throw new DatabaseReconciliationError("namespace precheck", error);
}
let environment: Record<string, Record<string, string>>;
try {
environment = await dependencies.reconcileDatabases(
workspace.project,
compose,
execution,
);
} catch (error) {
if (error instanceof DatabaseReconciliationError) throw error;
throw new DatabaseReconciliationError("database dependency", error);
}
let resources: KubernetesObject[];
try {
resources = await dependencies.listDatabaseResources(workspace.project);
} catch (error) {
throw new DatabaseReconciliationError("database resource listing", error);
}
try {
const claims = getComposePostgresClaims(compose);
const claimed = new Set(claims.map((claim) => claim.database));
for (const resource of resources) {
// Only a declared database may be shared. Never adopt its foreign
// workspace UID or apply its metadata through this field manager.
const existingOwner = resource.metadata?.labels?.[WORKSPACE_UID_LABEL];
if (resource.kind === "Database" && existingOwner &&
existingOwner !== workspace.uid &&
claimed.has(resource.metadata?.name ?? "")) {
if (resource.metadata?.namespace !== DATABASE_NAMESPACE) {
throw new Error("Database is outside expected namespace");
}
const claim = claims.find((item) => item.database === resource.metadata?.name);
const spec = (resource as KubernetesObject & {
spec?: { owner?: string; cluster?: { name?: string } };
}).spec;
if (spec?.owner !== claim?.username || spec?.cluster?.name !== DATABASE_CLUSTER) {
throw new Error("Database does not match the declared claim");
}
continue;
}
await ownExternalResources(workspace, DATABASE_NAMESPACE, [resource], execution);
}
} catch (error) {
throw new DatabaseReconciliationError("database resource ownership", error);
}
return environment;
},
async getDatabaseCredentials(workspace: Workspace, username: string) {
await assertSafe(workspace);
const databases = await dependencies.listDatabaseResources(
workspace.project,
);
const database = databases.find(
(resource) =>
resource.kind === "Database" &&
(resource as KubernetesObject & { spec?: { owner?: string } }).spec
?.owner === username,
);
if (!database) {
throw new Error(
`Database role ${username} is not managed by this workspace`,
);
}
assertResourceOwnership(workspace, database);
return dependencies.getDatabaseCredentials(username);
},
storageCredentialsMetadata(
compose: ComposeSpecification,
): CredentialMetadata[] {
return getComposeS3Claims(compose).map((claim) => ({
service: claim.service,
provider: "s3",
principal: claim.key,
resource: claim.bucket,
secretNamespace: STORAGE_NAMESPACE,
}));
},
async reconcileStorage(
workspace: Workspace,
compose: ComposeSpecification,
execution?: OperationExecution,
) {
throwIfExecutionAborted(execution);
await assertSafe(workspace, true);
const environment = await dependencies.reconcileStorage(
workspace.project,
compose,
execution,
);
await ownExternalResources(
workspace,
STORAGE_NAMESPACE,
await dependencies.listStorageResources(workspace.project),
execution,
);
return environment;
},
async getStorageCredentials(workspace: Workspace, claim: S3Claim) {
await assertSafe(workspace);
const resources = await dependencies.listStorageResources(
workspace.project,
);
const key = resources.find(
(resource) =>
resource.kind === "GarageKey" &&
resource.metadata?.name === claim.key,
);
const bucket = resources.find(
(resource) =>
resource.kind === "GarageBucket" &&
resource.metadata?.name === claim.bucket,
);
if (!key || !bucket) {
throw new Error(
`S3 claim ${claim.key}/${claim.bucket} is not managed by this workspace`,
);
}
assertResourceOwnership(workspace, key);
assertResourceOwnership(workspace, bucket);
return dependencies.getStorageCredentials(claim);
},
async planResources(
workspace: Workspace,
desired: KubernetesObject[],
): Promise<ResourcePlan> {
await assertSafe(workspace, true);
const labeled = sortResources(
desired.map((item) => labelDesired(workspace, item)),
);
const stale = await dependencies.findStaleResources(
workspace.project,
labeled,
);
return {
desired: labeled,
stale: sortResources(stale)
.reverse()
.map((item) => identity(workspace, item)),
};
},
async applyResources(
workspace: Workspace,
resources: KubernetesObject[],
execution?: OperationExecution,
): Promise<KubernetesObject[]> {
await assertSafe(workspace, true);
const applied: KubernetesObject[] = [];
for (const resource of sortResources(
resources.map((item) => labelDesired(workspace, item)),
)) {
const event = {
resource: resourceProgressIdentity({
apiVersion: resource.apiVersion!,
kind: resource.kind!,
name: resourceName(resource),
namespace: resource.metadata?.namespace,
}),
phase: "apply" as const,
};
try {
throwIfExecutionAborted(execution);
await execution?.emit?.({ ...event, state: "started" });
applied.push(await dependencies.applyResource(resource, execution));
await execution?.emit?.({ ...event, state: "succeeded" });
} catch (error) {
await execution?.emit?.({
...event,
state: execution?.signal?.aborted ? "aborted" : "failed",
});
throw error;
}
}
return applied;
},
async waitForResources(
workspace: Workspace,
deploymentTargets: string[],
timeoutMs?: number,
execution?: OperationExecution,
): Promise<void> {
const selected = await targets(workspace, deploymentTargets);
const deadline = Date.now() + (timeoutMs ?? 300_000);
const controller = new AbortController();
const cancel = () => controller.abort();
execution?.signal?.addEventListener("abort", cancel, { once: true });
if (execution?.signal?.aborted) cancel();
try {
await Promise.all(
selected.map(async (name) => {
const event = {
resource: {
apiVersion: "apps/v1",
kind: "Deployment",
name,
namespace: workspace.project,
},
phase: "wait" as const,
};
try {
throwIfExecutionAborted({ signal: controller.signal });
await execution?.emit?.({ ...event, state: "started" });
await dependencies.waitForDeployment(
workspace.project,
name,
Math.max(0, deadline - Date.now()),
{ ...execution, signal: controller.signal },
);
await execution?.emit?.({ ...event, state: "succeeded" });
} catch (error) {
await execution?.emit?.({
...event,
state: controller.signal.aborted ? "aborted" : "failed",
});
controller.abort();
throw error;
}
}),
);
} finally {
controller.abort();
execution?.signal?.removeEventListener("abort", cancel);
}
},
deleteResources,
planDown,
async down(
workspace: Workspace,
full = false,
execution?: OperationExecution,
): Promise<DownPlan> {
throwIfExecutionAborted(execution);
const plan = await planDown(workspace, full);
await deleteResources(workspace, plan.delete, execution);
return plan;
},
};
}