feat: harden self-managed reconciliation

server/management.ts

@@ -49,7 +49,6 @@ export const RESERVED_NAMESPACES = new Set([
   "kube-system",
   "kube-public",
   "kube-node-lease",
-  "kuber-system",
   DATABASE_NAMESPACE,
   STORAGE_NAMESPACE,
 ]);
@@ -99,6 +98,8 @@ export type CredentialMetadata = {
   secretName?: string;
 };
 
+export type OperationExecution = { signal?: AbortSignal };
+
 export type ManagementDependencies = {
   readNamespace(project: string): Promise<NamespaceRecord | undefined>;
   listDeployments(project: string): Promise<V1Deployment[]>;
@@ -106,18 +107,25 @@ export type ManagementDependencies = {
     project: string,
     name: string,
     replicas: number,
+    execution?: OperationExecution,
+  ): Promise<unknown>;
+  restartDeployment(
+    project: string,
+    name: string,
+    execution?: OperationExecution,
   ): Promise<unknown>;
-  restartDeployment(project: string, name: string): Promise<unknown>;
   waitForDeployment(
     project: string,
     name: string,
     timeoutMs?: number,
+    execution?: OperationExecution,
   ): Promise<void>;
   fetchGraphObjects(project: string): Promise<KubernetesObject[]>;
   planRollback(project: string, names?: string[]): Promise<RollbackCandidate[]>;
   rollbackDeployment(
     project: string,
     candidate: RollbackCandidate,
+    execution?: OperationExecution,
   ): Promise<unknown>;
   listProjectResources(project: string): Promise<KubernetesObject[]>;
   listDatabaseResources(project: string): Promise<KubernetesObject[]>;
@@ -126,16 +134,24 @@ export type ManagementDependencies = {
     project: string,
     desired: KubernetesObject[],
   ): Promise<KubernetesObject[]>;
-  applyResource(resource: KubernetesObject): Promise<KubernetesObject>;
-  deleteResource(identity: ResourceIdentity): Promise<void>;
+  applyResource(
+    resource: KubernetesObject,
+    execution?: OperationExecution,
+  ): Promise<KubernetesObject>;
+  deleteResource(
+    identity: ResourceIdentity,
+    execution?: OperationExecution,
+  ): Promise<void>;
   reconcileDatabases(
     project: string,
     compose: ComposeSpecification,
+    execution?: OperationExecution,
   ): Promise<Record<string, Record<string, string>>>;
   getDatabaseCredentials(username: string): Promise<RoleCredentials>;
   reconcileStorage(
     project: string,
     compose: ComposeSpecification,
+    execution?: OperationExecution,
   ): Promise<Record<string, Record<string, string>>>;
   getStorageCredentials(claim: S3Claim): Promise<Record<string, string>>;
 };
@@ -161,9 +177,11 @@ const defaultOperations: Omit<
   listStorageResources: listManagedStorageResources,
   findStaleResources: getStaleResources,
   applyResource,
-  reconcileDatabases: reconcilePostgresClaims,
+  reconcileDatabases: async (project, compose, execution) =>
+    reconcilePostgresClaims(project, compose, execution?.signal),
   getDatabaseCredentials: getRoleCredentials,
-  reconcileStorage: reconcileS3Claims,
+  reconcileStorage: async (project, compose, execution) =>
+    reconcileS3Claims(project, compose, execution?.signal),
   getStorageCredentials: getS3Credentials,
 };
 
@@ -191,6 +209,11 @@ function validateWorkspace(workspace: Workspace): void {
   }
 }
 
+function throwIfExecutionAborted(execution?: OperationExecution): void {
+  if (!execution?.signal?.aborted) return;
+  throw new Error("Workspace operation execution was cancelled");
+}
+
 function resourceName(resource: KubernetesObject): string {
   const name = resource.metadata?.name;
   if (!resource.apiVersion || !resource.kind || !name) {
@@ -378,10 +401,13 @@ export function createManagementService(dependencies: ManagementDependencies) {
     workspace: Workspace,
     namespace: string,
     resources: KubernetesObject[],
+    execution?: OperationExecution,
   ): Promise<void> {
     for (const resource of resources) {
+      throwIfExecutionAborted(execution);
       await dependencies.applyResource(
         labelExternal(workspace, resource, namespace),
+        execution,
       );
     }
   }
@@ -389,6 +415,7 @@ export function createManagementService(dependencies: ManagementDependencies) {
   async function deleteResources(
     workspace: Workspace,
     resources: ResourceIdentity[],
+    execution?: OperationExecution,
   ): Promise<void> {
     await assertSafe(workspace);
     for (const resource of resources) {
@@ -399,7 +426,8 @@ export function createManagementService(dependencies: ManagementDependencies) {
       }
     }
     for (const resource of resources) {
-      await dependencies.deleteResource(resource);
+      throwIfExecutionAborted(execution);
+      await dependencies.deleteResource(resource, execution);
     }
   }
 
@@ -472,7 +500,12 @@ export function createManagementService(dependencies: ManagementDependencies) {
       );
     },
 
-    async stop(workspace: Workspace, names?: string[]): Promise<string[]> {
+    async stop(
+      workspace: Workspace,
+      names?: string[],
+      execution?: OperationExecution,
+    ): Promise<string[]> {
+      throwIfExecutionAborted(execution);
       const selected = await targets(workspace, names);
       const selectedSet = new Set(selected);
       const hpas = (await dependencies.listProjectResources(workspace.project))
@@ -485,18 +518,24 @@ export function createManagementService(dependencies: ManagementDependencies) {
         )
         .map((resource) => identity(workspace, resource));
       if (hpas.length > 0) {
-        await deleteResources(workspace, hpas);
+        await deleteResources(workspace, hpas, execution);
       }
       for (const name of selected) {
-        await dependencies.scaleDeployment(workspace.project, name, 0);
+        throwIfExecutionAborted(execution);
+        await dependencies.scaleDeployment(workspace.project, name, 0, execution);
       }
       return selected;
     },
 
-    async restart(workspace: Workspace, names?: string[]): Promise<string[]> {
+    async restart(
+      workspace: Workspace,
+      names?: string[],
+      execution?: OperationExecution,
+    ): Promise<string[]> {
       const selected = await targets(workspace, names);
       for (const name of selected) {
-        await dependencies.restartDeployment(workspace.project, name);
+        throwIfExecutionAborted(execution);
+        await dependencies.restartDeployment(workspace.project, name, execution);
       }
       return selected;
     },
@@ -505,20 +544,25 @@ export function createManagementService(dependencies: ManagementDependencies) {
       workspace: Workspace,
       names?: string[],
       timeoutMs?: number,
+      execution?: OperationExecution,
     ): Promise<RollbackCandidate[]> {
+      throwIfExecutionAborted(execution);
       await assertSafe(workspace);
       const candidates = await dependencies.planRollback(
         workspace.project,
         names,
       );
       for (const candidate of candidates) {
-        await dependencies.rollbackDeployment(workspace.project, candidate);
+        throwIfExecutionAborted(execution);
+        await dependencies.rollbackDeployment(workspace.project, candidate, execution);
       }
       for (const candidate of candidates) {
+        throwIfExecutionAborted(execution);
         await dependencies.waitForDeployment(
           workspace.project,
           candidate.name,
           timeoutMs,
+          execution,
         );
       }
       return candidates;
@@ -540,16 +584,20 @@ export function createManagementService(dependencies: ManagementDependencies) {
     async reconcileDatabases(
       workspace: Workspace,
       compose: ComposeSpecification,
+      execution?: OperationExecution,
     ) {
+      throwIfExecutionAborted(execution);
       await assertSafe(workspace, true);
       const environment = await dependencies.reconcileDatabases(
         workspace.project,
         compose,
+        execution,
       );
       await ownExternalResources(
         workspace,
         DATABASE_NAMESPACE,
         await dependencies.listDatabaseResources(workspace.project),
+        execution,
       );
       return environment;
     },
@@ -589,16 +637,20 @@ export function createManagementService(dependencies: ManagementDependencies) {
     async reconcileStorage(
       workspace: Workspace,
       compose: ComposeSpecification,
+      execution?: OperationExecution,
     ) {
+      throwIfExecutionAborted(execution);
       await assertSafe(workspace, true);
       const environment = await dependencies.reconcileStorage(
         workspace.project,
         compose,
+        execution,
       );
       await ownExternalResources(
         workspace,
         STORAGE_NAMESPACE,
         await dependencies.listStorageResources(workspace.project),
+        execution,
       );
       return environment;
     },
@@ -651,13 +703,15 @@ export function createManagementService(dependencies: ManagementDependencies) {
     async applyResources(
       workspace: Workspace,
       resources: KubernetesObject[],
+      execution?: OperationExecution,
     ): Promise<KubernetesObject[]> {
       await assertSafe(workspace, true);
       const applied: KubernetesObject[] = [];
       for (const resource of sortResources(
         resources.map((item) => labelDesired(workspace, item)),
       )) {
-        applied.push(await dependencies.applyResource(resource));
+        throwIfExecutionAborted(execution);
+        applied.push(await dependencies.applyResource(resource, execution));
       }
       return applied;
     },
@@ -666,13 +720,16 @@ export function createManagementService(dependencies: ManagementDependencies) {
       workspace: Workspace,
       deploymentTargets: string[],
       timeoutMs?: number,
+      execution?: OperationExecution,
     ): Promise<void> {
       const selected = await targets(workspace, deploymentTargets);
       for (const name of selected) {
+        throwIfExecutionAborted(execution);
         await dependencies.waitForDeployment(
           workspace.project,
           name,
           timeoutMs,
+          execution,
         );
       }
     },
@@ -681,9 +738,14 @@ export function createManagementService(dependencies: ManagementDependencies) {
 
     planDown,
 
-    async down(workspace: Workspace, full = false): Promise<DownPlan> {
+    async down(
+      workspace: Workspace,
+      full = false,
+      execution?: OperationExecution,
+    ): Promise<DownPlan> {
+      throwIfExecutionAborted(execution);
       const plan = await planDown(workspace, full);
-      await deleteResources(workspace, plan.delete);
+      await deleteResources(workspace, plan.delete, execution);
       return plan;
     },
   };