import { AppsV1Api, BatchV1Api, CoordinationV1Api, CoreV1Api, KubeConfig, KubernetesObjectApi, PatchStrategy, type KubernetesObject, type V1Lease, type V1LeaseSpec, type V1PodTemplateSpec, type V1ReplicaSet, } from "@kubernetes/client-node"; import { createHash } from "node:crypto"; import { createKubernetesHttpLibrary } from "../lib/k8s-http"; import { sanitizeKubernetesObject } from "../lib/kubernetes-object"; import { type AuditEvent, type AuditPersistence } from "./audit-store"; import { managementDependencies, type ManagementDependencies, type OperationExecution, type ResourceIdentity, } from "./management"; import type { RollbackCandidate } from "../lib/rollback"; import { LABELS } from "../const"; import type { WorkspaceAdoptionResult, WorkspaceAdoptionService } from "./app"; import { WorkspaceAdoptionError } from "./app"; import { validateTrust, type TrustStore } from "./trust-store"; import { RESERVED_NAMESPACES, WORKSPACE_PROJECT_LABEL, WORKSPACE_UID_LABEL, } from "./management"; import { OperationConflictError, OperationNotFoundError, OperationValidationError, type Operation, type OperationPersistence, type WorkspaceLease, type WorkspaceLeaseProvider, } from "./operation-store"; import { PersistentWorkspaceStore, WorkspaceConflictError, WorkspaceNotFoundError, type Workspace, type WorkspacePersistence, type WorkspaceRevision, type WorkspaceStoreOptions, } from "./workspace-store"; export const KUBER_STATE_NAMESPACE = "kuber-system"; const FIELD_MANAGER = "kuber-server"; const TYPE_LABEL = "kuber.astrxl.dev/type"; const WORKSPACE_LABEL = "kuber.astrxl.dev/workspace"; const MANAGED_SELECTOR = "app.kubernetes.io/managed-by=kuber"; const ADOPTABLE_RESOURCES = [ { apiVersion: "v1", kind: "PersistentVolumeClaim" }, { apiVersion: "v1", kind: "Secret" }, { apiVersion: "v1", kind: "ConfigMap" }, { apiVersion: "v1", kind: "Service" }, { apiVersion: "apps/v1", kind: "Deployment" }, { apiVersion: "networking.k8s.io/v1", kind: "Ingress" }, { apiVersion: "traefik.io/v1alpha1", kind: "IngressRoute" }, ] as const; type DataObject = KubernetesObject & { data?: Record; stringData?: Record; type?: string; }; function isNotFound(error: unknown): boolean { return Boolean( error && typeof error === "object" && (("code" in error && error.code === 404) || ("statusCode" in error && error.statusCode === 404)), ); } function isConflict(error: unknown): boolean { return Boolean( error && typeof error === "object" && (("code" in error && error.code === 409) || ("statusCode" in error && error.statusCode === 409)), ); } function digestName(prefix: string, value: string): string { return `${prefix}-${createHash("sha256").update(value).digest("hex").slice(0, 48)}`; } function encode(value: unknown): string { return JSON.stringify(value); } function decodeSecretData(value: string | undefined): string | undefined { if (!value) return; try { return Buffer.from(value, "base64").toString("utf8"); } catch { return; } } function parsePayload(object: DataObject): T | undefined { const payload = object.kind === "Secret" ? decodeSecretData(object.data?.payload) : object.data?.payload; if (!payload) return; try { return JSON.parse(payload) as T; } catch { return; } } export function createKubernetesConfig(): KubeConfig { const config = new KubeConfig(); if (process.env.KUBERNETES_SERVICE_HOST) config.loadFromCluster(); else config.loadFromDefault(); const makeApiClient = config.makeApiClient.bind(config); const httpLibrary = createKubernetesHttpLibrary({ maxConcurrent: 4, minIntervalMs: 0, }); config.makeApiClient = ((apiClientType) => { const client = makeApiClient(apiClientType) as unknown as { api?: { configuration?: { httpApi?: typeof httpLibrary } }; configuration?: { httpApi?: typeof httpLibrary }; }; if (client.api?.configuration) client.api.configuration.httpApi = httpLibrary; if (client.configuration) client.configuration.httpApi = httpLibrary; return client; }) as typeof config.makeApiClient; return config; } export function createKubernetesClients(): { config: KubeConfig; objects: KubernetesObjectApi; apps: AppsV1Api; batch: BatchV1Api; core: CoreV1Api; coordination: CoordinationV1Api; } { const config = createKubernetesConfig(); return { config, objects: KubernetesObjectApi.makeApiClient(config), apps: config.makeApiClient(AppsV1Api), batch: config.makeApiClient(BatchV1Api), core: config.makeApiClient(CoreV1Api), coordination: config.makeApiClient(CoordinationV1Api), }; } /** * Minimal adapter over the coordination.k8s.io/v1 Lease API used by the lease * provider. Kept small and fakeable so the provider can be tested without a * cluster. */ export interface LeaseObjects { create(value: V1Lease): Promise; read(name: string, namespace: string): Promise; replace(value: V1Lease): Promise; delete( name: string, namespace: string, expectedResourceVersion?: string, ): Promise; } /** Wraps a CoordinationV1Api client in the LeaseObjects adapter. */ export function createKubernetesLeaseObjects( coordination: CoordinationV1Api, ): LeaseObjects { return { async create(value) { return coordination.createNamespacedLease({ namespace: value.metadata?.namespace ?? KUBER_STATE_NAMESPACE, body: value, }); }, async read(name, namespace) { try { return await coordination.readNamespacedLease({ name, namespace }); } catch (error) { if (isNotFound(error)) return; throw error; } }, replace(value) { return coordination.replaceNamespacedLease({ name: value.metadata?.name ?? "", namespace: value.metadata?.namespace ?? KUBER_STATE_NAMESPACE, body: value, }); }, async delete(name, namespace, expectedResourceVersion) { try { await coordination.deleteNamespacedLease({ name, namespace, ...(expectedResourceVersion && { body: { preconditions: { resourceVersion: expectedResourceVersion }, }, }), }); } catch (error) { if (isNotFound(error)) return; throw error; } }, }; } const KUBER_LEASE_API_VERSION = "coordination.k8s.io/v1"; const DEFAULT_LEASE_TTL_MS = 30_000; const MAX_LEASE_ACQUIRE_RETRIES = 5; /** * Kubernetes coordinates.k8s.io/v1 Lease acquireTime/renewTime are * metav1.MicroTime, which expect six fractional-second digits (for example * "2026-09-03T00:23:00.205000Z"). JavaScript Date#toISOString only emits three * digits (milliseconds), and kubernetes-client-node does not re-format * V1MicroTime when serializing, so the API server rejects the unpadded value. */ function microTimeString(ms: number): string { const iso = new Date(ms).toISOString(); const match = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d+))?Z$/.exec( iso, ); if (!match) return iso; const fraction = (match[2] ?? "").padEnd(6, "0"); return `${match[1]}.${fraction}Z`; } function leaseExpired(lease: V1Lease, nowMs: number): boolean { const renewTime = lease.spec?.renewTime ? Date.parse(String(lease.spec.renewTime)) : Number.NaN; const durationMs = (lease.spec?.leaseDurationSeconds ?? 0) * 1000; if (!Number.isFinite(renewTime)) return true; return renewTime + durationMs <= nowMs; } /** * Production WorkspaceLeaseProvider backed by coordination.k8s.io/v1 Lease * objects. Acquisition, renewal, and release are all optimistic: every mutation * carries the expected resourceVersion so the API server rejects lost-update * races. An expired lease may be taken over by any holder (expiry takeover). */ export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider { private readonly now: () => number; constructor( private readonly objects: LeaseObjects, private readonly namespace = KUBER_STATE_NAMESPACE, private readonly clock: () => number = Date.now, ) { this.now = clock; } private leaseName(workspaceId: string): string { return digestName("lease", workspaceId); } private leaseSpec( workspaceId: string, holder: string, ttlMs: number, ): V1Lease { return { apiVersion: KUBER_LEASE_API_VERSION, kind: "Lease", metadata: { name: this.leaseName(workspaceId), namespace: this.namespace, }, spec: { holderIdentity: holder, leaseDurationSeconds: Math.max(1, Math.round(ttlMs / 1000)), acquireTime: microTimeString( this.now(), ) as unknown as V1LeaseSpec["acquireTime"], renewTime: microTimeString( this.now(), ) as unknown as V1LeaseSpec["renewTime"], leaseTransitions: 0, }, }; } async acquire( workspaceId: string, holder: string, ttlMs = DEFAULT_LEASE_TTL_MS, ): Promise { if (!workspaceId || !holder || !Number.isFinite(ttlMs) || ttlMs <= 0) throw new OperationValidationError("Invalid workspace lease request"); const name = this.leaseName(workspaceId); for (let attempt = 0; attempt <= MAX_LEASE_ACQUIRE_RETRIES; attempt++) { const nowMs = this.now(); const lease = await this.objects.read(name, this.namespace); if (lease && !leaseExpired(lease, nowMs)) return; try { if (!lease) { const created = await this.objects.create( this.leaseSpec(workspaceId, holder, ttlMs), ); return this.wrap(created, workspaceId, holder); } const next: V1Lease = { ...this.leaseSpec(workspaceId, holder, ttlMs), metadata: { ...this.leaseSpec(workspaceId, holder, ttlMs).metadata, resourceVersion: lease.metadata?.resourceVersion, }, spec: { ...this.leaseSpec(workspaceId, holder, ttlMs).spec, leaseTransitions: (lease.spec?.leaseTransitions ?? 0) + 1, }, }; const replaced = await this.objects.replace(next); return this.wrap(replaced, workspaceId, holder); } catch (error) { if (isConflict(error)) { const raced = await this.objects.read(name, this.namespace); if (raced && !leaseExpired(raced, this.now())) return; continue; } throw error; } } return undefined; } private wrap( lease: V1Lease, workspaceId: string, holder: string, ): WorkspaceLease { const leaseDurationMs = (lease.spec?.leaseDurationSeconds ?? 30) * 1000; const expiresAt = ( Date.parse(String(lease.spec?.renewTime)) + leaseDurationMs ).toString(); const renew = async (ttlMs = 30_000): Promise => { if (!Number.isFinite(ttlMs) || ttlMs <= 0) return false; const name = lease.metadata?.name; if (!name) return false; const current = await this.objects.read(name, this.namespace); if (!current || leaseExpired(current, this.now())) return false; if (current.spec?.holderIdentity !== holder) return false; const next: V1Lease = { ...current, metadata: { ...current.metadata, resourceVersion: current.metadata?.resourceVersion, }, spec: { ...current.spec, holderIdentity: holder, leaseDurationSeconds: Math.max(1, Math.round(ttlMs / 1000)), renewTime: microTimeString( this.now(), ) as unknown as V1LeaseSpec["renewTime"], }, }; try { await this.objects.replace(next); return true; } catch (error) { if (isConflict(error)) return false; throw error; } }; const release = async (): Promise => { const name = lease.metadata?.name; if (!name) return; const current = await this.objects.read(name, this.namespace); if (current?.spec?.holderIdentity !== holder) return; const resourceVersion = current.metadata?.resourceVersion; if (!resourceVersion) return; try { await this.objects.delete(name, this.namespace, resourceVersion); } catch (error) { // A replace by a successor between read and delete invalidates the // resourceVersion precondition. Its lease must remain intact. if (isConflict(error) || isNotFound(error)) return; throw error; } }; return { workspaceId, holder, expiresAt, renew, release }; } } async function read( objects: KubernetesObjectApi, kind: "Secret" | "ConfigMap", name: string, ): Promise { try { return (await objects.read({ apiVersion: "v1", kind, metadata: { name, namespace: KUBER_STATE_NAMESPACE }, })) as DataObject; } catch (error) { if (isNotFound(error)) return; throw error; } } async function list( objects: KubernetesObjectApi, kind: "Secret" | "ConfigMap", type: string, workspaceId?: string, ): Promise { const selector = [ `${TYPE_LABEL}=${type}`, workspaceId && `${WORKSPACE_LABEL}=${workspaceId}`, ] .filter(Boolean) .join(","); const response = await objects.list( "v1", kind, KUBER_STATE_NAMESPACE, undefined, undefined, undefined, undefined, selector, ); return response.items.map((item) => ({ apiVersion: "v1", kind, ...item, })) as DataObject[]; } function stateObject( kind: "Secret" | "ConfigMap", name: string, type: string, value: unknown, workspaceId?: string, resourceVersion?: string, ): DataObject { const metadata = { name, namespace: KUBER_STATE_NAMESPACE, labels: { [TYPE_LABEL]: type, ...(workspaceId && { [WORKSPACE_LABEL]: workspaceId }), }, ...(resourceVersion && { resourceVersion }), }; return kind === "Secret" ? { apiVersion: "v1", kind, metadata, type: "Opaque", stringData: { payload: encode(value) }, } : { apiVersion: "v1", kind, metadata, data: { payload: encode(value) } }; } async function createObject( objects: KubernetesObjectApi, value: DataObject, ): Promise { await objects.create(value); } async function deleteObject( objects: KubernetesObjectApi, kind: "Secret" | "ConfigMap", name: string, ): Promise { try { await objects.delete({ apiVersion: "v1", kind, metadata: { name, namespace: KUBER_STATE_NAMESPACE }, }); return true; } catch (error) { if (isNotFound(error)) return false; throw error; } } export class KubernetesWorkspacePersistence implements WorkspacePersistence { constructor(private readonly objects = createKubernetesClients().objects) {} private workspaceName(id: string): string { return digestName("workspace", id); } private revisionName(id: string, revision: number): string { return digestName("revision", `${id}\0${revision}`); } async get(id: string): Promise { const object = await read(this.objects, "Secret", this.workspaceName(id)); const workspace = object && parsePayload(object); return workspace?.metadata.name === id ? workspace : undefined; } async list(): Promise { return (await list(this.objects, "Secret", "workspace")) .map((item) => parsePayload(item)) .filter((item): item is Workspace => item?.kind === "Workspace") .sort((a, b) => a.metadata.name.localeCompare(b.metadata.name)); } async create( workspace: Workspace, revision: WorkspaceRevision, ): Promise { const revisionName = this.revisionName( workspace.metadata.name, revision.spec.revision, ); let revisionCreated = false; try { await createObject( this.objects, stateObject( "Secret", revisionName, "workspace-revision", revision, workspace.metadata.name, ), ); revisionCreated = true; await createObject( this.objects, stateObject( "Secret", this.workspaceName(workspace.metadata.name), "workspace", workspace, workspace.metadata.name, ), ); } catch (error) { if (revisionCreated) await deleteObject(this.objects, "Secret", revisionName).catch( () => false, ); if (isConflict(error)) throw new WorkspaceConflictError("Workspace already exists"); throw error; } } async replace( workspace: Workspace, revision: WorkspaceRevision, expectedResourceVersion: string, ): Promise { const name = this.workspaceName(workspace.metadata.name); const currentObject = await read(this.objects, "Secret", name); const current = currentObject && parsePayload(currentObject); if (!current || !currentObject?.metadata?.resourceVersion) throw new WorkspaceNotFoundError("Workspace not found"); if (current.metadata.resourceVersion !== expectedResourceVersion) throw new WorkspaceConflictError("Workspace was concurrently modified"); const revisionName = this.revisionName( workspace.metadata.name, revision.spec.revision, ); let revisionCreated = false; try { try { await createObject( this.objects, stateObject( "Secret", revisionName, "workspace-revision", revision, workspace.metadata.name, ), ); revisionCreated = true; } catch (error) { if (!isConflict(error)) throw error; const existingObject = await read(this.objects, "Secret", revisionName); const existing = existingObject && parsePayload(existingObject); const matches = existing?.kind === "WorkspaceRevision" && existing.metadata.name === revision.metadata.name && existing.metadata.workspaceUid === revision.metadata.workspaceUid && existing.metadata.resourceVersion === revision.metadata.resourceVersion && JSON.stringify(existing.spec) === JSON.stringify(revision.spec); if (!matches) throw new WorkspaceConflictError("Workspace revision already exists"); } await this.objects.replace( stateObject( "Secret", name, "workspace", workspace, workspace.metadata.name, currentObject.metadata.resourceVersion, ), ); } catch (error) { if (revisionCreated) await deleteObject(this.objects, "Secret", revisionName).catch( () => false, ); if (isConflict(error)) throw new WorkspaceConflictError("Workspace was concurrently modified"); throw error; } } async getRevision( id: string, revision: number, ): Promise { const object = await read( this.objects, "Secret", this.revisionName(id, revision), ); const value = object && parsePayload(object); return value?.spec.workspaceId === id && value.spec.revision === revision ? value : undefined; } async listRevisions(id: string): Promise { return (await list(this.objects, "Secret", "workspace-revision", id)) .map((item) => parsePayload(item)) .filter( (item): item is WorkspaceRevision => item?.kind === "WorkspaceRevision" && item.spec.workspaceId === id, ) .sort((a, b) => a.spec.revision - b.spec.revision); } async delete(id: string): Promise { const deleted = await deleteObject( this.objects, "Secret", this.workspaceName(id), ); for (const revision of await list( this.objects, "Secret", "workspace-revision", id, )) { if (revision.metadata?.name) await deleteObject(this.objects, "Secret", revision.metadata.name); } return deleted; } adopt(workspaceId: string, workspaceUid: string) { return new KubernetesWorkspaceAdoptionService(this.objects).adopt( workspaceId, workspaceUid, ); } adoptPlatform(workspaceUid: string) { return new KubernetesWorkspaceAdoptionService(this.objects).adoptPlatform( workspaceUid, ); } } export class KubernetesWorkspaceStore extends PersistentWorkspaceStore { constructor( private readonly kubernetesPersistence = new KubernetesWorkspacePersistence(), options: WorkspaceStoreOptions = {}, ) { super(kubernetesPersistence, options); } delete(id: string): Promise { return this.kubernetesPersistence.delete(id); } adopt(workspaceId: string, workspaceUid: string) { return this.kubernetesPersistence.adopt(workspaceId, workspaceUid); } adoptPlatform(workspaceUid: string) { return this.kubernetesPersistence.adoptPlatform(workspaceUid); } } export class KubernetesWorkspaceAdoptionService implements WorkspaceAdoptionService { constructor(private readonly objects = createKubernetesClients().objects) {} private async adoptNamespace( workspaceId: string, workspaceUid: string, platform: boolean, ): Promise { if (!workspaceUid.trim()) throw new WorkspaceAdoptionError("Workspace UID is required"); if ( platform ? workspaceId !== KUBER_STATE_NAMESPACE : RESERVED_NAMESPACES.has(workspaceId) || workspaceId.startsWith("kube-") ) { throw new WorkspaceAdoptionError(`Namespace ${workspaceId} is reserved`); } let namespace: KubernetesObject; try { namespace = await this.objects.read({ apiVersion: "v1", kind: "Namespace", metadata: { name: workspaceId }, }); } catch (error) { if (isNotFound(error)) { return { workspaceId, workspaceUid, resourcesAdopted: 0 }; } throw error; } if ( namespace.metadata?.labels?.["app.kubernetes.io/managed-by"] !== LABELS["app.kubernetes.io/managed-by"] ) { throw new WorkspaceAdoptionError( `Namespace ${workspaceId} is not managed by kuber; refusing adoption`, ); } const namespaceOwner = namespace.metadata.labels?.[WORKSPACE_UID_LABEL]; if (namespaceOwner && namespaceOwner !== workspaceUid) { throw new WorkspaceAdoptionError( `Namespace ${workspaceId} belongs to another workspace`, ); } const resources: Array<{ apiVersion: string; kind: string; item: KubernetesObject; }> = []; for (const { apiVersion, kind } of ADOPTABLE_RESOURCES) { try { const result = await this.objects.list( apiVersion, kind, workspaceId, undefined, undefined, undefined, undefined, MANAGED_SELECTOR, ); resources.push( ...result.items.map((item) => ({ apiVersion, kind, item })), ); } catch (error) { if (!isNotFound(error)) throw error; } } for (const { kind, item } of resources) { const owner = item.metadata?.labels?.[WORKSPACE_UID_LABEL]; if (owner && owner !== workspaceUid) { throw new WorkspaceAdoptionError( `${kind}/${item.metadata?.name} belongs to another workspace`, ); } } const adoptionLabels = { ...LABELS, [WORKSPACE_PROJECT_LABEL]: workspaceId, [WORKSPACE_UID_LABEL]: workspaceUid, }; await this.objects.patch( { apiVersion: "v1", kind: "Namespace", metadata: { name: workspaceId, labels: { ...namespace.metadata?.labels, ...adoptionLabels }, }, }, undefined, undefined, undefined, undefined, PatchStrategy.MergePatch, ); for (const { apiVersion, kind, item } of resources) { await this.objects.patch( { apiVersion, kind, metadata: { name: item.metadata?.name, namespace: workspaceId, labels: { ...item.metadata?.labels, ...adoptionLabels }, }, }, undefined, undefined, undefined, undefined, PatchStrategy.MergePatch, ); } return { workspaceId, workspaceUid, resourcesAdopted: resources.length, }; } adopt(workspaceId: string, workspaceUid: string) { return this.adoptNamespace(workspaceId, workspaceUid, false); } adoptPlatform(workspaceUid: string) { return this.adoptNamespace(KUBER_STATE_NAMESPACE, workspaceUid, true); } } export class KubernetesOperationPersistence implements OperationPersistence { constructor(private readonly objects = createKubernetesClients().objects) {} async createIdempotent(operation: Operation) { const operationName = digestName( "operation", `${operation.spec.workspaceId}\0${operation.spec.idempotencyKey}`, ); const deterministic: Operation = { ...operation, metadata: { ...operation.metadata, name: operationName }, }; try { await createObject( this.objects, stateObject( "Secret", operationName, "operation", deterministic, deterministic.spec.workspaceId, ), ); return { operation: deterministic, created: true }; } catch (error) { if (!isConflict(error)) throw error; const existing = await this.get(operationName); if (!existing) throw new OperationConflictError( "Idempotent operation could not be recovered", ); return { operation: existing, created: false }; } } async get(id: string): Promise { const object = await read(this.objects, "Secret", id); const operation = object && parsePayload(object); return operation?.kind === "Operation" ? operation : undefined; } async list(workspaceId?: string): Promise { return (await list(this.objects, "Secret", "operation", workspaceId)) .map((item) => parsePayload(item)) .filter((item): item is Operation => item?.kind === "Operation") .sort((a, b) => a.metadata.creationTimestamp.localeCompare( b.metadata.creationTimestamp, ), ); } async replace( operation: Operation, expectedResourceVersion: string, ): Promise { const currentObject = await read( this.objects, "Secret", operation.metadata.name, ); const current = currentObject && parsePayload(currentObject); if (!current || !currentObject?.metadata?.resourceVersion) throw new OperationNotFoundError("Operation not found"); if (current.metadata.resourceVersion !== expectedResourceVersion) throw new OperationConflictError("Operation was concurrently modified"); try { await this.objects.replace( stateObject( "Secret", operation.metadata.name, "operation", operation, operation.spec.workspaceId, currentObject.metadata.resourceVersion, ), ); } catch (error) { if (isConflict(error)) throw new OperationConflictError("Operation was concurrently modified"); throw error; } } } export class KubernetesAuditPersistence implements AuditPersistence { constructor(private readonly objects = createKubernetesClients().objects) {} async append(event: AuditEvent): Promise { await createObject( this.objects, stateObject( "ConfigMap", event.metadata.name, "audit", event, event.spec.workspaceId, ), ); } async list(workspaceId?: string): Promise { return (await list(this.objects, "ConfigMap", "audit", workspaceId)) .map((item) => parsePayload(item)) .filter((item): item is AuditEvent => item?.kind === "AuditEvent") .sort((a, b) => a.metadata.creationTimestamp.localeCompare( b.metadata.creationTimestamp, ), ); } } export class KubernetesTrustStore implements TrustStore { constructor(private readonly objects = createKubernetesClients().objects) {} private name(project: string, fingerprint: string) { return digestName("trust", `${project}\0${fingerprint}`); } async grant(project: string, fingerprint: string): Promise { validateTrust(project, fingerprint); if (await this.has(project, fingerprint)) return; await createObject( this.objects, stateObject( "ConfigMap", this.name(project, fingerprint), "trust", { project, fingerprint }, project, ), ); } async list(project: string): Promise { return (await list(this.objects, "ConfigMap", "trust", project)) .map((item) => parsePayload<{ project: string; fingerprint: string }>(item), ) .filter((record): record is { project: string; fingerprint: string } => Boolean( record && record.project === project && /^[a-f0-9]{64}$/.test(record.fingerprint), ), ) .map((record) => record.fingerprint); } async has(project: string, fingerprint: string): Promise { const item = await read( this.objects, "ConfigMap", this.name(project, fingerprint), ); const record = item && parsePayload<{ project: string; fingerprint: string }>(item); return record?.project === project && record.fingerprint === fingerprint; } async revoke(project: string, fingerprint: string): Promise { if (!(await this.has(project, fingerprint))) return false; return deleteObject( this.objects, "ConfigMap", this.name(project, fingerprint), ); } } function resource(identity: ResourceIdentity): KubernetesObject { return { apiVersion: identity.apiVersion, kind: identity.kind, metadata: { name: identity.name, namespace: identity.namespace, uid: identity.uid, }, }; } async function sleepUntilExecutionCancelled( delayMs: number, execution?: OperationExecution, ): Promise { const signal = execution?.signal; if (!signal) { await Bun.sleep(delayMs); return; } if (signal.aborted) throw new Error("Workspace operation execution was cancelled"); await new Promise((resolve, reject) => { const timer = setTimeout(() => { signal.removeEventListener("abort", cancelSleep); resolve(); }, delayMs); const cancelSleep = () => { clearTimeout(timer); reject(new Error("Workspace operation execution was cancelled")); }; signal.addEventListener("abort", cancelSleep, { once: true }); }); } export function createKubernetesManagementDependencies( clients = createKubernetesClients(), ): ManagementDependencies { const { objects, apps } = clients; const revision = (replicaSet: V1ReplicaSet): number | undefined => { const value = Number( replicaSet.metadata?.annotations?.["deployment.kubernetes.io/revision"], ); return Number.isSafeInteger(value) && value > 0 ? value : undefined; }; const stripHash = (template: V1PodTemplateSpec): V1PodTemplateSpec => { const labels = { ...template.metadata?.labels }; delete labels["pod-template-hash"]; return { ...template, metadata: { ...template.metadata, labels: Object.keys(labels).length ? labels : undefined, }, }; }; const replicaSets = async (project: string, deploymentUid?: string) => // Deployment-created ReplicaSets inherit only the pod-template labels // (e.g. app, pod-template-hash), never the Deployment's metadata // managed-by label, so a managed selector here excludes every revision // and rollback reports "no previous release". List namespace-wide and // restrict by ownerReference instead. (await apps.listNamespacedReplicaSet({ namespace: project })).items.filter( (item) => item.metadata?.ownerReferences?.some( (owner) => owner.kind === "Deployment" && owner.uid === deploymentUid, ), ); const overrides: Partial = { applyResource: async (desired) => { let resource = desired as KubernetesObject & { spec?: { storageClassName?: string }; }; if ( desired.apiVersion === "v1" && desired.kind === "PersistentVolumeClaim" && desired.metadata?.name ) { try { const existing = (await objects.read({ apiVersion: "v1", kind: "PersistentVolumeClaim", metadata: { name: desired.metadata.name, namespace: desired.metadata.namespace, }, })) as KubernetesObject & { spec?: { storageClassName?: string }; status?: { phase?: string }; }; const spec = resource.spec; if ( existing.status?.phase === "Bound" && existing.spec?.storageClassName !== spec?.storageClassName ) { resource = { ...desired, spec: { ...spec, storageClassName: existing.spec?.storageClassName, }, }; } } catch (error) { if (!isNotFound(error)) throw error; } } return objects.patch( sanitizeKubernetesObject(resource), undefined, undefined, FIELD_MANAGER, true, PatchStrategy.ServerSideApply, ); }, listDeployments: async (project) => ( await apps.listNamespacedDeployment({ namespace: project, labelSelector: MANAGED_SELECTOR, }) ).items, scaleDeployment: async (project, name, replicas) => objects.patch({ apiVersion: "apps/v1", kind: "Deployment", metadata: { name, namespace: project }, spec: { replicas }, }), restartDeployment: async (project, name) => objects.patch({ apiVersion: "apps/v1", kind: "Deployment", metadata: { name, namespace: project }, spec: { template: { metadata: { annotations: { "kubectl.kubernetes.io/restartedAt": new Date().toISOString(), }, }, }, }, }), waitForDeployment: async ( project, name, timeoutMs = 300_000, execution?: OperationExecution, ) => { const started = Date.now(); while (Date.now() - started < timeoutMs) { if (execution?.signal?.aborted) throw new Error("Workspace operation execution was cancelled"); const deployment = await apps.readNamespacedDeployment({ namespace: project, name, }); const desired = deployment.spec?.replicas ?? 1; if ( (deployment.status?.observedGeneration ?? 0) >= (deployment.metadata?.generation ?? 0) && (deployment.status?.updatedReplicas ?? 0) === desired && (deployment.status?.availableReplicas ?? 0) === desired && (deployment.status?.unavailableReplicas ?? 0) === 0 ) return; if (execution?.signal?.aborted) throw new Error("Workspace operation execution was cancelled"); await sleepUntilExecutionCancelled(2_000, execution); } throw new Error(`Timed out waiting for deployment ${name} rollout`); }, planRollback: async (project, names) => { const deployments = await overrides.listDeployments!(project); const selected = names ? deployments.filter((item) => names.includes(item.metadata?.name ?? ""), ) : deployments; if (names) { const found = new Set(selected.map((item) => item.metadata?.name)); for (const name of names) { if (!found.has(name)) throw new Error( `No managed deployment named ${name} in ${project}`, ); } } const candidates: RollbackCandidate[] = []; for (const deployment of selected) { const name = deployment.metadata?.name; if (!name) continue; const revisions = (await replicaSets(project, deployment.metadata?.uid)) .map((item) => ({ item, revision: revision(item) })) .filter( (entry): entry is { item: V1ReplicaSet; revision: number } => entry.revision !== undefined, ) .sort((left, right) => right.revision - left.revision); const current = revisions[0]; const previous = revisions .slice(1) .find( (entry) => JSON.stringify(stripHash(entry.item.spec?.template ?? {})) !== JSON.stringify(stripHash(deployment.spec?.template ?? {})), ) ?? revisions[1]; const image = previous?.item.spec?.template?.spec?.containers?.[0]?.image; if (current && previous && image) { candidates.push({ name, currentRevision: current.revision, previousRevision: previous.revision, image, }); } } return candidates; }, rollbackDeployment: async (project, candidate) => { const deployment = await apps.readNamespacedDeployment({ namespace: project, name: candidate.name, }); const previous = ( await replicaSets(project, deployment.metadata?.uid) ).find((item) => revision(item) === candidate.previousRevision); if (!previous?.spec?.template) throw new Error( `Deployment ${candidate.name} has no ReplicaSet for revision ${candidate.previousRevision}`, ); return apps.patchNamespacedDeployment({ namespace: project, name: candidate.name, body: [ { op: "replace", path: "/spec/template", value: stripHash(previous.spec.template), }, ], }); }, }; return managementDependencies( { readNamespace: async (project) => { try { const namespace = await objects.read({ apiVersion: "v1", kind: "Namespace", metadata: { name: project }, }); return { uid: namespace.metadata?.uid, labels: namespace.metadata?.labels, }; } catch (error) { if (isNotFound(error)) return; throw error; } }, deleteResource: async (identity) => { // KubernetesObjectApi turns metadata.uid into a delete precondition. await objects.delete(resource(identity)); }, }, overrides, ); }