1298 lines
39 KiB
TypeScript
1298 lines
39 KiB
TypeScript
import {
|
|
AppsV1Api,
|
|
BatchV1Api,
|
|
CoordinationV1Api,
|
|
CoreV1Api,
|
|
KubeConfig,
|
|
KubernetesObjectApi,
|
|
PatchStrategy,
|
|
type KubernetesObject,
|
|
type V1Lease,
|
|
type V1LeaseSpec,
|
|
type V1PodTemplateSpec,
|
|
type V1ReplicaSet,
|
|
} from "@kubernetes/client-node";
|
|
import { createHash } from "node:crypto";
|
|
import { createKubernetesHttpLibrary } from "../lib/k8s-http";
|
|
import { sanitizeKubernetesObject } from "../lib/kubernetes-object";
|
|
import { type AuditEvent, type AuditPersistence } from "./audit-store";
|
|
import {
|
|
managementDependencies,
|
|
type ManagementDependencies,
|
|
type OperationExecution,
|
|
type ResourceIdentity,
|
|
} from "./management";
|
|
import type { RollbackCandidate } from "../lib/rollback";
|
|
import { LABELS } from "../const";
|
|
import type { WorkspaceAdoptionResult, WorkspaceAdoptionService } from "./app";
|
|
import { WorkspaceAdoptionError } from "./app";
|
|
import { validateTrust, type TrustStore } from "./trust-store";
|
|
import {
|
|
RESERVED_NAMESPACES,
|
|
WORKSPACE_PROJECT_LABEL,
|
|
WORKSPACE_UID_LABEL,
|
|
} from "./management";
|
|
import {
|
|
OperationConflictError,
|
|
OperationNotFoundError,
|
|
OperationValidationError,
|
|
type Operation,
|
|
type OperationPersistence,
|
|
type WorkspaceLease,
|
|
type WorkspaceLeaseProvider,
|
|
} from "./operation-store";
|
|
import {
|
|
PersistentWorkspaceStore,
|
|
WorkspaceConflictError,
|
|
WorkspaceNotFoundError,
|
|
type Workspace,
|
|
type WorkspacePersistence,
|
|
type WorkspaceRevision,
|
|
type WorkspaceStoreOptions,
|
|
} from "./workspace-store";
|
|
|
|
export const KUBER_STATE_NAMESPACE = "kuber-system";
|
|
const FIELD_MANAGER = "kuber-server";
|
|
const TYPE_LABEL = "kuber.astrxl.dev/type";
|
|
const WORKSPACE_LABEL = "kuber.astrxl.dev/workspace";
|
|
const MANAGED_SELECTOR = "app.kubernetes.io/managed-by=kuber";
|
|
const ADOPTABLE_RESOURCES = [
|
|
{ apiVersion: "v1", kind: "PersistentVolumeClaim" },
|
|
{ apiVersion: "v1", kind: "Secret" },
|
|
{ apiVersion: "v1", kind: "ConfigMap" },
|
|
{ apiVersion: "v1", kind: "Service" },
|
|
{ apiVersion: "apps/v1", kind: "Deployment" },
|
|
{ apiVersion: "networking.k8s.io/v1", kind: "Ingress" },
|
|
{ apiVersion: "traefik.io/v1alpha1", kind: "IngressRoute" },
|
|
] as const;
|
|
|
|
type DataObject = KubernetesObject & {
|
|
data?: Record<string, string>;
|
|
stringData?: Record<string, string>;
|
|
type?: string;
|
|
};
|
|
|
|
function isNotFound(error: unknown): boolean {
|
|
return Boolean(
|
|
error &&
|
|
typeof error === "object" &&
|
|
(("code" in error && error.code === 404) ||
|
|
("statusCode" in error && error.statusCode === 404)),
|
|
);
|
|
}
|
|
|
|
function isConflict(error: unknown): boolean {
|
|
return Boolean(
|
|
error &&
|
|
typeof error === "object" &&
|
|
(("code" in error && error.code === 409) ||
|
|
("statusCode" in error && error.statusCode === 409)),
|
|
);
|
|
}
|
|
|
|
function digestName(prefix: string, value: string): string {
|
|
return `${prefix}-${createHash("sha256").update(value).digest("hex").slice(0, 48)}`;
|
|
}
|
|
|
|
function encode(value: unknown): string {
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
function decodeSecretData(value: string | undefined): string | undefined {
|
|
if (!value) return;
|
|
try {
|
|
return Buffer.from(value, "base64").toString("utf8");
|
|
} catch {
|
|
return;
|
|
}
|
|
}
|
|
|
|
function parsePayload<T>(object: DataObject): T | undefined {
|
|
const payload =
|
|
object.kind === "Secret"
|
|
? decodeSecretData(object.data?.payload)
|
|
: object.data?.payload;
|
|
if (!payload) return;
|
|
try {
|
|
return JSON.parse(payload) as T;
|
|
} catch {
|
|
return;
|
|
}
|
|
}
|
|
|
|
export function createKubernetesConfig(): KubeConfig {
|
|
const config = new KubeConfig();
|
|
if (process.env.KUBERNETES_SERVICE_HOST) config.loadFromCluster();
|
|
else config.loadFromDefault();
|
|
const makeApiClient = config.makeApiClient.bind(config);
|
|
const httpLibrary = createKubernetesHttpLibrary({
|
|
maxConcurrent: 4,
|
|
minIntervalMs: 0,
|
|
});
|
|
config.makeApiClient = ((apiClientType) => {
|
|
const client = makeApiClient(apiClientType) as unknown as {
|
|
api?: { configuration?: { httpApi?: typeof httpLibrary } };
|
|
configuration?: { httpApi?: typeof httpLibrary };
|
|
};
|
|
if (client.api?.configuration)
|
|
client.api.configuration.httpApi = httpLibrary;
|
|
if (client.configuration) client.configuration.httpApi = httpLibrary;
|
|
return client;
|
|
}) as typeof config.makeApiClient;
|
|
return config;
|
|
}
|
|
|
|
export function createKubernetesClients(): {
|
|
config: KubeConfig;
|
|
objects: KubernetesObjectApi;
|
|
apps: AppsV1Api;
|
|
batch: BatchV1Api;
|
|
core: CoreV1Api;
|
|
coordination: CoordinationV1Api;
|
|
} {
|
|
const config = createKubernetesConfig();
|
|
return {
|
|
config,
|
|
objects: KubernetesObjectApi.makeApiClient(config),
|
|
apps: config.makeApiClient(AppsV1Api),
|
|
batch: config.makeApiClient(BatchV1Api),
|
|
core: config.makeApiClient(CoreV1Api),
|
|
coordination: config.makeApiClient(CoordinationV1Api),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Minimal adapter over the coordination.k8s.io/v1 Lease API used by the lease
|
|
* provider. Kept small and fakeable so the provider can be tested without a
|
|
* cluster.
|
|
*/
|
|
export interface LeaseObjects {
|
|
create(value: V1Lease): Promise<V1Lease>;
|
|
read(name: string, namespace: string): Promise<V1Lease | undefined>;
|
|
replace(value: V1Lease): Promise<V1Lease>;
|
|
delete(
|
|
name: string,
|
|
namespace: string,
|
|
expectedResourceVersion?: string,
|
|
): Promise<void>;
|
|
}
|
|
|
|
/** Wraps a CoordinationV1Api client in the LeaseObjects adapter. */
|
|
export function createKubernetesLeaseObjects(
|
|
coordination: CoordinationV1Api,
|
|
): LeaseObjects {
|
|
return {
|
|
async create(value) {
|
|
return coordination.createNamespacedLease({
|
|
namespace: value.metadata?.namespace ?? KUBER_STATE_NAMESPACE,
|
|
body: value,
|
|
});
|
|
},
|
|
async read(name, namespace) {
|
|
try {
|
|
return await coordination.readNamespacedLease({ name, namespace });
|
|
} catch (error) {
|
|
if (isNotFound(error)) return;
|
|
throw error;
|
|
}
|
|
},
|
|
replace(value) {
|
|
return coordination.replaceNamespacedLease({
|
|
name: value.metadata?.name ?? "",
|
|
namespace: value.metadata?.namespace ?? KUBER_STATE_NAMESPACE,
|
|
body: value,
|
|
});
|
|
},
|
|
async delete(name, namespace, expectedResourceVersion) {
|
|
try {
|
|
await coordination.deleteNamespacedLease({
|
|
name,
|
|
namespace,
|
|
...(expectedResourceVersion && {
|
|
body: {
|
|
preconditions: { resourceVersion: expectedResourceVersion },
|
|
},
|
|
}),
|
|
});
|
|
} catch (error) {
|
|
if (isNotFound(error)) return;
|
|
throw error;
|
|
}
|
|
},
|
|
};
|
|
}
|
|
|
|
const KUBER_LEASE_API_VERSION = "coordination.k8s.io/v1";
|
|
const DEFAULT_LEASE_TTL_MS = 30_000;
|
|
const MAX_LEASE_ACQUIRE_RETRIES = 5;
|
|
|
|
/**
|
|
* Kubernetes coordinates.k8s.io/v1 Lease acquireTime/renewTime are
|
|
* metav1.MicroTime, which expect six fractional-second digits (for example
|
|
* "2026-09-03T00:23:00.205000Z"). JavaScript Date#toISOString only emits three
|
|
* digits (milliseconds), and kubernetes-client-node does not re-format
|
|
* V1MicroTime when serializing, so the API server rejects the unpadded value.
|
|
*/
|
|
function microTimeString(ms: number): string {
|
|
const iso = new Date(ms).toISOString();
|
|
const match = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d+))?Z$/.exec(
|
|
iso,
|
|
);
|
|
if (!match) return iso;
|
|
const fraction = (match[2] ?? "").padEnd(6, "0");
|
|
return `${match[1]}.${fraction}Z`;
|
|
}
|
|
|
|
function leaseExpired(lease: V1Lease, nowMs: number): boolean {
|
|
const renewTime = lease.spec?.renewTime
|
|
? Date.parse(String(lease.spec.renewTime))
|
|
: Number.NaN;
|
|
const durationMs = (lease.spec?.leaseDurationSeconds ?? 0) * 1000;
|
|
if (!Number.isFinite(renewTime)) return true;
|
|
return renewTime + durationMs <= nowMs;
|
|
}
|
|
|
|
/**
|
|
* Production WorkspaceLeaseProvider backed by coordination.k8s.io/v1 Lease
|
|
* objects. Acquisition, renewal, and release are all optimistic: every mutation
|
|
* carries the expected resourceVersion so the API server rejects lost-update
|
|
* races. An expired lease may be taken over by any holder (expiry takeover).
|
|
*/
|
|
export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider {
|
|
private readonly now: () => number;
|
|
|
|
constructor(
|
|
private readonly objects: LeaseObjects,
|
|
private readonly namespace = KUBER_STATE_NAMESPACE,
|
|
private readonly clock: () => number = Date.now,
|
|
) {
|
|
this.now = clock;
|
|
}
|
|
|
|
private leaseName(workspaceId: string): string {
|
|
return digestName("lease", workspaceId);
|
|
}
|
|
|
|
private leaseSpec(
|
|
workspaceId: string,
|
|
holder: string,
|
|
ttlMs: number,
|
|
): V1Lease {
|
|
return {
|
|
apiVersion: KUBER_LEASE_API_VERSION,
|
|
kind: "Lease",
|
|
metadata: {
|
|
name: this.leaseName(workspaceId),
|
|
namespace: this.namespace,
|
|
},
|
|
spec: {
|
|
holderIdentity: holder,
|
|
leaseDurationSeconds: Math.max(1, Math.round(ttlMs / 1000)),
|
|
acquireTime: microTimeString(
|
|
this.now(),
|
|
) as unknown as V1LeaseSpec["acquireTime"],
|
|
renewTime: microTimeString(
|
|
this.now(),
|
|
) as unknown as V1LeaseSpec["renewTime"],
|
|
leaseTransitions: 0,
|
|
},
|
|
};
|
|
}
|
|
|
|
async acquire(
|
|
workspaceId: string,
|
|
holder: string,
|
|
ttlMs = DEFAULT_LEASE_TTL_MS,
|
|
): Promise<WorkspaceLease | undefined> {
|
|
if (!workspaceId || !holder || !Number.isFinite(ttlMs) || ttlMs <= 0)
|
|
throw new OperationValidationError("Invalid workspace lease request");
|
|
const name = this.leaseName(workspaceId);
|
|
|
|
for (let attempt = 0; attempt <= MAX_LEASE_ACQUIRE_RETRIES; attempt++) {
|
|
const nowMs = this.now();
|
|
|
|
const lease = await this.objects.read(name, this.namespace);
|
|
|
|
if (lease && !leaseExpired(lease, nowMs)) return;
|
|
|
|
try {
|
|
if (!lease) {
|
|
const created = await this.objects.create(
|
|
this.leaseSpec(workspaceId, holder, ttlMs),
|
|
);
|
|
return this.wrap(created, workspaceId, holder);
|
|
}
|
|
const next: V1Lease = {
|
|
...this.leaseSpec(workspaceId, holder, ttlMs),
|
|
metadata: {
|
|
...this.leaseSpec(workspaceId, holder, ttlMs).metadata,
|
|
resourceVersion: lease.metadata?.resourceVersion,
|
|
},
|
|
spec: {
|
|
...this.leaseSpec(workspaceId, holder, ttlMs).spec,
|
|
leaseTransitions: (lease.spec?.leaseTransitions ?? 0) + 1,
|
|
},
|
|
};
|
|
const replaced = await this.objects.replace(next);
|
|
return this.wrap(replaced, workspaceId, holder);
|
|
} catch (error) {
|
|
if (isConflict(error)) {
|
|
const raced = await this.objects.read(name, this.namespace);
|
|
if (raced && !leaseExpired(raced, this.now())) return;
|
|
continue;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
return undefined;
|
|
}
|
|
|
|
private wrap(
|
|
lease: V1Lease,
|
|
workspaceId: string,
|
|
holder: string,
|
|
): WorkspaceLease {
|
|
const leaseDurationMs = (lease.spec?.leaseDurationSeconds ?? 30) * 1000;
|
|
const expiresAt = (
|
|
Date.parse(String(lease.spec?.renewTime)) + leaseDurationMs
|
|
).toString();
|
|
|
|
const renew = async (ttlMs = 30_000): Promise<boolean> => {
|
|
if (!Number.isFinite(ttlMs) || ttlMs <= 0) return false;
|
|
const name = lease.metadata?.name;
|
|
if (!name) return false;
|
|
const current = await this.objects.read(name, this.namespace);
|
|
if (!current || leaseExpired(current, this.now())) return false;
|
|
if (current.spec?.holderIdentity !== holder) return false;
|
|
const next: V1Lease = {
|
|
...current,
|
|
metadata: {
|
|
...current.metadata,
|
|
resourceVersion: current.metadata?.resourceVersion,
|
|
},
|
|
spec: {
|
|
...current.spec,
|
|
holderIdentity: holder,
|
|
leaseDurationSeconds: Math.max(1, Math.round(ttlMs / 1000)),
|
|
renewTime: microTimeString(
|
|
this.now(),
|
|
) as unknown as V1LeaseSpec["renewTime"],
|
|
},
|
|
};
|
|
try {
|
|
await this.objects.replace(next);
|
|
return true;
|
|
} catch (error) {
|
|
if (isConflict(error)) return false;
|
|
throw error;
|
|
}
|
|
};
|
|
|
|
const release = async (): Promise<void> => {
|
|
const name = lease.metadata?.name;
|
|
if (!name) return;
|
|
const current = await this.objects.read(name, this.namespace);
|
|
if (current?.spec?.holderIdentity !== holder) return;
|
|
const resourceVersion = current.metadata?.resourceVersion;
|
|
if (!resourceVersion) return;
|
|
try {
|
|
await this.objects.delete(name, this.namespace, resourceVersion);
|
|
} catch (error) {
|
|
// A replace by a successor between read and delete invalidates the
|
|
// resourceVersion precondition. Its lease must remain intact.
|
|
if (isConflict(error) || isNotFound(error)) return;
|
|
throw error;
|
|
}
|
|
};
|
|
|
|
return { workspaceId, holder, expiresAt, renew, release };
|
|
}
|
|
}
|
|
|
|
async function read(
|
|
objects: KubernetesObjectApi,
|
|
kind: "Secret" | "ConfigMap",
|
|
name: string,
|
|
): Promise<DataObject | undefined> {
|
|
try {
|
|
return (await objects.read({
|
|
apiVersion: "v1",
|
|
kind,
|
|
metadata: { name, namespace: KUBER_STATE_NAMESPACE },
|
|
})) as DataObject;
|
|
} catch (error) {
|
|
if (isNotFound(error)) return;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function list(
|
|
objects: KubernetesObjectApi,
|
|
kind: "Secret" | "ConfigMap",
|
|
type: string,
|
|
workspaceId?: string,
|
|
): Promise<DataObject[]> {
|
|
const selector = [
|
|
`${TYPE_LABEL}=${type}`,
|
|
workspaceId && `${WORKSPACE_LABEL}=${workspaceId}`,
|
|
]
|
|
.filter(Boolean)
|
|
.join(",");
|
|
const response = await objects.list(
|
|
"v1",
|
|
kind,
|
|
KUBER_STATE_NAMESPACE,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
selector,
|
|
);
|
|
return response.items.map((item) => ({
|
|
apiVersion: "v1",
|
|
kind,
|
|
...item,
|
|
})) as DataObject[];
|
|
}
|
|
|
|
function stateObject(
|
|
kind: "Secret" | "ConfigMap",
|
|
name: string,
|
|
type: string,
|
|
value: unknown,
|
|
workspaceId?: string,
|
|
resourceVersion?: string,
|
|
): DataObject {
|
|
const metadata = {
|
|
name,
|
|
namespace: KUBER_STATE_NAMESPACE,
|
|
labels: {
|
|
[TYPE_LABEL]: type,
|
|
...(workspaceId && { [WORKSPACE_LABEL]: workspaceId }),
|
|
},
|
|
...(resourceVersion && { resourceVersion }),
|
|
};
|
|
return kind === "Secret"
|
|
? {
|
|
apiVersion: "v1",
|
|
kind,
|
|
metadata,
|
|
type: "Opaque",
|
|
stringData: { payload: encode(value) },
|
|
}
|
|
: { apiVersion: "v1", kind, metadata, data: { payload: encode(value) } };
|
|
}
|
|
|
|
async function createObject(
|
|
objects: KubernetesObjectApi,
|
|
value: DataObject,
|
|
): Promise<void> {
|
|
await objects.create(value);
|
|
}
|
|
|
|
async function deleteObject(
|
|
objects: KubernetesObjectApi,
|
|
kind: "Secret" | "ConfigMap",
|
|
name: string,
|
|
): Promise<boolean> {
|
|
try {
|
|
await objects.delete({
|
|
apiVersion: "v1",
|
|
kind,
|
|
metadata: { name, namespace: KUBER_STATE_NAMESPACE },
|
|
});
|
|
return true;
|
|
} catch (error) {
|
|
if (isNotFound(error)) return false;
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
export class KubernetesWorkspacePersistence implements WorkspacePersistence {
|
|
constructor(private readonly objects = createKubernetesClients().objects) {}
|
|
|
|
private workspaceName(id: string): string {
|
|
return digestName("workspace", id);
|
|
}
|
|
|
|
private revisionName(id: string, revision: number): string {
|
|
return digestName("revision", `${id}\0${revision}`);
|
|
}
|
|
|
|
async get(id: string): Promise<Workspace | undefined> {
|
|
const object = await read(this.objects, "Secret", this.workspaceName(id));
|
|
const workspace = object && parsePayload<Workspace>(object);
|
|
return workspace?.metadata.name === id ? workspace : undefined;
|
|
}
|
|
|
|
async list(): Promise<Workspace[]> {
|
|
return (await list(this.objects, "Secret", "workspace"))
|
|
.map((item) => parsePayload<Workspace>(item))
|
|
.filter((item): item is Workspace => item?.kind === "Workspace")
|
|
.sort((a, b) => a.metadata.name.localeCompare(b.metadata.name));
|
|
}
|
|
|
|
async create(
|
|
workspace: Workspace,
|
|
revision: WorkspaceRevision,
|
|
): Promise<void> {
|
|
const revisionName = this.revisionName(
|
|
workspace.metadata.name,
|
|
revision.spec.revision,
|
|
);
|
|
let revisionCreated = false;
|
|
try {
|
|
await createObject(
|
|
this.objects,
|
|
stateObject(
|
|
"Secret",
|
|
revisionName,
|
|
"workspace-revision",
|
|
revision,
|
|
workspace.metadata.name,
|
|
),
|
|
);
|
|
revisionCreated = true;
|
|
await createObject(
|
|
this.objects,
|
|
stateObject(
|
|
"Secret",
|
|
this.workspaceName(workspace.metadata.name),
|
|
"workspace",
|
|
workspace,
|
|
workspace.metadata.name,
|
|
),
|
|
);
|
|
} catch (error) {
|
|
if (revisionCreated)
|
|
await deleteObject(this.objects, "Secret", revisionName).catch(
|
|
() => false,
|
|
);
|
|
if (isConflict(error))
|
|
throw new WorkspaceConflictError("Workspace already exists");
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async replace(
|
|
workspace: Workspace,
|
|
revision: WorkspaceRevision,
|
|
expectedResourceVersion: string,
|
|
): Promise<void> {
|
|
const name = this.workspaceName(workspace.metadata.name);
|
|
const currentObject = await read(this.objects, "Secret", name);
|
|
const current = currentObject && parsePayload<Workspace>(currentObject);
|
|
if (!current || !currentObject?.metadata?.resourceVersion)
|
|
throw new WorkspaceNotFoundError("Workspace not found");
|
|
if (current.metadata.resourceVersion !== expectedResourceVersion)
|
|
throw new WorkspaceConflictError("Workspace was concurrently modified");
|
|
|
|
const revisionName = this.revisionName(
|
|
workspace.metadata.name,
|
|
revision.spec.revision,
|
|
);
|
|
let revisionCreated = false;
|
|
try {
|
|
try {
|
|
await createObject(
|
|
this.objects,
|
|
stateObject(
|
|
"Secret",
|
|
revisionName,
|
|
"workspace-revision",
|
|
revision,
|
|
workspace.metadata.name,
|
|
),
|
|
);
|
|
revisionCreated = true;
|
|
} catch (error) {
|
|
if (!isConflict(error)) throw error;
|
|
const existingObject = await read(this.objects, "Secret", revisionName);
|
|
const existing =
|
|
existingObject && parsePayload<WorkspaceRevision>(existingObject);
|
|
const matches =
|
|
existing?.kind === "WorkspaceRevision" &&
|
|
existing.metadata.name === revision.metadata.name &&
|
|
existing.metadata.workspaceUid === revision.metadata.workspaceUid &&
|
|
existing.metadata.resourceVersion ===
|
|
revision.metadata.resourceVersion &&
|
|
JSON.stringify(existing.spec) === JSON.stringify(revision.spec);
|
|
if (!matches)
|
|
throw new WorkspaceConflictError("Workspace revision already exists");
|
|
}
|
|
await this.objects.replace(
|
|
stateObject(
|
|
"Secret",
|
|
name,
|
|
"workspace",
|
|
workspace,
|
|
workspace.metadata.name,
|
|
currentObject.metadata.resourceVersion,
|
|
),
|
|
);
|
|
} catch (error) {
|
|
if (revisionCreated)
|
|
await deleteObject(this.objects, "Secret", revisionName).catch(
|
|
() => false,
|
|
);
|
|
if (isConflict(error))
|
|
throw new WorkspaceConflictError("Workspace was concurrently modified");
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async getRevision(
|
|
id: string,
|
|
revision: number,
|
|
): Promise<WorkspaceRevision | undefined> {
|
|
const object = await read(
|
|
this.objects,
|
|
"Secret",
|
|
this.revisionName(id, revision),
|
|
);
|
|
const value = object && parsePayload<WorkspaceRevision>(object);
|
|
return value?.spec.workspaceId === id && value.spec.revision === revision
|
|
? value
|
|
: undefined;
|
|
}
|
|
|
|
async listRevisions(id: string): Promise<WorkspaceRevision[]> {
|
|
return (await list(this.objects, "Secret", "workspace-revision", id))
|
|
.map((item) => parsePayload<WorkspaceRevision>(item))
|
|
.filter(
|
|
(item): item is WorkspaceRevision =>
|
|
item?.kind === "WorkspaceRevision" && item.spec.workspaceId === id,
|
|
)
|
|
.sort((a, b) => a.spec.revision - b.spec.revision);
|
|
}
|
|
|
|
async delete(id: string): Promise<boolean> {
|
|
const deleted = await deleteObject(
|
|
this.objects,
|
|
"Secret",
|
|
this.workspaceName(id),
|
|
);
|
|
for (const revision of await list(
|
|
this.objects,
|
|
"Secret",
|
|
"workspace-revision",
|
|
id,
|
|
)) {
|
|
if (revision.metadata?.name)
|
|
await deleteObject(this.objects, "Secret", revision.metadata.name);
|
|
}
|
|
return deleted;
|
|
}
|
|
|
|
adopt(workspaceId: string, workspaceUid: string) {
|
|
return new KubernetesWorkspaceAdoptionService(this.objects).adopt(
|
|
workspaceId,
|
|
workspaceUid,
|
|
);
|
|
}
|
|
|
|
adoptPlatform(workspaceUid: string) {
|
|
return new KubernetesWorkspaceAdoptionService(this.objects).adoptPlatform(
|
|
workspaceUid,
|
|
);
|
|
}
|
|
}
|
|
|
|
export class KubernetesWorkspaceStore extends PersistentWorkspaceStore {
|
|
constructor(
|
|
private readonly kubernetesPersistence = new KubernetesWorkspacePersistence(),
|
|
options: WorkspaceStoreOptions = {},
|
|
) {
|
|
super(kubernetesPersistence, options);
|
|
}
|
|
|
|
delete(id: string): Promise<boolean> {
|
|
return this.kubernetesPersistence.delete(id);
|
|
}
|
|
|
|
adopt(workspaceId: string, workspaceUid: string) {
|
|
return this.kubernetesPersistence.adopt(workspaceId, workspaceUid);
|
|
}
|
|
|
|
adoptPlatform(workspaceUid: string) {
|
|
return this.kubernetesPersistence.adoptPlatform(workspaceUid);
|
|
}
|
|
}
|
|
|
|
export class KubernetesWorkspaceAdoptionService implements WorkspaceAdoptionService {
|
|
constructor(private readonly objects = createKubernetesClients().objects) {}
|
|
|
|
private async adoptNamespace(
|
|
workspaceId: string,
|
|
workspaceUid: string,
|
|
platform: boolean,
|
|
): Promise<WorkspaceAdoptionResult> {
|
|
if (!workspaceUid.trim())
|
|
throw new WorkspaceAdoptionError("Workspace UID is required");
|
|
if (
|
|
platform
|
|
? workspaceId !== KUBER_STATE_NAMESPACE
|
|
: RESERVED_NAMESPACES.has(workspaceId) ||
|
|
workspaceId.startsWith("kube-")
|
|
) {
|
|
throw new WorkspaceAdoptionError(`Namespace ${workspaceId} is reserved`);
|
|
}
|
|
|
|
let namespace: KubernetesObject;
|
|
try {
|
|
namespace = await this.objects.read({
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: { name: workspaceId },
|
|
});
|
|
} catch (error) {
|
|
if (isNotFound(error)) {
|
|
return { workspaceId, workspaceUid, resourcesAdopted: 0 };
|
|
}
|
|
throw error;
|
|
}
|
|
if (
|
|
namespace.metadata?.labels?.["app.kubernetes.io/managed-by"] !==
|
|
LABELS["app.kubernetes.io/managed-by"]
|
|
) {
|
|
throw new WorkspaceAdoptionError(
|
|
`Namespace ${workspaceId} is not managed by kuber; refusing adoption`,
|
|
);
|
|
}
|
|
const namespaceOwner = namespace.metadata.labels?.[WORKSPACE_UID_LABEL];
|
|
if (namespaceOwner && namespaceOwner !== workspaceUid) {
|
|
throw new WorkspaceAdoptionError(
|
|
`Namespace ${workspaceId} belongs to another workspace`,
|
|
);
|
|
}
|
|
|
|
const resources: Array<{
|
|
apiVersion: string;
|
|
kind: string;
|
|
item: KubernetesObject;
|
|
}> = [];
|
|
for (const { apiVersion, kind } of ADOPTABLE_RESOURCES) {
|
|
try {
|
|
const result = await this.objects.list(
|
|
apiVersion,
|
|
kind,
|
|
workspaceId,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
MANAGED_SELECTOR,
|
|
);
|
|
resources.push(
|
|
...result.items.map((item) => ({ apiVersion, kind, item })),
|
|
);
|
|
} catch (error) {
|
|
if (!isNotFound(error)) throw error;
|
|
}
|
|
}
|
|
for (const { kind, item } of resources) {
|
|
const owner = item.metadata?.labels?.[WORKSPACE_UID_LABEL];
|
|
if (owner && owner !== workspaceUid) {
|
|
throw new WorkspaceAdoptionError(
|
|
`${kind}/${item.metadata?.name} belongs to another workspace`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const adoptionLabels = {
|
|
...LABELS,
|
|
[WORKSPACE_PROJECT_LABEL]: workspaceId,
|
|
[WORKSPACE_UID_LABEL]: workspaceUid,
|
|
};
|
|
await this.objects.patch(
|
|
{
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: {
|
|
name: workspaceId,
|
|
labels: { ...namespace.metadata?.labels, ...adoptionLabels },
|
|
},
|
|
},
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
PatchStrategy.MergePatch,
|
|
);
|
|
for (const { apiVersion, kind, item } of resources) {
|
|
await this.objects.patch(
|
|
{
|
|
apiVersion,
|
|
kind,
|
|
metadata: {
|
|
name: item.metadata?.name,
|
|
namespace: workspaceId,
|
|
labels: { ...item.metadata?.labels, ...adoptionLabels },
|
|
},
|
|
},
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
undefined,
|
|
PatchStrategy.MergePatch,
|
|
);
|
|
}
|
|
return {
|
|
workspaceId,
|
|
workspaceUid,
|
|
resourcesAdopted: resources.length,
|
|
};
|
|
}
|
|
|
|
adopt(workspaceId: string, workspaceUid: string) {
|
|
return this.adoptNamespace(workspaceId, workspaceUid, false);
|
|
}
|
|
|
|
adoptPlatform(workspaceUid: string) {
|
|
return this.adoptNamespace(KUBER_STATE_NAMESPACE, workspaceUid, true);
|
|
}
|
|
}
|
|
|
|
export class KubernetesOperationPersistence implements OperationPersistence {
|
|
constructor(private readonly objects = createKubernetesClients().objects) {}
|
|
|
|
async createIdempotent(operation: Operation) {
|
|
const operationName = digestName(
|
|
"operation",
|
|
`${operation.spec.workspaceId}\0${operation.spec.idempotencyKey}`,
|
|
);
|
|
const deterministic: Operation = {
|
|
...operation,
|
|
metadata: { ...operation.metadata, name: operationName },
|
|
};
|
|
try {
|
|
await createObject(
|
|
this.objects,
|
|
stateObject(
|
|
"Secret",
|
|
operationName,
|
|
"operation",
|
|
deterministic,
|
|
deterministic.spec.workspaceId,
|
|
),
|
|
);
|
|
return { operation: deterministic, created: true };
|
|
} catch (error) {
|
|
if (!isConflict(error)) throw error;
|
|
const existing = await this.get(operationName);
|
|
if (!existing)
|
|
throw new OperationConflictError(
|
|
"Idempotent operation could not be recovered",
|
|
);
|
|
return { operation: existing, created: false };
|
|
}
|
|
}
|
|
|
|
async get(id: string): Promise<Operation | undefined> {
|
|
const object = await read(this.objects, "Secret", id);
|
|
const operation = object && parsePayload<Operation>(object);
|
|
return operation?.kind === "Operation" ? operation : undefined;
|
|
}
|
|
|
|
async list(workspaceId?: string): Promise<Operation[]> {
|
|
return (await list(this.objects, "Secret", "operation", workspaceId))
|
|
.map((item) => parsePayload<Operation>(item))
|
|
.filter((item): item is Operation => item?.kind === "Operation")
|
|
.sort((a, b) =>
|
|
a.metadata.creationTimestamp.localeCompare(
|
|
b.metadata.creationTimestamp,
|
|
),
|
|
);
|
|
}
|
|
|
|
async replace(
|
|
operation: Operation,
|
|
expectedResourceVersion: string,
|
|
): Promise<void> {
|
|
const currentObject = await read(
|
|
this.objects,
|
|
"Secret",
|
|
operation.metadata.name,
|
|
);
|
|
const current = currentObject && parsePayload<Operation>(currentObject);
|
|
if (!current || !currentObject?.metadata?.resourceVersion)
|
|
throw new OperationNotFoundError("Operation not found");
|
|
if (current.metadata.resourceVersion !== expectedResourceVersion)
|
|
throw new OperationConflictError("Operation was concurrently modified");
|
|
try {
|
|
await this.objects.replace(
|
|
stateObject(
|
|
"Secret",
|
|
operation.metadata.name,
|
|
"operation",
|
|
operation,
|
|
operation.spec.workspaceId,
|
|
currentObject.metadata.resourceVersion,
|
|
),
|
|
);
|
|
} catch (error) {
|
|
if (isConflict(error))
|
|
throw new OperationConflictError("Operation was concurrently modified");
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
|
|
export class KubernetesAuditPersistence implements AuditPersistence {
|
|
constructor(private readonly objects = createKubernetesClients().objects) {}
|
|
|
|
async append(event: AuditEvent): Promise<void> {
|
|
await createObject(
|
|
this.objects,
|
|
stateObject(
|
|
"ConfigMap",
|
|
event.metadata.name,
|
|
"audit",
|
|
event,
|
|
event.spec.workspaceId,
|
|
),
|
|
);
|
|
}
|
|
|
|
async list(workspaceId?: string): Promise<AuditEvent[]> {
|
|
return (await list(this.objects, "ConfigMap", "audit", workspaceId))
|
|
.map((item) => parsePayload<AuditEvent>(item))
|
|
.filter((item): item is AuditEvent => item?.kind === "AuditEvent")
|
|
.sort((a, b) =>
|
|
a.metadata.creationTimestamp.localeCompare(
|
|
b.metadata.creationTimestamp,
|
|
),
|
|
);
|
|
}
|
|
}
|
|
|
|
export class KubernetesTrustStore implements TrustStore {
|
|
constructor(private readonly objects = createKubernetesClients().objects) {}
|
|
private name(project: string, fingerprint: string) {
|
|
return digestName("trust", `${project}\0${fingerprint}`);
|
|
}
|
|
async grant(project: string, fingerprint: string): Promise<void> {
|
|
validateTrust(project, fingerprint);
|
|
if (await this.has(project, fingerprint)) return;
|
|
await createObject(
|
|
this.objects,
|
|
stateObject(
|
|
"ConfigMap",
|
|
this.name(project, fingerprint),
|
|
"trust",
|
|
{ project, fingerprint },
|
|
project,
|
|
),
|
|
);
|
|
}
|
|
async list(project: string): Promise<string[]> {
|
|
return (await list(this.objects, "ConfigMap", "trust", project))
|
|
.map((item) =>
|
|
parsePayload<{ project: string; fingerprint: string }>(item),
|
|
)
|
|
.filter((record): record is { project: string; fingerprint: string } =>
|
|
Boolean(
|
|
record &&
|
|
record.project === project &&
|
|
/^[a-f0-9]{64}$/.test(record.fingerprint),
|
|
),
|
|
)
|
|
.map((record) => record.fingerprint);
|
|
}
|
|
async has(project: string, fingerprint: string): Promise<boolean> {
|
|
const item = await read(
|
|
this.objects,
|
|
"ConfigMap",
|
|
this.name(project, fingerprint),
|
|
);
|
|
const record =
|
|
item && parsePayload<{ project: string; fingerprint: string }>(item);
|
|
return record?.project === project && record.fingerprint === fingerprint;
|
|
}
|
|
async revoke(project: string, fingerprint: string): Promise<boolean> {
|
|
if (!(await this.has(project, fingerprint))) return false;
|
|
return deleteObject(
|
|
this.objects,
|
|
"ConfigMap",
|
|
this.name(project, fingerprint),
|
|
);
|
|
}
|
|
}
|
|
|
|
function resource(identity: ResourceIdentity): KubernetesObject {
|
|
return {
|
|
apiVersion: identity.apiVersion,
|
|
kind: identity.kind,
|
|
metadata: {
|
|
name: identity.name,
|
|
namespace: identity.namespace,
|
|
uid: identity.uid,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function sleepUntilExecutionCancelled(
|
|
delayMs: number,
|
|
execution?: OperationExecution,
|
|
): Promise<void> {
|
|
const signal = execution?.signal;
|
|
if (!signal) {
|
|
await Bun.sleep(delayMs);
|
|
return;
|
|
}
|
|
if (signal.aborted)
|
|
throw new Error("Workspace operation execution was cancelled");
|
|
await new Promise<void>((resolve, reject) => {
|
|
const timer = setTimeout(() => {
|
|
signal.removeEventListener("abort", cancelSleep);
|
|
resolve();
|
|
}, delayMs);
|
|
const cancelSleep = () => {
|
|
clearTimeout(timer);
|
|
reject(new Error("Workspace operation execution was cancelled"));
|
|
};
|
|
signal.addEventListener("abort", cancelSleep, { once: true });
|
|
});
|
|
}
|
|
|
|
export function createKubernetesManagementDependencies(
|
|
clients = createKubernetesClients(),
|
|
): ManagementDependencies {
|
|
const { objects, apps } = clients;
|
|
const revision = (replicaSet: V1ReplicaSet): number | undefined => {
|
|
const value = Number(
|
|
replicaSet.metadata?.annotations?.["deployment.kubernetes.io/revision"],
|
|
);
|
|
return Number.isSafeInteger(value) && value > 0 ? value : undefined;
|
|
};
|
|
const stripHash = (template: V1PodTemplateSpec): V1PodTemplateSpec => {
|
|
const labels = { ...template.metadata?.labels };
|
|
delete labels["pod-template-hash"];
|
|
return {
|
|
...template,
|
|
metadata: {
|
|
...template.metadata,
|
|
labels: Object.keys(labels).length ? labels : undefined,
|
|
},
|
|
};
|
|
};
|
|
const replicaSets = async (project: string, deploymentUid?: string) =>
|
|
// Deployment-created ReplicaSets inherit only the pod-template labels
|
|
// (e.g. app, pod-template-hash), never the Deployment's metadata
|
|
// managed-by label, so a managed selector here excludes every revision
|
|
// and rollback reports "no previous release". List namespace-wide and
|
|
// restrict by ownerReference instead.
|
|
(await apps.listNamespacedReplicaSet({ namespace: project })).items.filter(
|
|
(item) =>
|
|
item.metadata?.ownerReferences?.some(
|
|
(owner) => owner.kind === "Deployment" && owner.uid === deploymentUid,
|
|
),
|
|
);
|
|
const overrides: Partial<ManagementDependencies> = {
|
|
applyResource: async (desired) => {
|
|
let resource = desired as KubernetesObject & {
|
|
spec?: { storageClassName?: string };
|
|
};
|
|
if (
|
|
desired.apiVersion === "v1" &&
|
|
desired.kind === "PersistentVolumeClaim" &&
|
|
desired.metadata?.name
|
|
) {
|
|
try {
|
|
const existing = (await objects.read({
|
|
apiVersion: "v1",
|
|
kind: "PersistentVolumeClaim",
|
|
metadata: {
|
|
name: desired.metadata.name,
|
|
namespace: desired.metadata.namespace,
|
|
},
|
|
})) as KubernetesObject & {
|
|
spec?: { storageClassName?: string };
|
|
status?: { phase?: string };
|
|
};
|
|
const spec = resource.spec;
|
|
if (
|
|
existing.status?.phase === "Bound" &&
|
|
existing.spec?.storageClassName !== spec?.storageClassName
|
|
) {
|
|
resource = {
|
|
...desired,
|
|
spec: {
|
|
...spec,
|
|
storageClassName: existing.spec?.storageClassName,
|
|
},
|
|
};
|
|
}
|
|
} catch (error) {
|
|
if (!isNotFound(error)) throw error;
|
|
}
|
|
}
|
|
return objects.patch(
|
|
sanitizeKubernetesObject(resource),
|
|
undefined,
|
|
undefined,
|
|
FIELD_MANAGER,
|
|
true,
|
|
PatchStrategy.ServerSideApply,
|
|
);
|
|
},
|
|
listDeployments: async (project) =>
|
|
(
|
|
await apps.listNamespacedDeployment({
|
|
namespace: project,
|
|
labelSelector: MANAGED_SELECTOR,
|
|
})
|
|
).items,
|
|
scaleDeployment: async (project, name, replicas) =>
|
|
objects.patch({
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: { name, namespace: project },
|
|
spec: { replicas },
|
|
}),
|
|
restartDeployment: async (project, name) =>
|
|
objects.patch({
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: { name, namespace: project },
|
|
spec: {
|
|
template: {
|
|
metadata: {
|
|
annotations: {
|
|
"kubectl.kubernetes.io/restartedAt": new Date().toISOString(),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}),
|
|
waitForDeployment: async (
|
|
project,
|
|
name,
|
|
timeoutMs = 300_000,
|
|
execution?: OperationExecution,
|
|
) => {
|
|
const started = Date.now();
|
|
while (Date.now() - started < timeoutMs) {
|
|
if (execution?.signal?.aborted)
|
|
throw new Error("Workspace operation execution was cancelled");
|
|
const deployment = await apps.readNamespacedDeployment({
|
|
namespace: project,
|
|
name,
|
|
});
|
|
const desired = deployment.spec?.replicas ?? 1;
|
|
if (
|
|
(deployment.status?.observedGeneration ?? 0) >=
|
|
(deployment.metadata?.generation ?? 0) &&
|
|
(deployment.status?.updatedReplicas ?? 0) === desired &&
|
|
(deployment.status?.availableReplicas ?? 0) === desired &&
|
|
(deployment.status?.unavailableReplicas ?? 0) === 0
|
|
)
|
|
return;
|
|
if (execution?.signal?.aborted)
|
|
throw new Error("Workspace operation execution was cancelled");
|
|
await sleepUntilExecutionCancelled(2_000, execution);
|
|
}
|
|
throw new Error(`Timed out waiting for deployment ${name} rollout`);
|
|
},
|
|
planRollback: async (project, names) => {
|
|
const deployments = await overrides.listDeployments!(project);
|
|
const selected = names
|
|
? deployments.filter((item) =>
|
|
names.includes(item.metadata?.name ?? ""),
|
|
)
|
|
: deployments;
|
|
if (names) {
|
|
const found = new Set(selected.map((item) => item.metadata?.name));
|
|
for (const name of names) {
|
|
if (!found.has(name))
|
|
throw new Error(
|
|
`No managed deployment named ${name} in ${project}`,
|
|
);
|
|
}
|
|
}
|
|
const candidates: RollbackCandidate[] = [];
|
|
for (const deployment of selected) {
|
|
const name = deployment.metadata?.name;
|
|
if (!name) continue;
|
|
const revisions = (await replicaSets(project, deployment.metadata?.uid))
|
|
.map((item) => ({ item, revision: revision(item) }))
|
|
.filter(
|
|
(entry): entry is { item: V1ReplicaSet; revision: number } =>
|
|
entry.revision !== undefined,
|
|
)
|
|
.sort((left, right) => right.revision - left.revision);
|
|
const current = revisions[0];
|
|
const previous =
|
|
revisions
|
|
.slice(1)
|
|
.find(
|
|
(entry) =>
|
|
JSON.stringify(stripHash(entry.item.spec?.template ?? {})) !==
|
|
JSON.stringify(stripHash(deployment.spec?.template ?? {})),
|
|
) ?? revisions[1];
|
|
const image =
|
|
previous?.item.spec?.template?.spec?.containers?.[0]?.image;
|
|
if (current && previous && image) {
|
|
candidates.push({
|
|
name,
|
|
currentRevision: current.revision,
|
|
previousRevision: previous.revision,
|
|
image,
|
|
});
|
|
}
|
|
}
|
|
return candidates;
|
|
},
|
|
rollbackDeployment: async (project, candidate) => {
|
|
const deployment = await apps.readNamespacedDeployment({
|
|
namespace: project,
|
|
name: candidate.name,
|
|
});
|
|
const previous = (
|
|
await replicaSets(project, deployment.metadata?.uid)
|
|
).find((item) => revision(item) === candidate.previousRevision);
|
|
if (!previous?.spec?.template)
|
|
throw new Error(
|
|
`Deployment ${candidate.name} has no ReplicaSet for revision ${candidate.previousRevision}`,
|
|
);
|
|
return apps.patchNamespacedDeployment({
|
|
namespace: project,
|
|
name: candidate.name,
|
|
body: [
|
|
{
|
|
op: "replace",
|
|
path: "/spec/template",
|
|
value: stripHash(previous.spec.template),
|
|
},
|
|
],
|
|
});
|
|
},
|
|
};
|
|
return managementDependencies(
|
|
{
|
|
readNamespace: async (project) => {
|
|
try {
|
|
const namespace = await objects.read({
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: { name: project },
|
|
});
|
|
return {
|
|
uid: namespace.metadata?.uid,
|
|
labels: namespace.metadata?.labels,
|
|
};
|
|
} catch (error) {
|
|
if (isNotFound(error)) return;
|
|
throw error;
|
|
}
|
|
},
|
|
deleteResource: async (identity) => {
|
|
// KubernetesObjectApi turns metadata.uid into a delete precondition.
|
|
await objects.delete(resource(identity));
|
|
},
|
|
},
|
|
overrides,
|
|
);
|
|
}
|