astral/kuber
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden self-managed reconciliation

Browse Source
This commit is contained in:
dmgnr
2026-09-05 10:17:55 +00:00 Unverified
parent 8e9d207915
commit 321f4e807a
40 changed files with 4977 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/kubernetes-state.ts
+125 -26
View File
@@ -18,12 +18,14 @@ import { type AuditEvent, type AuditPersistence } from "./audit-store";
import {
managementDependencies,
type ManagementDependencies,
type OperationExecution,
type ResourceIdentity,
} from "./management";
import type { RollbackCandidate } from "../lib/rollback";
import { LABELS } from "../const";
import type { WorkspaceAdoptionResult, WorkspaceAdoptionService } from "./app";
import { WorkspaceAdoptionError } from "./app";
import { validateTrust, type TrustStore } from "./trust-store";
import {
RESERVED_NAMESPACES,
WORKSPACE_PROJECT_LABEL,
@@ -164,7 +166,11 @@ export interface LeaseObjects {
create(value: V1Lease): Promise<V1Lease>;
read(name: string, namespace: string): Promise<V1Lease | undefined>;
replace(value: V1Lease): Promise<V1Lease>;
delete(name: string, namespace: string): Promise<void>;
delete(
name: string,
namespace: string,
expectedResourceVersion?: string,
): Promise<void>;
}
/** Wraps a CoordinationV1Api client in the LeaseObjects adapter. */
@@ -193,9 +199,17 @@ export function createKubernetesLeaseObjects(
body: value,
});
},
async delete(name, namespace) {
async delete(name, namespace, expectedResourceVersion) {
try {
await coordination.deleteNamespacedLease({ name, namespace });
await coordination.deleteNamespacedLease({
name,
namespace,
...(expectedResourceVersion && {
body: {
preconditions: { resourceVersion: expectedResourceVersion },
},
}),
});
} catch (error) {
if (isNotFound(error)) return;
throw error;
@@ -217,8 +231,9 @@ const MAX_LEASE_ACQUIRE_RETRIES = 5;
*/
function microTimeString(ms: number): string {
const iso = new Date(ms).toISOString();
const match =
/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d+))?Z$/.exec(iso);
const match = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d+))?Z$/.exec(
iso,
);
if (!match) return iso;
const fraction = (match[2] ?? "").padEnd(6, "0");
return `${match[1]}.${fraction}Z`;
@@ -254,7 +269,11 @@ export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider
return digestName("lease", workspaceId);
}
private leaseSpec(workspaceId: string, holder: string, ttlMs: number): V1Lease {
private leaseSpec(
workspaceId: string,
holder: string,
ttlMs: number,
): V1Lease {
return {
apiVersion: KUBER_LEASE_API_VERSION,
kind: "Lease",
@@ -268,7 +287,9 @@ export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider
acquireTime: microTimeString(
this.now(),
) as unknown as V1LeaseSpec["acquireTime"],
renewTime: microTimeString(this.now()) as unknown as V1LeaseSpec["renewTime"],
renewTime: microTimeString(
this.now(),
) as unknown as V1LeaseSpec["renewTime"],
leaseTransitions: 0,
},
};
@@ -328,8 +349,7 @@ export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider
workspaceId: string,
holder: string,
): WorkspaceLease {
const leaseDurationMs =
(lease.spec?.leaseDurationSeconds ?? 30) * 1000;
const leaseDurationMs = (lease.spec?.leaseDurationSeconds ?? 30) * 1000;
const expiresAt = (
Date.parse(String(lease.spec?.renewTime)) + leaseDurationMs
).toString();
@@ -370,7 +390,16 @@ export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider
if (!name) return;
const current = await this.objects.read(name, this.namespace);
if (current?.spec?.holderIdentity !== holder) return;
await this.objects.delete(name, this.namespace);
const resourceVersion = current.metadata?.resourceVersion;
if (!resourceVersion) return;
try {
await this.objects.delete(name, this.namespace, resourceVersion);
} catch (error) {
// A replace by a successor between read and delete invalidates the
// resourceVersion precondition. Its lease must remain intact.
if (isConflict(error) || isNotFound(error)) return;
throw error;
}
};
return { workspaceId, holder, expiresAt, renew, release };
@@ -821,7 +850,7 @@ export class KubernetesWorkspaceAdoptionService implements WorkspaceAdoptionServ
export class KubernetesOperationPersistence implements OperationPersistence {
constructor(private readonly objects = createKubernetesClients().objects) {}
async createIdempotent(operation: Operation): Promise<Operation> {
async createIdempotent(operation: Operation) {
const operationName = digestName(
"operation",
`${operation.spec.workspaceId}\0${operation.spec.idempotencyKey}`,
@@ -841,7 +870,7 @@ export class KubernetesOperationPersistence implements OperationPersistence {
deterministic.spec.workspaceId,
),
);
return deterministic;
return { operation: deterministic, created: true };
} catch (error) {
if (!isConflict(error)) throw error;
const existing = await this.get(operationName);
@@ -849,7 +878,7 @@ export class KubernetesOperationPersistence implements OperationPersistence {
throw new OperationConflictError(
"Idempotent operation could not be recovered",
);
return existing;
return { operation: existing, created: false };
}
}
@@ -931,6 +960,44 @@ export class KubernetesAuditPersistence implements AuditPersistence {
}
}
export class KubernetesTrustStore implements TrustStore {
constructor(private readonly objects = createKubernetesClients().objects) {}
private name(project: string, fingerprint: string) {
return digestName("trust", `${project}\0${fingerprint}`);
}
async grant(project: string, fingerprint: string): Promise<void> {
validateTrust(project, fingerprint);
if (await this.has(project, fingerprint)) return;
await createObject(
this.objects,
stateObject(
"ConfigMap",
this.name(project, fingerprint),
"trust",
{ project, fingerprint },
project,
),
);
}
async list(project: string): Promise<string[]> {
return (await list(this.objects, "ConfigMap", "trust", project))
.map((item) => parsePayload<{ project: string; fingerprint: string }>(item))
.filter((record): record is { project: string; fingerprint: string } =>
Boolean(record && record.project === project && /^[a-f0-9]{64}$/.test(record.fingerprint)),
)
.map((record) => record.fingerprint);
}
async has(project: string, fingerprint: string): Promise<boolean> {
const item = await read(this.objects, "ConfigMap", this.name(project, fingerprint));
const record = item && parsePayload<{ project: string; fingerprint: string }>(item);
return record?.project === project && record.fingerprint === fingerprint;
}
async revoke(project: string, fingerprint: string): Promise<boolean> {
if (!(await this.has(project, fingerprint))) return false;
return deleteObject(this.objects, "ConfigMap", this.name(project, fingerprint));
}
}
function resource(identity: ResourceIdentity): KubernetesObject {
return {
apiVersion: identity.apiVersion,
@@ -943,6 +1010,30 @@ function resource(identity: ResourceIdentity): KubernetesObject {
};
}
async function sleepUntilExecutionCancelled(
delayMs: number,
execution?: OperationExecution,
): Promise<void> {
const signal = execution?.signal;
if (!signal) {
await Bun.sleep(delayMs);
return;
}
if (signal.aborted)
throw new Error("Workspace operation execution was cancelled");
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
signal.removeEventListener("abort", cancelSleep);
resolve();
}, delayMs);
const cancelSleep = () => {
clearTimeout(timer);
reject(new Error("Workspace operation execution was cancelled"));
};
signal.addEventListener("abort", cancelSleep, { once: true });
});
}
export function createKubernetesManagementDependencies(
clients = createKubernetesClients(),
): ManagementDependencies {
@@ -965,17 +1056,16 @@ export function createKubernetesManagementDependencies(
};
};
const replicaSets = async (project: string, deploymentUid?: string) =>
(
// Deployment-created ReplicaSets inherit only the pod-template labels
// (e.g. app, pod-template-hash), never the Deployment's metadata
// managed-by label, so a managed selector here excludes every revision
// and rollback reports "no previous release". List namespace-wide and
// restrict by ownerReference instead.
await apps.listNamespacedReplicaSet({ namespace: project })
).items.filter((item) =>
item.metadata?.ownerReferences?.some(
(owner) => owner.kind === "Deployment" && owner.uid === deploymentUid,
),
// Deployment-created ReplicaSets inherit only the pod-template labels
// (e.g. app, pod-template-hash), never the Deployment's metadata
// managed-by label, so a managed selector here excludes every revision
// and rollback reports "no previous release". List namespace-wide and
// restrict by ownerReference instead.
(await apps.listNamespacedReplicaSet({ namespace: project })).items.filter(
(item) =>
item.metadata?.ownerReferences?.some(
(owner) => owner.kind === "Deployment" && owner.uid === deploymentUid,
),
);
const overrides: Partial<ManagementDependencies> = {
listDeployments: async (project) =>
@@ -1007,9 +1097,16 @@ export function createKubernetesManagementDependencies(
},
},
}),
waitForDeployment: async (project, name, timeoutMs = 300_000) => {
waitForDeployment: async (
project,
name,
timeoutMs = 300_000,
execution?: OperationExecution,
) => {
const started = Date.now();
while (Date.now() - started < timeoutMs) {
if (execution?.signal?.aborted)
throw new Error("Workspace operation execution was cancelled");
const deployment = await apps.readNamespacedDeployment({
namespace: project,
name,
@@ -1023,7 +1120,9 @@ export function createKubernetesManagementDependencies(
(deployment.status?.unavailableReplicas ?? 0) === 0
)
return;
await Bun.sleep(2_000);
if (execution?.signal?.aborted)
throw new Error("Workspace operation execution was cancelled");
await sleepUntilExecutionCancelled(2_000, execution);
}
throw new Error(`Timed out waiting for deployment ${name} rollout`);
},