1216 lines
36 KiB
TypeScript
1216 lines
36 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { describe, expect, test } from "bun:test";
|
|
import type {
|
|
KubernetesObject,
|
|
KubernetesObjectApi,
|
|
V1Lease,
|
|
V1LeaseSpec,
|
|
} from "@kubernetes/client-node";
|
|
import {
|
|
createKubernetesManagementDependencies,
|
|
KubernetesOperationPersistence,
|
|
KubernetesWorkspaceAdoptionService,
|
|
KubernetesWorkspaceLeaseProvider,
|
|
KubernetesWorkspacePersistence,
|
|
type LeaseObjects,
|
|
} from "../../server/kubernetes-state";
|
|
import {
|
|
PersistentOperationStore,
|
|
type Operation,
|
|
} from "../../server/operation-store";
|
|
import type {
|
|
Workspace,
|
|
WorkspaceRevision,
|
|
} from "../../server/workspace-store";
|
|
import {
|
|
WORKSPACE_PROJECT_LABEL,
|
|
WORKSPACE_UID_LABEL,
|
|
} from "../../server/management";
|
|
|
|
type DataObject = KubernetesObject & {
|
|
data?: Record<string, string>;
|
|
stringData?: Record<string, string>;
|
|
spec?: unknown;
|
|
};
|
|
|
|
class FakeObjects {
|
|
readonly objects = new Map<string, DataObject>();
|
|
readonly patches: KubernetesObject[] = [];
|
|
readonly patchOptions: Array<{
|
|
fieldManager?: string;
|
|
force?: boolean;
|
|
strategy?: string;
|
|
}> = [];
|
|
|
|
key(value: KubernetesObject) {
|
|
return `${value.kind}:${value.metadata?.namespace ?? ""}:${value.metadata?.name}`;
|
|
}
|
|
|
|
async create(value: DataObject) {
|
|
const key = this.key(value);
|
|
if (this.objects.has(key)) throw { code: 409 };
|
|
const stored = structuredClone(value);
|
|
if (stored.stringData) {
|
|
stored.data = Object.fromEntries(
|
|
Object.entries(stored.stringData).map(([name, item]) => [
|
|
name,
|
|
Buffer.from(item).toString("base64"),
|
|
]),
|
|
);
|
|
delete stored.stringData;
|
|
}
|
|
stored.metadata = { ...stored.metadata, resourceVersion: "cluster-1" };
|
|
this.objects.set(key, stored);
|
|
return stored;
|
|
}
|
|
|
|
async read(value: KubernetesObject) {
|
|
const found = this.objects.get(this.key(value));
|
|
if (!found) throw { code: 404 };
|
|
return structuredClone(found);
|
|
}
|
|
|
|
async replace(value: DataObject) {
|
|
const key = this.key(value);
|
|
if (!this.objects.has(key)) throw { code: 404 };
|
|
const stored = structuredClone(value);
|
|
if (stored.stringData) {
|
|
stored.data = Object.fromEntries(
|
|
Object.entries(stored.stringData).map(([name, item]) => [
|
|
name,
|
|
Buffer.from(item).toString("base64"),
|
|
]),
|
|
);
|
|
delete stored.stringData;
|
|
}
|
|
stored.metadata = { ...stored.metadata, resourceVersion: "cluster-2" };
|
|
this.objects.set(key, stored);
|
|
return stored;
|
|
}
|
|
|
|
async delete(value: KubernetesObject) {
|
|
if (!this.objects.delete(this.key(value))) throw { code: 404 };
|
|
}
|
|
|
|
async list(
|
|
apiVersion: string,
|
|
kind: string,
|
|
namespace: string,
|
|
_pretty?: string,
|
|
_exact?: boolean,
|
|
_export?: boolean,
|
|
_fieldSelector?: string,
|
|
labelSelector?: string,
|
|
) {
|
|
const [label, expected] = labelSelector?.split("=") ?? [];
|
|
return {
|
|
items: [...this.objects.values()].filter(
|
|
(item) =>
|
|
item.apiVersion === apiVersion &&
|
|
item.kind === kind &&
|
|
item.metadata?.namespace === namespace &&
|
|
(!label || item.metadata.labels?.[label] === expected),
|
|
),
|
|
};
|
|
}
|
|
|
|
async patch(
|
|
value: KubernetesObject,
|
|
_pretty?: string,
|
|
_dryRun?: string,
|
|
fieldManager?: string,
|
|
force?: boolean,
|
|
strategy?: string,
|
|
) {
|
|
this.patches.push(structuredClone(value));
|
|
this.patchOptions.push({ fieldManager, force, strategy });
|
|
return value;
|
|
}
|
|
}
|
|
|
|
const timestamp = "2026-09-02T00:00:00.000Z";
|
|
|
|
function operation(name = "random"): Operation {
|
|
return {
|
|
apiVersion: "kuber.astrxl.dev/v2",
|
|
kind: "Operation",
|
|
metadata: {
|
|
name,
|
|
uid: name,
|
|
resourceVersion: "1",
|
|
creationTimestamp: timestamp,
|
|
},
|
|
spec: {
|
|
workspaceId: "demo",
|
|
action: "workspace.stop",
|
|
idempotencyKey: "same-key",
|
|
requestHash: "hash",
|
|
},
|
|
status: { state: "pending" },
|
|
};
|
|
}
|
|
|
|
function workspace(version: number): Workspace {
|
|
return {
|
|
apiVersion: "kuber.astrxl.dev/v2",
|
|
kind: "Workspace",
|
|
metadata: {
|
|
name: "demo",
|
|
uid: "workspace-uid",
|
|
resourceVersion: String(version),
|
|
creationTimestamp: timestamp,
|
|
},
|
|
spec: { source: { uri: "oci://demo", digest: `sha256:${version}` } },
|
|
status: { latestRevision: version },
|
|
};
|
|
}
|
|
|
|
function revision(version: number): WorkspaceRevision {
|
|
return {
|
|
apiVersion: "kuber.astrxl.dev/v2",
|
|
kind: "WorkspaceRevision",
|
|
metadata: {
|
|
name: `demo-r${version}`,
|
|
uid: `revision-${version}`,
|
|
workspaceUid: "workspace-uid",
|
|
resourceVersion: String(version),
|
|
creationTimestamp: timestamp,
|
|
},
|
|
spec: {
|
|
...workspace(version).spec,
|
|
workspaceId: "demo",
|
|
revision: version,
|
|
},
|
|
};
|
|
}
|
|
|
|
describe("Kubernetes state persistence", () => {
|
|
test("stores idempotency as one deterministic, recoverable operation object", async () => {
|
|
const fake = new FakeObjects();
|
|
const persistence = new KubernetesOperationPersistence(
|
|
fake as unknown as KubernetesObjectApi,
|
|
);
|
|
const first = await persistence.createIdempotent(operation("first"));
|
|
const second = await persistence.createIdempotent(operation("second"));
|
|
expect(first.created).toBe(true);
|
|
expect(second).toEqual({ operation: first.operation, created: false });
|
|
expect(fake.objects.size).toBe(1);
|
|
expect(first.operation.metadata.name).toBe(
|
|
`operation-${createHash("sha256")
|
|
.update("demo\0same-key")
|
|
.digest("hex")
|
|
.slice(0, 48)}`,
|
|
);
|
|
});
|
|
|
|
test("keeps operation progress in the Kubernetes-backed operation record", async () => {
|
|
const fake = new FakeObjects();
|
|
const store = new PersistentOperationStore(
|
|
new KubernetesOperationPersistence(
|
|
fake as unknown as KubernetesObjectApi,
|
|
),
|
|
() => new Date(timestamp),
|
|
() => "event-id",
|
|
);
|
|
const created = await store.create({
|
|
workspaceId: "demo",
|
|
action: "resources.apply",
|
|
idempotencyKey: "events",
|
|
});
|
|
await store.emit(created.metadata.name, {
|
|
resource: { apiVersion: "v1", kind: "ConfigMap", name: "settings" },
|
|
phase: "apply",
|
|
state: "succeeded",
|
|
});
|
|
const reloaded = new PersistentOperationStore(
|
|
new KubernetesOperationPersistence(
|
|
fake as unknown as KubernetesObjectApi,
|
|
),
|
|
);
|
|
expect(await reloaded.events(created.metadata.name)).toMatchObject({
|
|
items: [
|
|
expect.objectContaining({
|
|
sequence: 1,
|
|
data: expect.objectContaining({ phase: "apply" }),
|
|
}),
|
|
],
|
|
retainedFirstSequence: 1,
|
|
cursorGap: false,
|
|
});
|
|
});
|
|
|
|
test("recovers replacement from a matching precreated revision", async () => {
|
|
const fake = new FakeObjects();
|
|
const persistence = new KubernetesWorkspacePersistence(
|
|
fake as unknown as KubernetesObjectApi,
|
|
);
|
|
await persistence.create(workspace(1), revision(1));
|
|
const revisionName = `revision-${createHash("sha256")
|
|
.update("demo\0" + "2")
|
|
.digest("hex")
|
|
.slice(0, 48)}`;
|
|
await fake.create({
|
|
apiVersion: "v1",
|
|
kind: "Secret",
|
|
metadata: { name: revisionName, namespace: "kuber-system" },
|
|
stringData: { payload: JSON.stringify(revision(2)) },
|
|
});
|
|
await persistence.replace(workspace(2), revision(2), "1");
|
|
expect((await persistence.get("demo"))?.status.latestRevision).toBe(2);
|
|
});
|
|
|
|
test("adopts only kuber-managed resources and reserves platform adoption for its path", async () => {
|
|
const fake = new FakeObjects();
|
|
fake.objects.set("Namespace::demo", {
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: {
|
|
name: "demo",
|
|
labels: { "app.kubernetes.io/managed-by": "kuber" },
|
|
},
|
|
});
|
|
fake.objects.set("ConfigMap:demo:managed", {
|
|
apiVersion: "v1",
|
|
kind: "ConfigMap",
|
|
metadata: {
|
|
name: "managed",
|
|
namespace: "demo",
|
|
labels: { "app.kubernetes.io/managed-by": "kuber" },
|
|
},
|
|
});
|
|
fake.objects.set("ConfigMap:demo:external", {
|
|
apiVersion: "v1",
|
|
kind: "ConfigMap",
|
|
metadata: { name: "external", namespace: "demo" },
|
|
});
|
|
const adoption = new KubernetesWorkspaceAdoptionService(
|
|
fake as unknown as KubernetesObjectApi,
|
|
);
|
|
expect(await adoption.adopt("demo", "workspace-uid")).toMatchObject({
|
|
resourcesAdopted: 1,
|
|
});
|
|
expect(fake.patches).toHaveLength(2);
|
|
expect(
|
|
fake.patches.every(
|
|
(item) =>
|
|
item.metadata?.labels?.[WORKSPACE_UID_LABEL] === "workspace-uid",
|
|
),
|
|
).toBe(true);
|
|
await expect(adoption.adopt("kube-system", "uid")).rejects.toThrow(
|
|
"reserved",
|
|
);
|
|
});
|
|
|
|
test("adoption hydrates apiVersion/kind when list items lack them", async () => {
|
|
const fake = new FakeObjects();
|
|
fake.objects.set("Namespace::demo", {
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: {
|
|
name: "demo",
|
|
labels: { "app.kubernetes.io/managed-by": "kuber" },
|
|
},
|
|
});
|
|
fake.objects.set("Deployment:demo:managed", {
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: {
|
|
name: "managed",
|
|
namespace: "demo",
|
|
labels: { "app.kubernetes.io/managed-by": "kuber" },
|
|
},
|
|
});
|
|
// Emulate the KubernetesObjectApi behavior of returning list items without
|
|
// hydrated apiVersion/kind on the nested objects.
|
|
const dehydrating = {
|
|
list: async (
|
|
apiVersion: string,
|
|
kind: string,
|
|
namespace: string,
|
|
_pretty?: string,
|
|
_exact?: boolean,
|
|
_export?: boolean,
|
|
_fieldSelector?: string,
|
|
labelSelector?: string,
|
|
) => {
|
|
const [label, expected] = labelSelector?.split("=") ?? [];
|
|
const items = [...fake.objects.values()].filter(
|
|
(item) =>
|
|
item.apiVersion === apiVersion &&
|
|
item.kind === kind &&
|
|
item.metadata?.namespace === namespace &&
|
|
(!label || item.metadata.labels?.[label] === expected),
|
|
);
|
|
return {
|
|
items: items.map(({ kind: _k, apiVersion: _a, ...item }) => item),
|
|
};
|
|
},
|
|
read: (value: KubernetesObject) => fake.read(value),
|
|
patch: async (value: KubernetesObject) => {
|
|
fake.patches.push(structuredClone(value));
|
|
return value;
|
|
},
|
|
};
|
|
const adoption = new KubernetesWorkspaceAdoptionService(
|
|
dehydrating as unknown as KubernetesObjectApi,
|
|
);
|
|
expect(await adoption.adopt("demo", "workspace-uid")).toMatchObject({
|
|
resourcesAdopted: 1,
|
|
});
|
|
expect(fake.patches).toHaveLength(2);
|
|
const [namespacePatch, resourcePatch] = fake.patches;
|
|
expect(namespacePatch).toMatchObject({
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
});
|
|
expect(resourcePatch).toMatchObject({
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: { name: "managed", namespace: "demo" },
|
|
});
|
|
});
|
|
|
|
test("adoption merge-patches existing namespace and resource labels without affecting spec", async () => {
|
|
const fake = new FakeObjects();
|
|
const spec = {
|
|
ingressClassName: "nginx",
|
|
rules: [
|
|
{
|
|
host: "demo.example.test",
|
|
http: {
|
|
paths: [
|
|
{
|
|
path: "/",
|
|
pathType: "Prefix",
|
|
backend: {
|
|
service: { name: "web", port: { number: 80 } },
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
],
|
|
};
|
|
fake.objects.set("Namespace::demo", {
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: {
|
|
name: "demo",
|
|
labels: {
|
|
"app.kubernetes.io/managed-by": "kuber",
|
|
"example.test/namespace-label": "preserve-me",
|
|
},
|
|
},
|
|
});
|
|
fake.objects.set("Ingress:demo:web", {
|
|
apiVersion: "networking.k8s.io/v1",
|
|
kind: "Ingress",
|
|
metadata: {
|
|
name: "web",
|
|
namespace: "demo",
|
|
labels: {
|
|
"app.kubernetes.io/managed-by": "kuber",
|
|
"example.test/resource-label": "preserve-me",
|
|
},
|
|
},
|
|
spec,
|
|
});
|
|
|
|
const adoption = new KubernetesWorkspaceAdoptionService(
|
|
fake as unknown as KubernetesObjectApi,
|
|
);
|
|
await adoption.adopt("demo", "workspace-uid");
|
|
|
|
const namespacePatch = fake.patches.find(
|
|
(patch) => patch.kind === "Namespace",
|
|
);
|
|
const ingressPatch = fake.patches.find((patch) => patch.kind === "Ingress");
|
|
expect(namespacePatch).toMatchObject({
|
|
apiVersion: "v1",
|
|
kind: "Namespace",
|
|
metadata: {
|
|
name: "demo",
|
|
labels: {
|
|
"app.kubernetes.io/managed-by": "kuber",
|
|
"example.test/namespace-label": "preserve-me",
|
|
[WORKSPACE_PROJECT_LABEL]: "demo",
|
|
[WORKSPACE_UID_LABEL]: "workspace-uid",
|
|
},
|
|
},
|
|
});
|
|
expect(namespacePatch).not.toHaveProperty("spec");
|
|
expect(ingressPatch).toMatchObject({
|
|
apiVersion: "networking.k8s.io/v1",
|
|
kind: "Ingress",
|
|
metadata: {
|
|
name: "web",
|
|
namespace: "demo",
|
|
labels: {
|
|
"app.kubernetes.io/managed-by": "kuber",
|
|
"example.test/resource-label": "preserve-me",
|
|
[WORKSPACE_PROJECT_LABEL]: "demo",
|
|
[WORKSPACE_UID_LABEL]: "workspace-uid",
|
|
},
|
|
},
|
|
});
|
|
expect(ingressPatch).not.toHaveProperty("spec");
|
|
expect(fake.objects.get("Ingress:demo:web")?.spec).toEqual(spec);
|
|
expect(fake.patchOptions).toEqual([
|
|
{ strategy: "application/merge-patch+json" },
|
|
{ strategy: "application/merge-patch+json" },
|
|
]);
|
|
});
|
|
});
|
|
|
|
class FakeLeaseStore implements LeaseObjects {
|
|
readonly leases = new Map<string, V1Lease>();
|
|
private rv = 0;
|
|
beforeDelete?: () => void;
|
|
|
|
private key(name: string, namespace: string) {
|
|
return `${namespace}/${name}`;
|
|
}
|
|
|
|
private persisted(value: V1Lease): V1Lease {
|
|
const stored = structuredClone(value) as V1Lease;
|
|
stored.metadata = {
|
|
...(stored.metadata ?? {}),
|
|
name: value.metadata?.name ?? "",
|
|
namespace: value.metadata?.namespace ?? "kuber-system",
|
|
resourceVersion: String(++this.rv),
|
|
};
|
|
return stored;
|
|
}
|
|
|
|
async create(value: V1Lease) {
|
|
const key = this.key(
|
|
value.metadata!.name!,
|
|
value.metadata!.namespace ?? "kuber-system",
|
|
);
|
|
if (this.leases.has(key)) throw { code: 409 };
|
|
const stored = this.persisted(value);
|
|
this.leases.set(key, stored);
|
|
return structuredClone(stored);
|
|
}
|
|
|
|
async read(name: string, namespace: string) {
|
|
const found = this.leases.get(this.key(name, namespace));
|
|
return found ? structuredClone(found) : undefined;
|
|
}
|
|
|
|
async replace(value: V1Lease) {
|
|
const key = this.key(
|
|
value.metadata!.name!,
|
|
value.metadata!.namespace ?? "kuber-system",
|
|
);
|
|
const current = this.leases.get(key);
|
|
if (!current) throw { code: 404 };
|
|
if (current.metadata?.resourceVersion !== value.metadata?.resourceVersion)
|
|
throw { code: 409 };
|
|
const stored = this.persisted(value);
|
|
this.leases.set(key, stored);
|
|
return structuredClone(stored);
|
|
}
|
|
|
|
async delete(
|
|
name: string,
|
|
namespace: string,
|
|
expectedResourceVersion?: string,
|
|
) {
|
|
this.beforeDelete?.();
|
|
this.beforeDelete = undefined;
|
|
const key = this.key(name, namespace);
|
|
const current = this.leases.get(key);
|
|
if (!current) throw { code: 404 };
|
|
if (
|
|
expectedResourceVersion &&
|
|
current.metadata?.resourceVersion !== expectedResourceVersion
|
|
)
|
|
throw { code: 409 };
|
|
this.leases.delete(key);
|
|
}
|
|
}
|
|
|
|
describe("Kubernetes workspace lease provider", () => {
|
|
test("acquire creates a lease and blocks other holders while valid", async () => {
|
|
const fake = new FakeLeaseStore();
|
|
const provider = new KubernetesWorkspaceLeaseProvider(fake, "kuber-system");
|
|
const lease = await provider.acquire("demo", "worker-a", 1000);
|
|
expect(lease).toBeDefined();
|
|
expect(lease!.workspaceId).toBe("demo");
|
|
expect(lease!.holder).toBe("worker-a");
|
|
expect(fake.leases.size).toBe(1);
|
|
expect(await provider.acquire("demo", "worker-b", 1000)).toBeUndefined();
|
|
});
|
|
|
|
test("renews optimistically and refuses after expiry or holder change", async () => {
|
|
const fake = new FakeLeaseStore();
|
|
let now = 0;
|
|
const provider = new KubernetesWorkspaceLeaseProvider(
|
|
fake,
|
|
"kuber-system",
|
|
() => now,
|
|
);
|
|
const lease = await provider.acquire("demo", "worker-a", 1000);
|
|
expect(await lease!.renew(1000)).toBe(true);
|
|
|
|
const second = await provider.acquire("demo", "worker-b", 1000);
|
|
expect(second).toBeUndefined();
|
|
|
|
now = 1500;
|
|
expect(await lease!.renew(1000)).toBe(false);
|
|
});
|
|
|
|
test("takes over an expired lease from a different holder", async () => {
|
|
const fake = new FakeLeaseStore();
|
|
let now = 0;
|
|
const provider = new KubernetesWorkspaceLeaseProvider(
|
|
fake,
|
|
"kuber-system",
|
|
() => now,
|
|
);
|
|
const first = await provider.acquire("demo", "worker-a", 1000);
|
|
expect(first).toBeDefined();
|
|
|
|
now = 1500;
|
|
const taken = await provider.acquire("demo", "worker-b", 1000);
|
|
expect(taken).toBeDefined();
|
|
expect(taken!.holder).toBe("worker-b");
|
|
const stored = [...fake.leases.values()][0]!;
|
|
expect(stored.spec?.holderIdentity).toBe("worker-b");
|
|
expect(stored.spec?.leaseTransitions).toBe(1);
|
|
|
|
expect(await first!.renew(1000)).toBe(false);
|
|
});
|
|
|
|
test("release only removes a lease still held by the owner", async () => {
|
|
const fake = new FakeLeaseStore();
|
|
const provider = new KubernetesWorkspaceLeaseProvider(fake, "kuber-system");
|
|
const lease = await provider.acquire("demo", "worker-a", 1000);
|
|
await lease!.release();
|
|
expect(fake.leases.size).toBe(0);
|
|
expect(await provider.acquire("demo", "worker-b", 1000)).toBeDefined();
|
|
});
|
|
|
|
test("stale release cannot delete a successor that takes over after expiry", async () => {
|
|
const fake = new FakeLeaseStore();
|
|
let now = 0;
|
|
const provider = new KubernetesWorkspaceLeaseProvider(
|
|
fake,
|
|
"kuber-system",
|
|
() => now,
|
|
);
|
|
const stale = await provider.acquire("demo", "worker-a", 1000);
|
|
now = 1500;
|
|
fake.beforeDelete = () => {
|
|
const current = [...fake.leases.values()][0]!;
|
|
fake.leases.set(
|
|
`${current.metadata!.namespace}/${current.metadata!.name}`,
|
|
{
|
|
...current,
|
|
metadata: {
|
|
...current.metadata,
|
|
resourceVersion: "successor-version",
|
|
},
|
|
spec: { ...current.spec, holderIdentity: "worker-b" },
|
|
},
|
|
);
|
|
};
|
|
|
|
await stale!.release();
|
|
|
|
expect([...fake.leases.values()][0]?.spec?.holderIdentity).toBe("worker-b");
|
|
});
|
|
|
|
test("writes acquireTime and renewTime as microsecond MicroTime strings", async () => {
|
|
const fake = new FakeLeaseStore();
|
|
const provider = new KubernetesWorkspaceLeaseProvider(
|
|
fake,
|
|
"kuber-system",
|
|
() => Date.parse("2026-09-03T00:23:00.205Z"),
|
|
);
|
|
const lease = await provider.acquire("demo", "worker-a", 1000);
|
|
const microRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/;
|
|
for (const stored of fake.leases.values()) {
|
|
expect(String(stored.spec?.acquireTime)).toMatch(microRegex);
|
|
expect(String(stored.spec?.renewTime)).toMatch(microRegex);
|
|
}
|
|
await lease!.renew(1000);
|
|
for (const stored of fake.leases.values()) {
|
|
expect(String(stored.spec?.renewTime)).toMatch(microRegex);
|
|
}
|
|
});
|
|
|
|
test("retries repeated expired-lease conflicts and eventually acquires", async () => {
|
|
const now = Date.parse("2026-09-03T00:00:00.000Z");
|
|
const fake = new TakeoverContentionStore(now, 3);
|
|
const provider = new KubernetesWorkspaceLeaseProvider(
|
|
fake,
|
|
"kuber-system",
|
|
() => now,
|
|
);
|
|
|
|
const lease = await provider.acquire("demo", "worker-a", 1000);
|
|
|
|
expect(lease).toBeDefined();
|
|
expect(lease!.holder).toBe("worker-a");
|
|
expect(fake.conflicts).toBe(3);
|
|
});
|
|
|
|
test("exhausts bounded retries and returns undefined under sustained contention", async () => {
|
|
const now = Date.parse("2026-09-03T00:00:00.000Z");
|
|
const fake = new SustainedContentionStore(now);
|
|
const provider = new KubernetesWorkspaceLeaseProvider(
|
|
fake,
|
|
"kuber-system",
|
|
() => now,
|
|
);
|
|
|
|
const lease = await provider.acquire("demo", "worker-a", 1000);
|
|
|
|
expect(lease).toBeUndefined();
|
|
expect(fake.replaceAttempts).toBeGreaterThan(0);
|
|
expect(fake.replaceAttempts).toBeLessThanOrEqual(6);
|
|
});
|
|
});
|
|
|
|
class TakeoverContentionStore implements LeaseObjects {
|
|
private rv = 0;
|
|
private lease: V1Lease | undefined;
|
|
conflicts = 0;
|
|
|
|
constructor(
|
|
private readonly now: number,
|
|
private readonly conflictsBeforeSuccess: number,
|
|
) {
|
|
this.lease = this.expiredLease();
|
|
}
|
|
|
|
private expiredLease(): V1Lease {
|
|
return {
|
|
apiVersion: "coordination.k8s.io/v1",
|
|
kind: "Lease",
|
|
metadata: {
|
|
name: "lease-demo",
|
|
namespace: "kuber-system",
|
|
resourceVersion: String(++this.rv),
|
|
},
|
|
spec: {
|
|
holderIdentity: "contender",
|
|
leaseDurationSeconds: 1,
|
|
renewTime: new Date(
|
|
this.now - 5000,
|
|
).toISOString() as unknown as V1LeaseSpec["renewTime"],
|
|
},
|
|
};
|
|
}
|
|
|
|
async create(value: V1Lease): Promise<V1Lease> {
|
|
if (this.lease) throw { code: 409 };
|
|
this.lease = structuredClone(value);
|
|
return structuredClone(this.lease);
|
|
}
|
|
|
|
async read(): Promise<V1Lease | undefined> {
|
|
return this.lease ? structuredClone(this.lease) : undefined;
|
|
}
|
|
|
|
async replace(value: V1Lease): Promise<V1Lease> {
|
|
if (!this.lease) throw { code: 404 };
|
|
if (this.conflicts < this.conflictsBeforeSuccess) {
|
|
this.conflicts++;
|
|
this.lease = this.expiredLease();
|
|
throw { code: 409 };
|
|
}
|
|
this.lease = structuredClone(value);
|
|
return structuredClone(this.lease);
|
|
}
|
|
|
|
async delete(): Promise<void> {
|
|
this.lease = undefined;
|
|
}
|
|
}
|
|
|
|
class SustainedContentionStore implements LeaseObjects {
|
|
replaceAttempts = 0;
|
|
|
|
constructor(private readonly now: number) {}
|
|
|
|
async create(_value: V1Lease): Promise<V1Lease> {
|
|
throw { code: 409 };
|
|
}
|
|
|
|
async read(): Promise<V1Lease | undefined> {
|
|
return {
|
|
apiVersion: "coordination.k8s.io/v1",
|
|
kind: "Lease",
|
|
metadata: {
|
|
name: "lease-demo",
|
|
namespace: "kuber-system",
|
|
resourceVersion: "1",
|
|
},
|
|
spec: {
|
|
holderIdentity: "contender",
|
|
leaseDurationSeconds: 1,
|
|
renewTime: new Date(
|
|
this.now - 5000,
|
|
).toISOString() as unknown as V1LeaseSpec["renewTime"],
|
|
},
|
|
};
|
|
}
|
|
|
|
async replace(_value: V1Lease): Promise<V1Lease> {
|
|
this.replaceAttempts++;
|
|
throw { code: 409 };
|
|
}
|
|
|
|
async delete(): Promise<void> {}
|
|
}
|
|
|
|
class DehydratingListFake extends FakeObjects {
|
|
override async list(
|
|
apiVersion: string,
|
|
kind: string,
|
|
namespace: string,
|
|
_pretty?: string,
|
|
_exact?: boolean,
|
|
_export?: boolean,
|
|
_fieldSelector?: string,
|
|
labelSelector?: string,
|
|
) {
|
|
const selectors = (labelSelector ?? "")
|
|
.split(",")
|
|
.map((part) => part.split("="))
|
|
.filter(([label]) => label) as Array<[string, string]>;
|
|
const items = [...this.objects.values()].filter(
|
|
(item) =>
|
|
item.apiVersion === apiVersion &&
|
|
item.kind === kind &&
|
|
item.metadata?.namespace === namespace &&
|
|
selectors.every(
|
|
([label, expected]) => item.metadata?.labels?.[label] === expected,
|
|
),
|
|
);
|
|
return {
|
|
items: items.map(({ apiVersion: _a, kind: _k, ...rest }) => rest),
|
|
};
|
|
}
|
|
}
|
|
|
|
describe("list normalization for dehydrated Kubernetes list items", () => {
|
|
test("lists Secret-backed operations despite missing apiVersion/kind on items", async () => {
|
|
const fake = new DehydratingListFake();
|
|
const persistence = new KubernetesOperationPersistence(
|
|
fake as unknown as KubernetesObjectApi,
|
|
);
|
|
const op = operation("op-1");
|
|
await persistence.createIdempotent(op);
|
|
|
|
const listed = await persistence.list();
|
|
expect(listed).toHaveLength(1);
|
|
expect(listed[0]!.kind).toBe("Operation");
|
|
expect(listed[0]!.spec.workspaceId).toBe("demo");
|
|
});
|
|
|
|
test("lists Secret-backed workspaces despite missing apiVersion/kind on items", async () => {
|
|
const fake = new DehydratingListFake();
|
|
const persistence = new KubernetesWorkspacePersistence(
|
|
fake as unknown as KubernetesObjectApi,
|
|
);
|
|
await persistence.create(workspace(1), revision(1));
|
|
|
|
const listed = await persistence.list();
|
|
expect(listed).toHaveLength(1);
|
|
expect(listed[0]!.metadata.name).toBe("demo");
|
|
expect(listed[0]!.kind).toBe("Workspace");
|
|
});
|
|
|
|
test("lists Secret-backed workspace revisions despite missing apiVersion/kind on items", async () => {
|
|
const fake = new DehydratingListFake();
|
|
const persistence = new KubernetesWorkspacePersistence(
|
|
fake as unknown as KubernetesObjectApi,
|
|
);
|
|
await persistence.create(workspace(1), revision(1));
|
|
|
|
const listed = await persistence.listRevisions("demo");
|
|
expect(listed).toHaveLength(1);
|
|
expect(listed[0]!.spec.revision).toBe(1);
|
|
expect(listed[0]!.kind).toBe("WorkspaceRevision");
|
|
});
|
|
|
|
test("preserves explicit apiVersion/kind when list items already carry them", async () => {
|
|
const fake = new FakeObjects();
|
|
const persistence = new KubernetesOperationPersistence(
|
|
fake as unknown as KubernetesObjectApi,
|
|
);
|
|
const op = operation("op-explicit");
|
|
await persistence.createIdempotent(op);
|
|
|
|
const listed = await persistence.list();
|
|
expect(listed).toHaveLength(1);
|
|
expect(listed[0]!.apiVersion).toBe("kuber.astrxl.dev/v2");
|
|
expect(listed[0]!.kind).toBe("Operation");
|
|
});
|
|
});
|
|
|
|
function replicaSet(
|
|
name: string,
|
|
namespace: string,
|
|
ownerUid: string,
|
|
revisionNumber: number,
|
|
image: string,
|
|
labels: Record<string, string> = {},
|
|
hasManagedBy = false,
|
|
): Record<string, unknown> {
|
|
return {
|
|
apiVersion: "apps/v1",
|
|
kind: "ReplicaSet",
|
|
metadata: {
|
|
name,
|
|
namespace,
|
|
...(hasManagedBy && {
|
|
labels: { ...labels, "app.kubernetes.io/managed-by": "kuber" },
|
|
}),
|
|
...(!hasManagedBy && Object.keys(labels).length && { labels }),
|
|
annotations: {
|
|
"deployment.kubernetes.io/revision": String(revisionNumber),
|
|
},
|
|
...(ownerUid
|
|
? { ownerReferences: [{ kind: "Deployment", name, uid: ownerUid }] }
|
|
: {}),
|
|
},
|
|
spec: {
|
|
template: {
|
|
metadata: {
|
|
labels: { app: name, "pod-template-hash": `hash${revisionNumber}` },
|
|
},
|
|
spec: { containers: [{ name: "app", image }] },
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
function deployment(
|
|
name: string,
|
|
namespace: string,
|
|
uid: string,
|
|
image: string,
|
|
): Record<string, unknown> {
|
|
return {
|
|
apiVersion: "apps/v1",
|
|
kind: "Deployment",
|
|
metadata: {
|
|
name,
|
|
namespace,
|
|
uid,
|
|
labels: { "app.kubernetes.io/managed-by": "kuber" },
|
|
},
|
|
spec: {
|
|
template: {
|
|
metadata: { labels: { app: name } },
|
|
spec: { containers: [{ name: "app", image }] },
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
class FakeApps {
|
|
constructor(
|
|
private readonly deployments: KubernetesObject[],
|
|
private readonly replicaSets: KubernetesObject[],
|
|
) {}
|
|
|
|
async listNamespacedDeployment({ namespace }: { namespace: string }) {
|
|
return {
|
|
items: this.deployments.filter(
|
|
(d) => d.metadata?.namespace === namespace,
|
|
),
|
|
};
|
|
}
|
|
|
|
async listNamespacedReplicaSet({ namespace }: { namespace: string }) {
|
|
return {
|
|
items: this.replicaSets.filter(
|
|
(rs) => rs.metadata?.namespace === namespace,
|
|
),
|
|
};
|
|
}
|
|
}
|
|
|
|
test("production rollout wait reports current pod crashes promptly", async () => {
|
|
const current = {
|
|
metadata: {
|
|
uid: "rs-current",
|
|
namespace: "shop",
|
|
ownerReferences: [{ kind: "Deployment", uid: "deployment-uid" }],
|
|
},
|
|
spec: {
|
|
template: {
|
|
metadata: { labels: { app: "web", "pod-template-hash": "current" } },
|
|
spec: { containers: [{ name: "web", image: "web:v2" }] },
|
|
},
|
|
},
|
|
};
|
|
const clients = managementClients([], []);
|
|
clients.apps.readNamespacedDeployment = async () =>
|
|
({
|
|
metadata: { name: "web", uid: "deployment-uid", generation: 2 },
|
|
spec: {
|
|
replicas: 1,
|
|
template: {
|
|
metadata: { labels: { app: "web" } },
|
|
spec: { containers: [{ name: "web", image: "web:v2" }] },
|
|
},
|
|
},
|
|
status: {
|
|
observedGeneration: 2,
|
|
updatedReplicas: 1,
|
|
availableReplicas: 0,
|
|
unavailableReplicas: 1,
|
|
},
|
|
}) as never;
|
|
clients.apps.listNamespacedReplicaSet = async () =>
|
|
({ items: [current] }) as never;
|
|
clients.core.listNamespacedPod = async () =>
|
|
({
|
|
items: [
|
|
{
|
|
metadata: {
|
|
name: "web-pod",
|
|
ownerReferences: [{ kind: "ReplicaSet", uid: "rs-current" }],
|
|
},
|
|
status: {
|
|
containerStatuses: [
|
|
{
|
|
name: "web",
|
|
restartCount: 3,
|
|
state: { waiting: { reason: "CrashLoopBackOff" } },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
],
|
|
}) as never;
|
|
await expect(
|
|
createKubernetesManagementDependencies(clients).waitForDeployment(
|
|
"shop",
|
|
"web",
|
|
1000,
|
|
),
|
|
).rejects.toThrow(
|
|
"Deployment web rollout failed: pod web-pod, container web: CrashLoopBackOff",
|
|
);
|
|
});
|
|
|
|
type ManagementClientsType = NonNullable<
|
|
Parameters<typeof createKubernetesManagementDependencies>[0]
|
|
>;
|
|
|
|
const managementClients = (
|
|
deployments: Array<Record<string, unknown>>,
|
|
replicaSets: Array<Record<string, unknown>>,
|
|
objects = {} as KubernetesObjectApi,
|
|
): ManagementClientsType => {
|
|
const apps = new FakeApps(deployments, replicaSets);
|
|
return {
|
|
apps: apps as unknown as ManagementClientsType["apps"],
|
|
objects,
|
|
config: {} as ManagementClientsType["config"],
|
|
batch: {} as ManagementClientsType["batch"],
|
|
core: {} as ManagementClientsType["core"],
|
|
coordination: {} as ManagementClientsType["coordination"],
|
|
};
|
|
};
|
|
|
|
type PersistentVolumeClaim = KubernetesObject & {
|
|
spec: { storageClassName?: string };
|
|
status?: { phase?: string };
|
|
};
|
|
|
|
function persistentVolumeClaim(
|
|
storageClassName = "kuber-longhorn-test",
|
|
): PersistentVolumeClaim {
|
|
return {
|
|
apiVersion: "v1",
|
|
kind: "PersistentVolumeClaim",
|
|
metadata: { name: "kimaki-kimaki-data", namespace: "kimaki" },
|
|
spec: { storageClassName },
|
|
};
|
|
}
|
|
|
|
function managementObjects(existing?: KubernetesObject) {
|
|
const applied: PersistentVolumeClaim[] = [];
|
|
return {
|
|
applied,
|
|
objects: {
|
|
async read() {
|
|
if (!existing) throw { code: 404 };
|
|
return structuredClone(existing);
|
|
},
|
|
async patch(resource: KubernetesObject) {
|
|
applied.push(structuredClone(resource) as PersistentVolumeClaim);
|
|
return resource;
|
|
},
|
|
} as unknown as KubernetesObjectApi,
|
|
};
|
|
}
|
|
|
|
describe("management PVC storage classes", () => {
|
|
test("sends the existing storage class for a Bound claim with a different class", async () => {
|
|
const existing = {
|
|
...persistentVolumeClaim("longhorn"),
|
|
status: { phase: "Bound" },
|
|
};
|
|
const fake = managementObjects(existing);
|
|
const dependencies = createKubernetesManagementDependencies(
|
|
managementClients([], [], fake.objects),
|
|
);
|
|
const desired = persistentVolumeClaim();
|
|
|
|
await dependencies.applyResource(desired);
|
|
|
|
expect(fake.applied).toEqual([
|
|
{
|
|
...persistentVolumeClaim(),
|
|
spec: { storageClassName: "longhorn" },
|
|
},
|
|
]);
|
|
expect(desired).toEqual(persistentVolumeClaim());
|
|
});
|
|
|
|
test("retains a matching storage class for a Bound claim", async () => {
|
|
const existing = {
|
|
...persistentVolumeClaim(),
|
|
status: { phase: "Bound" },
|
|
};
|
|
const fake = managementObjects(existing);
|
|
const dependencies = createKubernetesManagementDependencies(
|
|
managementClients([], [], fake.objects),
|
|
);
|
|
const desired = persistentVolumeClaim();
|
|
|
|
await dependencies.applyResource(desired);
|
|
|
|
expect(fake.applied).toEqual([desired]);
|
|
});
|
|
|
|
test("retains the desired storage class for an absent claim", async () => {
|
|
const fake = managementObjects();
|
|
const dependencies = createKubernetesManagementDependencies(
|
|
managementClients([], [], fake.objects),
|
|
);
|
|
const desired = persistentVolumeClaim();
|
|
|
|
await dependencies.applyResource(desired);
|
|
|
|
expect(fake.applied).toEqual([desired]);
|
|
});
|
|
|
|
test("retains a different storage class for a Pending claim", async () => {
|
|
const existing = {
|
|
...persistentVolumeClaim("longhorn"),
|
|
status: { phase: "Pending" },
|
|
};
|
|
const fake = managementObjects(existing);
|
|
const dependencies = createKubernetesManagementDependencies(
|
|
managementClients([], [], fake.objects),
|
|
);
|
|
const desired = persistentVolumeClaim();
|
|
|
|
await dependencies.applyResource(desired);
|
|
|
|
expect(fake.applied).toEqual([desired]);
|
|
});
|
|
});
|
|
|
|
describe("rollback ReplicaSet discovery", () => {
|
|
test("discovers owned revisions without the managed-by selector", async () => {
|
|
const depUid = "deployment-uid";
|
|
const deployments = [deployment("web", "demo", depUid, "img:v2")];
|
|
// ReplicaSets inherit only pod-template labels, never the deployment's
|
|
// managed-by metadata label.
|
|
const replicaSets = [
|
|
replicaSet("web-1", "demo", depUid, 1, "img:v1", { app: "web" }),
|
|
replicaSet("web-2", "demo", depUid, 2, "img:v2", { app: "web" }),
|
|
];
|
|
const deps = createKubernetesManagementDependencies(
|
|
managementClients(deployments, replicaSets),
|
|
);
|
|
const candidates = await deps.planRollback("demo");
|
|
expect(candidates).toHaveLength(1);
|
|
expect(candidates[0]).toMatchObject({
|
|
name: "web",
|
|
currentRevision: 2,
|
|
previousRevision: 1,
|
|
image: "img:v1",
|
|
});
|
|
});
|
|
|
|
test("excludes ReplicaSets owned by another deployment", async () => {
|
|
const depUid = "deployment-uid";
|
|
const otherUid = "other-deployment-uid";
|
|
const deployments = [deployment("web", "demo", depUid, "img:v2")];
|
|
const replicaSets = [
|
|
replicaSet("web-1", "demo", depUid, 1, "img:v1", { app: "web" }),
|
|
replicaSet("web-2", "demo", depUid, 2, "img:v2", { app: "web" }),
|
|
replicaSet("other-1", "demo", otherUid, 1, "img:other", { app: "other" }),
|
|
];
|
|
const deps = createKubernetesManagementDependencies(
|
|
managementClients(deployments, replicaSets),
|
|
);
|
|
const candidates = await deps.planRollback("demo");
|
|
expect(candidates).toHaveLength(1);
|
|
const candidate = candidates[0]!;
|
|
expect(candidate).toMatchObject({
|
|
name: "web",
|
|
currentRevision: 2,
|
|
previousRevision: 1,
|
|
image: "img:v1",
|
|
});
|
|
});
|
|
|
|
test("excludes orphan ReplicaSets with no Deployment owner reference", async () => {
|
|
const depUid = "deployment-uid";
|
|
const deployments = [deployment("web", "demo", depUid, "img:v1")];
|
|
const replicaSets = [
|
|
replicaSet("orphan-1", "demo", "", 1, "img:orphan", { app: "orphan" }),
|
|
];
|
|
const deps = createKubernetesManagementDependencies(
|
|
managementClients(deployments, replicaSets),
|
|
);
|
|
const candidates = await deps.planRollback("demo");
|
|
expect(candidates).toHaveLength(0);
|
|
});
|
|
|
|
test("excludes unrelated ReplicaSets with an unrelated managed-by label", async () => {
|
|
const depUid = "deployment-uid";
|
|
const deployments = [deployment("web", "demo", depUid, "img:v2")];
|
|
const replicaSets = [
|
|
replicaSet("web-1", "demo", depUid, 1, "img:v1", { app: "web" }),
|
|
replicaSet("web-2", "demo", depUid, 2, "img:v2", { app: "web" }),
|
|
// An unrelated RS that happens to carry the managed-by label (e.g. a
|
|
// standalone non-Deployment object) must still be ignored.
|
|
replicaSet(
|
|
"standalone",
|
|
"demo",
|
|
"other-uid",
|
|
1,
|
|
"img:standalone",
|
|
{ app: "standalone" },
|
|
true,
|
|
),
|
|
];
|
|
const deps = createKubernetesManagementDependencies(
|
|
managementClients(deployments, replicaSets),
|
|
);
|
|
const candidates = await deps.planRollback("demo");
|
|
expect(candidates).toHaveLength(1);
|
|
expect(candidates[0]).toMatchObject({
|
|
name: "web",
|
|
currentRevision: 2,
|
|
previousRevision: 1,
|
|
image: "img:v1",
|
|
});
|
|
});
|
|
});
|