import { describe, expect, test } from "bun:test"; import { Listr } from "listr2"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { ApiRequestInit, ApiRequestOptions } from "../../lib/api"; import { KuberApiError } from "../../lib/api"; import type { ApiRequester } from "../../lib/build"; import { ensureWorkspace, reconcileResources, runLiveResourceOperation, runUp, workspaceAdoptionRoute, } from "../../command/up"; import { provideContext } from "../../lib/context"; import { resolveTrustIdentity, updateTrust } from "../../lib/trust"; import { workspaceManifestDigest } from "../../lib/workspace"; const manifest = { version: 1 as const, files: [] }; const snapshot = { manifest, digest: workspaceManifestDigest(manifest), blobs: [], }; describe("up API pipeline", () => { test("forwards the build timeout through the trusted requester", async () => { const root = await mkdtemp(join(tmpdir(), "kuber-up-api-")); const previousCwd = process.cwd(); const previousConfigHome = process.env.XDG_CONFIG_HOME; const configHome = join(root, "config"); const calls: Array<{ path: string; init?: ApiRequestInit; options?: ApiRequestOptions; }> = []; try { await writeFile( join(root, "compose.yml"), "services:\n web:\n build: .\n", ); await writeFile( join(root, ".kuberrc.ts"), 'export default { project: "shop" };\n', ); const git = Bun.spawn(["git", "init", "-q", root]); expect(await git.exited).toBe(0); process.chdir(root); process.env.XDG_CONFIG_HOME = configHome; const identity = await resolveTrustIdentity("shop", root); await updateTrust((records) => [...records, identity]); const request: ApiRequester = async ( path: string, init?: ApiRequestInit, options?: ApiRequestOptions, ) => { calls.push({ path, init, options }); if (path === "/snapshots/negotiate") return { workspace: snapshot.digest, missing: [], ready: true } as T; if (path === "/builds") { const buildRequest = init?.json as { id?: unknown } | undefined; if (typeof buildRequest?.id !== "string") throw new Error("Expected build request ID"); return { version: 1, id: buildRequest.id, state: "succeeded", createdAt: "2026-01-01T00:00:00Z", } as T; } if (path.includes("/events")) return [] as T; if (path.endsWith("/result")) return { image: "registry.server/kuber/shop-web", digest: `sha256:${"a".repeat(64)}`, reference: `registry.server/kuber/shop-web@sha256:${"a".repeat(64)}`, } as T; if (path === "/workspaces/shop") throw new KuberApiError("missing", 404); if (path === "/workspaces") return { metadata: { name: "shop", uid: "workspace", resourceVersion: "1" }, } as T; if (path.endsWith("/adopt")) return { resourcesAdopted: 0 } as T; if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; return {} as T; }; await provideContext(() => runUp(true, request)); const build = calls.find(({ path }) => path === "/builds"); if (!build) throw new Error("Expected build submission"); expect(build.options).toEqual({ timeoutMs: 300_000 }); expect( new Headers(build.init?.headers).get("x-kuber-trust-project"), ).toBe("shop"); } finally { process.chdir(previousCwd); if (previousConfigHome === undefined) delete process.env.XDG_CONFIG_HOME; else process.env.XDG_CONFIG_HOME = previousConfigHome; await rm(root, { recursive: true, force: true }); } }); test("creates workspace metadata without embedding source blobs", async () => { const calls: Array<{ path: string; init?: ApiRequestInit }> = []; const request: ApiRequester = async ( path: string, init?: ApiRequestInit, ) => { calls.push({ path, init }); if (!init) throw new KuberApiError("missing", 404); return { metadata: { name: "shop", uid: "uid", resourceVersion: "1" }, } as T; }; await ensureWorkspace( "shop", { services: { web: { image: "nginx" } } }, snapshot, request, ); expect(calls.map(({ path }) => path)).toEqual([ "/workspaces/shop", "/workspaces", ]); const body = calls[1]!.init?.json as Record; expect(body).toMatchObject({ id: "shop", source: { uri: `cas://${snapshot.digest}`, digest: snapshot.digest }, }); expect(JSON.stringify(body)).not.toContain('"blob"'); expect(JSON.stringify(body)).not.toContain('"files"'); }); test("updates workspace metadata with the current resource-version ETag", async () => { const calls: Array<{ path: string; init?: ApiRequestInit }> = []; const request: ApiRequester = async ( path: string, init?: ApiRequestInit, ) => { calls.push({ path, init }); return { metadata: { name: "shop", uid: "uid", resourceVersion: init ? "8" : "7", }, } as T; }; await ensureWorkspace("shop", { services: {} }, snapshot, request); expect(calls[1]?.init?.method).toBe("PUT"); expect(new Headers(calls[1]?.init?.headers).get("if-match")).toBe('"7"'); }); test("plans, applies, runs the hook, waits, then deletes stale identities", async () => { const order: string[] = []; const desired = [ { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: "web" } }, ]; const stale = [ { apiVersion: "v1", kind: "Secret", name: "old", uid: "secret-uid", workspaceUid: "workspace-uid", }, ]; const request: ApiRequester = async (path: string) => { order.push(path.split("/").at(-1)!); if (path.endsWith("/plan")) return { desired, stale } as T; return {} as T; }; await reconcileResources( "shop", desired, 42_000, () => { order.push("hook"); }, request, ); expect(order).toEqual(["plan", "apply", "hook", "wait", "delete"]); }); test("resumes an interrupted reconcile operation by its persisted ID", async () => { const calls: string[] = []; const applyIdempotencyKeys: Array = []; let applyAttempts = 0; let operationPolls = 0; const request: ApiRequester = async ( path: string, init?: ApiRequestInit, ) => { calls.push(path); if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; if (path.endsWith("/apply")) { applyIdempotencyKeys.push( new Headers(init?.headers).get("idempotency-key"), ); applyAttempts++; if (applyAttempts === 1) throw new TypeError("fetch failed"); return { operationId: "operation-apply", operation: { status: { state: "running" } }, } as T; } if (path === "/operations/operation-apply") { operationPolls++; if (operationPolls === 1) throw new TypeError("connection reset"); return { status: { state: "succeeded" } } as T; } throw new Error(`Unexpected request: ${path}`); }; await reconcileResources("shop", [], 1, undefined, request, { sleep: async () => {}, }); expect(calls).toEqual([ "/workspaces/shop/resources/plan", "/workspaces/shop/resources/apply", "/workspaces/shop/resources/apply", "/operations/operation-apply/events?after=0", "/operations/operation-apply", "/operations/operation-apply", "/operations/operation-apply/events?after=0", ]); expect(applyIdempotencyKeys[0]).toBeTruthy(); expect(applyIdempotencyKeys[1]).toBe(applyIdempotencyKeys[0]); }); test("restarts after an interrupted apply is persisted before its ID is received", async () => { const applyRequests: Array<{ key: string | null; json: unknown }> = []; const request: ApiRequester = async ( path: string, init?: ApiRequestInit, ) => { if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; if (path.endsWith("/apply")) { applyRequests.push({ key: new Headers(init?.headers).get("idempotency-key"), json: init?.json, }); if (applyRequests.length === 1) throw new TypeError("fetch failed"); if (applyRequests.length === 2) throw new KuberApiError("operation interrupted", 500, { title: "Operation interrupted", status: 500, code: "OPERATION_INTERRUPTED", }); return { operationId: "operation-apply", operation: { status: { state: "succeeded" } }, } as T; } throw new Error(`Unexpected request: ${path}`); }; await reconcileResources("shop", [], 1, undefined, request, { sleep: async () => {}, }); expect(applyRequests).toHaveLength(3); expect(applyRequests[0]?.key).toBeTruthy(); expect(applyRequests[1]?.key).toBe(applyRequests[0]?.key); expect(applyRequests[2]?.key).toBeTruthy(); expect(applyRequests[2]?.key).not.toBe(applyRequests[0]?.key); expect(applyRequests[2]?.json).toEqual(applyRequests[0]?.json); }); test("polls persisted resource progress without duplicating events after reconnect", async () => { const progress: string[] = []; let poll = 0; const request: ApiRequester = async (path: string) => { if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; if (path.endsWith("/apply")) return { operationId: "operation-apply", operation: { status: { state: "running" } }, } as T; if (path.includes("/events")) { poll++; return { items: poll === 1 ? [ { sequence: 1, data: null, }, { sequence: 2, data: { resource: { apiVersion: "v1", kind: "Service", name: "web", }, phase: "apply", state: "started", }, }, ] : [ { sequence: 2, data: { resource: { apiVersion: "v1", kind: "Service", name: "web", }, phase: "apply", state: "started", }, }, { sequence: 3, data: { resource: { apiVersion: "v1", kind: "Service", name: "web", }, phase: "apply", state: "succeeded", }, }, ], } as T; } if (path === "/operations/operation-apply") { return { status: { state: poll > 1 ? "succeeded" : "running" } } as T; } throw new Error(`Unexpected request: ${path}`); }; await reconcileResources("shop", [], 1, undefined, request, { sleep: async () => {}, onEvent: (event) => progress.push(`${event.sequence}:${event.data.state}`), }); expect(progress).toEqual(["2:started", "3:succeeded"]); }); test("fails visibly when retained operation progress has a cursor gap", async () => { let applyAttempts = 0; let operationPolls = 0; const request: ApiRequester = async (path: string) => { if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; if (path.endsWith("/apply")) { applyAttempts += 1; return { operationId: "operation-apply", operation: { status: { state: "running" } }, } as T; } if (path.includes("/events")) return { retainedFirstSequence: 3, cursorGap: true, items: [], } as T; if (path === "/operations/operation-apply") { operationPolls += 1; return { status: { state: "succeeded" } } as T; } throw new Error(`Unexpected request: ${path}`); }; await expect( reconcileResources("shop", [], 1, undefined, request, { sleep: async () => {}, }), ).rejects.toThrow("progress history was truncated"); expect(applyAttempts).toBe(1); expect(operationPolls).toBe(0); }); test("starts live resource subtasks before the operation and updates them from progress", async () => { let finish!: () => void; let child: { title: string; output: string } | undefined; const completed = new Promise((resolve) => (finish = resolve)); const listr = new Listr([ { title: "Apply resources", task: async (_ctx, task) => runLiveResourceOperation( task, "apply", [ { apiVersion: "v1", kind: "Service", name: "web", }, ], async (onEvent) => { expect(child?.title).toBe("Apply Service/web"); onEvent({ sequence: 1, data: { resource: { apiVersion: "v1", kind: "Service", name: "web", }, phase: "apply", state: "started", }, }); expect(child?.output).toBe("started"); await completed; }, { onTaskStarted: (_target, activeTask) => (child = activeTask) }, ), }, ]); const run = listr.run(); await Bun.sleep(0); finish(); await run; }); test("resubmits with a fresh key after server restart interruption", async () => { const applyRequests: Array<{ key: string | null; json: unknown }> = []; let operationPolls = 0; const request: ApiRequester = async ( path: string, init?: ApiRequestInit, ) => { if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; if (path.endsWith("/apply")) { applyRequests.push({ key: new Headers(init?.headers).get("idempotency-key"), json: init?.json, }); return { operationId: `operation-apply-${applyRequests.length}`, operation: { status: { state: "running" } }, } as T; } if (path === "/operations/operation-apply-1") { operationPolls++; if (operationPolls === 1) return { status: { state: "failed", error: { code: "OPERATION_INTERRUPTED", message: "server restarted", }, }, } as T; return { status: { state: "succeeded" } } as T; } if (path === "/operations/operation-apply-2") return { status: { state: "succeeded" } } as T; throw new Error(`Unexpected request: ${path}`); }; await reconcileResources("shop", [], 1, undefined, request, { sleep: async () => {}, }); expect(applyRequests).toHaveLength(2); expect(applyRequests[0]?.key).toBeTruthy(); expect(applyRequests[1]?.key).toBeTruthy(); expect(applyRequests[1]?.key).not.toBe(applyRequests[0]?.key); expect(applyRequests[1]?.json).toEqual(applyRequests[0]?.json); }); test.each([ ["failed", "OPERATION_FAILED"], ["cancelled", "OPERATION_CANCELLED"], ])("does not retry arbitrary %s operations", async (state, code) => { let applyAttempts = 0; let operationPolls = 0; const request: ApiRequester = async (path: string) => { if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; if (path.endsWith("/apply")) { applyAttempts++; return { operationId: "operation-apply", operation: { status: { state: "running" } }, } as T; } if (path === "/operations/operation-apply") { operationPolls++; return { status: { state, error: { code, message: "terminal" } }, } as T; } throw new Error(`Unexpected request: ${path}`); }; await expect( reconcileResources("shop", [], 1, undefined, request, { sleep: async () => {}, }), ).rejects.toMatchObject({ code }); expect(applyAttempts).toBe(1); expect(operationPolls).toBe(1); }); test("bounds restart recovery sleeps by the resume deadline", async () => { let currentTime = 0; const sleeps: Array<{ startedAt: number; milliseconds: number }> = []; let operationNumber = 0; const request: ApiRequester = async (path: string) => { if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; if (path.endsWith("/apply")) { operationNumber++; return { operationId: `operation-${operationNumber}`, operation: { status: { state: "running" } }, } as T; } return { status: { state: "failed", error: { code: "OPERATION_INTERRUPTED", message: "restarted" }, }, } as T; }; await expect( reconcileResources("shop", [], 0, undefined, request, { now: () => currentTime, sleep: async (milliseconds) => { sleeps.push({ startedAt: currentTime, milliseconds }); currentTime += milliseconds; }, }), ).rejects.toThrow("Timed out while reconnecting to resume the operation"); expect(sleeps).not.toHaveLength(0); expect( sleeps.every( ({ startedAt, milliseconds }) => startedAt + milliseconds <= 60_000, ), ).toBe(true); expect(currentTime).toBe(60_000); }); test("does not restart after an apply interruption reaches the resume deadline", async () => { let currentTime = 0; let applyAttempts = 0; const request: ApiRequester = async (path: string) => { if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; applyAttempts++; if (applyAttempts === 1) throw new TypeError("connection reset"); throw new KuberApiError("operation interrupted", 500, { title: "Operation interrupted", status: 500, code: "OPERATION_INTERRUPTED", }); }; await expect( reconcileResources("shop", [], 0, undefined, request, { now: () => currentTime, sleep: async (milliseconds) => { currentTime += milliseconds; if (currentTime < 60_000) currentTime = 60_000; }, }), ).rejects.toMatchObject({ code: "OPERATION_INTERRUPTED", status: 500, }); expect(applyAttempts).toBe(2); expect(currentTime).toBe(60_000); }); test("fails immediately for a typed 503 operation-store error", async () => { const unavailable = new KuberApiError( "Operation storage is not configured", 503, { title: "Service unavailable", status: 503, code: "OPERATION_STORE_UNAVAILABLE", }, ); const calls: string[] = []; const sleeps: number[] = []; const request: ApiRequester = async (path: string) => { calls.push(path); if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; throw unavailable; }; await expect( reconcileResources("shop", [], 1, undefined, request, { sleep: async (milliseconds) => { sleeps.push(milliseconds); }, }), ).rejects.toBe(unavailable); expect(calls).toEqual([ "/workspaces/shop/resources/plan", "/workspaces/shop/resources/apply", ]); expect(sleeps).toEqual([]); }); test("never sleeps past the operation resume deadline", async () => { let currentTime = 0; const sleeps: Array<{ startedAt: number; milliseconds: number }> = []; const request: ApiRequester = async (path: string) => { if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; throw new TypeError("connection reset"); }; await expect( reconcileResources("shop", [], 0, undefined, request, { now: () => currentTime, sleep: async (milliseconds) => { sleeps.push({ startedAt: currentTime, milliseconds }); currentTime += milliseconds; }, }), ).rejects.toThrow("connection reset"); expect(sleeps).not.toHaveLength(0); expect( sleeps.every( ({ startedAt, milliseconds }) => startedAt + milliseconds <= 60_000, ), ).toBe(true); expect(sleeps.at(-1)).toEqual({ startedAt: 57_750, milliseconds: 2_250, }); expect(currentTime).toBe(60_000); }); test("fails closed with the precise missing adoption route", async () => { const request: ApiRequester = async (path: string) => { if (path.endsWith("/plan")) throw new Error( "Namespace shop is external; refusing workspace mutation", ); return {} as T; }; await expect( reconcileResources("shop", [], 1, undefined, request), ).rejects.toThrow(`POST ${workspaceAdoptionRoute("shop")}`); }); });