feat: harden self-managed reconciliation

2026-09-05 10:17:55 +00:00
parent 8e9d207915
commit 321f4e807a
40 changed files with 4977 additions and 404 deletions
@@ -0,0 +1,97 @@
+import { afterEach, describe, expect, spyOn, test } from "bun:test";
+import { rm } from "node:fs/promises";
+import { KuberApiError, type ApiRequestInit } from "../../lib/api";
+import { getTrustPath, readTrust, updateTrust } from "../../lib/trust";
+import { grantTrust, revokeTrust, statusTrust } from "../../command/trust";
+
+const originalConfig = process.env.XDG_CONFIG_HOME;
+const identity = { project: "demo", fingerprint: "a".repeat(64) };
+
+afterEach(async () => {
+  const path = getTrustPath();
+  if (originalConfig === undefined) delete process.env.XDG_CONFIG_HOME;
+  else process.env.XDG_CONFIG_HOME = originalConfig;
+  await rm(path.slice(0, path.lastIndexOf("/")), {
+    recursive: true,
+    force: true,
+  });
+});
+
+function requester(
+  calls: Array<{ path: string; init?: ApiRequestInit }>,
+  response: unknown = undefined,
+) {
+  return async <T>(path: string, init?: ApiRequestInit): Promise<T> => {
+    calls.push({ path, init });
+    return response as T;
+  };
+}
+
+describe("trust command", () => {
+  test("grants, reports, and revokes the current namespace fingerprint", async () => {
+    process.env.XDG_CONFIG_HOME = `/tmp/kuber-trust-command-${crypto.randomUUID()}`;
+    const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
+    const output = spyOn(console, "log").mockImplementation(() => {});
+
+    await grantTrust(identity, requester(calls));
+    expect(calls).toEqual([
+      {
+        path: "/workspaces/demo/trust",
+        init: { method: "POST", json: { fingerprint: identity.fingerprint } },
+      },
+    ]);
+    expect(await readTrust()).toEqual([identity]);
+
+    calls.length = 0;
+    await statusTrust(
+      identity,
+      requester(calls, { fingerprints: [identity.fingerprint] }),
+    );
+    expect(calls).toEqual([
+      { path: "/workspaces/demo/trust", init: undefined },
+    ]);
+    expect(output.mock.calls.map(([line]) => line)).toEqual([
+      "Trusted this directory for namespace demo",
+      "Namespace: demo",
+      "Local: trusted",
+      "Server: registered",
+    ]);
+
+    calls.length = 0;
+    await revokeTrust(identity, requester(calls));
+    expect(calls).toEqual([
+      {
+        path: `/workspaces/demo/trust?fingerprint=${identity.fingerprint}`,
+        init: { method: "DELETE" },
+      },
+    ]);
+    expect(await readTrust()).toEqual([]);
+    output.mockRestore();
+  });
+
+  test("removes local trust when the server registration is already absent", async () => {
+    process.env.XDG_CONFIG_HOME = `/tmp/kuber-trust-command-${crypto.randomUUID()}`;
+    await updateTrust(() => [identity]);
+    const output = spyOn(console, "log").mockImplementation(() => {});
+
+    await revokeTrust(identity, async () => {
+      throw new KuberApiError("not found", 404);
+    });
+
+    expect(await readTrust()).toEqual([]);
+    expect(output).toHaveBeenCalledWith("Revoked trust for namespace demo");
+    output.mockRestore();
+  });
+
+  test("removes local trust before reporting a remote revoke failure", async () => {
+    process.env.XDG_CONFIG_HOME = `/tmp/kuber-trust-command-${crypto.randomUUID()}`;
+    await updateTrust(() => [identity]);
+
+    await expect(
+      revokeTrust(identity, async () => {
+        throw new KuberApiError("unavailable", 503);
+      }),
+    ).rejects.toThrow("Removed local trust for namespace demo");
+    expect(await readTrust()).toEqual([]);
+  });
+});
@@ -1,12 +1,18 @@
 import { describe, expect, test } from "bun:test";
-import type { ApiRequestInit } from "../../lib/api";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { ApiRequestInit, ApiRequestOptions } from "../../lib/api";
 import { KuberApiError } from "../../lib/api";
 import type { ApiRequester } from "../../lib/build";
 import {
  ensureWorkspace,
  reconcileResources,
+  runUp,
  workspaceAdoptionRoute,
 } from "../../command/up";
+import { provideContext } from "../../lib/context";
+import { resolveTrustIdentity, updateTrust } from "../../lib/trust";
 import { workspaceManifestDigest } from "../../lib/workspace";

 const manifest = { version: 1 as const, files: [] };
@@ -17,6 +23,87 @@ const snapshot = {
 };

 describe("up API pipeline", () => {
+  test("forwards the build timeout through the trusted requester", async () => {
+    const root = await mkdtemp(join(tmpdir(), "kuber-up-api-"));
+    const previousCwd = process.cwd();
+    const previousConfigHome = process.env.XDG_CONFIG_HOME;
+    const configHome = join(root, "config");
+    const calls: Array<{
+      path: string;
+      init?: ApiRequestInit;
+      options?: ApiRequestOptions;
+    }> = [];
+
+    try {
+      await writeFile(
+        join(root, "compose.yml"),
+        "services:\n  web:\n    build: .\n",
+      );
+      await writeFile(
+        join(root, ".kuberrc.ts"),
+        'export default { project: "shop" };\n',
+      );
+      const git = Bun.spawn(["git", "init", "-q", root]);
+      expect(await git.exited).toBe(0);
+
+      process.chdir(root);
+      process.env.XDG_CONFIG_HOME = configHome;
+      const identity = await resolveTrustIdentity("shop", root);
+      await updateTrust((records) => [...records, identity]);
+
+      const request: ApiRequester = async <T>(
+        path: string,
+        init?: ApiRequestInit,
+        options?: ApiRequestOptions,
+      ) => {
+        calls.push({ path, init, options });
+        if (path === "/snapshots/negotiate")
+          return { workspace: snapshot.digest, missing: [], ready: true } as T;
+        if (path === "/builds") {
+          const buildRequest = init?.json as { id?: unknown } | undefined;
+          if (typeof buildRequest?.id !== "string")
+            throw new Error("Expected build request ID");
+          return {
+            version: 1,
+            id: buildRequest.id,
+            state: "succeeded",
+            createdAt: "2026-01-01T00:00:00Z",
+          } as T;
+        }
+        if (path.includes("/events")) return [] as T;
+        if (path.endsWith("/result"))
+          return {
+            image: "registry.server/kuber/shop-web",
+            digest: `sha256:${"a".repeat(64)}`,
+            reference: `registry.server/kuber/shop-web@sha256:${"a".repeat(64)}`,
+          } as T;
+        if (path === "/workspaces/shop")
+          throw new KuberApiError("missing", 404);
+        if (path === "/workspaces")
+          return {
+            metadata: { name: "shop", uid: "workspace", resourceVersion: "1" },
+          } as T;
+        if (path.endsWith("/adopt")) return { resourcesAdopted: 0 } as T;
+        if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
+        return {} as T;
+      };
+
+      await provideContext(() => runUp(true, request));
+
+      const build = calls.find(({ path }) => path === "/builds");
+      if (!build) throw new Error("Expected build submission");
+      expect(build.options).toEqual({ timeoutMs: 300_000 });
+      expect(
+        new Headers(build.init?.headers).get("x-kuber-trust-project"),
+      ).toBe("shop");
+    } finally {
+      process.chdir(previousCwd);
+      if (previousConfigHome === undefined) delete process.env.XDG_CONFIG_HOME;
+      else process.env.XDG_CONFIG_HOME = previousConfigHome;
+      await rm(root, { recursive: true, force: true });
+    }
+  });
+
  test("creates workspace metadata without embedding source blobs", async () => {
    const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
    const request: ApiRequester = async <T>(
@@ -101,6 +188,305 @@ describe("up API pipeline", () => {
    expect(order).toEqual(["plan", "apply", "hook", "wait", "delete"]);
  });

+  test("resumes an interrupted reconcile operation by its persisted ID", async () => {
+    const calls: string[] = [];
+    const applyIdempotencyKeys: Array<string | null> = [];
+    let applyAttempts = 0;
+    let operationPolls = 0;
+    const request: ApiRequester = async <T>(
+      path: string,
+      init?: ApiRequestInit,
+    ) => {
+      calls.push(path);
+      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
+      if (path.endsWith("/apply")) {
+        applyIdempotencyKeys.push(
+          new Headers(init?.headers).get("idempotency-key"),
+        );
+        applyAttempts++;
+        if (applyAttempts === 1) throw new TypeError("fetch failed");
+        return {
+          operationId: "operation-apply",
+          operation: { status: { state: "running" } },
+        } as T;
+      }
+      if (path === "/operations/operation-apply") {
+        operationPolls++;
+        if (operationPolls === 1) throw new TypeError("connection reset");
+        return { status: { state: "succeeded" } } as T;
+      }
+      throw new Error(`Unexpected request: ${path}`);
+    };
+
+    await reconcileResources("shop", [], 1, undefined, request, {
+      sleep: async () => {},
+    });
+
+    expect(calls).toEqual([
+      "/workspaces/shop/resources/plan",
+      "/workspaces/shop/resources/apply",
+      "/workspaces/shop/resources/apply",
+      "/operations/operation-apply",
+      "/operations/operation-apply",
+    ]);
+    expect(applyIdempotencyKeys[0]).toBeTruthy();
+    expect(applyIdempotencyKeys[1]).toBe(applyIdempotencyKeys[0]);
+  });
+
+  test("restarts after an interrupted apply is persisted before its ID is received", async () => {
+    const applyRequests: Array<{ key: string | null; json: unknown }> = [];
+    const request: ApiRequester = async <T>(
+      path: string,
+      init?: ApiRequestInit,
+    ) => {
+      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
+      if (path.endsWith("/apply")) {
+        applyRequests.push({
+          key: new Headers(init?.headers).get("idempotency-key"),
+          json: init?.json,
+        });
+        if (applyRequests.length === 1) throw new TypeError("fetch failed");
+        if (applyRequests.length === 2)
+          throw new KuberApiError("operation interrupted", 500, {
+            title: "Operation interrupted",
+            status: 500,
+            code: "OPERATION_INTERRUPTED",
+          });
+        return {
+          operationId: "operation-apply",
+          operation: { status: { state: "succeeded" } },
+        } as T;
+      }
+      throw new Error(`Unexpected request: ${path}`);
+    };
+
+    await reconcileResources("shop", [], 1, undefined, request, {
+      sleep: async () => {},
+    });
+
+    expect(applyRequests).toHaveLength(3);
+    expect(applyRequests[0]?.key).toBeTruthy();
+    expect(applyRequests[1]?.key).toBe(applyRequests[0]?.key);
+    expect(applyRequests[2]?.key).toBeTruthy();
+    expect(applyRequests[2]?.key).not.toBe(applyRequests[0]?.key);
+    expect(applyRequests[2]?.json).toEqual(applyRequests[0]?.json);
+  });
+
+  test("resubmits with a fresh key after server restart interruption", async () => {
+    const applyRequests: Array<{ key: string | null; json: unknown }> = [];
+    let operationPolls = 0;
+    const request: ApiRequester = async <T>(
+      path: string,
+      init?: ApiRequestInit,
+    ) => {
+      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
+      if (path.endsWith("/apply")) {
+        applyRequests.push({
+          key: new Headers(init?.headers).get("idempotency-key"),
+          json: init?.json,
+        });
+        return {
+          operationId: `operation-apply-${applyRequests.length}`,
+          operation: { status: { state: "running" } },
+        } as T;
+      }
+      if (path === "/operations/operation-apply-1") {
+        operationPolls++;
+        if (operationPolls === 1)
+          return {
+            status: {
+              state: "failed",
+              error: {
+                code: "OPERATION_INTERRUPTED",
+                message: "server restarted",
+              },
+            },
+          } as T;
+        return { status: { state: "succeeded" } } as T;
+      }
+      if (path === "/operations/operation-apply-2")
+        return { status: { state: "succeeded" } } as T;
+      throw new Error(`Unexpected request: ${path}`);
+    };
+
+    await reconcileResources("shop", [], 1, undefined, request, {
+      sleep: async () => {},
+    });
+
+    expect(applyRequests).toHaveLength(2);
+    expect(applyRequests[0]?.key).toBeTruthy();
+    expect(applyRequests[1]?.key).toBeTruthy();
+    expect(applyRequests[1]?.key).not.toBe(applyRequests[0]?.key);
+    expect(applyRequests[1]?.json).toEqual(applyRequests[0]?.json);
+  });
+
+  test.each([
+    ["failed", "OPERATION_FAILED"],
+    ["cancelled", "OPERATION_CANCELLED"],
+  ])("does not retry arbitrary %s operations", async (state, code) => {
+    let applyAttempts = 0;
+    let operationPolls = 0;
+    const request: ApiRequester = async <T>(path: string) => {
+      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
+      if (path.endsWith("/apply")) {
+        applyAttempts++;
+        return {
+          operationId: "operation-apply",
+          operation: { status: { state: "running" } },
+        } as T;
+      }
+      if (path === "/operations/operation-apply") {
+        operationPolls++;
+        return {
+          status: { state, error: { code, message: "terminal" } },
+        } as T;
+      }
+      throw new Error(`Unexpected request: ${path}`);
+    };
+
+    await expect(
+      reconcileResources("shop", [], 1, undefined, request, {
+        sleep: async () => {},
+      }),
+    ).rejects.toMatchObject({ code });
+    expect(applyAttempts).toBe(1);
+    expect(operationPolls).toBe(1);
+  });
+
+  test("bounds restart recovery sleeps by the resume deadline", async () => {
+    let currentTime = 0;
+    const sleeps: Array<{ startedAt: number; milliseconds: number }> = [];
+    let operationNumber = 0;
+    const request: ApiRequester = async <T>(path: string) => {
+      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
+      if (path.endsWith("/apply")) {
+        operationNumber++;
+        return {
+          operationId: `operation-${operationNumber}`,
+          operation: { status: { state: "running" } },
+        } as T;
+      }
+      return {
+        status: {
+          state: "failed",
+          error: { code: "OPERATION_INTERRUPTED", message: "restarted" },
+        },
+      } as T;
+    };
+
+    await expect(
+      reconcileResources("shop", [], 0, undefined, request, {
+        now: () => currentTime,
+        sleep: async (milliseconds) => {
+          sleeps.push({ startedAt: currentTime, milliseconds });
+          currentTime += milliseconds;
+        },
+      }),
+    ).rejects.toThrow("Timed out while reconnecting to resume the operation");
+
+    expect(sleeps).not.toHaveLength(0);
+    expect(
+      sleeps.every(
+        ({ startedAt, milliseconds }) => startedAt + milliseconds <= 60_000,
+      ),
+    ).toBe(true);
+    expect(currentTime).toBe(60_000);
+  });
+
+  test("does not restart after an apply interruption reaches the resume deadline", async () => {
+    let currentTime = 0;
+    let applyAttempts = 0;
+    const request: ApiRequester = async <T>(path: string) => {
+      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
+      applyAttempts++;
+      if (applyAttempts === 1) throw new TypeError("connection reset");
+      throw new KuberApiError("operation interrupted", 500, {
+        title: "Operation interrupted",
+        status: 500,
+        code: "OPERATION_INTERRUPTED",
+      });
+    };
+
+    await expect(
+      reconcileResources("shop", [], 0, undefined, request, {
+        now: () => currentTime,
+        sleep: async (milliseconds) => {
+          currentTime += milliseconds;
+          if (currentTime < 60_000) currentTime = 60_000;
+        },
+      }),
+    ).rejects.toMatchObject({
+      code: "OPERATION_INTERRUPTED",
+      status: 500,
+    });
+    expect(applyAttempts).toBe(2);
+    expect(currentTime).toBe(60_000);
+  });
+
+  test("fails immediately for a typed 503 operation-store error", async () => {
+    const unavailable = new KuberApiError(
+      "Operation storage is not configured",
+      503,
+      {
+        title: "Service unavailable",
+        status: 503,
+        code: "OPERATION_STORE_UNAVAILABLE",
+      },
+    );
+    const calls: string[] = [];
+    const sleeps: number[] = [];
+    const request: ApiRequester = async <T>(path: string) => {
+      calls.push(path);
+      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
+      throw unavailable;
+    };
+
+    await expect(
+      reconcileResources("shop", [], 1, undefined, request, {
+        sleep: async (milliseconds) => {
+          sleeps.push(milliseconds);
+        },
+      }),
+    ).rejects.toBe(unavailable);
+
+    expect(calls).toEqual([
+      "/workspaces/shop/resources/plan",
+      "/workspaces/shop/resources/apply",
+    ]);
+    expect(sleeps).toEqual([]);
+  });
+
+  test("never sleeps past the operation resume deadline", async () => {
+    let currentTime = 0;
+    const sleeps: Array<{ startedAt: number; milliseconds: number }> = [];
+    const request: ApiRequester = async <T>(path: string) => {
+      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
+      throw new TypeError("connection reset");
+    };
+
+    await expect(
+      reconcileResources("shop", [], 0, undefined, request, {
+        now: () => currentTime,
+        sleep: async (milliseconds) => {
+          sleeps.push({ startedAt: currentTime, milliseconds });
+          currentTime += milliseconds;
+        },
+      }),
+    ).rejects.toThrow("connection reset");
+
+    expect(sleeps).not.toHaveLength(0);
+    expect(
+      sleeps.every(
+        ({ startedAt, milliseconds }) => startedAt + milliseconds <= 60_000,
+      ),
+    ).toBe(true);
+    expect(sleeps.at(-1)).toEqual({
+      startedAt: 57_750,
+      milliseconds: 2_250,
+    });
+    expect(currentTime).toBe(60_000);
+  });
+
  test("fails closed with the precise missing adoption route", async () => {
    const request: ApiRequester = async <T>(path: string) => {
      if (path.endsWith("/plan"))