import { describe, expect, spyOn, test } from "bun:test";
import { cleanupExpiredSessions, createApp } from "../../server/app";
import { hashToken, MemoryAuthStore } from "../../server/auth";
import { MemoryAuditStore } from "../../server/audit-store";
import type { ManagementService } from "../../server/management";
import {
  MemoryOperationStore,
  MemoryWorkspaceLeaseProvider,
} from "../../server/operation-store";
import { MemoryWorkspaceStore } from "../../server/workspace-store";
import { MemoryTrustStore } from "../../server/trust-store";
import type { BuildController } from "../../server/build-controller";

function request(
  path: string,
  init: RequestInit = {},
  token?: string,
): Request {
  const headers = new Headers(init.headers);
  if (token) headers.set("authorization", `Bearer ${token}`);
  return new Request(`https://kuber.astrxl.dev${path}`, { ...init, headers });
}

describe("kuber API authentication", () => {
  test("logs every request start and completion without changing responses", async () => {
    const logs: Record<string, unknown>[] = [];
    const app = createApp({
      store: new MemoryAuthStore(),
      requestId: () => "request-log-id",
      logger: {
        error: () => {},
        log: (entry) => logs.push(entry),
      },
    });

    const response = await app(request("/api/v2/health"));

    expect(response.status).toBe(200);
    expect(logs).toEqual(
      expect.arrayContaining([
        expect.objectContaining({
          event: "kuber.server.request.start",
          requestId: "request-log-id",
          method: "GET",
          pathname: "/api/v2/health",
        }),
        expect.objectContaining({
          event: "kuber.server.request.end",
          requestId: "request-log-id",
          status: 200,
          durationMs: expect.any(Number),
        }),
      ]),
    );
  });

  test("ignores request logging failures", async () => {
    const app = createApp({
      store: new MemoryAuthStore(),
      logger: {
        error: () => {},
        log: () => {
          throw new Error("logger unavailable");
        },
      },
    });

    expect((await app(request("/api/v2/health"))).status).toBe(200);
  });

  test("logs in, resolves identity, and revokes the session", async () => {
    const store = new MemoryAuthStore();
    await store.putUser({
      username: "dmgnr",
      passwordHash: "stored-hash",
      roles: ["admin"],
    });
    const app = createApp({
      store,
      now: () => Date.parse("2026-09-02T00:00:00.000Z"),
      verifyPassword: async (password, hash) =>
        password === "correct" && hash === "stored-hash",
    });

    const login = await app(
      request("/api/v2/login", {
        method: "POST",
        headers: { "content-type": "application/json" },
        body: JSON.stringify({
          username: "dmgnr",
          password: "correct",
          persistent: false,
        }),
      }),
    );
    expect(login.status).toBe(200);
    const session = (await login.json()) as {
      token: string;
      expiresAt: string;
    };
    expect(session.expiresAt).toBe("2026-09-03T00:00:00.000Z");

    const me = await app(request("/api/v2/me", {}, session.token));
    expect(await me.json()).toEqual({ username: "dmgnr", roles: ["admin"] });

    const logout = await app(
      request("/api/v2/logout", { method: "POST" }, session.token),
    );
    expect(logout.status).toBe(204);
    expect((await app(request("/api/v2/me", {}, session.token))).status).toBe(
      401,
    );
  });

  test("rejects invalid credentials and unauthenticated requests", async () => {
    const store = new MemoryAuthStore();
    await store.putUser({
      username: "dmgnr",
      passwordHash: "stored-hash",
      roles: ["admin"],
    });
    const app = createApp({
      store,
      verifyPassword: async () => false,
    });

    const login = await app(
      request("/api/v2/login", {
        method: "POST",
        headers: { "content-type": "application/json" },
        body: JSON.stringify({ username: "dmgnr", password: "wrong" }),
      }),
    );
    expect(login.status).toBe(401);
    expect((await app(request("/api/v2/me"))).status).toBe(401);
  });

  test("provides an unauthenticated health endpoint", async () => {
    const app = createApp({ store: new MemoryAuthStore() });
    const response = await app(request("/api/v2/health"));
    expect(response.status).toBe(200);
    expect(await response.json()).toEqual({ status: "ok" });
  });

  test("provides an explicit expired-session startup cleanup helper", async () => {
    const store = new MemoryAuthStore();
    await store.putUser({
      username: "user",
      passwordHash: "hash",
      roles: ["viewer"],
    });
    await store.putSession({
      tokenHash: hashToken("expired"),
      username: "user",
      authVersion: 1,
      expiresAt: "2026-09-01T00:00:00.000Z",
    });
    expect(
      await cleanupExpiredSessions(
        store,
        Date.parse("2026-09-02T00:00:00.000Z"),
      ),
    ).toBe(1);
  });

  test("rate limits repeated failed logins", async () => {
    const app = createApp({
      store: new MemoryAuthStore(),
      now: () => 0,
    });
    const login = () =>
      app(
        request("/api/v2/login", {
          method: "POST",
          body: JSON.stringify({ username: "missing", password: "wrong" }),
        }),
      );

    for (let attempt = 0; attempt < 5; attempt += 1) {
      expect((await login()).status).toBe(401);
    }
    const limited = await login();
    expect(limited.status).toBe(429);
    expect(limited.headers.get("retry-after")).toBe("300");
  });
});

describe("operation response safety", () => {
  test("database and storage reconcile concurrently while conflicting scopes fail fast", async () => {
    const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-uid" });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore();
    const leases = new MemoryWorkspaceLeaseProvider();
    const entered = new Set<string>();
    let release!: () => void;
    const blocked = new Promise<void>((resolve) => (release = resolve));
    let bothEntered!: () => void;
    const bothStarted = new Promise<void>((resolve) => (bothEntered = resolve));
    const reconcile = (name: string) => async () => {
      entered.add(name);
      if (entered.size === 2) bothEntered();
      await blocked;
      return {};
    };
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      leases,
      management: {
        reconcileDatabases: reconcile("database"),
        reconcileStorage: reconcile("storage"),
        stop: async () => [],
      } as unknown as ManagementService,
    });
    const post = (path: string, key: string, body: unknown = { compose: { services: {} } }) =>
      app(request(`/api/v2/workspaces/demo/${path}`, {
        method: "POST",
        headers: { "idempotency-key": key },
        body: JSON.stringify(body),
      }, "token"));

    const database = post("databases", "database-one");
    const storage = post("storage", "storage-one");
    await bothStarted;
    expect(entered).toEqual(new Set(["database", "storage"]));

    const sameScope = await post("databases", "database-two");
    expect(sameScope.status).toBe(409);
    const broad = await post("lifecycle", "lifecycle", { action: "stop" });
    expect(broad.status).toBe(409);

    release();
    expect((await database).status).toBe(200);
    expect((await storage).status).toBe(200);
  });

  test("redacts database and storage reconciliation results immediately", async () => {
    const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-uid" });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const seen: unknown[] = [];
    const databaseResult = {
      credentials: { password: "database-password" },
      env: [{ name: "DB_PASSWORD", value: "database-password" }],
    };
    const storageResult = {
      credentials: { secretKey: "storage-secret" },
      env: [{ name: "STORAGE_SECRET", value: "storage-secret" }],
    };
    const management = {
      reconcileDatabases: async (_workspace: unknown, compose: unknown) => {
        seen.push(compose);
        return databaseResult;
      },
      reconcileStorage: async (_workspace: unknown, compose: unknown) => {
        seen.push(compose);
        return storageResult;
      },
    } as unknown as ManagementService;
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore: new MemoryOperationStore(undefined, () =>
        crypto.randomUUID(),
      ),
      management,
    });
    const compose = {
      services: {},
      secret: "internal-compose-secret",
    };
    for (const [path, key, secret] of [
      ["databases", "db-once", "database-password"],
      ["storage", "storage-once", "storage-secret"],
    ] as const) {
      const result = await app(
        request(
          `/api/v2/workspaces/demo/${path}`,
          {
            method: "POST",
            headers: { "idempotency-key": key },
            body: JSON.stringify({ compose }),
          },
          "token",
        ),
      );
      expect(result.status).toBe(200);
      const body = JSON.stringify(await result.json());
      expect(body).not.toContain(secret);
      expect(body).not.toContain("internal-compose-secret");
      expect(body).toContain('"redacted":true');
    }
    expect(seen).toHaveLength(2);
    expect(JSON.stringify(seen[0])).toContain("internal-compose-secret");
    expect(databaseResult.credentials.password).toBe("database-password");
    expect(storageResult.credentials.secretKey).toBe("storage-secret");
  });

  test("redacts failed reconciliation errors immediately and when retrieved", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(undefined, () =>
      crypto.randomUUID(),
    );
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      management: {
        reconcileDatabases: async () => {
          throw new Error(
            'database provider failed DB_PASSWORD=database-password response={"data":{"token":"kube-secret"}}',
          );
        },
        reconcileStorage: async () => {
          throw new Error(
            'storage provider failed STORAGE_SECRET=storage-secret response={"data":{"password":"storage-kube-secret"}}',
          );
        },
      } as unknown as ManagementService,
    });
    const path = "/api/v2/workspaces/demo/databases";
    const init = {
      method: "POST",
      headers: { "idempotency-key": "failed-reconcile" },
      body: JSON.stringify({ compose: { services: {} } }),
    };
    const immediate = await app(request(path, init, "token"));
    const immediateBody = JSON.stringify(await immediate.json());
    expect(immediate.status).toBe(500);
    expect(immediateBody).not.toContain("database-password");
    expect(immediateBody).not.toContain("kube-secret");
    expect(immediateBody).toContain("DATABASE_RECONCILE_FAILED");
    expect(immediateBody).toContain("during operation execution: Unexpected failure; check server logs using the operation ID");

    const operationId = (await operationStore.list())[0]!.metadata.name;
    const retrieved = await app(
      request(`/api/v2/operations/${operationId}`, {}, "token"),
    );
    const retrievedBody = JSON.stringify(await retrieved.json());
    expect(retrievedBody).not.toContain("database-password");
    expect(retrievedBody).not.toContain("kube-secret");

    const storageImmediate = await app(
      request(
        "/api/v2/workspaces/demo/storage",
        {
          ...init,
          headers: { "idempotency-key": "failed-storage" },
        },
        "token",
      ),
    );
    const storageBody = JSON.stringify(await storageImmediate.json());
    expect(storageImmediate.status).toBe(500);
    expect(storageBody).not.toContain("storage-secret");
    expect(storageBody).not.toContain("storage-kube-secret");
    const storageId = (await operationStore.list())[1]!.metadata.name;
    const storageRetrieved = await app(
      request(`/api/v2/operations/${storageId}`, {}, "token"),
    );
    expect(JSON.stringify(await storageRetrieved.json())).not.toContain(
      "storage-kube-secret",
    );

    const idempotent = await app(request(path, init, "token"));
    const idempotentBody = JSON.stringify(await idempotent.json());
    expect(idempotentBody).not.toContain("database-password");
    expect(idempotentBody).not.toContain("kube-secret");
  });
});

describe("API key route expiry", () => {
  test("omitted expiry creates a non-expiring key that authenticates and survives cleanup", async () => {
    const store = new MemoryAuthStore();
    await store.putUser({
      username: "admin",
      passwordHash: "hash",
      roles: ["admin"],
    });
    await store.putUser({
      username: "ci",
      passwordHash: "hash",
      roles: ["viewer"],
    });
    await store.putSession({
      tokenHash: hashToken("admin-token"),
      username: "admin",
      authVersion: 1,
      expiresAt: "2027-01-01T00:00:00.000Z",
    });
    let time = Date.parse("2026-10-05T00:00:00.000Z");
    const app = createApp({ store, now: () => time });
    const created = await app(
      request(
        "/api/v2/users/ci/keys",
        {
          method: "POST",
          body: JSON.stringify({ capabilities: ["kubernetes:read"] }),
        },
        "admin-token",
      ),
    );
    expect(created.status).toBe(201);
    const key = (await created.json()) as {
      id: string;
      token: string;
      expiresAt?: string;
    };
    expect(key).not.toHaveProperty("expiresAt");
    expect((await store.listApiKeys("ci"))[0]).not.toHaveProperty("expiresAt");
    const listed = await app(
      request("/api/v2/users/ci/keys", {}, "admin-token"),
    );
    const list = (await listed.json()) as { items: Record<string, unknown>[] };
    expect(list.items).toEqual([expect.objectContaining({ id: key.id })]);
    expect(list.items[0]).not.toHaveProperty("expiresAt");
    time = Date.parse("2028-01-01T00:00:00.000Z");
    expect(await cleanupExpiredSessions(store, time)).toBe(1);
    expect((await app(request("/api/v2/me", {}, key.token))).status).toBe(200);
  });

  test("explicit finite expiry is enforced and malformed expiry is rejected", async () => {
    const store = new MemoryAuthStore();
    await store.putUser({
      username: "admin",
      passwordHash: "hash",
      roles: ["admin"],
    });
    await store.putUser({
      username: "ci",
      passwordHash: "hash",
      roles: ["viewer"],
    });
    await store.putSession({
      tokenHash: hashToken("admin-token"),
      username: "admin",
      authVersion: 1,
      expiresAt: "2027-01-01T00:00:00.000Z",
    });
    let time = Date.parse("2026-10-05T00:00:00.000Z");
    const app = createApp({ store, now: () => time });
    const create = (expiresAt: unknown) =>
      app(
        request(
          "/api/v2/users/ci/keys",
          {
            method: "POST",
            body: JSON.stringify({
              capabilities: ["kubernetes:read"],
              expiresAt,
            }),
          },
          "admin-token",
        ),
      );
    const expiry = "2026-10-06T00:00:00.000Z";
    const created = await create(expiry);
    expect(created.status).toBe(201);
    const key = (await created.json()) as { token: string; expiresAt: string };
    expect(key.expiresAt).toBe(expiry);
    expect((await app(request("/api/v2/me", {}, key.token))).status).toBe(200);
    for (const invalid of [
      null,
      0,
      "",
      "2026-10-05T00:00:00.000Z",
      "2027-10-06T00:00:00.000Z",
    ])
      expect((await create(invalid)).status).toBe(400);
    time = Date.parse(expiry);
    expect((await app(request("/api/v2/me", {}, key.token))).status).toBe(401);
  });

  test("a finite parent API key cannot delegate a non-expiring child", async () => {
    const store = new MemoryAuthStore();
    await store.putUser({
      username: "ci",
      passwordHash: "hash",
      roles: ["viewer"],
    });
    await store.createApiKey({
      id: "finite-parent-key-1",
      tokenHash: hashToken("parent-token"),
      username: "ci",
      capabilities: ["users:write", "kubernetes:read"],
      expiresAt: "2027-01-01T00:00:00.000Z",
    });
    const app = createApp({
      store,
      now: () => Date.parse("2026-10-05T00:00:00.000Z"),
    });
    const created = await app(
      request(
        "/api/v2/users/ci/keys",
        {
          method: "POST",
          body: JSON.stringify({ capabilities: ["kubernetes:read"] }),
        },
        "parent-token",
      ),
    );
    expect(created.status).toBe(403);
    expect((await created.json()) as { code: string }).toMatchObject({
      code: "API_KEY_DELEGATION_FORBIDDEN",
    });
  });
});

async function authenticatedStore(role: "viewer" | "operator" | "admin") {
  const store = new MemoryAuthStore();
  await store.putUser({ username: role, passwordHash: "hash", roles: [role] });
  await store.putSession({
    tokenHash: hashToken("token"),
    username: role,
    authVersion: 1,
    expiresAt: "2030-01-01T00:00:00.000Z",
  });
  return store;
}

describe("kuber v2 HTTP routes", () => {
  test("streams build status and logs, supports resume cursors, and keeps version headers", async () => {
    const controller = {
      getBuildProject: async () => "demo",
      getBuildStatus: async () => ({ state: "running", phase: "building" }),
      getBuildEvents: async (_id: string, after = 0) => [
        { type: "log", sequence: 4, message: "build output" },
      ].filter((event) => event.sequence > after),
      reconcileBuild: async () => ({ state: "running", phase: "building" }),
    } as unknown as BuildController;
    const app = createApp({ store: await authenticatedStore("operator"), builds: controller });
    const abort = new AbortController();
    const response = await app(request(
      "/api/v2/builds/build-1/events?after=2",
      { headers: { accept: "text/event-stream" }, signal: abort.signal },
      "token",
    ));
    expect(response.status).toBe(200);
    expect(response.headers.get("content-type")).toContain("text/event-stream");
    expect(response.headers.get("x-kuber-version")).toBeTruthy();
    const reader = response.body!.getReader();
    let body = "";
    while (!body.includes("event: status")) {
      const { done, value } = await reader.read();
      if (done) break;
      body += new TextDecoder().decode(value);
    }
    expect(body).toContain("id: 4\nevent: log");
    expect(body).toContain("build output");
    expect(body).toContain("event: status");
    abort.abort();
    await reader.cancel();

    const resumedAbort = new AbortController();
    const resumed = await app(request(
      "/api/v2/builds/build-1/events",
      { headers: { accept: "text/event-stream", "last-event-id": "4" }, signal: resumedAbort.signal },
      "token",
    ));
    const resumedReader = resumed.body!.getReader();
    const resumedChunk = await resumedReader.read();
    expect(new TextDecoder().decode(resumedChunk.value)).not.toContain("id: 4");
    resumedAbort.abort();
    await resumedReader.cancel();
  });

  test("checks authentication and workspace scope before opening a build stream", async () => {
    const controller = {
      getBuildProject: async () => "other",
      getBuildStatus: async () => ({ state: "running", phase: "building" }),
      getBuildEvents: async () => [],
      reconcileBuild: async () => ({ state: "running", phase: "building" }),
    } as unknown as BuildController;
    const app = createApp({ store: await authenticatedStore("operator"), builds: controller });
    const unauthenticated = await app(request(
      "/api/v2/builds/build-1/events",
      { headers: { accept: "text/event-stream" } },
    ));
    expect(unauthenticated.status).toBe(401);

    const store = new MemoryAuthStore();
    await store.putUser({ username: "scoped", passwordHash: "hash", roles: ["operator"] });
    await store.createApiKey({
      id: "scoped-key-12345678", username: "scoped", tokenHash: hashToken("scoped-token"),
      capabilities: ["kubernetes:write"], workspace: "demo",
    });
    const scopedApp = createApp({ store, builds: controller });
    const forbidden = await scopedApp(request(
      "/api/v2/builds/build-1/events",
      { headers: { accept: "text/event-stream" } },
      "scoped-token",
    ));
    expect(forbidden.status).toBe(403);
    expect(forbidden.headers.get("content-type")).not.toContain("text/event-stream");
  });

  test("rejects malformed SSE cursors before opening the stream", async () => {
    const controller = {
      getBuildProject: async () => "demo",
      getBuildStatus: async () => ({ state: "running", phase: "building" }),
      getBuildEvents: async () => [],
      reconcileBuild: async () => ({ state: "running", phase: "building" }),
    } as unknown as BuildController;
    const app = createApp({ store: await authenticatedStore("operator"), builds: controller });
    const response = await app(request(
      "/api/v2/builds/build-1/events?after=1.5",
      { headers: { accept: "text/event-stream" } },
      "token",
    ));
    expect(response.status).toBe(400);
    expect(response.headers.get("content-type")).toContain("application/problem+json");
  });

  test("grants, lists, revokes, and enforces namespace trust for applies", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const trustStore = new MemoryTrustStore();
    const fingerprint = "a".repeat(64);
    const headers = {
      "x-kuber-trust-project": "demo",
      "x-kuber-trust-fingerprint": fingerprint,
    };
    const apply = (app: ReturnType<typeof createApp>) =>
      app(
        request(
          "/api/v2/workspaces/demo/resources/apply",
          {
            method: "POST",
            headers,
            body: JSON.stringify({ resources: [] }),
          },
          "token",
        ),
      );
    const unavailable = await createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore: new MemoryOperationStore(),
      management: {
        applyResources: async () => [],
      } as unknown as ManagementService,
    });
    const unavailableApply = await apply(unavailable);
    expect(unavailableApply.status).toBe(503);
    expect(await unavailableApply.json()).toMatchObject({
      code: "TRUST_STORE_UNAVAILABLE",
    });

    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore: new MemoryOperationStore(),
      trustStore,
      management: {
        applyResources: async () => [],
      } as unknown as ManagementService,
    });
    const untrustedApply = (headers?: RequestInit["headers"]) =>
      app(
        request(
          "/api/v2/workspaces/demo/resources/apply",
          {
            method: "POST",
            headers,
            body: JSON.stringify({ resources: [] }),
          },
          "token",
        ),
      );

    expect((await untrustedApply()).status).toBe(428);
    expect((await apply(app)).status).toBe(403);
    expect(
      (
        await app(
          request(
            "/api/v2/workspaces/demo/trust",
            { method: "POST", body: JSON.stringify({ fingerprint }) },
            "token",
          ),
        )
      ).status,
    ).toBe(204);
    const status = await app(
      request("/api/v2/workspaces/demo/trust", {}, "token"),
    );
    expect(status.status).toBe(200);
    expect(await status.json()).toEqual({ fingerprints: [fingerprint] });
    expect((await apply(app)).status).toBe(200);
    expect(
      (
        await app(
          request(
            `/api/v2/workspaces/demo/trust?fingerprint=${fingerprint}`,
            { method: "DELETE" },
            "token",
          ),
        )
      ).status,
    ).toBe(204);
    expect((await apply(app)).status).toBe(403);
  });

  test("returns OPERATION_CONFLICT when an idempotency key is reused for another body", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const trustStore = new MemoryTrustStore();
    const fingerprint = "c".repeat(64);
    await trustStore.grant("demo", fingerprint);
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      trustStore,
      operationStore: new MemoryOperationStore(),
      management: {
        applyResources: async () => [],
      } as unknown as ManagementService,
    });
    const apply = (resources: unknown[]) =>
      app(
        request(
          "/api/v2/workspaces/demo/resources/apply",
          {
            method: "POST",
            headers: {
              "idempotency-key": "same-apply",
              "x-kuber-trust-project": "demo",
              "x-kuber-trust-fingerprint": fingerprint,
            },
            body: JSON.stringify({ resources }),
          },
          "token",
        ),
      );

    expect((await apply([])).status).toBe(200);
    const conflict = await apply([
      { apiVersion: "v1", kind: "Service", metadata: { name: "web" } },
    ]);
    expect(conflict.status).toBe(409);
    expect(await conflict.json()).toMatchObject({ code: "OPERATION_CONFLICT" });
  });

  test("starts preferred resource operations before returning so progress can be polled", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const trustStore = new MemoryTrustStore();
    const fingerprint = "b".repeat(64);
    let start!: () => void;
    let finish!: () => void;
    let complete!: () => void;
    const started = new Promise<void>((resolve) => (start = resolve));
    const unblock = new Promise<void>((resolve) => (finish = resolve));
    const completed = new Promise<void>((resolve) => (complete = resolve));
    const management = {
      applyResources: async (
        _workspace: unknown,
        _resources: unknown,
        execution: {
          emit?: (event: {
            resource: { apiVersion: string; kind: string; name: string };
            phase: "apply";
            state: "started" | "succeeded";
          }) => Promise<void>;
        },
      ) => {
        await execution.emit?.({
          resource: { apiVersion: "v1", kind: "Service", name: "web" },
          phase: "apply",
          state: "started",
        });
        start();
        await unblock;
        await execution.emit?.({
          resource: { apiVersion: "v1", kind: "Service", name: "web" },
          phase: "apply",
          state: "succeeded",
        });
        complete();
        return [];
      },
    } as unknown as ManagementService;
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore: new MemoryOperationStore(),
      trustStore,
      management,
    });
    await app(
      request(
        "/api/v2/workspaces/demo/trust",
        { method: "POST", body: JSON.stringify({ fingerprint }) },
        "token",
      ),
    );

    const submitted = await app(
      request(
        "/api/v2/workspaces/demo/resources/apply",
        {
          method: "POST",
          headers: {
            "idempotency-key": "progress-once",
            prefer: "respond-async",
            "x-kuber-trust-project": "demo",
            "x-kuber-trust-fingerprint": fingerprint,
          },
          body: JSON.stringify({ resources: [] }),
        },
        "token",
      ),
    );
    expect(submitted.status).toBe(202);
    const { operationId } = (await submitted.json()) as { operationId: string };
    await started;
    const events = await app(
      request(`/api/v2/operations/${operationId}/events?after=0`, {}, "token"),
    );
    expect(await events.json()).toMatchObject({
      items: [{ sequence: 1, data: { state: "started" } }],
    });
    finish();
    await completed;
  });

  test("uses exact origins, request IDs, and problem+json errors", async () => {
    const app = createApp({
      store: new MemoryAuthStore(),
      allowedOrigins: ["https://console.example"],
      requestId: () => "generated-id",
    });
    const denied = await app(
      request("/api/v2/health", {
        headers: { origin: "https://console.example.evil" },
      }),
    );
    expect(denied.status).toBe(403);
    expect(denied.headers.get("content-type")).toContain(
      "application/problem+json",
    );
    expect(denied.headers.get("x-request-id")).toBe("generated-id");
    expect(await denied.json()).toMatchObject({
      code: "ORIGIN_NOT_ALLOWED",
      requestId: "generated-id",
    });

    const allowed = await app(
      request("/api/v2/health", {
        headers: {
          origin: "https://console.example",
          "x-request-id": "caller-id",
        },
      }),
    );
    expect(allowed.headers.get("access-control-allow-origin")).toBe(
      "https://console.example",
    );
    expect(allowed.headers.get("x-request-id")).toBe("caller-id");
  });

  test("enforces capabilities and supports user CRUD with revocation", async () => {
    const viewerStore = await authenticatedStore("viewer");
    const viewerApp = createApp({ store: viewerStore });
    expect(
      (await viewerApp(request("/api/v2/users", {}, "token"))).status,
    ).toBe(403);

    const store = await authenticatedStore("admin");
    const auditStore = new MemoryAuditStore();
    const app = createApp({
      store,
      auditStore,
      hashPassword: async (password) => `hashed:${password}`,
    });
    const created = await app(
      request(
        "/api/v2/users",
        {
          method: "POST",
          body: JSON.stringify({
            username: "alice",
            password: "secret",
            roles: ["operator"],
          }),
        },
        "token",
      ),
    );
    expect(created.status).toBe(201);
    expect(await created.json()).toEqual({
      username: "alice",
      roles: ["operator"],
      disabled: false,
    });
    expect((await store.getUser("alice"))?.passwordHash).toBe("hashed:secret");

    const revoke = await app(
      request(
        "/api/v2/users/alice/sessions/revoke",
        { method: "POST" },
        "token",
      ),
    );
    expect(revoke.status).toBe(200);
    expect(await revoke.json()).toEqual({ username: "alice", revoked: 0 });
    expect((await auditStore.list()).map((event) => event.spec.action)).toEqual(
      ["user.create", "sessions.revoke"],
    );
  });

  test("provides workspace ETags and idempotent synchronous operations", async () => {
    const store = await authenticatedStore("operator");
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
      now: () => new Date("2026-09-02T00:00:00.000Z"),
    });
    const operationStore = new MemoryOperationStore(
      () => new Date("2026-09-02T00:00:00.000Z"),
      () => "operation-uid",
    );
    let stops = 0;
    const management = {
      stop: async () => {
        stops += 1;
        return ["api"];
      },
    } as unknown as ManagementService;
    const app = createApp({
      store,
      workspaceStore,
      operationStore,
      management,
    });

    const created = await app(
      request(
        "/api/v2/workspaces",
        {
          method: "POST",
          body: JSON.stringify({
            id: "demo",
            source: { uri: "oci://example/demo", digest: "sha256:abc" },
          }),
        },
        "token",
      ),
    );
    expect(created.status).toBe(201);
    expect(created.headers.get("etag")).toBe('"1"');

    const missingPrecondition = await app(
      request(
        "/api/v2/workspaces/demo",
        {
          method: "PUT",
          body: JSON.stringify({
            source: { uri: "oci://example/demo", digest: "sha256:def" },
          }),
        },
        "token",
      ),
    );
    expect(missingPrecondition.status).toBe(428);

    const stop = () =>
      app(
        request(
          "/api/v2/workspaces/demo/lifecycle",
          {
            method: "POST",
            headers: { "idempotency-key": "stop-once" },
            body: JSON.stringify({ action: "stop", services: ["api"] }),
          },
          "token",
        ),
      );
    expect((await stop()).status).toBe(200);
    expect((await stop()).status).toBe(200);
    expect(stops).toBe(1);
    expect(await operationStore.list("demo")).toHaveLength(1);
  });

  test("logs raw diagnostics for correlated generic failures", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const logs: unknown[] = [];
    const providerError = Object.assign(
      new Error(
        'provider failed password=top-secret config={"compose":"private"}',
      ),
      {
        name: "KubernetesError",
        stack:
          "KubernetesError: provider failed\n    at provider (test.ts:1:1)",
        body: {
          status: "Failure",
          reason: "InternalError",
          code: 500,
          message: "contains top-secret",
        },
      },
    );
    spyOn(workspaceStore, "update").mockRejectedValue(providerError);
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      requestId: () => "request-123",
      logger: { error: (entry) => logs.push(entry) },
    });

    const result = await app(
      request(
        "/api/v2/workspaces/demo",
        {
          method: "PUT",
          headers: { "if-match": '"1"' },
          body: JSON.stringify({
            source: { uri: "cas://secret-source", digest: "secret-digest" },
            config: { compose: "private-config", password: "top-secret" },
          }),
        },
        "token",
      ),
    );

    expect(result.status).toBe(500);
    expect(await result.json()).toMatchObject({
      code: "INTERNAL_ERROR",
      requestId: "request-123",
      detail: "The request could not be completed",
    });
    expect(logs).toEqual([
      {
        event: "request.failed",
        requestId: "request-123",
        method: "PUT",
        pathname: "/api/v2/workspaces/demo",
        workspaceId: "demo",
        status: 500,
        code: "INTERNAL_ERROR",
        errorName: "KubernetesError",
        message:
          'provider failed password=top-secret config={"compose":"private"}',
        stack:
          "KubernetesError: provider failed\n    at provider (test.ts:1:1)",
        kubernetesStatus: {
          status: "Failure",
          reason: "InternalError",
          code: 500,
        },
      },
    ]);
  });

  test("lists persisted operation events with capability and workspace scope checks", async () => {
    const store = await authenticatedStore("operator");
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(
      undefined,
      () => "event-id",
    );
    const operation = await operationStore.create({
      workspaceId: "demo",
      action: "resources.apply",
      idempotencyKey: "event-key",
    });
    await operationStore.emit(operation.metadata.name, {
      resource: { apiVersion: "v1", kind: "Service", name: "web" },
      phase: "apply",
      state: "succeeded",
    });
    await store.createApiKey({
      id: "demo-reader-key-01",
      tokenHash: hashToken("demo-reader-token"),
      username: "operator",
      capabilities: ["kubernetes:read"],
      workspace: "demo",
      expiresAt: "2030-01-01T00:00:00.000Z",
    });
    await store.createApiKey({
      id: "other-reader-key-1",
      tokenHash: hashToken("other-reader-token"),
      username: "operator",
      capabilities: ["kubernetes:read"],
      workspace: "other",
      expiresAt: "2030-01-01T00:00:00.000Z",
    });
    const app = createApp({ store, workspaceStore, operationStore });
    const path = `/api/v2/operations/${operation.metadata.name}/events?after=0`;
    const allowed = await app(request(path, {}, "demo-reader-token"));
    expect(allowed.status).toBe(200);
    expect(await allowed.json()).toMatchObject({
      items: [
        {
          sequence: 1,
          data: { resource: { kind: "Service", name: "web" }, phase: "apply" },
        },
      ],
      nextCursor: 1,
      retainedFirstSequence: 1,
      cursorGap: false,
    });
    expect((await app(request(path, {}, "other-reader-token"))).status).toBe(
      403,
    );
    expect(
      (await app(request(`${path}x`, {}, "demo-reader-token"))).status,
    ).toBe(400);
  });

  test("reports operation event cursor gaps after retained history is truncated", async () => {
    const operationStore = new MemoryOperationStore(
      undefined,
      () => "retained",
    );
    const operation = await operationStore.create({
      workspaceId: "demo",
      action: "resources.apply",
      idempotencyKey: "retained-events",
    });
    for (let sequence = 0; sequence < 257; sequence += 1) {
      await operationStore.emit(operation.metadata.name, {
        resource: { apiVersion: "v1", kind: "ConfigMap", name: "config" },
        phase: "apply",
        state: "succeeded",
      });
    }
    const app = createApp({
      store: await authenticatedStore("operator"),
      operationStore,
    });

    const response = await app(
      request(
        `/api/v2/operations/${operation.metadata.name}/events?after=0`,
        {},
        "token",
      ),
    );
    expect(response.status).toBe(200);
    const events = (await response.json()) as {
      retainedFirstSequence: number;
      cursorGap: boolean;
      items: { sequence: number }[];
    };
    expect(events.retainedFirstSequence).toBe(2);
    expect(events.cursorGap).toBe(true);
    expect(events.items[0]?.sequence).toBe(2);
  });

  test("rejects JSON bodies over the configured limit", async () => {
    const app = createApp({
      store: await authenticatedStore("admin"),
      workspaceStore: new MemoryWorkspaceStore(),
      jsonBodyLimit: 32,
    });
    const result = await app(
      request(
        "/api/v2/workspaces",
        { method: "POST", body: JSON.stringify({ value: "x".repeat(64) }) },
        "token",
      ),
    );
    expect(result.status).toBe(413);
    expect(await result.json()).toMatchObject({ code: "BODY_TOO_LARGE" });
  });

  test("does not expose request internals or corrupt success when auditing fails", async () => {
    const logs: unknown[] = [];
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(
      undefined,
      () => "operation-uid",
    );
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      management: { stop: async () => ["api"] } as unknown as ManagementService,
      auditStore: {
        append: async () => {
          throw new Error("audit unavailable");
        },
        list: async () => [],
      },
      requestId: () => "audit-request-123",
      logger: { error: (entry) => logs.push(entry) },
    });
    const result = await app(
      request(
        "/api/v2/workspaces/demo/lifecycle",
        {
          method: "POST",
          headers: { "idempotency-key": "private-request" },
          body: JSON.stringify({ action: "stop", password: "do-not-store" }),
        },
        "token",
      ),
    );
    expect(result.status).toBe(200);
    const body = (await result.json()) as Record<string, any>;
    expect(body.operation.spec).toEqual({
      workspaceId: "demo",
      action: "workspace.stop",
    });
    expect(body.operation.status.state).toBe("succeeded");
    expect(logs).toEqual([
      {
        event: "operation.audit.failed",
        requestId: "audit-request-123",
        method: "POST",
        pathname: "/api/v2/workspaces/demo/lifecycle",
        workspaceId: "demo",
        operationId: "operation-operation-uid",
        status: 500,
        code: "AUDIT_APPEND_FAILED",
        errorName: "Error",
        message: "Successful operation audit event could not be appended",
      },
    ]);
    expect(JSON.stringify(logs)).not.toContain("audit unavailable");
    expect(JSON.stringify(logs)).not.toContain("private-request");
    expect(JSON.stringify(logs)).not.toContain("do-not-store");
    expect(
      await operationStore.get("operation-operation-uid"),
    ).not.toHaveProperty("spec.request");
  });

  test("fails and identifies operations that cannot acquire the workspace lease", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(undefined, () => "blocked");
    const leases = new MemoryWorkspaceLeaseProvider();
    await leases.acquire("demo", "other");
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      leases,
      management: { stop: async () => [] } as unknown as ManagementService,
    });
    const result = await app(
      request(
        "/api/v2/workspaces/demo/lifecycle",
        { method: "POST", body: JSON.stringify({ action: "stop" }) },
        "token",
      ),
    );
    expect(result.status).toBe(409);
    expect(await result.json()).toMatchObject({
      code: "WORKSPACE_BUSY",
      operationId: "operation-blocked",
    });
    expect((await operationStore.get("operation-blocked"))?.status.state).toBe(
      "pending",
    );
  });

  test("a stalled observation-only wait does not block mutation and remains idempotent", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore();
    const leases = new MemoryWorkspaceLeaseProvider();
    let waitStarted!: () => void;
    let finishWait!: () => void;
    const started = new Promise<void>((resolve) => (waitStarted = resolve));
    const blocked = new Promise<void>((resolve) => (finishWait = resolve));
    let waits = 0;
    let stops = 0;
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      leases,
      management: {
        waitForResources: async () => {
          waits += 1;
          waitStarted();
          await blocked;
        },
        stop: async () => {
          stops += 1;
          return ["api"];
        },
      } as unknown as ManagementService,
    });
    const waitRequest = () =>
      app(
        request(
          "/api/v2/workspaces/demo/resources/wait",
          {
            method: "POST",
            headers: { "idempotency-key": "wait-once", prefer: "respond-async" },
            body: JSON.stringify({ deployments: ["api"] }),
          },
          "token",
        ),
      );

    const submitted = await waitRequest();
    expect(submitted.status).toBe(202);
    await started;
    const duplicate = await waitRequest();
    expect(duplicate.status).toBe(202);
    expect(waits).toBe(1);

    const stop = await app(
      request(
        "/api/v2/workspaces/demo/lifecycle",
        {
          method: "POST",
          headers: { "idempotency-key": "stop-during-wait" },
          body: JSON.stringify({ action: "stop" }),
        },
        "token",
      ),
    );
    expect(stop.status).toBe(200);
    expect(stops).toBe(1);
    finishWait();
  });

  test("keeps a concurrent idempotent operation pending when its lease acquisition is denied", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(
      undefined,
      () => "same-key",
    );
    let acquireCount = 0;
    let releaseCount = 0;
    let allowFirstAcquire!: () => void;
    let signalFirstAcquire!: () => void;
    const firstAcquireStarted = new Promise<void>((resolve) => {
      signalFirstAcquire = resolve;
    });
    const leases = {
      acquire: async () => {
        acquireCount += 1;
        if (acquireCount === 2) return undefined;
        if (acquireCount === 1) {
          signalFirstAcquire();
          await new Promise<void>((resolve) => {
            allowFirstAcquire = resolve;
          });
        }
        return {
          workspaceId: "demo",
          holder: "operation-same-key",
          expiresAt: new Date().toISOString(),
          renew: async () => true,
          release: async () => {
            releaseCount += 1;
          },
        };
      },
    };
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      leases,
      management: { stop: async () => ["api"] } as unknown as ManagementService,
    });
    const stop = () =>
      app(
        request(
          "/api/v2/workspaces/demo/lifecycle",
          {
            method: "POST",
            headers: { "idempotency-key": "same-key" },
            body: JSON.stringify({ action: "stop" }),
          },
          "token",
        ),
      );

    const owner = stop();
    await firstAcquireStarted;
    const duplicate = await stop();
    expect(duplicate.status).toBe(409);
    expect((await operationStore.get("operation-same-key"))?.status.state).toBe(
      "pending",
    );

    allowFirstAcquire();
    expect((await owner).status).toBe(200);
    expect((await operationStore.get("operation-same-key"))?.status.state).toBe(
      "succeeded",
    );
    expect(acquireCount).toBe(4);
    expect(releaseCount).toBe(3);
  });

  test("lets the lease winner claim execution even when a duplicate created the operation", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(
      undefined,
      () => "same-key",
    );
    let acquireCount = 0;
    let releaseFirstAcquire!: () => void;
    let firstAcquireStarted!: () => void;
    const firstAcquire = new Promise<void>((resolve) => {
      firstAcquireStarted = resolve;
    });
    const leases = {
      acquire: async () => {
        acquireCount += 1;
        if (acquireCount === 1) {
          firstAcquireStarted();
          await new Promise<void>((resolve) => {
            releaseFirstAcquire = resolve;
          });
          return undefined;
        }
        return {
          workspaceId: "demo",
          holder: "operation-same-key",
          expiresAt: new Date().toISOString(),
          renew: async () => true,
          release: async () => {},
        };
      },
    };
    let executions = 0;
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      leases,
      management: {
        stop: async () => {
          executions += 1;
          return ["api"];
        },
      } as unknown as ManagementService,
    });
    const stop = () =>
      app(
        request(
          "/api/v2/workspaces/demo/lifecycle",
          {
            method: "POST",
            headers: { "idempotency-key": "same-key" },
            body: JSON.stringify({ action: "stop" }),
          },
          "token",
        ),
      );

    const creator = stop();
    await firstAcquire;
    expect((await stop()).status).toBe(200);
    releaseFirstAcquire();
    expect((await creator).status).toBe(409);
    expect(executions).toBe(1);
    expect((await operationStore.get("operation-same-key"))?.status.state).toBe(
      "succeeded",
    );
  });

  test("releases a lease when the initial execution claim fails", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(
      undefined,
      () => "claim-fails",
    );
    spyOn(operationStore, "claimExecution").mockRejectedValue(
      new Error("claim unavailable"),
    );
    let releases = 0;
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      leases: {
        acquire: async () => ({
          workspaceId: "demo",
          holder: "operation-claim-fails",
          expiresAt: new Date().toISOString(),
          renew: async () => true,
          release: async () => {
            releases += 1;
          },
        }),
      },
      management: { stop: async () => [] } as unknown as ManagementService,
    });

    expect(
      (
        await app(
          request(
            "/api/v2/workspaces/demo/lifecycle",
            { method: "POST", body: JSON.stringify({ action: "stop" }) },
            "token",
          ),
        )
      ).status,
    ).toBe(500);
    expect(releases).toBe(3);
  });

  test("fails a claimed operation before execution when lease ownership is lost", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(
      undefined,
      () => "lost-lease",
    );
    let executions = 0;
    let releases = 0;
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      leases: {
        acquire: async () => ({
          workspaceId: "demo",
          holder: "operation-lost-lease",
          expiresAt: new Date().toISOString(),
          renew: async () => false,
          release: async () => {
            releases += 1;
          },
        }),
      },
      management: {
        stop: async () => {
          executions += 1;
          return [];
        },
      } as unknown as ManagementService,
    });

    const result = await app(
      request(
        "/api/v2/workspaces/demo/lifecycle",
        { method: "POST", body: JSON.stringify({ action: "stop" }) },
        "token",
      ),
    );
    expect(result.status).toBe(409);
    expect(executions).toBe(0);
    expect(releases).toBe(3);
    expect(await result.json()).toMatchObject({
      code: "WORKSPACE_LEASE_LOST",
    });
    expect(
      (await operationStore.get("operation-lost-lease"))?.status,
    ).toMatchObject({
      state: "failed",
      error: { code: "WORKSPACE_LEASE_LOST" },
    });
  });

  test("aborts blocked execution and fails the operation when renewal loses the lease", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(
      undefined,
      () => "blocked-loss",
    );
    const originalSetTimeout = globalThis.setTimeout;
    let scheduledRenewal!: () => void;
    globalThis.setTimeout = ((callback: Parameters<typeof setTimeout>[0]) => {
      scheduledRenewal = callback as () => void;
      return 0 as unknown as ReturnType<typeof setTimeout>;
    }) as typeof setTimeout;
    try {
      let executionStarted!: () => void;
      const started = new Promise<void>((resolve) => {
        executionStarted = resolve;
      });
      let observedAbort = false;
      let renewals = 0;
      const app = createApp({
        store: await authenticatedStore("operator"),
        workspaceStore,
        operationStore,
        leases: {
          acquire: async () => ({
            workspaceId: "demo",
            holder: "operation-blocked-loss",
            expiresAt: new Date().toISOString(),
            renew: async () => ++renewals <= 3,
            release: async () => {},
          }),
        },
        management: {
          stop: async (
            _workspace: { project: string; uid: string },
            _names?: string[],
            execution?: { signal?: AbortSignal },
          ) => {
            executionStarted();
            await new Promise<void>((resolve) => {
              execution?.signal?.addEventListener(
                "abort",
                () => {
                  observedAbort = true;
                  resolve();
                },
                { once: true },
              );
            });
            return [];
          },
        } as unknown as ManagementService,
      });
      const pending = app(
        request(
          "/api/v2/workspaces/demo/lifecycle",
          { method: "POST", body: JSON.stringify({ action: "stop" }) },
          "token",
        ),
      );
      await started;
      scheduledRenewal();
      const result = await pending;

      expect(observedAbort).toBe(true);
      expect(result.status).toBe(409);
      expect(await result.json()).toMatchObject({
        code: "WORKSPACE_LEASE_LOST",
      });
      expect(
        (await operationStore.get("operation-blocked-loss"))?.status,
      ).toMatchObject({
        state: "failed",
        error: { code: "WORKSPACE_LEASE_LOST" },
      });
    } finally {
      globalThis.setTimeout = originalSetTimeout;
    }
  });

  test("renews the workspace lease while an operation remains in flight", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    let scheduledRenewal: (() => void) | undefined;
    const originalSetTimeout = globalThis.setTimeout;
    const originalClearTimeout = globalThis.clearTimeout;
    const clearedTimers: unknown[] = [];
    globalThis.setTimeout = ((callback: Parameters<typeof setTimeout>[0]) => {
      scheduledRenewal = callback as () => void;
      return 0 as unknown as ReturnType<typeof setTimeout>;
    }) as typeof setTimeout;
    globalThis.clearTimeout = ((timer: ReturnType<typeof setTimeout>) => {
      clearedTimers.push(timer);
    }) as typeof clearTimeout;
    try {
      let releaseOperation!: () => void;
      let operationStarted!: () => void;
      const started = new Promise<void>((resolve) => {
        operationStarted = resolve;
      });
      const completed = new Promise<string[]>((resolve) => {
        releaseOperation = () => resolve(["api"]);
      });
      let renewals = 0;
      const app = createApp({
        store: await authenticatedStore("operator"),
        workspaceStore,
        operationStore: new MemoryOperationStore(undefined, () => "renewing"),
        leases: {
          acquire: async () => ({
            workspaceId: "demo",
            holder: "operation-renewing",
            expiresAt: new Date().toISOString(),
            renew: async () => {
              renewals += 1;
              return true;
            },
            release: async () => {},
          }),
        },
        management: {
          stop: async () => {
            operationStarted();
            return completed;
          },
        } as unknown as ManagementService,
      });
      const response = app(
        request(
          "/api/v2/workspaces/demo/lifecycle",
          { method: "POST", body: JSON.stringify({ action: "stop" }) },
          "token",
        ),
      );
      await started;
      expect(renewals).toBe(3);
      scheduledRenewal?.();
      await Promise.resolve();
      await Promise.resolve();
      expect(renewals).toBe(6);
      releaseOperation();
      expect((await response).status).toBe(200);
      expect(renewals).toBe(9);
      expect(clearedTimers).toEqual([0]);
    } finally {
      globalThis.setTimeout = originalSetTimeout;
      globalThis.clearTimeout = originalClearTimeout;
    }
  });

  test("reuses a failed reconciliation without attempting a second failure transition", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(
      undefined,
      () => "reconcile",
    );
    let reconciliations = 0;
    const management = {
      reconcileDatabases: async () => {
        reconciliations += 1;
        const [operation] = await operationStore.list("demo");
        await operationStore.transition(operation!.metadata.name, "failed", {
          error: {
            code: "RECONCILE_FAILED",
            message: "Database reconciliation failed",
          },
        });
        return {};
      },
    } as unknown as ManagementService;
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      management,
    });
    const reconcile = () =>
      app(
        request(
          "/api/v2/workspaces/demo/databases",
          {
            method: "POST",
            headers: { "idempotency-key": "reconcile-once" },
            body: JSON.stringify({ compose: {} }),
          },
          "token",
        ),
      );

    for (const result of [await reconcile(), await reconcile()]) {
      expect(result.status).toBe(500);
      expect(await result.json()).toMatchObject({
        code: "RECONCILE_FAILED",
        detail: "Database reconciliation failed",
      });
    }
    expect(reconciliations).toBe(1);
    expect(
      (await operationStore.get("operation-reconcile"))?.status.state,
    ).toBe("failed");
  });

  test("returns safe database phase and claim details on first and repeated failures", async () => {
    const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-uid" });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(undefined, () => "db-failure");
    const { DatabaseReconciliationError } = await import("../../lib/database");
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      management: {
        reconcileDatabases: async () => {
          throw new DatabaseReconciliationError(
            "database apply",
            new Error("Forbidden: DB_PASSWORD=private-value"),
            { service: "web", username: "web_role", database: "web_db", secretName: "web" },
          );
        },
      } as unknown as ManagementService,
    });
    const reconcile = () => app(request(
      "/api/v2/workspaces/demo/databases",
      { method: "POST", headers: { "idempotency-key": "db-failure" }, body: JSON.stringify({ compose: {} }) },
      "token",
    ));
    for (const result of [await reconcile(), await reconcile()]) {
      expect(result.status).toBe(500);
      const problem = await result.json() as { code: string; detail: string };
      expect(problem.code).toBe("DATABASE_RECONCILE_FAILED");
      expect(problem.detail).toContain("database apply for requested database claim: Forbidden");
      expect(JSON.stringify(problem)).not.toContain("private-value");
    }
    expect((await operationStore.get("operation-db-failure"))?.status.error?.message)
      .toContain("database apply for requested database claim");
  });

  test("never exposes syntactically valid secret-like database claim identifiers", async () => {
    const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-uid" });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(undefined, () => "db-secret-identifier");
    const { DatabaseReconciliationError } = await import("../../lib/database");
    const secretLike = ["DB_PASSWORD_private123", "svc_api_token_abc", "role_secret_key_123"];
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      management: {
        reconcileDatabases: async () => {
          throw new DatabaseReconciliationError(
            "database apply",
            Object.assign(new Error("Forbidden"), { statusCode: 403 }),
            { service: secretLike[1]!, username: secretLike[2]!, database: secretLike[0]!, secretName: "safe-secret-name" },
          );
        },
      } as unknown as ManagementService,
    });
    const response = await app(request(
      "/api/v2/workspaces/demo/databases",
      { method: "POST", headers: { "idempotency-key": "db-secret-identifier" }, body: JSON.stringify({ compose: {} }) },
      "token",
    ));
    const responseBody = JSON.stringify(await response.json());
    const operation = await operationStore.get("operation-db-secret-identifier");

    expect(response.status).toBe(500);
    expect(responseBody).toContain("database apply for requested database claim: Forbidden (HTTP 403)");
    expect(operation?.status.error?.message).toContain("database apply for requested database claim");
    for (const identifier of secretLike) {
      expect(responseBody).not.toContain(identifier);
      expect(JSON.stringify(operation)).not.toContain(identifier);
    }
  });

  test("logs allowlisted database failure metadata with the operation ID, never provider data", async () => {
    const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-uid" });
    await workspaceStore.create({ id: "demo", source: { uri: "oci://example/demo", digest: "sha256:abc" } });
    const operationStore = new MemoryOperationStore(undefined, () => "diagnostic");
    const logs: Record<string, unknown>[] = [];
    const { DatabaseReconciliationError } = await import("../../lib/database");
    const secretLike = "DB_PASSWORD_private123";
    const cause = Object.assign(new Error(`token=${secretLike}`), {
      name: secretLike,
      code: "ECONNREFUSED",
      statusCode: 503,
      body: { message: secretLike, headers: { authorization: secretLike } },
      stack: `Error: ${secretLike}\n    at ${secretLike} (/tmp/${secretLike}/lib/database.ts:22:7)`,
    });
    const app = createApp({
      store: await authenticatedStore("operator"), workspaceStore, operationStore,
      logger: { error: (entry) => logs.push(entry) },
      management: { reconcileDatabases: async () => {
        throw new DatabaseReconciliationError("database apply", cause, {
          service: secretLike, username: secretLike, database: secretLike, secretName: secretLike,
        });
      } } as unknown as ManagementService,
    });
    const response = await app(request("/api/v2/workspaces/demo/databases", {
      method: "POST", body: JSON.stringify({ compose: {} }),
    }, "token"));
    expect(response.status).toBe(500);
    expect(await response.json()).toMatchObject({
      code: "DATABASE_RECONCILE_FAILED", operationId: "operation-diagnostic",
    });
    const diagnostic = logs.filter((entry) => entry.event === "operation.database_reconcile.failed");
    expect(diagnostic).toHaveLength(1);
    expect(diagnostic[0]).toMatchObject({
      operationId: "operation-diagnostic", phase: "database apply",
      errorClass: "UnknownError", providerCode: "ECONNREFUSED", providerStatus: 503,
      topFrame: "lib/database.ts:22:7",
    });
    expect(JSON.stringify(diagnostic)).not.toContain(secretLike);
    expect(JSON.stringify(await operationStore.get("operation-diagnostic"))).not.toContain(secretLike);
  });

  test("retains a safe diagnosis for an ordinary database provider error through HTTP, persistence and polling", async () => {
    const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-uid" });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const operationStore = new MemoryOperationStore(undefined, () => "ordinary-db-error");
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      operationStore,
      management: {
        reconcileDatabases: async () => {
          throw Object.assign(new Error("provider failed: DATABASE_URL=postgresql://admin:private@db.example/app"), {
            statusCode: 403,
            body: { reason: "Forbidden", message: "password=private", code: 403 },
          });
        },
      } as unknown as ManagementService,
    });
    const reconcile = () => app(request(
      "/api/v2/workspaces/demo/databases",
      { method: "POST", headers: { "idempotency-key": "ordinary-db-error" }, body: JSON.stringify({ compose: {} }) },
      "token",
    ));
    const expected = "Database reconciliation failed during operation execution: Forbidden (HTTP 403)";
    for (const result of [await reconcile(), await reconcile()]) {
      expect(result.status).toBe(500);
      expect(await result.json()).toMatchObject({
        code: "DATABASE_RECONCILE_FAILED",
        detail: expected,
        operationId: "operation-ordinary-db-error",
      });
    }
    const persisted = await operationStore.get("operation-ordinary-db-error");
    expect(persisted?.status.error).toEqual({ code: "DATABASE_RECONCILE_FAILED", message: expected });
    const polled = await app(request("/api/v2/operations/operation-ordinary-db-error", {}, "token"));
    expect(polled.status).toBe(200);
    expect(await polled.json()).toMatchObject({ status: { error: { message: expected } } });
    expect(JSON.stringify(persisted)).not.toContain("private");
  });

  test("routes workspace adoption and keeps platform adoption admin-only", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const adopted: string[] = [];
    const adoption = {
      adopt: async (workspaceId: string, workspaceUid: string) => {
        adopted.push(`${workspaceId}:${workspaceUid}`);
        return { workspaceId, workspaceUid, resourcesAdopted: 2 };
      },
      adoptPlatform: async (workspaceUid: string) => ({
        workspaceId: "kuber-system",
        workspaceUid,
        resourcesAdopted: 1,
      }),
    };
    const operatorApp = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      adoption,
    });
    const regular = await operatorApp(
      request("/api/v2/workspaces/demo/adopt", { method: "POST" }, "token"),
    );
    expect(regular.status).toBe(200);
    expect(adopted).toEqual(["demo:workspace-uid"]);
    expect(
      (
        await operatorApp(
          request(
            "/api/v2/platform/kuber-system/adopt",
            {
              method: "POST",
              body: JSON.stringify({ workspaceUid: "platform" }),
            },
            "token",
          ),
        )
      ).status,
    ).toBe(403);

    const adminApp = createApp({
      store: await authenticatedStore("admin"),
      workspaceStore,
      adoption,
    });
    const platform = await adminApp(
      request(
        "/api/v2/platform/kuber-system/adopt",
        { method: "POST", body: JSON.stringify({ workspaceUid: "platform" }) },
        "token",
      ),
    );
    expect(platform.status).toBe(200);
  });

  test("audits failed workspace adoption with request correlation", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const auditStore = new MemoryAuditStore();
    const logs: unknown[] = [];
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      auditStore,
      requestId: () => "adopt-request-123",
      logger: { error: (entry) => logs.push(entry) },
      adoption: {
        adopt: async () => {
          throw new Error("provider adoption failed");
        },
        adoptPlatform: async () => ({
          workspaceId: "kuber-system",
          workspaceUid: "platform",
          resourcesAdopted: 0,
        }),
      },
    });

    const result = await app(
      request("/api/v2/workspaces/demo/adopt", { method: "POST" }, "token"),
    );

    expect(result.status).toBe(500);
    expect(await result.json()).toMatchObject({
      code: "INTERNAL_ERROR",
      detail: "The request could not be completed",
      requestId: "adopt-request-123",
    });
    expect(await auditStore.list("demo")).toMatchObject([
      {
        spec: {
          action: "workspace.adopt",
          outcome: "failure",
          workspaceId: "demo",
          details: {
            route: "/api/v2/workspaces/demo/adopt",
            requestId: "adopt-request-123",
          },
        },
      },
    ]);
    expect(logs).toMatchObject([
      {
        event: "request.failed",
        requestId: "adopt-request-123",
        method: "POST",
        pathname: "/api/v2/workspaces/demo/adopt",
        workspaceId: "demo",
        status: 500,
        code: "INTERNAL_ERROR",
        errorName: "Error",
        message: "provider adoption failed",
      },
    ]);
  });

  test("forwards Kubernetes adoption errors with request correlation", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      requestId: () => "adopt-kubernetes-request-123",
      adoption: {
        adopt: async () => {
          throw {
            statusCode: 422,
            body: {
              apiVersion: "v1",
              kind: "Status",
              status: "Failure",
              reason: "Invalid",
              message:
                'Ingress.networking.k8s.io "web" is invalid: spec: Required value',
              code: 422,
            },
          };
        },
        adoptPlatform: async () => ({
          workspaceId: "kuber-system",
          workspaceUid: "platform",
          resourcesAdopted: 0,
        }),
      },
    });

    const result = await app(
      request("/api/v2/workspaces/demo/adopt", { method: "POST" }, "token"),
    );

    expect(result.status).toBe(422);
    expect(await result.json()).toMatchObject({
      code: "KUBERNETES_ERROR",
      detail:
        'Ingress.networking.k8s.io "web" is invalid: spec: Required value',
      requestId: "adopt-kubernetes-request-123",
    });
  });

  test("does not forward arbitrary adoption errors as Kubernetes errors", async () => {
    const workspaceStore = new MemoryWorkspaceStore({
      uid: () => "workspace-uid",
    });
    await workspaceStore.create({
      id: "demo",
      source: { uri: "oci://example/demo", digest: "sha256:abc" },
    });
    const app = createApp({
      store: await authenticatedStore("operator"),
      workspaceStore,
      adoption: {
        adopt: async () => {
          throw {
            statusCode: 422,
            body: { message: "arbitrary provider failure" },
          };
        },
        adoptPlatform: async () => ({
          workspaceId: "kuber-system",
          workspaceUid: "platform",
          resourcesAdopted: 0,
        }),
      },
    });

    const result = await app(
      request("/api/v2/workspaces/demo/adopt", { method: "POST" }, "token"),
    );

    expect(result.status).toBe(500);
    expect(await result.json()).toMatchObject({
      code: "INTERNAL_ERROR",
      detail: "The request could not be completed",
    });
  });
});