import { describe, expect, test } from "bun:test";
import { Listr } from "listr2";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { ApiRequestInit, ApiRequestOptions } from "../../lib/api";
import { KuberApiError } from "../../lib/api";
import type { ApiRequester } from "../../lib/build";
import {
  ensureWorkspace,
  reconcileResources,
  runLiveResourceOperation,
  runUp,
  workspaceAdoptionRoute,
} from "../../command/up";
import { provideContext } from "../../lib/context";
import { resolveTrustIdentity, updateTrust } from "../../lib/trust";
import { workspaceManifestDigest } from "../../lib/workspace";

const manifest = { version: 1 as const, files: [] };
const snapshot = {
  manifest,
  digest: workspaceManifestDigest(manifest),
  blobs: [],
};

describe("up API pipeline", () => {
  test("forwards the build timeout through the trusted requester", async () => {
    const root = await mkdtemp(join(tmpdir(), "kuber-up-api-"));
    const previousCwd = process.cwd();
    const previousConfigHome = process.env.XDG_CONFIG_HOME;
    const configHome = join(root, "config");
    const calls: Array<{
      path: string;
      init?: ApiRequestInit;
      options?: ApiRequestOptions;
    }> = [];

    try {
      await writeFile(
        join(root, "compose.yml"),
        "services:\n  web:\n    build: .\n",
      );
      await writeFile(
        join(root, ".kuberrc.ts"),
        'export default { project: "shop" };\n',
      );
      process.chdir(root);
      process.env.XDG_CONFIG_HOME = configHome;
      const identity = await resolveTrustIdentity("shop", root);
      await updateTrust((records) => [...records, identity]);

      const request: ApiRequester = async <T>(
        path: string,
        init?: ApiRequestInit,
        options?: ApiRequestOptions,
      ) => {
        calls.push({ path, init, options });
        if (path === "/snapshots/negotiate")
          return { workspace: snapshot.digest, missing: [], ready: true } as T;
        if (path === "/builds") {
          const buildRequest = init?.json as { id?: unknown } | undefined;
          if (typeof buildRequest?.id !== "string")
            throw new Error("Expected build request ID");
          return {
            version: 1,
            id: buildRequest.id,
            state: "succeeded",
            createdAt: "2026-01-01T00:00:00Z",
          } as T;
        }
        if (path.includes("/events")) return [] as T;
        if (path.endsWith("/result"))
          return {
            image: "registry.server/kuber/shop-web",
            digest: `sha256:${"a".repeat(64)}`,
            reference: `registry.server/kuber/shop-web@sha256:${"a".repeat(64)}`,
          } as T;
        if (path === "/workspaces/shop")
          throw new KuberApiError("missing", 404);
        if (path === "/workspaces")
          return {
            metadata: { name: "shop", uid: "workspace", resourceVersion: "1" },
          } as T;
        if (path.endsWith("/adopt")) return { resourcesAdopted: 0 } as T;
        if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
        return {} as T;
      };

      await provideContext(() => runUp(true, request));

      const build = calls.find(({ path }) => path === "/builds");
      if (!build) throw new Error("Expected build submission");
      expect(build.options).toEqual({ timeoutMs: 300_000 });
      expect(
        new Headers(build.init?.headers).get("x-kuber-trust-project"),
      ).toBe("shop");
    } finally {
      process.chdir(previousCwd);
      if (previousConfigHome === undefined) delete process.env.XDG_CONFIG_HOME;
      else process.env.XDG_CONFIG_HOME = previousConfigHome;
      await rm(root, { recursive: true, force: true });
    }
  });

  test("adopts before overlapping database and storage reconciliation, then renders merged env", async () => {
    const root = await mkdtemp(join(tmpdir(), "kuber-up-api-"));
    const previousCwd = process.cwd();
    const order: string[] = [];
    const started: string[] = [];
    let bothReady!: () => void;
    const bothStarted = new Promise<void>((resolve) => {
      bothReady = resolve;
    });
    let releaseDatabase!: () => void;
    let releaseStorage!: () => void;
    const database = new Promise<void>((resolve) => {
      releaseDatabase = resolve;
    });
    const storage = new Promise<void>((resolve) => {
      releaseStorage = resolve;
    });
    try {
      await writeFile(
        join(root, "compose.yml"),
        "services:\n  app:\n    image: nginx\n    volumes:\n      - postgresql:app\n      - s3:assets\n",
      );
      await writeFile(
        join(root, ".kuberrc.ts"),
        'export default { project: "shop" };\n',
      );
      expect(await Bun.spawn(["git", "init", "-q", root]).exited).toBe(0);
      process.chdir(root);
      const trust = await resolveTrustIdentity("shop", root);
      const request: ApiRequester = async <T>(path: string) => {
        order.push(path);
        if (path === "/workspaces/shop") throw new KuberApiError("missing", 404);
        if (path === "/workspaces")
          return {
            metadata: { name: "shop", uid: "workspace", resourceVersion: "1" },
          } as T;
        if (path.endsWith("/adopt")) return { resourcesAdopted: 0 } as T;
        if (path.endsWith("/databases")) {
          started.push("database");
          if (started.length === 2) bothReady();
          await database;
          return { app: { DATABASE_URL: "postgresql://app", SHARED: "database" } } as T;
        }
        if (path.endsWith("/storage")) {
          started.push("storage");
          if (started.length === 2) bothReady();
          await storage;
          return { app: { AWS_ACCESS_KEY_ID: "key", SHARED: "storage" } } as T;
        }
        if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
        return {} as T;
      };
      const run = provideContext(() => runUp(false, request, { trust }));
      try {
        await bothStarted;
        expect(started).toEqual(["database", "storage"]);
        expect(order.indexOf("/workspaces/shop/adopt")).toBeLessThan(
          order.indexOf("/workspaces/shop/databases"),
        );
        expect(order).not.toContain("/workspaces/shop/resources/plan");
        releaseStorage();
        await Promise.resolve();
        expect(order).not.toContain("/workspaces/shop/resources/plan");
        releaseDatabase();
        const result = await run;
        expect(result.serviceEnv).toEqual({
          app: {
            DATABASE_URL: "postgresql://app",
            SHARED: "storage",
            AWS_ACCESS_KEY_ID: "key",
          },
        });
        expect(order.indexOf("/workspaces/shop/resources/plan")).toBeGreaterThan(
          order.indexOf("/workspaces/shop/storage"),
        );
      } finally {
        releaseStorage();
        releaseDatabase();
        await run.catch(() => {});
      }
    } finally {
      process.chdir(previousCwd);
      await rm(root, { recursive: true, force: true });
    }
  });

  test("creates workspace metadata without embedding source blobs", async () => {
    const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
    const request: ApiRequester = async <T>(
      path: string,
      init?: ApiRequestInit,
    ) => {
      calls.push({ path, init });
      if (!init) throw new KuberApiError("missing", 404);
      return {
        metadata: { name: "shop", uid: "uid", resourceVersion: "1" },
      } as T;
    };
    await ensureWorkspace(
      "shop",
      { services: { web: { image: "nginx" } } },
      snapshot,
      request,
    );

    expect(calls.map(({ path }) => path)).toEqual([
      "/workspaces/shop",
      "/workspaces",
    ]);
    const body = calls[1]!.init?.json as Record<string, unknown>;
    expect(body).toMatchObject({
      id: "shop",
      source: { uri: `cas://${snapshot.digest}`, digest: snapshot.digest },
    });
    expect(JSON.stringify(body)).not.toContain('"blob"');
    expect(JSON.stringify(body)).not.toContain('"files"');
  });

  test("updates workspace metadata with the current resource-version ETag", async () => {
    const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
    const request: ApiRequester = async <T>(
      path: string,
      init?: ApiRequestInit,
    ) => {
      calls.push({ path, init });
      return {
        metadata: {
          name: "shop",
          uid: "uid",
          resourceVersion: init ? "8" : "7",
        },
      } as T;
    };
    await ensureWorkspace("shop", { services: {} }, snapshot, request);
    expect(calls[1]?.init?.method).toBe("PUT");
    expect(new Headers(calls[1]?.init?.headers).get("if-match")).toBe('"7"');
  });

  test("retries one conflict using a fresh ETag", async () => {
    const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
    let gets = 0;
    const request: ApiRequester = async <T>(
      path: string,
      init?: ApiRequestInit,
    ) => {
      calls.push({ path, init });
      if (!init) {
        gets += 1;
        return {
          metadata: {
            name: "shop",
            uid: "uid",
            resourceVersion: gets === 1 ? "7" : "8",
          },
          spec: { source: { uri: "cas://old", digest: "old" }, config: {} },
        } as T;
      }
      if (calls.length === 2)
        throw new KuberApiError("conflict", 409, {
          title: "Conflict",
          status: 409,
          code: "WORKSPACE_CONFLICT",
        });
      return {
        metadata: { name: "shop", uid: "uid", resourceVersion: "9" },
      } as T;
    };

    await ensureWorkspace("shop", { services: {} }, snapshot, request);

    expect(calls.map(({ path }) => path)).toEqual([
      "/workspaces/shop",
      "/workspaces/shop",
      "/workspaces/shop",
      "/workspaces/shop",
    ]);
    expect(new Headers(calls[3]?.init?.headers).get("if-match")).toBe('"8"');
  });

  test("accepts a fresh workspace that already matches after a conflict", async () => {
    let gets = 0;
    let puts = 0;
    const desired = {
      source: { uri: `cas://${snapshot.digest}`, digest: snapshot.digest },
      config: { compose: { services: {} } },
    };
    const request: ApiRequester = async <T>(
      _path: string,
      init?: ApiRequestInit,
    ) => {
      if (!init) {
        gets += 1;
        return {
          metadata: { name: "shop", uid: "uid", resourceVersion: "8" },
          spec:
            gets === 1
              ? { ...desired, source: { uri: "cas://old", digest: "old" } }
              : desired,
        } as T;
      }
      puts += 1;
      throw new KuberApiError("conflict", 409, {
        title: "Conflict",
        status: 409,
        code: "WORKSPACE_CONFLICT",
      });
    };

    await ensureWorkspace("shop", { services: {} }, snapshot, request);

    expect({ gets, puts }).toEqual({ gets: 2, puts: 1 });
  });

  test("recovers an ambiguous transport failure after the update committed", async () => {
    let gets = 0;
    let puts = 0;
    const desired = {
      source: { uri: `cas://${snapshot.digest}`, digest: snapshot.digest },
      config: { compose: { services: {} } },
    };
    const request: ApiRequester = async <T>(
      _path: string,
      init?: ApiRequestInit,
    ) => {
      if (!init) {
        gets += 1;
        return {
          metadata: { name: "shop", uid: "uid", resourceVersion: "8" },
          spec:
            gets === 1
              ? { ...desired, source: { uri: "cas://old", digest: "old" } }
              : desired,
        } as T;
      }
      puts += 1;
      throw new TypeError("connection reset");
    };

    await ensureWorkspace("shop", { services: {} }, snapshot, request);

    expect({ gets, puts }).toEqual({ gets: 2, puts: 1 });
  });

  test("recovers an ambiguous transport failure with one fresh-ETag retry", async () => {
    const etags: string[] = [];
    let gets = 0;
    const request: ApiRequester = async <T>(
      _path: string,
      init?: ApiRequestInit,
    ) => {
      if (!init) {
        gets += 1;
        return {
          metadata: {
            name: "shop",
            uid: "uid",
            resourceVersion: gets === 1 ? "7" : "8",
          },
          spec: { source: { uri: "cas://old", digest: "old" }, config: {} },
        } as T;
      }
      etags.push(new Headers(init.headers).get("if-match")!);
      if (etags.length === 1) throw new TypeError("connection reset");
      return {
        metadata: { name: "shop", uid: "uid", resourceVersion: "9" },
      } as T;
    };

    await ensureWorkspace("shop", { services: {} }, snapshot, request);

    expect(etags).toEqual(['"7"', '"8"']);
  });

  test("does not retry a generic workspace update failure", async () => {
    const failure = new KuberApiError("failed", 500, {
      title: "Internal server error",
      status: 500,
      code: "INTERNAL_ERROR",
    });
    let calls = 0;
    const request: ApiRequester = async <T>() => {
      calls += 1;
      if (calls === 1)
        return {
          metadata: { name: "shop", uid: "uid", resourceVersion: "7" },
        } as T;
      throw failure;
    };

    await expect(
      ensureWorkspace("shop", { services: {} }, snapshot, request),
    ).rejects.toBe(failure);
    expect(calls).toBe(2);
  });

  test("plans, applies, runs the hook, waits, then deletes stale identities", async () => {
    const order: string[] = [];
    const desired = [
      { apiVersion: "apps/v1", kind: "Deployment", metadata: { name: "web" } },
    ];
    const stale = [
      {
        apiVersion: "v1",
        kind: "Secret",
        name: "old",
        uid: "secret-uid",
        workspaceUid: "workspace-uid",
      },
    ];
    const request: ApiRequester = async <T>(path: string) => {
      order.push(path.split("/").at(-1)!);
      if (path.endsWith("/plan")) return { desired, stale } as T;
      return {} as T;
    };

    await reconcileResources(
      "shop",
      desired,
      42_000,
      () => {
        order.push("hook");
      },
      request,
    );
    expect(order).toEqual(["plan", "apply", "hook", "wait", "delete"]);
  });

  test("resumes an interrupted reconcile operation by its persisted ID", async () => {
    const calls: string[] = [];
    const applyIdempotencyKeys: Array<string | null> = [];
    let applyAttempts = 0;
    let operationPolls = 0;
    const request: ApiRequester = async <T>(
      path: string,
      init?: ApiRequestInit,
    ) => {
      calls.push(path);
      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
      if (path.endsWith("/apply")) {
        applyIdempotencyKeys.push(
          new Headers(init?.headers).get("idempotency-key"),
        );
        applyAttempts++;
        if (applyAttempts === 1) throw new TypeError("fetch failed");
        return {
          operationId: "operation-apply",
          operation: { status: { state: "running" } },
        } as T;
      }
      if (path === "/operations/operation-apply") {
        operationPolls++;
        if (operationPolls === 1) throw new TypeError("connection reset");
        return { status: { state: "succeeded" } } as T;
      }
      throw new Error(`Unexpected request: ${path}`);
    };

    await reconcileResources("shop", [], 1, undefined, request, {
      sleep: async () => {},
    });

    expect(calls).toEqual([
      "/workspaces/shop/resources/plan",
      "/workspaces/shop/resources/apply",
      "/workspaces/shop/resources/apply",
      "/operations/operation-apply/events?after=0",
      "/operations/operation-apply",
      "/operations/operation-apply",
      "/operations/operation-apply/events?after=0",
    ]);
    expect(applyIdempotencyKeys[0]).toBeTruthy();
    expect(applyIdempotencyKeys[1]).toBe(applyIdempotencyKeys[0]);
  });

  test("clears the production retry timer and does not retry when cancelled", async () => {
    const controller = new AbortController();
    const originalSetTimeout = globalThis.setTimeout;
    const originalClearTimeout = globalThis.clearTimeout;
    const timer = {} as ReturnType<typeof setTimeout>;
    let timerCallback: (() => void) | undefined;
    let timerDelay: number | undefined;
    let timerCleared = false;
    let applyAttempts = 0;
    let startedTimer!: () => void;
    const timerStarted = new Promise<void>((resolve) => {
      startedTimer = resolve;
    });
    globalThis.setTimeout = ((callback: () => void, milliseconds?: number) => {
      timerCallback = callback;
      timerDelay = milliseconds;
      startedTimer();
      return timer;
    }) as typeof setTimeout;
    globalThis.clearTimeout = ((handle: ReturnType<typeof setTimeout>) => {
      if (handle === timer) timerCleared = true;
    }) as typeof clearTimeout;

    try {
      const request: ApiRequester = async <T>(path: string) => {
        if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
        if (path.endsWith("/apply")) {
          applyAttempts += 1;
          throw new TypeError("connection reset");
        }
        throw new Error(`Unexpected request: ${path}`);
      };
      const cancelled = new DOMException("Cancelled", "AbortError");
      const run = reconcileResources("shop", [], 1, undefined, request, {
        signal: controller.signal,
      });

      await timerStarted;
      controller.abort(cancelled);

      await expect(run).rejects.toBe(cancelled);
      expect(timerDelay).toBe(250);
      expect(timerCleared).toBe(true);
      expect(timerCallback).toBeDefined();
      expect(applyAttempts).toBe(1);
    } finally {
      globalThis.setTimeout = originalSetTimeout;
      globalThis.clearTimeout = originalClearTimeout;
    }
  });

  test("restarts after an interrupted apply is persisted before its ID is received", async () => {
    const applyRequests: Array<{ key: string | null; json: unknown }> = [];
    const request: ApiRequester = async <T>(
      path: string,
      init?: ApiRequestInit,
    ) => {
      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
      if (path.endsWith("/apply")) {
        applyRequests.push({
          key: new Headers(init?.headers).get("idempotency-key"),
          json: init?.json,
        });
        if (applyRequests.length === 1) throw new TypeError("fetch failed");
        if (applyRequests.length === 2)
          throw new KuberApiError("operation interrupted", 500, {
            title: "Operation interrupted",
            status: 500,
            code: "OPERATION_INTERRUPTED",
          });
        return {
          operationId: "operation-apply",
          operation: { status: { state: "succeeded" } },
        } as T;
      }
      throw new Error(`Unexpected request: ${path}`);
    };

    await reconcileResources("shop", [], 1, undefined, request, {
      sleep: async () => {},
    });

    expect(applyRequests).toHaveLength(3);
    expect(applyRequests[0]?.key).toBeTruthy();
    expect(applyRequests[1]?.key).toBe(applyRequests[0]?.key);
    expect(applyRequests[2]?.key).toBeTruthy();
    expect(applyRequests[2]?.key).not.toBe(applyRequests[0]?.key);
    expect(applyRequests[2]?.json).toEqual(applyRequests[0]?.json);
  });

  test("polls persisted resource progress without duplicating events after reconnect", async () => {
    const progress: string[] = [];
    let poll = 0;
    const request: ApiRequester = async <T>(path: string) => {
      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
      if (path.endsWith("/apply"))
        return {
          operationId: "operation-apply",
          operation: { status: { state: "running" } },
        } as T;
      if (path.includes("/events")) {
        poll++;
        return {
          items:
            poll === 1
              ? [
                  {
                    sequence: 1,
                    data: null,
                  },
                  {
                    sequence: 2,
                    data: {
                      resource: {
                        apiVersion: "v1",
                        kind: "Service",
                        name: "web",
                      },
                      phase: "apply",
                      state: "started",
                    },
                  },
                ]
              : [
                  {
                    sequence: 2,
                    data: {
                      resource: {
                        apiVersion: "v1",
                        kind: "Service",
                        name: "web",
                      },
                      phase: "apply",
                      state: "started",
                    },
                  },
                  {
                    sequence: 3,
                    data: {
                      resource: {
                        apiVersion: "v1",
                        kind: "Service",
                        name: "web",
                      },
                      phase: "apply",
                      state: "succeeded",
                    },
                  },
                ],
        } as T;
      }
      if (path === "/operations/operation-apply") {
        return { status: { state: poll > 1 ? "succeeded" : "running" } } as T;
      }
      throw new Error(`Unexpected request: ${path}`);
    };

    await reconcileResources("shop", [], 1, undefined, request, {
      sleep: async () => {},
      onEvent: (event) =>
        progress.push(`${event.sequence}:${event.data.state}`),
    });
    expect(progress).toEqual(["2:started", "3:succeeded"]);
  });

  test("fails visibly when retained operation progress has a cursor gap", async () => {
    let applyAttempts = 0;
    let operationPolls = 0;
    const request: ApiRequester = async <T>(path: string) => {
      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
      if (path.endsWith("/apply")) {
        applyAttempts += 1;
        return {
          operationId: "operation-apply",
          operation: { status: { state: "running" } },
        } as T;
      }
      if (path.includes("/events"))
        return {
          retainedFirstSequence: 3,
          cursorGap: true,
          items: [],
        } as T;
      if (path === "/operations/operation-apply") {
        operationPolls += 1;
        return { status: { state: "succeeded" } } as T;
      }
      throw new Error(`Unexpected request: ${path}`);
    };

    await expect(
      reconcileResources("shop", [], 1, undefined, request, {
        sleep: async () => {},
      }),
    ).rejects.toThrow("progress history was truncated");
    expect(applyAttempts).toBe(1);
    expect(operationPolls).toBe(0);
  });

  test("attaches concurrent live resource subtasks before driving multi-target operations", async () => {
    const children = new Map<string, { title: string; output: string }>();
    let operationCalls = 0;
    const listr = new Listr([
      {
        title: "Apply resources",
        task: (_ctx, task) =>
          runLiveResourceOperation(
            task,
            "apply",
            [
              {
                apiVersion: "v1",
                kind: "Service",
                name: "web",
              },
              {
                apiVersion: "apps/v1",
                kind: "Deployment",
                name: "api",
              },
            ],
            async (onEvent) => {
              operationCalls += 1;
              expect(children.size).toBe(2);
              onEvent({
                sequence: 1,
                data: {
                  resource: {
                    apiVersion: "v1",
                    kind: "Service",
                    name: "web",
                  },
                  phase: "apply",
                  state: "started",
                },
              });
              onEvent({
                sequence: 2,
                data: {
                  resource: {
                    apiVersion: "apps/v1",
                    kind: "Deployment",
                    name: "api",
                  },
                  phase: "apply",
                  state: "succeeded",
                },
              });
              expect(children.get("web")?.output).toBe("started");
              expect(children.get("api")?.output).toBe("succeeded");
            },
            {
              onTaskStarted: (target, activeTask) =>
                children.set(target.name, activeTask),
            },
          ),
      },
    ]);

    await listr.run();

    expect(operationCalls).toBe(1);
    expect(listr.tasks[0]?.subtasks).toHaveLength(2);
    expect(listr.tasks[0]?.subtasks.map((task) => task.title)).toEqual([
      "Apply Service/web",
      "Apply Deployment/api",
    ]);
  }, 1_000);

  test("renders a single live resource target and runs zero-target operations directly", async () => {
    const single = new Listr([
      {
        title: "Apply resources",
        task: (_ctx, task) =>
          runLiveResourceOperation(
            task,
            "apply",
            [{ apiVersion: "v1", kind: "Service", name: "web" }],
            async (onEvent) => {
              onEvent({
                sequence: 1,
                data: {
                  resource: { apiVersion: "v1", kind: "Service", name: "web" },
                  phase: "apply",
                  state: "succeeded",
                },
              });
            },
          ),
      },
    ]);
    await single.run();
    expect(single.tasks[0]?.subtasks.map(({ title }) => title)).toEqual([
      "Apply Service/web",
    ]);

    let called = false;
    await runLiveResourceOperation(
      { signal: new AbortController().signal } as never,
      "delete",
      [],
      async () => {
        called = true;
      },
    );
    expect(called).toBe(true);
  });

  test("cancels a multi-resource live operation without leaving child tasks waiting", async () => {
    const children: Array<{ signal: AbortSignal }> = [];
    let phaseTask: { cancel: () => void } | undefined;
    let operationObservedAbort = false;
    let startOperation: (() => void) | undefined;
    const operationStarted = new Promise<void>((resolve) => {
      startOperation = resolve;
    });
    const listr = new Listr([
      {
        title: "Apply resources",
        task: (_ctx, task) => {
          phaseTask = task;
          return runLiveResourceOperation(
            task,
            "apply",
            [
              { apiVersion: "v1", kind: "Service", name: "web" },
              { apiVersion: "apps/v1", kind: "Deployment", name: "api" },
            ],
            async (_onEvent, signal) => {
              startOperation?.();
              await new Promise<void>((_resolve, reject) => {
                signal.addEventListener(
                  "abort",
                  () => {
                    operationObservedAbort = true;
                    reject(signal.reason);
                  },
                  { once: true },
                );
              });
            },
            { onTaskStarted: (_target, child) => children.push(child) },
          );
        },
      },
    ]);
    const originalExit = process.exit;
    const exitCodes: Array<number | undefined> = [];
    process.exit = ((code?: number) => {
      exitCodes.push(code);
      return undefined as never;
    }) as typeof process.exit;

    try {
      const run = listr.run();
      await operationStarted;
      phaseTask?.cancel();
      await Promise.race([
        run,
        Bun.sleep(100).then(() => {
          throw new Error("Cancelled live operation did not settle promptly");
        }),
      ]);
    } finally {
      process.exit = originalExit;
    }

    expect(operationObservedAbort).toBe(true);
    expect(exitCodes).toEqual([127]);
    expect(children).toHaveLength(2);
    expect(children.every(({ signal }) => signal.aborted)).toBe(true);
    expect(
      listr.tasks[0]?.subtasks.every(({ state }) => state === "CANCELLED"),
    ).toBe(true);
  }, 1_000);

  test("resubmits with a fresh key after server restart interruption", async () => {
    const applyRequests: Array<{ key: string | null; json: unknown }> = [];
    let operationPolls = 0;
    const request: ApiRequester = async <T>(
      path: string,
      init?: ApiRequestInit,
    ) => {
      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
      if (path.endsWith("/apply")) {
        applyRequests.push({
          key: new Headers(init?.headers).get("idempotency-key"),
          json: init?.json,
        });
        return {
          operationId: `operation-apply-${applyRequests.length}`,
          operation: { status: { state: "running" } },
        } as T;
      }
      if (path === "/operations/operation-apply-1") {
        operationPolls++;
        if (operationPolls === 1)
          return {
            status: {
              state: "failed",
              error: {
                code: "OPERATION_INTERRUPTED",
                message: "server restarted",
              },
            },
          } as T;
        return { status: { state: "succeeded" } } as T;
      }
      if (path === "/operations/operation-apply-2")
        return { status: { state: "succeeded" } } as T;
      throw new Error(`Unexpected request: ${path}`);
    };

    await reconcileResources("shop", [], 1, undefined, request, {
      sleep: async () => {},
    });

    expect(applyRequests).toHaveLength(2);
    expect(applyRequests[0]?.key).toBeTruthy();
    expect(applyRequests[1]?.key).toBeTruthy();
    expect(applyRequests[1]?.key).not.toBe(applyRequests[0]?.key);
    expect(applyRequests[1]?.json).toEqual(applyRequests[0]?.json);
  });

  test.each([
    ["failed", "OPERATION_FAILED"],
    ["cancelled", "OPERATION_CANCELLED"],
  ])("does not retry arbitrary %s operations", async (state, code) => {
    let applyAttempts = 0;
    let operationPolls = 0;
    const request: ApiRequester = async <T>(path: string) => {
      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
      if (path.endsWith("/apply")) {
        applyAttempts++;
        return {
          operationId: "operation-apply",
          operation: { status: { state: "running" } },
        } as T;
      }
      if (path === "/operations/operation-apply") {
        operationPolls++;
        return {
          status: { state, error: { code, message: "terminal" } },
        } as T;
      }
      throw new Error(`Unexpected request: ${path}`);
    };

    await expect(
      reconcileResources("shop", [], 1, undefined, request, {
        sleep: async () => {},
      }),
    ).rejects.toMatchObject({ code });
    expect(applyAttempts).toBe(1);
    expect(operationPolls).toBe(1);
  });

  test("bounds restart recovery sleeps by the resume deadline", async () => {
    let currentTime = 0;
    const sleeps: Array<{ startedAt: number; milliseconds: number }> = [];
    let operationNumber = 0;
    const request: ApiRequester = async <T>(path: string) => {
      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
      if (path.endsWith("/apply")) {
        operationNumber++;
        return {
          operationId: `operation-${operationNumber}`,
          operation: { status: { state: "running" } },
        } as T;
      }
      return {
        status: {
          state: "failed",
          error: { code: "OPERATION_INTERRUPTED", message: "restarted" },
        },
      } as T;
    };

    await expect(
      reconcileResources("shop", [], 0, undefined, request, {
        now: () => currentTime,
        sleep: async (milliseconds) => {
          sleeps.push({ startedAt: currentTime, milliseconds });
          currentTime += milliseconds;
        },
      }),
    ).rejects.toThrow("Timed out while reconnecting to resume the operation");

    expect(sleeps).not.toHaveLength(0);
    expect(
      sleeps.every(
        ({ startedAt, milliseconds }) => startedAt + milliseconds <= 60_000,
      ),
    ).toBe(true);
    expect(currentTime).toBe(60_000);
  });

  test("does not restart after an apply interruption reaches the resume deadline", async () => {
    let currentTime = 0;
    let applyAttempts = 0;
    const request: ApiRequester = async <T>(path: string) => {
      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
      applyAttempts++;
      if (applyAttempts === 1) throw new TypeError("connection reset");
      throw new KuberApiError("operation interrupted", 500, {
        title: "Operation interrupted",
        status: 500,
        code: "OPERATION_INTERRUPTED",
      });
    };

    await expect(
      reconcileResources("shop", [], 0, undefined, request, {
        now: () => currentTime,
        sleep: async (milliseconds) => {
          currentTime += milliseconds;
          if (currentTime < 60_000) currentTime = 60_000;
        },
      }),
    ).rejects.toMatchObject({
      code: "OPERATION_INTERRUPTED",
      status: 500,
    });
    expect(applyAttempts).toBe(2);
    expect(currentTime).toBe(60_000);
  });

  test("fails immediately for a typed 503 operation-store error", async () => {
    const unavailable = new KuberApiError(
      "Operation storage is not configured",
      503,
      {
        title: "Service unavailable",
        status: 503,
        code: "OPERATION_STORE_UNAVAILABLE",
      },
    );
    const calls: string[] = [];
    const sleeps: number[] = [];
    const request: ApiRequester = async <T>(path: string) => {
      calls.push(path);
      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
      throw unavailable;
    };

    await expect(
      reconcileResources("shop", [], 1, undefined, request, {
        sleep: async (milliseconds) => {
          sleeps.push(milliseconds);
        },
      }),
    ).rejects.toBe(unavailable);

    expect(calls).toEqual([
      "/workspaces/shop/resources/plan",
      "/workspaces/shop/resources/apply",
    ]);
    expect(sleeps).toEqual([]);
  });

  test("never sleeps past the operation resume deadline", async () => {
    let currentTime = 0;
    const sleeps: Array<{ startedAt: number; milliseconds: number }> = [];
    const request: ApiRequester = async <T>(path: string) => {
      if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
      throw new TypeError("connection reset");
    };

    await expect(
      reconcileResources("shop", [], 0, undefined, request, {
        now: () => currentTime,
        sleep: async (milliseconds) => {
          sleeps.push({ startedAt: currentTime, milliseconds });
          currentTime += milliseconds;
        },
      }),
    ).rejects.toThrow("connection reset");

    expect(sleeps).not.toHaveLength(0);
    expect(
      sleeps.every(
        ({ startedAt, milliseconds }) => startedAt + milliseconds <= 60_000,
      ),
    ).toBe(true);
    expect(sleeps.at(-1)).toEqual({
      startedAt: 57_750,
      milliseconds: 2_250,
    });
    expect(currentTime).toBe(60_000);
  });

  test("fails closed with the precise missing adoption route", async () => {
    const request: ApiRequester = async <T>(path: string) => {
      if (path.endsWith("/plan"))
        throw new Error(
          "Namespace shop is external; refusing workspace mutation",
        );
      return {} as T;
    };
    await expect(
      reconcileResources("shop", [], 1, undefined, request),
    ).rejects.toThrow(`POST ${workspaceAdoptionRoute("shop")}`);
  });
});