kuber/tests/server/operation-store.test.ts

import { describe, expect, test } from "bun:test";
import {
  MemoryOperationStore,
  MemoryWorkspaceLeaseProvider,
  OperationConflictError,
  OperationValidationError,
  recoverStaleOperations,
  sanitizeOperationResult,
  sanitizeOperationError,
} from "../../server/operation-store";

describe("operation store", () => {
  test("deduplicates matching requests and rejects key reuse", async () => {
    let uid = 0;
    const store = new MemoryOperationStore(undefined, () => `uid-${++uid}`);
    const input = {
      workspaceId: "demo",
      action: "deploy",
      idempotencyKey: "request-1",
      request: { revision: 2 },
    };
    const first = await store.create(input);
    expect(first.spec).not.toHaveProperty("request");
    expect((await store.create(input)).metadata.uid).toBe(first.metadata.uid);
    expect(
      (await store.create({ ...input, request: { revision: 2 } })).metadata.uid,
    ).toBe(first.metadata.uid);
    expect(
      store.create({ ...input, request: { revision: 3 } }),
    ).rejects.toBeInstanceOf(OperationConflictError);
  });

  test("hashes object requests canonically and redacts persisted results", async () => {
    const store = new MemoryOperationStore();
    const operation = await store.create({
      workspaceId: "demo",
      action: "resources.apply",
      idempotencyKey: "canonical",
      request: { z: 1, nested: { b: 2, a: 1 } },
    });
    expect(
      (
        await store.create({
          workspaceId: "demo",
          action: "resources.apply",
          idempotencyKey: "canonical",
          request: { nested: { a: 1, b: 2 }, z: 1 },
        })
      ).metadata.name,
    ).toBe(operation.metadata.name);
    await store.transition(operation.metadata.name, "running");
    const completed = await store.transition(
      operation.metadata.name,
      "succeeded",
      {
        result: {
          kind: "Secret",
          data: { password: "encoded" },
          metadata: { name: "credentials" },
        },
      },
    );
    expect(completed.status.result).toEqual({
      kind: "Secret",
      data: "[REDACTED]",
      metadata: { name: "credentials" },
    });
    expect(
      sanitizeOperationResult({ DATABASE_URL: "postgres://secret" }),
    ).toEqual({ DATABASE_URL: "[REDACTED]" });
  });

  test("redacts credential-bearing values while preserving ordinary ones", () => {
    expect(
      sanitizeOperationResult({
        endpoint: "https://alice:[email protected]:5432/mydb",
        awsKey: "AKIAIOSFODNN7EXAMPLE",
        keyMaterial: "-----BEGIN OPENSSH PRIVATE KEY-----",
        registry: "https://registry.example.com/v2/app",
        note: "used [email protected] for the AKIA lookup",
        uid: "abc-123-def",
      }),
    ).toEqual({
      endpoint: "https://[REDACTED]@db.example.com:5432/mydb",
      awsKey: "[REDACTED]",
      keyMaterial: "[REDACTED]",
      registry: "https://registry.example.com/v2/app",
      note: "used [email protected] for the AKIA lookup",
      uid: "abc-123-def",
    });
    expect(
      sanitizeOperationResult([
        "checkout https://[email protected]/org/repo.git",
        "AKIAIOSFODNN7EXAMPLE",
      ]),
    ).toEqual(["checkout https://[email protected]/org/repo.git", "[REDACTED]"]);
  });

  test("sanitizes operation failures while preserving their codes", () => {
    expect(
      sanitizeOperationError(
        {
          code: "PROVIDER_FAILED",
          message:
            'database failed: DB_PASSWORD=database-password response={"data":{"token":"kube-secret"}}',
        },
        "deploy",
      ),
    ).toEqual({
      code: "PROVIDER_FAILED",
      message:
        'database failed: DB_PASSWORD=[REDACTED] response={"data":{"token":"[REDACTED]"}}',
    });
    expect(
      sanitizeOperationError(
        { code: "RECONCILE_FAILED", message: "secret: db-password" },
        "databases.reconcile",
      ),
    ).toEqual({
      code: "RECONCILE_FAILED",
      message: "Database reconciliation failed",
    });
  });

  test("enforces the operation state machine", async () => {
    const store = new MemoryOperationStore();
    const operation = await store.create({
      workspaceId: "demo",
      action: "deploy",
      idempotencyKey: "request-2",
    });
    const running = await store.transition(operation.metadata.name, "running");
    expect(running.status.startedAt).toBeDefined();
    const succeeded = await store.transition(
      operation.metadata.name,
      "succeeded",
      { result: { ready: true } },
    );
    expect(succeeded.metadata.resourceVersion).toBe("3");
    expect(
      store.transition(operation.metadata.name, "failed", {
        error: { code: "late", message: "late" },
      }),
    ).rejects.toBeInstanceOf(OperationConflictError);

    const failed = await store.create({
      workspaceId: "demo",
      action: "delete",
      idempotencyKey: "request-3",
    });
    expect(
      store.transition(failed.metadata.name, "failed"),
    ).rejects.toBeInstanceOf(OperationValidationError);
  });

  test("provides exclusive, renewable per-workspace leases", async () => {
    let now = 0;
    const leases = new MemoryWorkspaceLeaseProvider(() => now);
    const first = await leases.acquire("demo", "worker-a", 100);
    expect(first).toBeDefined();
    expect(await leases.acquire("demo", "worker-b", 100)).toBeUndefined();
    expect(await first!.renew()).toBe(true);
    await first!.release();
    expect(await leases.acquire("demo", "worker-b", 100)).toBeDefined();
    now = 101;
  });

  test("startup recovery fails stale pending and running operations", async () => {
    const store = new MemoryOperationStore();
    const pending = await store.create({
      workspaceId: "demo",
      action: "deploy",
      idempotencyKey: "recover-1",
    });
    const running = await store.create({
      workspaceId: "demo",
      action: "stop",
      idempotencyKey: "recover-2",
    });
    await store.transition(running.metadata.name, "running");
    const done = await store.create({
      workspaceId: "demo",
      action: "restart",
      idempotencyKey: "recover-3",
    });
    await store.transition(done.metadata.name, "running");
    await store.transition(done.metadata.name, "succeeded", {
      result: { ok: true },
    });

    const recovered = await recoverStaleOperations(store);
    expect(recovered).toBe(2);

    const after = await store.list();
    const byId = new Map(after.map((o) => [o.metadata.name, o]));
    expect(byId.get(pending.metadata.name)?.status.state).toBe("failed");
    expect(byId.get(running.metadata.name)?.status.state).toBe("failed");
    expect(byId.get(pending.metadata.name)?.status.error?.code).toBe(
      "OPERATION_INTERRUPTED",
    );
    expect(byId.get(done.metadata.name)?.status.state).toBe("succeeded");

    expect(await recoverStaleOperations(store)).toBe(0);
  });
});