feat: add CI deployment and live progress

2026-09-05 12:09:16 +00:00
parent 321f4e807a
commit aa02826dbb
27 changed files with 2711 additions and 180 deletions
@@ -1,5 +1,5 @@
 import { defineCommand } from "citty";
-import { Listr } from "listr2";
+import { Listr, type ListrTaskWrapper } from "listr2";
 import { randomUUID } from "node:crypto";
 import type { ComposeSpecification } from "../schema/docker.d";
 import type { KuberResource } from "../types";
@@ -24,6 +24,7 @@ import {
  requireLocalTrust,
  resolveTrustIdentity,
  trustHeaders,
+  type TrustIdentity,
 } from "../lib/trust";

 type Workspace = {
@@ -54,9 +55,44 @@ type OperationResponse = {
  operation?: { status?: OperationStatus };
 };

+export type ResourceOperationEvent = {
+  sequence: number;
+  data: {
+    resource: {
+      apiVersion: string;
+      kind: string;
+      name: string;
+      namespace?: string;
+    };
+    phase: "apply" | "wait" | "delete";
+    state: "started" | "succeeded" | "failed" | "aborted";
+  };
+};
+
 export type OperationResumeOptions = {
  now?: () => number;
  sleep?: (milliseconds: number) => Promise<void>;
+  onEvent?: (event: ResourceOperationEvent) => void;
+};
+
+type OperationEventsResponse = {
+  items?: ResourceOperationEvent[];
+  retainedFirstSequence?: unknown;
+  cursorGap?: unknown;
+};
+
+type ResourceOperationTarget = {
+  apiVersion: string;
+  kind: string;
+  name: string;
+  namespace?: string;
+};
+
+export type LiveResourceOperationOptions = {
+  onTaskStarted?: (
+    target: ResourceOperationTarget,
+    task: ListrTaskWrapper<ResourceOperationTarget, any, any>,
+  ) => void;
 };

 type UpContext = {
@@ -78,6 +114,14 @@ const RECOVERABLE_API_ERROR_CODES = new Set([
  "HTTP_504",
 ]);

+class OperationProgressCursorGapError extends Error {
+  constructor() {
+    super(
+      "Operation progress history was truncated; refusing to report an incomplete resource stream",
+    );
+  }
+}
+
 export function workspaceAdoptionRoute(project: string): string {
  return `/workspaces/${encodeURIComponent(project)}/adopt`;
 }
@@ -99,6 +143,44 @@ export function getDeploymentNames(resources: KubernetesResource[]): string[] {
    .filter((name): name is string => Boolean(name));
 }

+export function resourceOperationEventTitle(
+  event: ResourceOperationEvent,
+): string {
+  const { resource, phase, state } = event.data;
+  return `${phase === "apply" ? "Applied" : phase === "wait" ? "Waited for" : "Deleted"} ${resource.kind}/${resource.name}: ${state}`;
+}
+
+function isResourceOperationEvent(
+  event: unknown,
+): event is ResourceOperationEvent {
+  if (!event || typeof event !== "object") return false;
+  const { sequence, data } = event as {
+    sequence?: unknown;
+    data?: unknown;
+  };
+  if (!Number.isSafeInteger(sequence) || !data || typeof data !== "object")
+    return false;
+  const { resource, phase, state } = data as {
+    resource?: unknown;
+    phase?: unknown;
+    state?: unknown;
+  };
+  if (!resource || typeof resource !== "object") return false;
+  const identity = resource as Record<string, unknown>;
+  return (
+    typeof identity.apiVersion === "string" &&
+    typeof identity.kind === "string" &&
+    typeof identity.name === "string" &&
+    (identity.namespace === undefined ||
+      typeof identity.namespace === "string") &&
+    (phase === "apply" || phase === "wait" || phase === "delete") &&
+    (state === "started" ||
+      state === "succeeded" ||
+      state === "failed" ||
+      state === "aborted")
+  );
+}
+
 function workspaceInput(
  compose: ComposeSpecification,
  snapshot: WorkspaceSnapshot,
@@ -235,6 +317,17 @@ function isInterruptedOperation(status: OperationStatus): boolean {
  );
 }

+function hasOperationEventCursorGap(
+  events: OperationEventsResponse,
+  after: number,
+): boolean {
+  return (
+    events.cursorGap === true ||
+    (Number.isSafeInteger(events.retainedFirstSequence) &&
+      (events.retainedFirstSequence as number) > after + 1)
+  );
+}
+
 function operationResumeDeadline(
  rolloutTimeoutMs: number,
  now: number,
@@ -255,10 +348,12 @@ async function resumeManagedOperation(
  const deadline = operationResumeDeadline(rolloutTimeoutMs, now());
  const headers = new Headers(init.headers);
  headers.set("idempotency-key", randomUUID());
+  headers.set("prefer", "respond-async");
  let operationInit = { ...init, headers };
  let operationId: string | undefined;
  let backoffMs = OPERATION_RESUME_INITIAL_BACKOFF_MS;
  let restartRetryPending = false;
+  let eventCursor = 0;

  for (;;) {
    if (restartRetryPending && now() >= deadline)
@@ -285,10 +380,37 @@ async function resumeManagedOperation(
        status = operationStatusFromResponse(response);
      }

+      if (operationId) {
+        try {
+          const events = await managementRequest<OperationEventsResponse>(
+            project,
+            request,
+            `/operations/${encodeURIComponent(operationId)}/events?after=${eventCursor}`,
+            {},
+          );
+          if (hasOperationEventCursorGap(events, eventCursor))
+            throw new OperationProgressCursorGapError();
+          for (const event of events.items ?? []) {
+            if (
+              !Number.isSafeInteger(event.sequence) ||
+              event.sequence <= eventCursor
+            )
+              continue;
+            eventCursor = event.sequence;
+            if (!isResourceOperationEvent(event)) continue;
+            options.onEvent?.(event);
+          }
+        } catch (error) {
+          if (error instanceof OperationProgressCursorGapError) throw error;
+          // Event polling is additive; never delay resumable operation status polling.
+        }
+      }
+
      if (!operationId || !status || status.state === "succeeded") return;
      if (isInterruptedOperation(status)) {
        if (now() >= deadline) throw operationFailure(operationId, status);
        operationId = undefined;
+        eventCursor = 0;
        restartRetryPending = true;
        headers.set("idempotency-key", randomUUID());
        operationInit = { ...init, headers };
@@ -301,6 +423,7 @@ async function resumeManagedOperation(
      ) {
        if (now() >= deadline) throw adoptionHint(project, error);
        operationId = undefined;
+        eventCursor = 0;
        restartRetryPending = true;
        headers.set("idempotency-key", randomUUID());
        operationInit = { ...init, headers };
@@ -319,23 +442,31 @@ async function resumeManagedOperation(
  }
 }

-export async function reconcileResources(
+async function planResources(
  project: string,
  resources: KubernetesResource[],
-  rolloutTimeoutMs: number,
-  postApply:
-    | ((resources: KubernetesResource[]) => void | Promise<void>)
-    | undefined,
  request: ApiRequester = apiRequest,
-  resumeOptions: OperationResumeOptions = {},
 ): Promise<ResourcePlan> {
  const workspacePath = `/workspaces/${encodeURIComponent(project)}`;
-  const plan = await managementRequest<ResourcePlan>(
+  return managementRequest<ResourcePlan>(
    project,
    request,
    `${workspacePath}/resources/plan`,
    { method: "POST", json: { resources } },
  );
+}
+
+async function applyResourcePlan(
+  project: string,
+  plan: ResourcePlan,
+  rolloutTimeoutMs: number,
+  postApply:
+    | ((resources: KubernetesResource[]) => void | Promise<void>)
+    | undefined,
+  request: ApiRequester,
+  resumeOptions: OperationResumeOptions,
+): Promise<void> {
+  const workspacePath = `/workspaces/${encodeURIComponent(project)}`;
  await resumeManagedOperation(
    project,
    request,
@@ -376,17 +507,117 @@ export async function reconcileResources(
      resumeOptions,
    );
  }
+}
+
+export async function reconcileResources(
+  project: string,
+  resources: KubernetesResource[],
+  rolloutTimeoutMs: number,
+  postApply:
+    | ((resources: KubernetesResource[]) => void | Promise<void>)
+    | undefined,
+  request: ApiRequester = apiRequest,
+  resumeOptions: OperationResumeOptions = {},
+): Promise<ResourcePlan> {
+  const plan = await planResources(project, resources, request);
+  await applyResourcePlan(
+    project,
+    plan,
+    rolloutTimeoutMs,
+    postApply,
+    request,
+    resumeOptions,
+  );
  return plan;
 }

+function resourceOperationTarget(resource: {
+  apiVersion?: string;
+  kind?: string;
+  name?: string;
+  namespace?: string;
+  metadata?: { name?: string; namespace?: string };
+}): ResourceOperationTarget | undefined {
+  const name = resource.name ?? resource.metadata?.name;
+  if (!resource.apiVersion || !resource.kind || !name) return;
+  return {
+    apiVersion: resource.apiVersion,
+    kind: resource.kind,
+    name,
+    namespace: resource.namespace ?? resource.metadata?.namespace,
+  };
+}
+
+function resourceOperationKey(resource: ResourceOperationTarget): string {
+  return `${resource.apiVersion}\0${resource.kind}\0${resource.namespace ?? ""}\0${resource.name}`;
+}
+
+function resourceOperationTaskTitle(
+  phase: ResourceOperationEvent["data"]["phase"],
+  resource: ResourceOperationTarget,
+): string {
+  const action =
+    phase === "apply" ? "Apply" : phase === "wait" ? "Wait for" : "Delete";
+  return `${action} ${resource.kind}/${resource.name}`;
+}
+
+export async function runLiveResourceOperation(
+  task: ListrTaskWrapper<any, any, any>,
+  phase: ResourceOperationEvent["data"]["phase"],
+  targets: ResourceOperationTarget[],
+  operation: (
+    onEvent: (event: ResourceOperationEvent) => void,
+  ) => Promise<void>,
+  options: LiveResourceOperationOptions = {},
+): Promise<void> {
+  if (targets.length === 0) return operation(() => {});
+  const active = new Map<
+    string,
+    ListrTaskWrapper<ResourceOperationTarget, any, any>
+  >();
+  const complete: Array<() => void> = [];
+  let started = 0;
+  let markStarted!: () => void;
+  const ready = new Promise<void>((resolve) => {
+    markStarted = resolve;
+  });
+  const children = task.newListr<ResourceOperationTarget>(
+    targets.map((target) => ({
+      title: resourceOperationTaskTitle(phase, target),
+      task: (_target, child) => {
+        active.set(resourceOperationKey(target), child);
+        options.onTaskStarted?.(target, child);
+        started += 1;
+        if (started === targets.length) markStarted();
+        return new Promise<void>((resolve) => complete.push(resolve));
+      },
+    })),
+  );
+  const childRun = children.run();
+  await ready;
+  try {
+    await operation((event) => {
+      const resource = event.data.resource;
+      const child = active.get(resourceOperationKey(resource));
+      if (!child) return;
+      child.output = event.data.state;
+      task.output = resourceOperationEventTitle(event);
+    });
+  } finally {
+    for (const resolve of complete) resolve();
+    await childRun;
+  }
+}
+
 export async function runUp(
  build: boolean,
  request: ApiRequester = apiRequest,
+  options: { trust?: TrustIdentity } = {},
 ) {
  const { project, compose, cwd, config, hookContext: getHookContext } = ctx();
-  const trusted = await requireLocalTrust(
-    await resolveTrustIdentity(project, cwd),
-  );
+  const trusted =
+    options.trust ??
+    (await requireLocalTrust(await resolveTrustIdentity(project, cwd)));
  const baseRequest = request;
  request = async <T>(
    path: string,
@@ -539,20 +770,104 @@ export async function runUp(
      {
        title: "Reconcile resources",
        task: async (taskCtx, task) => {
-          taskCtx.plan = await reconcileResources(
+          taskCtx.plan = await planResources(
            project,
            taskCtx.resources!,
-            config.rolloutTimeoutMs,
-            config.postApply
-              ? async (resources) =>
-                  config.postApply?.(
-                    resources as KuberResource[],
-                    await getHookContext(),
-                  )
-              : undefined,
            request,
          );
-          task.output = `${taskCtx.plan.desired.length} applied, ${taskCtx.plan.stale.length} stale deleted`;
+          const plan = taskCtx.plan;
+          const desired = plan.desired
+            .map(resourceOperationTarget)
+            .filter((resource): resource is ResourceOperationTarget =>
+              Boolean(resource),
+            );
+          const deployments = plan.desired
+            .filter((resource) => resource.kind === "Deployment")
+            .map(resourceOperationTarget)
+            .filter((resource): resource is ResourceOperationTarget =>
+              Boolean(resource),
+            );
+          const stale = plan.stale
+            .map(resourceOperationTarget)
+            .filter((resource): resource is ResourceOperationTarget =>
+              Boolean(resource),
+            );
+          const resourcePath = `${workspacePath}/resources`;
+          return task.newListr([
+            {
+              title: "Apply resources",
+              task: async (_ctx, phaseTask) =>
+                runLiveResourceOperation(
+                  phaseTask,
+                  "apply",
+                  desired,
+                  (onEvent) =>
+                    resumeManagedOperation(
+                      project,
+                      request,
+                      `${resourcePath}/apply`,
+                      { method: "POST", json: { resources: plan.desired } },
+                      config.rolloutTimeoutMs,
+                      { onEvent },
+                    ),
+                ),
+            },
+            {
+              title: "Run post-apply hook",
+              skip: !config.postApply,
+              task: async () =>
+                config.postApply?.(
+                  plan.desired as KuberResource[],
+                  await getHookContext(),
+                ),
+            },
+            {
+              title: "Wait for deployments",
+              skip: deployments.length === 0,
+              task: async (_ctx, phaseTask) =>
+                runLiveResourceOperation(
+                  phaseTask,
+                  "wait",
+                  deployments,
+                  (onEvent) =>
+                    resumeManagedOperation(
+                      project,
+                      request,
+                      `${resourcePath}/wait`,
+                      {
+                        method: "POST",
+                        json: {
+                          deployments: deployments.map(
+                            (deployment) => deployment.name,
+                          ),
+                          timeoutMs: config.rolloutTimeoutMs,
+                        },
+                      },
+                      config.rolloutTimeoutMs,
+                      { onEvent },
+                    ),
+                ),
+            },
+            {
+              title: "Delete stale resources",
+              skip: stale.length === 0,
+              task: async (_ctx, phaseTask) =>
+                runLiveResourceOperation(
+                  phaseTask,
+                  "delete",
+                  stale,
+                  (onEvent) =>
+                    resumeManagedOperation(
+                      project,
+                      request,
+                      `${resourcePath}/delete`,
+                      { method: "POST", json: { resources: plan.stale } },
+                      config.rolloutTimeoutMs,
+                      { onEvent },
+                    ),
+                ),
+            },
+          ]);
        },
      },
    ],