feat: harden self-managed reconciliation

2026-09-05 10:17:55 +00:00
parent 8e9d207915
commit 321f4e807a
40 changed files with 4977 additions and 404 deletions
@@ -1,8 +1,14 @@
 import { defineCommand } from "citty";
 import { Listr } from "listr2";
+import { randomUUID } from "node:crypto";
 import type { ComposeSpecification } from "../schema/docker.d";
 import type { KuberResource } from "../types";
-import { apiRequest, KuberApiError, type ApiRequestInit } from "../lib/api";
+import {
+  apiRequest,
+  KuberApiError,
+  type ApiRequestInit,
+  type ApiRequestOptions,
+} from "../lib/api";
 import {
  buildServices,
  getRepoRoot,
@@ -14,6 +20,11 @@ import { ctx } from "../lib/context";
 import { getComposePostgresClaims } from "../lib/database";
 import { getComposeS3Claims } from "../lib/storage";
 import { enumerateWorkspace, type WorkspaceSnapshot } from "../lib/workspace";
+import {
+  requireLocalTrust,
+  resolveTrustIdentity,
+  trustHeaders,
+} from "../lib/trust";

 type Workspace = {
  metadata: { name: string; uid: string; resourceVersion: string };
@@ -33,6 +44,21 @@ type ResourcePlan = {
  stale: ResourceIdentity[];
 };

+type OperationStatus = {
+  state?: unknown;
+  error?: { code?: unknown; message?: unknown };
+};
+
+type OperationResponse = {
+  operationId?: unknown;
+  operation?: { status?: OperationStatus };
+};
+
+export type OperationResumeOptions = {
+  now?: () => number;
+  sleep?: (milliseconds: number) => Promise<void>;
+};
+
 type UpContext = {
  compose?: ComposeSpecification;
  snapshot?: WorkspaceSnapshot;
@@ -43,6 +69,14 @@ type UpContext = {
 };

 export const WORKSPACE_ADOPTION_METHOD = "POST";
+const OPERATION_RESUME_INITIAL_BACKOFF_MS = 250;
+const OPERATION_RESUME_MAX_BACKOFF_MS = 5_000;
+const OPERATION_RESUME_GRACE_MS = 60_000;
+const RECOVERABLE_API_ERROR_CODES = new Set([
+  "HTTP_502",
+  "HTTP_503",
+  "HTTP_504",
+]);

 export function workspaceAdoptionRoute(project: string): string {
  return `/workspaces/${encodeURIComponent(project)}/adopt`;
@@ -141,6 +175,150 @@ async function managementRequest<T>(
  }
 }

+function operationIdFromResponse(response: unknown): string | undefined {
+  if (!response || typeof response !== "object") return;
+  const operationId = (response as OperationResponse).operationId;
+  return typeof operationId === "string" && operationId
+    ? operationId
+    : undefined;
+}
+
+function operationStatusFromResponse(
+  response: unknown,
+): OperationStatus | undefined {
+  if (!response || typeof response !== "object") return;
+  const status = (response as OperationResponse).operation?.status;
+  return status && typeof status === "object" ? status : undefined;
+}
+
+function operationFailure(
+  operationId: string,
+  status: OperationStatus,
+): KuberApiError {
+  const error = status.error;
+  const cancelled = status.state === "cancelled";
+  const message =
+    typeof error?.message === "string"
+      ? error.message
+      : cancelled
+        ? "The operation was cancelled"
+        : "The operation failed";
+  const code =
+    typeof error?.code === "string"
+      ? error.code
+      : cancelled
+        ? "OPERATION_CANCELLED"
+        : "OPERATION_FAILED";
+  return new KuberApiError(message, cancelled ? 409 : 500, {
+    title: cancelled ? "Operation cancelled" : "Operation failed",
+    status: cancelled ? 409 : 500,
+    code,
+    operationId,
+  });
+}
+
+function isRecoverableConnectionInterruption(error: unknown): boolean {
+  if (error instanceof KuberApiError)
+    return RECOVERABLE_API_ERROR_CODES.has(error.code);
+  if (error instanceof DOMException && error.name === "AbortError")
+    return false;
+  if (error instanceof Error && error.name === "AbortError") return false;
+  return (
+    error instanceof TypeError ||
+    (error instanceof Error && error.name === "TimeoutError")
+  );
+}
+
+function isInterruptedOperation(status: OperationStatus): boolean {
+  return (
+    status.state === "failed" && status.error?.code === "OPERATION_INTERRUPTED"
+  );
+}
+
+function operationResumeDeadline(
+  rolloutTimeoutMs: number,
+  now: number,
+): number {
+  return now + Math.max(rolloutTimeoutMs, 0) + OPERATION_RESUME_GRACE_MS;
+}
+
+async function resumeManagedOperation(
+  project: string,
+  request: ApiRequester,
+  path: string,
+  init: ApiRequestInit,
+  rolloutTimeoutMs: number,
+  options: OperationResumeOptions,
+): Promise<void> {
+  const now = options.now ?? Date.now;
+  const sleep = options.sleep ?? ((milliseconds) => Bun.sleep(milliseconds));
+  const deadline = operationResumeDeadline(rolloutTimeoutMs, now());
+  const headers = new Headers(init.headers);
+  headers.set("idempotency-key", randomUUID());
+  let operationInit = { ...init, headers };
+  let operationId: string | undefined;
+  let backoffMs = OPERATION_RESUME_INITIAL_BACKOFF_MS;
+  let restartRetryPending = false;
+
+  for (;;) {
+    if (restartRetryPending && now() >= deadline)
+      throw new Error("Timed out while reconnecting to resume the operation");
+    try {
+      let status: OperationStatus | undefined;
+      if (operationId) {
+        const operation = await managementRequest<{ status: OperationStatus }>(
+          project,
+          request,
+          `/operations/${encodeURIComponent(operationId)}`,
+          {},
+        );
+        status = operation.status;
+      } else {
+        restartRetryPending = false;
+        const response = await managementRequest<OperationResponse>(
+          project,
+          request,
+          path,
+          operationInit,
+        );
+        operationId = operationIdFromResponse(response);
+        status = operationStatusFromResponse(response);
+      }
+
+      if (!operationId || !status || status.state === "succeeded") return;
+      if (isInterruptedOperation(status)) {
+        if (now() >= deadline) throw operationFailure(operationId, status);
+        operationId = undefined;
+        restartRetryPending = true;
+        headers.set("idempotency-key", randomUUID());
+        operationInit = { ...init, headers };
+      } else if (status.state === "failed" || status.state === "cancelled")
+        throw operationFailure(operationId, status);
+    } catch (error) {
+      if (
+        error instanceof KuberApiError &&
+        error.code === "OPERATION_INTERRUPTED"
+      ) {
+        if (now() >= deadline) throw adoptionHint(project, error);
+        operationId = undefined;
+        restartRetryPending = true;
+        headers.set("idempotency-key", randomUUID());
+        operationInit = { ...init, headers };
+      } else {
+        if (!isRecoverableConnectionInterruption(error))
+          throw adoptionHint(project, error);
+        if (now() >= deadline) throw adoptionHint(project, error);
+      }
+    }
+
+    const remainingMs = deadline - now();
+    if (remainingMs <= 0)
+      throw new Error("Timed out while reconnecting to resume the operation");
+    await sleep(Math.min(backoffMs, remainingMs));
+    backoffMs = Math.min(backoffMs * 2, OPERATION_RESUME_MAX_BACKOFF_MS);
+  }
+}
+
 export async function reconcileResources(
  project: string,
  resources: KubernetesResource[],
@@ -149,6 +327,7 @@ export async function reconcileResources(
    | ((resources: KubernetesResource[]) => void | Promise<void>)
    | undefined,
  request: ApiRequester = apiRequest,
+  resumeOptions: OperationResumeOptions = {},
 ): Promise<ResourcePlan> {
  const workspacePath = `/workspaces/${encodeURIComponent(project)}`;
  const plan = await managementRequest<ResourcePlan>(
@@ -157,7 +336,7 @@ export async function reconcileResources(
    `${workspacePath}/resources/plan`,
    { method: "POST", json: { resources } },
  );
-  await managementRequest(
+  await resumeManagedOperation(
    project,
    request,
    `${workspacePath}/resources/apply`,
@@ -165,12 +344,14 @@ export async function reconcileResources(
      method: "POST",
      json: { resources: plan.desired },
    },
+    rolloutTimeoutMs,
+    resumeOptions,
  );
  await postApply?.(plan.desired);

  const deployments = getDeploymentNames(plan.desired);
  if (deployments.length > 0) {
-    await managementRequest(
+    await resumeManagedOperation(
      project,
      request,
      `${workspacePath}/resources/wait`,
@@ -178,10 +359,12 @@ export async function reconcileResources(
        method: "POST",
        json: { deployments, timeoutMs: rolloutTimeoutMs },
      },
+      rolloutTimeoutMs,
+      resumeOptions,
    );
  }
  if (plan.stale.length > 0) {
-    await managementRequest(
+    await resumeManagedOperation(
      project,
      request,
      `${workspacePath}/resources/delete`,
@@ -189,6 +372,8 @@ export async function reconcileResources(
        method: "POST",
        json: { resources: plan.stale },
      },
+      rolloutTimeoutMs,
+      resumeOptions,
    );
  }
  return plan;
@@ -199,6 +384,20 @@ export async function runUp(
  request: ApiRequester = apiRequest,
 ) {
  const { project, compose, cwd, config, hookContext: getHookContext } = ctx();
+  const trusted = await requireLocalTrust(
+    await resolveTrustIdentity(project, cwd),
+  );
+  const baseRequest = request;
+  request = async <T>(
+    path: string,
+    init: ApiRequestInit = {},
+    options?: ApiRequestOptions,
+  ) => {
+    const headers = new Headers(init.headers);
+    for (const [key, value] of Object.entries(trustHeaders(trusted)))
+      headers.set(key, value);
+    return baseRequest<T>(path, { ...init, headers }, options);
+  };
  const workspacePath = `/workspaces/${encodeURIComponent(project)}`;

  const taskCtx = await new Listr<UpContext>(