kuber/lib/rollout.ts

import type {
  V1Deployment,
  V1Pod,
  V1ReplicaSet,
  V1ContainerStatus,
} from "@kubernetes/client-node";

const safe = (value: string | undefined) =>
  (value ?? "unknown").replace(/[^a-zA-Z0-9._/-]/g, "_").slice(0, 100);

function containsTemplate(actual: unknown, desired: unknown): boolean {
  if (Array.isArray(desired))
    return (
      Array.isArray(actual) &&
      actual.length === desired.length &&
      desired.every((value, index) => containsTemplate(actual[index], value))
    );
  if (desired && typeof desired === "object")
    return (
      Object.keys(desired).length === 0 ||
      Boolean(
        actual &&
        typeof actual === "object" &&
        Object.entries(desired).every(([key, value]) =>
          containsTemplate((actual as Record<string, unknown>)[key], value),
        ),
      )
    );
  return actual === desired;
}

export function rolloutReady(deployment: V1Deployment): boolean {
  const desired = deployment.spec?.replicas ?? 1;
  return (
    (deployment.status?.observedGeneration ?? 0) >=
      (deployment.metadata?.generation ?? 0) &&
    (deployment.status?.updatedReplicas ?? 0) === desired &&
    (deployment.status?.availableReplicas ?? 0) === desired &&
    (deployment.status?.unavailableReplicas ?? 0) === 0
  );
}

export function rolloutConditionFailure(
  deployment: V1Deployment,
): string | undefined {
  if (
    (deployment.status?.observedGeneration ?? 0) <
    (deployment.metadata?.generation ?? 0)
  )
    return;
  const condition = deployment.status?.conditions?.find(
    (item) =>
      item.type === "Progressing" &&
      item.status === "False" &&
      item.reason === "ProgressDeadlineExceeded",
  );
  if (condition)
    return `Deployment ${safe(deployment.metadata?.name)} rollout failed: ${safe(condition.reason)}`;
}

// An older ReplicaSet may still have crashing pods while its replacement is being
// created. Require both Deployment ownership and a match against the desired template.
export function currentReplicaSet(
  deployment: V1Deployment,
  sets: V1ReplicaSet[],
): V1ReplicaSet | undefined {
  const uid = deployment.metadata?.uid;
  const template = deployment.spec?.template;
  if (!uid || !template) return;
  const matches = sets.filter((set) => {
    if (
      !set.metadata?.ownerReferences?.some(
        (owner) => owner.kind === "Deployment" && owner.uid === uid,
      )
    )
      return false;
    const actual = set.spec?.template;
    if (!actual) return false;
    return (
      containsTemplate(
        actual.metadata?.labels,
        template.metadata?.labels ?? {},
      ) &&
      containsTemplate(
        actual.metadata?.annotations,
        template.metadata?.annotations ?? {},
      ) &&
      containsTemplate(actual.spec, template.spec)
    );
  });
  return matches.sort(
    (a, b) =>
      Number(
        b.metadata?.annotations?.["deployment.kubernetes.io/revision"] ?? 0,
      ) -
      Number(
        a.metadata?.annotations?.["deployment.kubernetes.io/revision"] ?? 0,
      ),
  )[0];
}

const failureReasons = new Set([
  "CrashLoopBackOff",
  "ImagePullBackOff",
  "ErrImagePull",
  "InvalidImageName",
  "CreateContainerConfigError",
  "CreateContainerError",
  "RunContainerError",
  "CreateContainerRuntimeError",
  "ErrImageNeverPull",
  "OOMKilled",
]);

function containerFailure(status: V1ContainerStatus): string | undefined {
  const waiting = status.state?.waiting;
  if (waiting?.reason && failureReasons.has(waiting.reason))
    return waiting.reason;
  const terminated = status.state?.terminated ?? status.lastState?.terminated;
  if (terminated?.reason === "OOMKilled") return "OOMKilled";
  if (
    terminated &&
    terminated.exitCode !== 0 &&
    (status.restartCount ?? 0) >= 2
  )
    return terminated.reason || "RepeatedNonzeroExit";
}

export function rolloutPodFailure(
  deployment: V1Deployment,
  replicaSet: V1ReplicaSet,
  pods: V1Pod[],
): string | undefined {
  const uid = replicaSet.metadata?.uid;
  if (!uid) return;
  for (const pod of pods) {
    if (
      !pod.metadata?.ownerReferences?.some(
        (owner) => owner.kind === "ReplicaSet" && owner.uid === uid,
      )
    )
      continue;
    for (const status of [
      ...(pod.status?.initContainerStatuses ?? []),
      ...(pod.status?.containerStatuses ?? []),
    ]) {
      const reason = containerFailure(status);
      if (reason)
        return `Deployment ${safe(deployment.metadata?.name)} rollout failed: pod ${safe(pod.metadata?.name)}, container ${safe(status.name)}: ${safe(reason)} (exit ${status.state?.terminated?.exitCode ?? status.lastState?.terminated?.exitCode ?? "unknown"}, restarts ${status.restartCount ?? 0})`;
    }
  }
}

export async function waitForRollout(
  name: string,
  deadline: number,
  read: () => Promise<V1Deployment>,
  listSets: () => Promise<V1ReplicaSet[]>,
  listPods: () => Promise<V1Pod[]>,
  signal?: AbortSignal,
): Promise<void> {
  const cancelled = () =>
    new Error("Workspace operation execution was cancelled");
  const timedOut = () =>
    new Error(`Timed out waiting for deployment ${safe(name)} rollout`);
  async function bounded<T>(operation: () => Promise<T>): Promise<T> {
    if (signal?.aborted) throw cancelled();
    const remaining = deadline - Date.now();
    if (remaining <= 0) throw timedOut();
    return new Promise<T>((resolve, reject) => {
      const timer = setTimeout(
        () => finish(() => reject(timedOut())),
        remaining,
      );
      const abort = () => finish(() => reject(cancelled()));
      const finish = (settle: () => void) => {
        clearTimeout(timer);
        signal?.removeEventListener("abort", abort);
        settle();
      };
      signal?.addEventListener("abort", abort, { once: true });
      if (signal?.aborted) return abort();
      Promise.resolve()
        .then(operation)
        .then(
          (value) => finish(() => resolve(value)),
          (error) => finish(() => reject(error)),
        );
    });
  }
  while (true) {
    const deployment = await bounded(read);
    const failure = rolloutConditionFailure(deployment);
    if (failure) throw new Error(failure);
    if (rolloutReady(deployment)) return;
    const current = currentReplicaSet(deployment, await bounded(listSets));
    if (current) {
      const podFailure = rolloutPodFailure(
        deployment,
        current,
        await bounded(listPods),
      );
      if (podFailure) throw new Error(podFailure);
    }
    await bounded(
      () =>
        new Promise<void>((resolve) =>
          setTimeout(resolve, Math.min(2_000, deadline - Date.now())),
        ),
    );
  }
}