import type { V1Deployment, V1Pod, V1ReplicaSet, V1ContainerStatus, } from "@kubernetes/client-node"; const safe = (value: string | undefined) => (value ?? "unknown").replace(/[^a-zA-Z0-9._/-]/g, "_").slice(0, 100); function containsTemplate(actual: unknown, desired: unknown): boolean { if (Array.isArray(desired)) return ( Array.isArray(actual) && actual.length === desired.length && desired.every((value, index) => containsTemplate(actual[index], value)) ); if (desired && typeof desired === "object") return ( Object.keys(desired).length === 0 || Boolean( actual && typeof actual === "object" && Object.entries(desired).every(([key, value]) => containsTemplate((actual as Record)[key], value), ), ) ); return actual === desired; } export function rolloutReady(deployment: V1Deployment): boolean { const desired = deployment.spec?.replicas ?? 1; return ( (deployment.status?.observedGeneration ?? 0) >= (deployment.metadata?.generation ?? 0) && (deployment.status?.updatedReplicas ?? 0) === desired && (deployment.status?.availableReplicas ?? 0) === desired && (deployment.status?.unavailableReplicas ?? 0) === 0 ); } export function rolloutConditionFailure( deployment: V1Deployment, ): string | undefined { if ( (deployment.status?.observedGeneration ?? 0) < (deployment.metadata?.generation ?? 0) ) return; const condition = deployment.status?.conditions?.find( (item) => item.type === "Progressing" && item.status === "False" && item.reason === "ProgressDeadlineExceeded", ); if (condition) return `Deployment ${safe(deployment.metadata?.name)} rollout failed: ${safe(condition.reason)}`; } // An older ReplicaSet may still have crashing pods while its replacement is being // created. Require both Deployment ownership and a match against the desired template. export function currentReplicaSet( deployment: V1Deployment, sets: V1ReplicaSet[], ): V1ReplicaSet | undefined { const uid = deployment.metadata?.uid; const template = deployment.spec?.template; if (!uid || !template) return; const matches = sets.filter((set) => { if ( !set.metadata?.ownerReferences?.some( (owner) => owner.kind === "Deployment" && owner.uid === uid, ) ) return false; const actual = set.spec?.template; if (!actual) return false; return ( containsTemplate( actual.metadata?.labels, template.metadata?.labels ?? {}, ) && containsTemplate( actual.metadata?.annotations, template.metadata?.annotations ?? {}, ) && containsTemplate(actual.spec, template.spec) ); }); return matches.sort( (a, b) => Number( b.metadata?.annotations?.["deployment.kubernetes.io/revision"] ?? 0, ) - Number( a.metadata?.annotations?.["deployment.kubernetes.io/revision"] ?? 0, ), )[0]; } const failureReasons = new Set([ "CrashLoopBackOff", "ImagePullBackOff", "ErrImagePull", "InvalidImageName", "CreateContainerConfigError", "CreateContainerError", "RunContainerError", "CreateContainerRuntimeError", "ErrImageNeverPull", "OOMKilled", ]); function containerFailure(status: V1ContainerStatus): string | undefined { const waiting = status.state?.waiting; if (waiting?.reason && failureReasons.has(waiting.reason)) return waiting.reason; const terminated = status.state?.terminated ?? status.lastState?.terminated; if (terminated?.reason === "OOMKilled") return "OOMKilled"; if ( terminated && terminated.exitCode !== 0 && (status.restartCount ?? 0) >= 2 ) return terminated.reason || "RepeatedNonzeroExit"; } export function rolloutPodFailure( deployment: V1Deployment, replicaSet: V1ReplicaSet, pods: V1Pod[], ): string | undefined { const uid = replicaSet.metadata?.uid; if (!uid) return; for (const pod of pods) { if ( !pod.metadata?.ownerReferences?.some( (owner) => owner.kind === "ReplicaSet" && owner.uid === uid, ) ) continue; for (const status of [ ...(pod.status?.initContainerStatuses ?? []), ...(pod.status?.containerStatuses ?? []), ]) { const reason = containerFailure(status); if (reason) return `Deployment ${safe(deployment.metadata?.name)} rollout failed: pod ${safe(pod.metadata?.name)}, container ${safe(status.name)}: ${safe(reason)} (exit ${status.state?.terminated?.exitCode ?? status.lastState?.terminated?.exitCode ?? "unknown"}, restarts ${status.restartCount ?? 0})`; } } } export async function waitForRollout( name: string, deadline: number, read: () => Promise, listSets: () => Promise, listPods: () => Promise, signal?: AbortSignal, ): Promise { const cancelled = () => new Error("Workspace operation execution was cancelled"); const timedOut = () => new Error(`Timed out waiting for deployment ${safe(name)} rollout`); async function bounded(operation: () => Promise): Promise { if (signal?.aborted) throw cancelled(); const remaining = deadline - Date.now(); if (remaining <= 0) throw timedOut(); return new Promise((resolve, reject) => { const timer = setTimeout( () => finish(() => reject(timedOut())), remaining, ); const abort = () => finish(() => reject(cancelled())); const finish = (settle: () => void) => { clearTimeout(timer); signal?.removeEventListener("abort", abort); settle(); }; signal?.addEventListener("abort", abort, { once: true }); if (signal?.aborted) return abort(); Promise.resolve() .then(operation) .then( (value) => finish(() => resolve(value)), (error) => finish(() => reject(error)), ); }); } while (true) { const deployment = await bounded(read); const failure = rolloutConditionFailure(deployment); if (failure) throw new Error(failure); if (rolloutReady(deployment)) return; const current = currentReplicaSet(deployment, await bounded(listSets)); if (current) { const podFailure = rolloutPodFailure( deployment, current, await bounded(listPods), ); if (podFailure) throw new Error(podFailure); } await bounded( () => new Promise((resolve) => setTimeout(resolve, Math.min(2_000, deadline - Date.now())), ), ); } }