1192 lines
38 KiB
TypeScript
1192 lines
38 KiB
TypeScript
import { defineCommand } from "citty";
|
|
import { Listr, type ListrTaskWrapper } from "listr2";
|
|
import { randomUUID } from "node:crypto";
|
|
import { isDeepStrictEqual } from "node:util";
|
|
import type { ComposeSpecification } from "../schema/docker.d";
|
|
import type { KuberResource } from "../types";
|
|
import {
|
|
apiRequest,
|
|
KuberApiError,
|
|
type ApiRequestInit,
|
|
type ApiRequestOptions,
|
|
} from "../lib/api";
|
|
import {
|
|
buildServices,
|
|
getRepoRoot,
|
|
resolveBuildImages,
|
|
type ApiRequester,
|
|
} from "../lib/build";
|
|
import { composeToKubernetes, type KubernetesResource } from "../lib/convert";
|
|
import { ctx } from "../lib/context";
|
|
import { getComposePostgresClaims } from "../lib/database";
|
|
import { getComposeS3Claims } from "../lib/storage";
|
|
import { enumerateWorkspace, type WorkspaceSnapshot } from "../lib/workspace";
|
|
import {
|
|
requireLocalTrust,
|
|
resolveTrustIdentity,
|
|
trustHeaders,
|
|
type TrustIdentity,
|
|
} from "../lib/trust";
|
|
|
|
type Workspace = {
|
|
metadata: { name: string; uid: string; resourceVersion: string };
|
|
spec?: { source?: unknown; config?: unknown };
|
|
};
|
|
|
|
type ResourceIdentity = {
|
|
apiVersion: string;
|
|
kind: string;
|
|
name: string;
|
|
namespace?: string;
|
|
uid: string;
|
|
workspaceUid: string;
|
|
};
|
|
|
|
type ResourcePlan = {
|
|
desired: KubernetesResource[];
|
|
stale: ResourceIdentity[];
|
|
};
|
|
|
|
type OperationStatus = {
|
|
state?: unknown;
|
|
error?: { code?: unknown; message?: unknown };
|
|
};
|
|
|
|
type OperationResponse = {
|
|
operationId?: unknown;
|
|
operation?: { status?: OperationStatus };
|
|
};
|
|
|
|
export type ResourceOperationEvent = {
|
|
sequence: number;
|
|
data: {
|
|
resource: {
|
|
apiVersion: string;
|
|
kind: string;
|
|
name: string;
|
|
namespace?: string;
|
|
};
|
|
phase: "apply" | "wait" | "delete";
|
|
state: "started" | "succeeded" | "failed" | "aborted";
|
|
};
|
|
};
|
|
|
|
export type OperationResumeOptions = {
|
|
now?: () => number;
|
|
sleep?: (milliseconds: number) => Promise<void>;
|
|
onEvent?: (event: ResourceOperationEvent) => void;
|
|
signal?: AbortSignal;
|
|
};
|
|
|
|
type OperationEventsResponse = {
|
|
items?: ResourceOperationEvent[];
|
|
retainedFirstSequence?: unknown;
|
|
cursorGap?: unknown;
|
|
};
|
|
|
|
type ResourceOperationTarget = {
|
|
apiVersion: string;
|
|
kind: string;
|
|
name: string;
|
|
namespace?: string;
|
|
};
|
|
|
|
export type LiveResourceOperationOptions = {
|
|
onTaskStarted?: (
|
|
target: ResourceOperationTarget,
|
|
task: ListrTaskWrapper<ResourceOperationTarget, any, any>,
|
|
) => void;
|
|
};
|
|
|
|
type UpContext = {
|
|
compose?: ComposeSpecification;
|
|
snapshot?: WorkspaceSnapshot;
|
|
workspaceRoot?: string;
|
|
buildImages?: Record<string, string>;
|
|
serviceEnv?: Record<string, Record<string, string>>;
|
|
databaseEnv?: Record<string, Record<string, string>>;
|
|
storageEnv?: Record<string, Record<string, string>>;
|
|
resources?: KubernetesResource[];
|
|
plan?: ResourcePlan;
|
|
};
|
|
|
|
export const WORKSPACE_ADOPTION_METHOD = "POST";
|
|
const OPERATION_RESUME_INITIAL_BACKOFF_MS = 250;
|
|
const OPERATION_RESUME_MAX_BACKOFF_MS = 5_000;
|
|
const OPERATION_RESUME_GRACE_MS = 60_000;
|
|
const RESOURCE_PLAN_TIMEOUT_MS = 120_000;
|
|
const RECOVERABLE_API_ERROR_CODES = new Set([
|
|
"HTTP_502",
|
|
"HTTP_503",
|
|
"HTTP_504",
|
|
]);
|
|
|
|
class OperationProgressCursorGapError extends Error {
|
|
constructor() {
|
|
super(
|
|
"Operation progress history was truncated; refusing to report an incomplete resource stream",
|
|
);
|
|
}
|
|
}
|
|
|
|
export function workspaceAdoptionRoute(project: string): string {
|
|
return `/workspaces/${encodeURIComponent(project)}/adopt`;
|
|
}
|
|
|
|
export function mergeServiceEnv(
|
|
current: Record<string, Record<string, string>> | undefined,
|
|
next: Record<string, Record<string, string>>,
|
|
): Record<string, Record<string, string>> {
|
|
const merged = { ...current };
|
|
for (const [service, environment] of Object.entries(next))
|
|
merged[service] = { ...merged[service], ...environment };
|
|
return merged;
|
|
}
|
|
|
|
export function getDeploymentNames(resources: KubernetesResource[]): string[] {
|
|
return resources
|
|
.filter((resource) => resource.kind === "Deployment")
|
|
.map((resource) => resource.metadata?.name)
|
|
.filter((name): name is string => Boolean(name));
|
|
}
|
|
|
|
export function resourceOperationEventTitle(
|
|
event: ResourceOperationEvent,
|
|
): string {
|
|
const { resource, phase, state } = event.data;
|
|
return `${phase === "apply" ? "Applied" : phase === "wait" ? "Waited for" : "Deleted"} ${resource.kind}/${resource.name}: ${state}`;
|
|
}
|
|
|
|
function isResourceOperationEvent(
|
|
event: unknown,
|
|
): event is ResourceOperationEvent {
|
|
if (!event || typeof event !== "object") return false;
|
|
const { sequence, data } = event as {
|
|
sequence?: unknown;
|
|
data?: unknown;
|
|
};
|
|
if (!Number.isSafeInteger(sequence) || !data || typeof data !== "object")
|
|
return false;
|
|
const { resource, phase, state } = data as {
|
|
resource?: unknown;
|
|
phase?: unknown;
|
|
state?: unknown;
|
|
};
|
|
if (!resource || typeof resource !== "object") return false;
|
|
const identity = resource as Record<string, unknown>;
|
|
return (
|
|
typeof identity.apiVersion === "string" &&
|
|
typeof identity.kind === "string" &&
|
|
typeof identity.name === "string" &&
|
|
(identity.namespace === undefined ||
|
|
typeof identity.namespace === "string") &&
|
|
(phase === "apply" || phase === "wait" || phase === "delete") &&
|
|
(state === "started" ||
|
|
state === "succeeded" ||
|
|
state === "failed" ||
|
|
state === "aborted")
|
|
);
|
|
}
|
|
|
|
function workspaceInput(
|
|
compose: ComposeSpecification,
|
|
snapshot: WorkspaceSnapshot,
|
|
) {
|
|
return {
|
|
source: {
|
|
uri: `cas://${snapshot.digest}`,
|
|
digest: snapshot.digest,
|
|
},
|
|
config: { compose },
|
|
};
|
|
}
|
|
|
|
function workspaceMatchesInput(
|
|
workspace: Workspace,
|
|
input: ReturnType<typeof workspaceInput>,
|
|
): boolean {
|
|
return (
|
|
isDeepStrictEqual(workspace.spec?.source, input.source) &&
|
|
isDeepStrictEqual(workspace.spec?.config, input.config)
|
|
);
|
|
}
|
|
|
|
function isWorkspaceUpdateAmbiguous(error: unknown): boolean {
|
|
if (error instanceof KuberApiError) return error.status === 409;
|
|
if (error instanceof DOMException && error.name === "AbortError")
|
|
return false;
|
|
if (error instanceof Error && error.name === "AbortError") return false;
|
|
return (
|
|
error instanceof TypeError ||
|
|
(error instanceof Error && error.name === "TimeoutError")
|
|
);
|
|
}
|
|
|
|
export async function ensureWorkspace(
|
|
project: string,
|
|
compose: ComposeSpecification,
|
|
snapshot: WorkspaceSnapshot,
|
|
request: ApiRequester = apiRequest,
|
|
): Promise<Workspace> {
|
|
const path = `/workspaces/${encodeURIComponent(project)}`;
|
|
const input = workspaceInput(compose, snapshot);
|
|
const update = (workspace: Workspace) =>
|
|
request<Workspace>(path, {
|
|
method: "PUT",
|
|
headers: { "if-match": `"${workspace.metadata.resourceVersion}"` },
|
|
json: input,
|
|
});
|
|
const recover = async (originalError: unknown): Promise<Workspace> => {
|
|
try {
|
|
const fresh = await request<Workspace>(path);
|
|
if (workspaceMatchesInput(fresh, input)) return fresh;
|
|
return await update(fresh);
|
|
} catch {
|
|
throw originalError;
|
|
}
|
|
};
|
|
let current: Workspace;
|
|
try {
|
|
current = await request<Workspace>(path);
|
|
} catch (error) {
|
|
if (!(error instanceof KuberApiError) || error.status !== 404) throw error;
|
|
try {
|
|
return await request<Workspace>("/workspaces", {
|
|
method: "POST",
|
|
json: { id: project, ...input },
|
|
});
|
|
} catch (createError) {
|
|
if (!isWorkspaceUpdateAmbiguous(createError)) throw createError;
|
|
return recover(createError);
|
|
}
|
|
}
|
|
try {
|
|
return await update(current);
|
|
} catch (updateError) {
|
|
if (!isWorkspaceUpdateAmbiguous(updateError)) throw updateError;
|
|
return recover(updateError);
|
|
}
|
|
}
|
|
|
|
function operationEnvironment(
|
|
response: Record<string, unknown>,
|
|
): Record<string, Record<string, string>> {
|
|
return Object.fromEntries(
|
|
Object.entries(response).filter(
|
|
([key, value]) =>
|
|
key !== "operation" &&
|
|
key !== "operationId" &&
|
|
value !== null &&
|
|
typeof value === "object" &&
|
|
!Array.isArray(value),
|
|
),
|
|
) as Record<string, Record<string, string>>;
|
|
}
|
|
|
|
function adoptionHint(project: string, error: unknown): Error {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
if (!/namespace .* (?:external|different-workspace)/i.test(message))
|
|
return error instanceof Error ? error : new Error(message);
|
|
return new Error(
|
|
`${message}. Safe adoption requires ${WORKSPACE_ADOPTION_METHOD} ${workspaceAdoptionRoute(project)} with namespace UID and ownership preconditions.`,
|
|
{ cause: error },
|
|
);
|
|
}
|
|
|
|
async function managementRequest<T>(
|
|
project: string,
|
|
request: ApiRequester,
|
|
path: string,
|
|
init: ApiRequestInit,
|
|
options?: ApiRequestOptions,
|
|
): Promise<T> {
|
|
try {
|
|
return await request<T>(path, init, options);
|
|
} catch (error) {
|
|
throw adoptionHint(project, error);
|
|
}
|
|
}
|
|
|
|
function operationIdFromResponse(response: unknown): string | undefined {
|
|
if (!response || typeof response !== "object") return;
|
|
const operationId = (response as OperationResponse).operationId;
|
|
return typeof operationId === "string" && operationId
|
|
? operationId
|
|
: undefined;
|
|
}
|
|
|
|
function operationStatusFromResponse(
|
|
response: unknown,
|
|
): OperationStatus | undefined {
|
|
if (!response || typeof response !== "object") return;
|
|
const status = (response as OperationResponse).operation?.status;
|
|
return status && typeof status === "object" ? status : undefined;
|
|
}
|
|
|
|
function operationFailure(
|
|
operationId: string,
|
|
status: OperationStatus,
|
|
): KuberApiError {
|
|
const error = status.error;
|
|
const cancelled = status.state === "cancelled";
|
|
const message =
|
|
typeof error?.message === "string"
|
|
? error.message
|
|
: cancelled
|
|
? "The operation was cancelled"
|
|
: "The operation failed";
|
|
const code =
|
|
typeof error?.code === "string"
|
|
? error.code
|
|
: cancelled
|
|
? "OPERATION_CANCELLED"
|
|
: "OPERATION_FAILED";
|
|
return new KuberApiError(message, cancelled ? 409 : 500, {
|
|
title: cancelled ? "Operation cancelled" : "Operation failed",
|
|
status: cancelled ? 409 : 500,
|
|
code,
|
|
operationId,
|
|
});
|
|
}
|
|
|
|
function isRecoverableConnectionInterruption(error: unknown): boolean {
|
|
if (error instanceof KuberApiError)
|
|
return RECOVERABLE_API_ERROR_CODES.has(error.code);
|
|
if (error instanceof DOMException && error.name === "AbortError")
|
|
return false;
|
|
if (error instanceof Error && error.name === "AbortError") return false;
|
|
return (
|
|
error instanceof TypeError ||
|
|
(error instanceof Error && error.name === "TimeoutError")
|
|
);
|
|
}
|
|
|
|
function isInterruptedOperation(status: OperationStatus): boolean {
|
|
return (
|
|
status.state === "failed" && status.error?.code === "OPERATION_INTERRUPTED"
|
|
);
|
|
}
|
|
|
|
function hasOperationEventCursorGap(
|
|
events: OperationEventsResponse,
|
|
after: number,
|
|
): boolean {
|
|
return (
|
|
events.cursorGap === true ||
|
|
(Number.isSafeInteger(events.retainedFirstSequence) &&
|
|
(events.retainedFirstSequence as number) > after + 1)
|
|
);
|
|
}
|
|
|
|
function operationResumeDeadline(
|
|
rolloutTimeoutMs: number,
|
|
now: number,
|
|
): number {
|
|
return now + Math.max(rolloutTimeoutMs, 0) + OPERATION_RESUME_GRACE_MS;
|
|
}
|
|
|
|
function abortReason(signal: AbortSignal): unknown {
|
|
return (
|
|
signal.reason ?? new DOMException("The operation was aborted", "AbortError")
|
|
);
|
|
}
|
|
|
|
function throwIfAborted(signal: AbortSignal | undefined): void {
|
|
if (signal?.aborted) throw abortReason(signal);
|
|
}
|
|
|
|
function abortableDelay(
|
|
milliseconds: number,
|
|
signal: AbortSignal | undefined,
|
|
): Promise<void> {
|
|
throwIfAborted(signal);
|
|
return new Promise<void>((resolve, reject) => {
|
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
const cleanup = () => {
|
|
if (timeout !== undefined) clearTimeout(timeout);
|
|
signal?.removeEventListener("abort", abort);
|
|
};
|
|
const complete = () => {
|
|
cleanup();
|
|
resolve();
|
|
};
|
|
const abort = () => {
|
|
cleanup();
|
|
reject(abortReason(signal!));
|
|
};
|
|
timeout = setTimeout(complete, milliseconds);
|
|
signal?.addEventListener("abort", abort, { once: true });
|
|
if (signal?.aborted) abort();
|
|
});
|
|
}
|
|
|
|
async function resumeManagedOperation(
|
|
project: string,
|
|
request: ApiRequester,
|
|
path: string,
|
|
init: ApiRequestInit,
|
|
rolloutTimeoutMs: number,
|
|
options: OperationResumeOptions,
|
|
): Promise<void> {
|
|
const now = options.now ?? Date.now;
|
|
const deadline = operationResumeDeadline(rolloutTimeoutMs, now());
|
|
const headers = new Headers(init.headers);
|
|
headers.set("idempotency-key", randomUUID());
|
|
headers.set("prefer", "respond-async");
|
|
let operationInit = {
|
|
...init,
|
|
headers,
|
|
signal: options.signal ?? init.signal,
|
|
};
|
|
let operationId: string | undefined;
|
|
let backoffMs = OPERATION_RESUME_INITIAL_BACKOFF_MS;
|
|
let restartRetryPending = false;
|
|
let eventCursor = 0;
|
|
|
|
for (;;) {
|
|
throwIfAborted(options.signal);
|
|
if (restartRetryPending && now() >= deadline)
|
|
throw new Error("Timed out while reconnecting to resume the operation");
|
|
try {
|
|
let status: OperationStatus | undefined;
|
|
if (operationId) {
|
|
const operation = await managementRequest<{ status: OperationStatus }>(
|
|
project,
|
|
request,
|
|
`/operations/${encodeURIComponent(operationId)}`,
|
|
{ signal: options.signal },
|
|
);
|
|
status = operation.status;
|
|
} else {
|
|
restartRetryPending = false;
|
|
const response = await managementRequest<OperationResponse>(
|
|
project,
|
|
request,
|
|
path,
|
|
operationInit,
|
|
);
|
|
operationId = operationIdFromResponse(response);
|
|
status = operationStatusFromResponse(response);
|
|
}
|
|
|
|
if (operationId) {
|
|
try {
|
|
const events = await managementRequest<OperationEventsResponse>(
|
|
project,
|
|
request,
|
|
`/operations/${encodeURIComponent(operationId)}/events?after=${eventCursor}`,
|
|
{ signal: options.signal },
|
|
);
|
|
if (hasOperationEventCursorGap(events, eventCursor))
|
|
throw new OperationProgressCursorGapError();
|
|
for (const event of events.items ?? []) {
|
|
if (
|
|
!Number.isSafeInteger(event.sequence) ||
|
|
event.sequence <= eventCursor
|
|
)
|
|
continue;
|
|
eventCursor = event.sequence;
|
|
if (!isResourceOperationEvent(event)) continue;
|
|
options.onEvent?.(event);
|
|
}
|
|
} catch (error) {
|
|
if (error instanceof OperationProgressCursorGapError) throw error;
|
|
throwIfAborted(options.signal);
|
|
// Event polling is additive; never delay resumable operation status polling.
|
|
}
|
|
}
|
|
|
|
if (!operationId || !status || status.state === "succeeded") return;
|
|
if (isInterruptedOperation(status)) {
|
|
if (now() >= deadline) throw operationFailure(operationId, status);
|
|
operationId = undefined;
|
|
eventCursor = 0;
|
|
restartRetryPending = true;
|
|
headers.set("idempotency-key", randomUUID());
|
|
operationInit = {
|
|
...init,
|
|
headers,
|
|
signal: options.signal ?? init.signal,
|
|
};
|
|
} else if (status.state === "failed" || status.state === "cancelled")
|
|
throw operationFailure(operationId, status);
|
|
} catch (error) {
|
|
throwIfAborted(options.signal);
|
|
if (
|
|
error instanceof KuberApiError &&
|
|
error.code === "OPERATION_INTERRUPTED"
|
|
) {
|
|
if (now() >= deadline) throw adoptionHint(project, error);
|
|
operationId = undefined;
|
|
eventCursor = 0;
|
|
restartRetryPending = true;
|
|
headers.set("idempotency-key", randomUUID());
|
|
operationInit = {
|
|
...init,
|
|
headers,
|
|
signal: options.signal ?? init.signal,
|
|
};
|
|
} else {
|
|
if (!isRecoverableConnectionInterruption(error))
|
|
throw adoptionHint(project, error);
|
|
if (now() >= deadline) throw adoptionHint(project, error);
|
|
}
|
|
}
|
|
|
|
const remainingMs = deadline - now();
|
|
if (remainingMs <= 0)
|
|
throw new Error("Timed out while reconnecting to resume the operation");
|
|
const delayMs = Math.min(backoffMs, remainingMs);
|
|
if (options.sleep) {
|
|
throwIfAborted(options.signal);
|
|
await options.sleep(delayMs);
|
|
throwIfAborted(options.signal);
|
|
} else await abortableDelay(delayMs, options.signal);
|
|
backoffMs = Math.min(backoffMs * 2, OPERATION_RESUME_MAX_BACKOFF_MS);
|
|
}
|
|
}
|
|
|
|
async function planResources(
|
|
project: string,
|
|
resources: KubernetesResource[],
|
|
request: ApiRequester = apiRequest,
|
|
signal?: AbortSignal,
|
|
): Promise<ResourcePlan> {
|
|
const workspacePath = `/workspaces/${encodeURIComponent(project)}`;
|
|
return managementRequest<ResourcePlan>(
|
|
project,
|
|
request,
|
|
`${workspacePath}/resources/plan`,
|
|
{ method: "POST", json: { resources }, signal },
|
|
{ timeoutMs: RESOURCE_PLAN_TIMEOUT_MS },
|
|
);
|
|
}
|
|
|
|
async function applyResourcePlan(
|
|
project: string,
|
|
plan: ResourcePlan,
|
|
rolloutTimeoutMs: number,
|
|
postApply:
|
|
| ((resources: KubernetesResource[]) => void | Promise<void>)
|
|
| undefined,
|
|
request: ApiRequester,
|
|
resumeOptions: OperationResumeOptions,
|
|
): Promise<void> {
|
|
const workspacePath = `/workspaces/${encodeURIComponent(project)}`;
|
|
await resumeManagedOperation(
|
|
project,
|
|
request,
|
|
`${workspacePath}/resources/apply`,
|
|
{
|
|
method: "POST",
|
|
json: { resources: plan.desired },
|
|
},
|
|
rolloutTimeoutMs,
|
|
resumeOptions,
|
|
);
|
|
await postApply?.(plan.desired);
|
|
|
|
const deployments = getDeploymentNames(plan.desired);
|
|
if (deployments.length > 0) {
|
|
await resumeManagedOperation(
|
|
project,
|
|
request,
|
|
`${workspacePath}/resources/wait`,
|
|
{
|
|
method: "POST",
|
|
json: { deployments, timeoutMs: rolloutTimeoutMs },
|
|
},
|
|
rolloutTimeoutMs,
|
|
resumeOptions,
|
|
);
|
|
}
|
|
if (plan.stale.length > 0) {
|
|
await resumeManagedOperation(
|
|
project,
|
|
request,
|
|
`${workspacePath}/resources/delete`,
|
|
{
|
|
method: "POST",
|
|
json: { resources: plan.stale },
|
|
},
|
|
rolloutTimeoutMs,
|
|
resumeOptions,
|
|
);
|
|
}
|
|
}
|
|
|
|
export async function reconcileResources(
|
|
project: string,
|
|
resources: KubernetesResource[],
|
|
rolloutTimeoutMs: number,
|
|
postApply:
|
|
| ((resources: KubernetesResource[]) => void | Promise<void>)
|
|
| undefined,
|
|
request: ApiRequester = apiRequest,
|
|
resumeOptions: OperationResumeOptions = {},
|
|
): Promise<ResourcePlan> {
|
|
const plan = await planResources(
|
|
project,
|
|
resources,
|
|
request,
|
|
resumeOptions.signal,
|
|
);
|
|
await applyResourcePlan(
|
|
project,
|
|
plan,
|
|
rolloutTimeoutMs,
|
|
postApply,
|
|
request,
|
|
resumeOptions,
|
|
);
|
|
return plan;
|
|
}
|
|
|
|
function resourceOperationTarget(resource: {
|
|
apiVersion?: string;
|
|
kind?: string;
|
|
name?: string;
|
|
namespace?: string;
|
|
metadata?: { name?: string; namespace?: string };
|
|
}): ResourceOperationTarget | undefined {
|
|
const name = resource.name ?? resource.metadata?.name;
|
|
if (!resource.apiVersion || !resource.kind || !name) return;
|
|
return {
|
|
apiVersion: resource.apiVersion,
|
|
kind: resource.kind,
|
|
name,
|
|
namespace: resource.namespace ?? resource.metadata?.namespace,
|
|
};
|
|
}
|
|
|
|
function resourceOperationKey(resource: ResourceOperationTarget): string {
|
|
return `${resource.apiVersion}\0${resource.kind}\0${resource.namespace ?? ""}\0${resource.name}`;
|
|
}
|
|
|
|
function resourceOperationTaskTitle(
|
|
phase: ResourceOperationEvent["data"]["phase"],
|
|
resource: ResourceOperationTarget,
|
|
): string {
|
|
const action =
|
|
phase === "apply" ? "Apply" : phase === "wait" ? "Wait for" : "Delete";
|
|
return `${action} ${resource.kind}/${resource.name}`;
|
|
}
|
|
|
|
export function runLiveResourceOperation(
|
|
task: ListrTaskWrapper<any, any, any>,
|
|
phase: ResourceOperationEvent["data"]["phase"],
|
|
targets: ResourceOperationTarget[],
|
|
operation: (
|
|
onEvent: (event: ResourceOperationEvent) => void,
|
|
signal: AbortSignal,
|
|
) => Promise<void>,
|
|
options: LiveResourceOperationOptions = {},
|
|
): Listr<ResourceOperationTarget, any, any> | Promise<void> {
|
|
if (targets.length === 0) return operation(() => {}, task.signal);
|
|
const active = new Map<
|
|
string,
|
|
ListrTaskWrapper<ResourceOperationTarget, any, any>
|
|
>();
|
|
const complete = new Set<() => void>();
|
|
let started = 0;
|
|
return task.newListr<ResourceOperationTarget>(
|
|
targets.map((target) => ({
|
|
title: resourceOperationTaskTitle(phase, target),
|
|
task: async (_target, child) => {
|
|
active.set(resourceOperationKey(target), child);
|
|
options.onTaskStarted?.(target, child);
|
|
started += 1;
|
|
if (started !== targets.length)
|
|
return new Promise<void>((resolve) => {
|
|
const completeTask = () => {
|
|
task.signal.removeEventListener("abort", completeTask);
|
|
complete.delete(completeTask);
|
|
resolve();
|
|
};
|
|
complete.add(completeTask);
|
|
if (task.signal.aborted) completeTask();
|
|
else
|
|
task.signal.addEventListener("abort", completeTask, {
|
|
once: true,
|
|
});
|
|
});
|
|
try {
|
|
await operation((event) => {
|
|
const resource = event.data.resource;
|
|
const activeTask = active.get(resourceOperationKey(resource));
|
|
if (!activeTask) return;
|
|
activeTask.output = event.data.state;
|
|
task.output = resourceOperationEventTitle(event);
|
|
}, task.signal);
|
|
} finally {
|
|
for (const resolve of [...complete]) resolve();
|
|
}
|
|
},
|
|
})),
|
|
{ concurrent: true },
|
|
);
|
|
}
|
|
|
|
export async function runUp(
|
|
build: boolean,
|
|
request: ApiRequester = apiRequest,
|
|
options: { trust?: TrustIdentity } = {},
|
|
) {
|
|
const { project, compose, cwd, config, hookContext: getHookContext } = ctx();
|
|
const trusted =
|
|
options.trust ??
|
|
(await requireLocalTrust(await resolveTrustIdentity(project, cwd)));
|
|
const baseRequest = request;
|
|
request = async <T>(
|
|
path: string,
|
|
init: ApiRequestInit = {},
|
|
options?: ApiRequestOptions,
|
|
) => {
|
|
const headers = new Headers(init.headers);
|
|
for (const [key, value] of Object.entries(trustHeaders(trusted)))
|
|
headers.set(key, value);
|
|
return baseRequest<T>(path, { ...init, headers }, options);
|
|
};
|
|
const workspacePath = `/workspaces/${encodeURIComponent(project)}`;
|
|
|
|
const taskCtx = await new Listr<UpContext>(
|
|
[
|
|
{
|
|
title: "Read compose",
|
|
task: async (taskCtx, task) => {
|
|
taskCtx.compose = await compose();
|
|
if (build)
|
|
await config.preBuild?.(taskCtx.compose, await getHookContext());
|
|
task.output = `${Object.keys(taskCtx.compose.services ?? {}).length} services`;
|
|
},
|
|
},
|
|
{
|
|
title: "Snapshot workspace",
|
|
task: async (taskCtx, task) => {
|
|
let workspaceRoot: string;
|
|
try {
|
|
workspaceRoot = await getRepoRoot(cwd);
|
|
} catch (error) {
|
|
workspaceRoot = cwd;
|
|
}
|
|
taskCtx.workspaceRoot = workspaceRoot;
|
|
taskCtx.snapshot = await enumerateWorkspace(workspaceRoot);
|
|
task.output = `${taskCtx.snapshot.manifest.files.length} files`;
|
|
},
|
|
},
|
|
{
|
|
title: "Build images",
|
|
rendererOptions: { bottomBar: Infinity, persistentOutput: true },
|
|
enabled: async () =>
|
|
build &&
|
|
Object.values((await compose()).services ?? {}).some(
|
|
(service) => service.build,
|
|
),
|
|
task: async (taskCtx, task) => {
|
|
const services = Object.entries(taskCtx.compose!.services ?? {})
|
|
.filter(([, service]) => service.build)
|
|
.map(([name]) => name);
|
|
const children = new Map<
|
|
string,
|
|
ListrTaskWrapper<UpContext, any, any>
|
|
>();
|
|
const pending = new Map<string, (error?: unknown) => void>();
|
|
const completed = new Map<string, Promise<unknown>>(
|
|
services.map((name) => [
|
|
name,
|
|
new Promise((resolve) => pending.set(name, resolve)),
|
|
]),
|
|
);
|
|
let settleUpload!: (error?: unknown) => void;
|
|
const uploaded = new Promise<unknown>((resolve) => {
|
|
settleUpload = resolve;
|
|
});
|
|
let uploadChild: ListrTaskWrapper<UpContext, any, any> | undefined;
|
|
let uploadedBytes = 0;
|
|
let totalBytes = 0;
|
|
const formatBytes = (bytes: number): string => {
|
|
if (bytes < 1024) return `${bytes} B`;
|
|
const unit = bytes < 1024 ** 2 ? "KiB" : "MiB";
|
|
const divisor = unit === "KiB" ? 1024 : 1024 ** 2;
|
|
return `${Number((bytes / divisor).toFixed(1))} ${unit}`;
|
|
};
|
|
const uploadTitle = (
|
|
bytes: number,
|
|
total: number,
|
|
finished = false,
|
|
): string =>
|
|
`Uploading context: ${formatBytes(bytes)}/${formatBytes(total)} (${finished ? 100 : total === 0 ? 0 : Math.min(99, Math.floor((100 * bytes) / total))}%)`;
|
|
const finishUpload = (error?: unknown) => {
|
|
if (!error && uploadChild)
|
|
uploadChild.title = uploadTitle(uploadedBytes, totalBytes, true);
|
|
settleUpload(error);
|
|
};
|
|
let result: Awaited<ReturnType<typeof buildServices>> | undefined;
|
|
let operationFailure: Error | undefined;
|
|
let operation: Promise<void> | undefined;
|
|
const start = () => {
|
|
operation ??= buildServices(
|
|
project,
|
|
taskCtx.compose!,
|
|
cwd,
|
|
{
|
|
upload: (bytes, total) => {
|
|
uploadedBytes = bytes;
|
|
totalBytes = total;
|
|
if (uploadChild)
|
|
uploadChild.title = uploadTitle(bytes, total);
|
|
},
|
|
uploadSettled: finishUpload,
|
|
service: (name) => {
|
|
let stream:
|
|
| ReturnType<
|
|
ListrTaskWrapper<UpContext, any, any>["stdout"]
|
|
>
|
|
| undefined;
|
|
return {
|
|
progress: (message) => {
|
|
const child = children.get(name);
|
|
if (!child) return;
|
|
const phase =
|
|
/^Build (queued|creating|starting|running|done)\b/.exec(
|
|
message,
|
|
)?.[1];
|
|
if (phase) child.title = `Build ${name}: ${phase}`;
|
|
// A failed status includes the full error; the parent bottom bar owns it.
|
|
},
|
|
get stream() {
|
|
const child = children.get(name);
|
|
if (child) stream ??= child.stdout();
|
|
return stream;
|
|
},
|
|
};
|
|
},
|
|
settled: (name, error) => {
|
|
pending.get(name)?.(error);
|
|
pending.delete(name);
|
|
},
|
|
},
|
|
{
|
|
...config,
|
|
request,
|
|
snapshot: taskCtx.snapshot,
|
|
workspaceRoot: taskCtx.workspaceRoot,
|
|
signal: task.signal,
|
|
},
|
|
).then(
|
|
(value) => {
|
|
result = value;
|
|
},
|
|
(error: unknown) => {
|
|
const failure =
|
|
error instanceof Error ? error : new Error(String(error));
|
|
operationFailure = failure;
|
|
task.output = failure.stack ?? failure.message;
|
|
finishUpload(failure);
|
|
for (const resolve of pending.values()) resolve(failure);
|
|
pending.clear();
|
|
},
|
|
);
|
|
return operation;
|
|
};
|
|
return task.newListr<UpContext>(
|
|
[
|
|
{
|
|
title: uploadTitle(0, 0),
|
|
exitOnError: false,
|
|
task: async (_ctx, child) => {
|
|
uploadChild = child;
|
|
if (children.size === services.length) start();
|
|
const error = await uploaded;
|
|
if (error) {
|
|
child.title = "Uploading context: failed";
|
|
throw error;
|
|
}
|
|
},
|
|
},
|
|
...services.map((name) => ({
|
|
title: `Build ${name}`,
|
|
rendererOptions: { outputBar: 10, persistentOutput: true },
|
|
exitOnError: false,
|
|
task: async (
|
|
_ctx: UpContext,
|
|
child: ListrTaskWrapper<UpContext, any, any>,
|
|
) => {
|
|
children.set(name, child);
|
|
child.title = `Build ${name}: queued`;
|
|
if (children.size === services.length && uploadChild) start();
|
|
const error = await completed.get(name);
|
|
if (error) {
|
|
child.title = `Build ${name}: failed`;
|
|
throw error;
|
|
}
|
|
child.title = `Build ${name}: done`;
|
|
},
|
|
})),
|
|
{
|
|
task: async () => {
|
|
await Promise.all(completed.values());
|
|
await operation;
|
|
if (!result)
|
|
throw operationFailure ?? new Error("Build images failed");
|
|
taskCtx.buildImages = result.images;
|
|
await config.postBuild?.(result, await getHookContext());
|
|
task.title = `Built ${result.built.length} image${result.built.length === 1 ? "" : "s"}`;
|
|
},
|
|
},
|
|
],
|
|
{ concurrent: true },
|
|
);
|
|
},
|
|
},
|
|
{
|
|
title: "Resolve images",
|
|
enabled: async () =>
|
|
!build &&
|
|
Object.values((await compose()).services ?? {}).some(
|
|
(service) => service.build,
|
|
),
|
|
task: async (taskCtx, task) => {
|
|
taskCtx.buildImages = await resolveBuildImages(
|
|
project,
|
|
taskCtx.compose!,
|
|
{
|
|
...config,
|
|
request,
|
|
},
|
|
);
|
|
task.output = `${Object.keys(taskCtx.buildImages).length} images`;
|
|
},
|
|
},
|
|
{
|
|
title: "Update workspace",
|
|
task: async (taskCtx, task) => {
|
|
const workspace = await ensureWorkspace(
|
|
project,
|
|
taskCtx.compose!,
|
|
taskCtx.snapshot!,
|
|
request,
|
|
);
|
|
const adopted = await request<{ resourcesAdopted: number }>(
|
|
workspaceAdoptionRoute(project),
|
|
{
|
|
method: WORKSPACE_ADOPTION_METHOD,
|
|
json: { workspaceUid: workspace.metadata.uid },
|
|
},
|
|
);
|
|
task.output = adopted.resourcesAdopted
|
|
? `Adopted ${adopted.resourcesAdopted} existing resources`
|
|
: "Workspace ready";
|
|
},
|
|
},
|
|
{
|
|
title: "Reconcile backing services",
|
|
rendererOptions: { bottomBar: 1, persistentOutput: true },
|
|
task: (taskCtx, task) =>
|
|
task.newListr(
|
|
[
|
|
{
|
|
title: "Reconcile databases",
|
|
enabled: () =>
|
|
getComposePostgresClaims(taskCtx.compose!).length > 0,
|
|
task: async (_ctx, child) => {
|
|
let response: Record<string, unknown>;
|
|
try {
|
|
response = await managementRequest<Record<string, unknown>>(
|
|
project,
|
|
request,
|
|
`${workspacePath}/databases`,
|
|
{ method: "POST", json: { compose: taskCtx.compose } },
|
|
);
|
|
} catch (error) {
|
|
task.output =
|
|
error instanceof Error ? error.message : String(error);
|
|
throw error;
|
|
}
|
|
taskCtx.databaseEnv = operationEnvironment(response);
|
|
child.output = `${Object.keys(taskCtx.databaseEnv).length} services`;
|
|
},
|
|
},
|
|
{
|
|
title: "Reconcile S3 storage",
|
|
enabled: () => getComposeS3Claims(taskCtx.compose!).length > 0,
|
|
task: async (_ctx, child) => {
|
|
let response: Record<string, unknown>;
|
|
try {
|
|
response = await managementRequest<Record<string, unknown>>(
|
|
project,
|
|
request,
|
|
`${workspacePath}/storage`,
|
|
{ method: "POST", json: { compose: taskCtx.compose } },
|
|
);
|
|
} catch (error) {
|
|
task.output =
|
|
error instanceof Error ? error.message : String(error);
|
|
throw error;
|
|
}
|
|
taskCtx.storageEnv = operationEnvironment(response);
|
|
child.output = `${Object.keys(taskCtx.storageEnv).length} services`;
|
|
},
|
|
},
|
|
],
|
|
{ concurrent: true },
|
|
),
|
|
},
|
|
{
|
|
title: "Render manifests",
|
|
task: async (taskCtx, task) => {
|
|
taskCtx.serviceEnv = mergeServiceEnv(
|
|
mergeServiceEnv(taskCtx.serviceEnv, taskCtx.databaseEnv ?? {}),
|
|
taskCtx.storageEnv ?? {},
|
|
);
|
|
taskCtx.resources = await composeToKubernetes(
|
|
project,
|
|
taskCtx.compose!,
|
|
cwd,
|
|
taskCtx.serviceEnv,
|
|
taskCtx.buildImages,
|
|
);
|
|
await config.postRender?.(
|
|
taskCtx.resources as KuberResource[],
|
|
await getHookContext(),
|
|
);
|
|
task.output = `${taskCtx.resources.length} resources`;
|
|
},
|
|
},
|
|
{
|
|
title: "Reconcile resources",
|
|
task: async (taskCtx, task) => {
|
|
taskCtx.plan = await planResources(
|
|
project,
|
|
taskCtx.resources!,
|
|
request,
|
|
task.signal,
|
|
);
|
|
const plan = taskCtx.plan;
|
|
const desired = plan.desired
|
|
.map(resourceOperationTarget)
|
|
.filter((resource): resource is ResourceOperationTarget =>
|
|
Boolean(resource),
|
|
);
|
|
const deployments = plan.desired
|
|
.filter((resource) => resource.kind === "Deployment")
|
|
.map(resourceOperationTarget)
|
|
.filter((resource): resource is ResourceOperationTarget =>
|
|
Boolean(resource),
|
|
);
|
|
const stale = plan.stale
|
|
.map(resourceOperationTarget)
|
|
.filter((resource): resource is ResourceOperationTarget =>
|
|
Boolean(resource),
|
|
);
|
|
const resourcePath = `${workspacePath}/resources`;
|
|
return task.newListr([
|
|
{
|
|
title: "Apply resources",
|
|
task: async (_ctx, phaseTask) =>
|
|
runLiveResourceOperation(
|
|
phaseTask,
|
|
"apply",
|
|
desired,
|
|
(onEvent, signal) =>
|
|
resumeManagedOperation(
|
|
project,
|
|
request,
|
|
`${resourcePath}/apply`,
|
|
{ method: "POST", json: { resources: plan.desired } },
|
|
config.rolloutTimeoutMs,
|
|
{ onEvent, signal },
|
|
),
|
|
),
|
|
},
|
|
{
|
|
title: "Run post-apply hook",
|
|
skip: !config.postApply,
|
|
task: async () =>
|
|
config.postApply?.(
|
|
plan.desired as KuberResource[],
|
|
await getHookContext(),
|
|
),
|
|
},
|
|
{
|
|
title: "Wait for deployments",
|
|
skip: deployments.length === 0,
|
|
task: async (_ctx, phaseTask) =>
|
|
runLiveResourceOperation(
|
|
phaseTask,
|
|
"wait",
|
|
deployments,
|
|
(onEvent, signal) =>
|
|
resumeManagedOperation(
|
|
project,
|
|
request,
|
|
`${resourcePath}/wait`,
|
|
{
|
|
method: "POST",
|
|
json: {
|
|
deployments: deployments.map(
|
|
(deployment) => deployment.name,
|
|
),
|
|
timeoutMs: config.rolloutTimeoutMs,
|
|
},
|
|
},
|
|
config.rolloutTimeoutMs,
|
|
{ onEvent, signal },
|
|
),
|
|
),
|
|
},
|
|
{
|
|
title: "Delete stale resources",
|
|
skip: stale.length === 0,
|
|
task: async (_ctx, phaseTask) =>
|
|
runLiveResourceOperation(
|
|
phaseTask,
|
|
"delete",
|
|
stale,
|
|
(onEvent, signal) =>
|
|
resumeManagedOperation(
|
|
project,
|
|
request,
|
|
`${resourcePath}/delete`,
|
|
{ method: "POST", json: { resources: plan.stale } },
|
|
config.rolloutTimeoutMs,
|
|
{ onEvent, signal },
|
|
),
|
|
),
|
|
},
|
|
]);
|
|
},
|
|
},
|
|
],
|
|
{
|
|
renderer: "default",
|
|
// Only redraw when a task changes; a queued build should not generate
|
|
// another frame on each spinner tick. Keep the renderer's task tree.
|
|
rendererOptions: {
|
|
collapseErrors: false,
|
|
collapseSubtasks: false,
|
|
showErrorMessage: false,
|
|
lazy: true,
|
|
},
|
|
},
|
|
).run();
|
|
|
|
return taskCtx;
|
|
}
|
|
|
|
export const up = defineCommand({
|
|
meta: { name: "up", description: "Create and start deployments" },
|
|
args: {
|
|
build: {
|
|
type: "boolean",
|
|
default: true,
|
|
alias: "b",
|
|
description: "Build images before starting deployments",
|
|
negativeDescription: "Don't build an image, even if it's policy",
|
|
},
|
|
},
|
|
async run({ args }) {
|
|
await runUp(args.build);
|
|
},
|
|
});
|