580 lines
17 KiB
TypeScript
580 lines
17 KiB
TypeScript
import { defineCommand } from "citty";
|
|
import { Listr } from "listr2";
|
|
import { randomUUID } from "node:crypto";
|
|
import type { ComposeSpecification } from "../schema/docker.d";
|
|
import type { KuberResource } from "../types";
|
|
import {
|
|
apiRequest,
|
|
KuberApiError,
|
|
type ApiRequestInit,
|
|
type ApiRequestOptions,
|
|
} from "../lib/api";
|
|
import {
|
|
buildServices,
|
|
getRepoRoot,
|
|
resolveBuildImages,
|
|
type ApiRequester,
|
|
} from "../lib/build";
|
|
import { composeToKubernetes, type KubernetesResource } from "../lib/convert";
|
|
import { ctx } from "../lib/context";
|
|
import { getComposePostgresClaims } from "../lib/database";
|
|
import { getComposeS3Claims } from "../lib/storage";
|
|
import { enumerateWorkspace, type WorkspaceSnapshot } from "../lib/workspace";
|
|
import {
|
|
requireLocalTrust,
|
|
resolveTrustIdentity,
|
|
trustHeaders,
|
|
} from "../lib/trust";
|
|
|
|
type Workspace = {
|
|
metadata: { name: string; uid: string; resourceVersion: string };
|
|
};
|
|
|
|
type ResourceIdentity = {
|
|
apiVersion: string;
|
|
kind: string;
|
|
name: string;
|
|
namespace?: string;
|
|
uid: string;
|
|
workspaceUid: string;
|
|
};
|
|
|
|
type ResourcePlan = {
|
|
desired: KubernetesResource[];
|
|
stale: ResourceIdentity[];
|
|
};
|
|
|
|
type OperationStatus = {
|
|
state?: unknown;
|
|
error?: { code?: unknown; message?: unknown };
|
|
};
|
|
|
|
type OperationResponse = {
|
|
operationId?: unknown;
|
|
operation?: { status?: OperationStatus };
|
|
};
|
|
|
|
export type OperationResumeOptions = {
|
|
now?: () => number;
|
|
sleep?: (milliseconds: number) => Promise<void>;
|
|
};
|
|
|
|
type UpContext = {
|
|
compose?: ComposeSpecification;
|
|
snapshot?: WorkspaceSnapshot;
|
|
buildImages?: Record<string, string>;
|
|
serviceEnv?: Record<string, Record<string, string>>;
|
|
resources?: KubernetesResource[];
|
|
plan?: ResourcePlan;
|
|
};
|
|
|
|
export const WORKSPACE_ADOPTION_METHOD = "POST";
|
|
const OPERATION_RESUME_INITIAL_BACKOFF_MS = 250;
|
|
const OPERATION_RESUME_MAX_BACKOFF_MS = 5_000;
|
|
const OPERATION_RESUME_GRACE_MS = 60_000;
|
|
const RECOVERABLE_API_ERROR_CODES = new Set([
|
|
"HTTP_502",
|
|
"HTTP_503",
|
|
"HTTP_504",
|
|
]);
|
|
|
|
export function workspaceAdoptionRoute(project: string): string {
|
|
return `/workspaces/${encodeURIComponent(project)}/adopt`;
|
|
}
|
|
|
|
export function mergeServiceEnv(
|
|
current: Record<string, Record<string, string>> | undefined,
|
|
next: Record<string, Record<string, string>>,
|
|
): Record<string, Record<string, string>> {
|
|
const merged = { ...current };
|
|
for (const [service, environment] of Object.entries(next))
|
|
merged[service] = { ...merged[service], ...environment };
|
|
return merged;
|
|
}
|
|
|
|
export function getDeploymentNames(resources: KubernetesResource[]): string[] {
|
|
return resources
|
|
.filter((resource) => resource.kind === "Deployment")
|
|
.map((resource) => resource.metadata?.name)
|
|
.filter((name): name is string => Boolean(name));
|
|
}
|
|
|
|
function workspaceInput(
|
|
compose: ComposeSpecification,
|
|
snapshot: WorkspaceSnapshot,
|
|
) {
|
|
return {
|
|
source: {
|
|
uri: `cas://${snapshot.digest}`,
|
|
digest: snapshot.digest,
|
|
},
|
|
config: { compose },
|
|
};
|
|
}
|
|
|
|
export async function ensureWorkspace(
|
|
project: string,
|
|
compose: ComposeSpecification,
|
|
snapshot: WorkspaceSnapshot,
|
|
request: ApiRequester = apiRequest,
|
|
): Promise<Workspace> {
|
|
const path = `/workspaces/${encodeURIComponent(project)}`;
|
|
const input = workspaceInput(compose, snapshot);
|
|
let current: Workspace;
|
|
try {
|
|
current = await request<Workspace>(path);
|
|
} catch (error) {
|
|
if (!(error instanceof KuberApiError) || error.status !== 404) throw error;
|
|
return request<Workspace>("/workspaces", {
|
|
method: "POST",
|
|
json: { id: project, ...input },
|
|
});
|
|
}
|
|
return request<Workspace>(path, {
|
|
method: "PUT",
|
|
headers: { "if-match": `"${current.metadata.resourceVersion}"` },
|
|
json: input,
|
|
});
|
|
}
|
|
|
|
function operationEnvironment(
|
|
response: Record<string, unknown>,
|
|
): Record<string, Record<string, string>> {
|
|
return Object.fromEntries(
|
|
Object.entries(response).filter(
|
|
([key, value]) =>
|
|
key !== "operation" &&
|
|
key !== "operationId" &&
|
|
value !== null &&
|
|
typeof value === "object" &&
|
|
!Array.isArray(value),
|
|
),
|
|
) as Record<string, Record<string, string>>;
|
|
}
|
|
|
|
function adoptionHint(project: string, error: unknown): Error {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
if (!/namespace .* (?:external|different-workspace)/i.test(message))
|
|
return error instanceof Error ? error : new Error(message);
|
|
return new Error(
|
|
`${message}. Safe adoption requires ${WORKSPACE_ADOPTION_METHOD} ${workspaceAdoptionRoute(project)} with namespace UID and ownership preconditions.`,
|
|
{ cause: error },
|
|
);
|
|
}
|
|
|
|
async function managementRequest<T>(
|
|
project: string,
|
|
request: ApiRequester,
|
|
path: string,
|
|
init: ApiRequestInit,
|
|
): Promise<T> {
|
|
try {
|
|
return await request<T>(path, init);
|
|
} catch (error) {
|
|
throw adoptionHint(project, error);
|
|
}
|
|
}
|
|
|
|
function operationIdFromResponse(response: unknown): string | undefined {
|
|
if (!response || typeof response !== "object") return;
|
|
const operationId = (response as OperationResponse).operationId;
|
|
return typeof operationId === "string" && operationId
|
|
? operationId
|
|
: undefined;
|
|
}
|
|
|
|
function operationStatusFromResponse(
|
|
response: unknown,
|
|
): OperationStatus | undefined {
|
|
if (!response || typeof response !== "object") return;
|
|
const status = (response as OperationResponse).operation?.status;
|
|
return status && typeof status === "object" ? status : undefined;
|
|
}
|
|
|
|
function operationFailure(
|
|
operationId: string,
|
|
status: OperationStatus,
|
|
): KuberApiError {
|
|
const error = status.error;
|
|
const cancelled = status.state === "cancelled";
|
|
const message =
|
|
typeof error?.message === "string"
|
|
? error.message
|
|
: cancelled
|
|
? "The operation was cancelled"
|
|
: "The operation failed";
|
|
const code =
|
|
typeof error?.code === "string"
|
|
? error.code
|
|
: cancelled
|
|
? "OPERATION_CANCELLED"
|
|
: "OPERATION_FAILED";
|
|
return new KuberApiError(message, cancelled ? 409 : 500, {
|
|
title: cancelled ? "Operation cancelled" : "Operation failed",
|
|
status: cancelled ? 409 : 500,
|
|
code,
|
|
operationId,
|
|
});
|
|
}
|
|
|
|
function isRecoverableConnectionInterruption(error: unknown): boolean {
|
|
if (error instanceof KuberApiError)
|
|
return RECOVERABLE_API_ERROR_CODES.has(error.code);
|
|
if (error instanceof DOMException && error.name === "AbortError")
|
|
return false;
|
|
if (error instanceof Error && error.name === "AbortError") return false;
|
|
return (
|
|
error instanceof TypeError ||
|
|
(error instanceof Error && error.name === "TimeoutError")
|
|
);
|
|
}
|
|
|
|
function isInterruptedOperation(status: OperationStatus): boolean {
|
|
return (
|
|
status.state === "failed" && status.error?.code === "OPERATION_INTERRUPTED"
|
|
);
|
|
}
|
|
|
|
function operationResumeDeadline(
|
|
rolloutTimeoutMs: number,
|
|
now: number,
|
|
): number {
|
|
return now + Math.max(rolloutTimeoutMs, 0) + OPERATION_RESUME_GRACE_MS;
|
|
}
|
|
|
|
async function resumeManagedOperation(
|
|
project: string,
|
|
request: ApiRequester,
|
|
path: string,
|
|
init: ApiRequestInit,
|
|
rolloutTimeoutMs: number,
|
|
options: OperationResumeOptions,
|
|
): Promise<void> {
|
|
const now = options.now ?? Date.now;
|
|
const sleep = options.sleep ?? ((milliseconds) => Bun.sleep(milliseconds));
|
|
const deadline = operationResumeDeadline(rolloutTimeoutMs, now());
|
|
const headers = new Headers(init.headers);
|
|
headers.set("idempotency-key", randomUUID());
|
|
let operationInit = { ...init, headers };
|
|
let operationId: string | undefined;
|
|
let backoffMs = OPERATION_RESUME_INITIAL_BACKOFF_MS;
|
|
let restartRetryPending = false;
|
|
|
|
for (;;) {
|
|
if (restartRetryPending && now() >= deadline)
|
|
throw new Error("Timed out while reconnecting to resume the operation");
|
|
try {
|
|
let status: OperationStatus | undefined;
|
|
if (operationId) {
|
|
const operation = await managementRequest<{ status: OperationStatus }>(
|
|
project,
|
|
request,
|
|
`/operations/${encodeURIComponent(operationId)}`,
|
|
{},
|
|
);
|
|
status = operation.status;
|
|
} else {
|
|
restartRetryPending = false;
|
|
const response = await managementRequest<OperationResponse>(
|
|
project,
|
|
request,
|
|
path,
|
|
operationInit,
|
|
);
|
|
operationId = operationIdFromResponse(response);
|
|
status = operationStatusFromResponse(response);
|
|
}
|
|
|
|
if (!operationId || !status || status.state === "succeeded") return;
|
|
if (isInterruptedOperation(status)) {
|
|
if (now() >= deadline) throw operationFailure(operationId, status);
|
|
operationId = undefined;
|
|
restartRetryPending = true;
|
|
headers.set("idempotency-key", randomUUID());
|
|
operationInit = { ...init, headers };
|
|
} else if (status.state === "failed" || status.state === "cancelled")
|
|
throw operationFailure(operationId, status);
|
|
} catch (error) {
|
|
if (
|
|
error instanceof KuberApiError &&
|
|
error.code === "OPERATION_INTERRUPTED"
|
|
) {
|
|
if (now() >= deadline) throw adoptionHint(project, error);
|
|
operationId = undefined;
|
|
restartRetryPending = true;
|
|
headers.set("idempotency-key", randomUUID());
|
|
operationInit = { ...init, headers };
|
|
} else {
|
|
if (!isRecoverableConnectionInterruption(error))
|
|
throw adoptionHint(project, error);
|
|
if (now() >= deadline) throw adoptionHint(project, error);
|
|
}
|
|
}
|
|
|
|
const remainingMs = deadline - now();
|
|
if (remainingMs <= 0)
|
|
throw new Error("Timed out while reconnecting to resume the operation");
|
|
await sleep(Math.min(backoffMs, remainingMs));
|
|
backoffMs = Math.min(backoffMs * 2, OPERATION_RESUME_MAX_BACKOFF_MS);
|
|
}
|
|
}
|
|
|
|
export async function reconcileResources(
|
|
project: string,
|
|
resources: KubernetesResource[],
|
|
rolloutTimeoutMs: number,
|
|
postApply:
|
|
| ((resources: KubernetesResource[]) => void | Promise<void>)
|
|
| undefined,
|
|
request: ApiRequester = apiRequest,
|
|
resumeOptions: OperationResumeOptions = {},
|
|
): Promise<ResourcePlan> {
|
|
const workspacePath = `/workspaces/${encodeURIComponent(project)}`;
|
|
const plan = await managementRequest<ResourcePlan>(
|
|
project,
|
|
request,
|
|
`${workspacePath}/resources/plan`,
|
|
{ method: "POST", json: { resources } },
|
|
);
|
|
await resumeManagedOperation(
|
|
project,
|
|
request,
|
|
`${workspacePath}/resources/apply`,
|
|
{
|
|
method: "POST",
|
|
json: { resources: plan.desired },
|
|
},
|
|
rolloutTimeoutMs,
|
|
resumeOptions,
|
|
);
|
|
await postApply?.(plan.desired);
|
|
|
|
const deployments = getDeploymentNames(plan.desired);
|
|
if (deployments.length > 0) {
|
|
await resumeManagedOperation(
|
|
project,
|
|
request,
|
|
`${workspacePath}/resources/wait`,
|
|
{
|
|
method: "POST",
|
|
json: { deployments, timeoutMs: rolloutTimeoutMs },
|
|
},
|
|
rolloutTimeoutMs,
|
|
resumeOptions,
|
|
);
|
|
}
|
|
if (plan.stale.length > 0) {
|
|
await resumeManagedOperation(
|
|
project,
|
|
request,
|
|
`${workspacePath}/resources/delete`,
|
|
{
|
|
method: "POST",
|
|
json: { resources: plan.stale },
|
|
},
|
|
rolloutTimeoutMs,
|
|
resumeOptions,
|
|
);
|
|
}
|
|
return plan;
|
|
}
|
|
|
|
export async function runUp(
|
|
build: boolean,
|
|
request: ApiRequester = apiRequest,
|
|
) {
|
|
const { project, compose, cwd, config, hookContext: getHookContext } = ctx();
|
|
const trusted = await requireLocalTrust(
|
|
await resolveTrustIdentity(project, cwd),
|
|
);
|
|
const baseRequest = request;
|
|
request = async <T>(
|
|
path: string,
|
|
init: ApiRequestInit = {},
|
|
options?: ApiRequestOptions,
|
|
) => {
|
|
const headers = new Headers(init.headers);
|
|
for (const [key, value] of Object.entries(trustHeaders(trusted)))
|
|
headers.set(key, value);
|
|
return baseRequest<T>(path, { ...init, headers }, options);
|
|
};
|
|
const workspacePath = `/workspaces/${encodeURIComponent(project)}`;
|
|
|
|
const taskCtx = await new Listr<UpContext>(
|
|
[
|
|
{
|
|
title: "Read compose",
|
|
task: async (taskCtx, task) => {
|
|
taskCtx.compose = await compose();
|
|
if (build)
|
|
await config.preBuild?.(taskCtx.compose, await getHookContext());
|
|
task.output = `${Object.keys(taskCtx.compose.services ?? {}).length} services`;
|
|
},
|
|
},
|
|
{
|
|
title: "Snapshot workspace",
|
|
task: async (taskCtx, task) => {
|
|
taskCtx.snapshot = await enumerateWorkspace(await getRepoRoot(cwd));
|
|
task.output = `${taskCtx.snapshot.manifest.files.length} files`;
|
|
},
|
|
},
|
|
{
|
|
title: "Build images",
|
|
rendererOptions: { outputBar: 10, persistentOutput: true },
|
|
enabled: async () =>
|
|
build &&
|
|
Object.values((await compose()).services ?? {}).some(
|
|
(service) => service.build,
|
|
),
|
|
task: async (taskCtx, task) => {
|
|
const result = await buildServices(
|
|
project,
|
|
taskCtx.compose!,
|
|
cwd,
|
|
{
|
|
progress: (message) => {
|
|
task.output = message;
|
|
},
|
|
stream: task.stdout(),
|
|
},
|
|
{ ...config, request, snapshot: taskCtx.snapshot },
|
|
);
|
|
taskCtx.buildImages = result.images;
|
|
await config.postBuild?.(result, await getHookContext());
|
|
task.output = `Built ${result.built.length} image${result.built.length === 1 ? "" : "s"}`;
|
|
},
|
|
},
|
|
{
|
|
title: "Resolve images",
|
|
enabled: async () =>
|
|
!build &&
|
|
Object.values((await compose()).services ?? {}).some(
|
|
(service) => service.build,
|
|
),
|
|
task: async (taskCtx, task) => {
|
|
taskCtx.buildImages = await resolveBuildImages(
|
|
project,
|
|
taskCtx.compose!,
|
|
{
|
|
...config,
|
|
request,
|
|
},
|
|
);
|
|
task.output = `${Object.keys(taskCtx.buildImages).length} images`;
|
|
},
|
|
},
|
|
{
|
|
title: "Update workspace",
|
|
task: async (taskCtx, task) => {
|
|
const workspace = await ensureWorkspace(
|
|
project,
|
|
taskCtx.compose!,
|
|
taskCtx.snapshot!,
|
|
request,
|
|
);
|
|
const adopted = await request<{ resourcesAdopted: number }>(
|
|
workspaceAdoptionRoute(project),
|
|
{
|
|
method: WORKSPACE_ADOPTION_METHOD,
|
|
json: { workspaceUid: workspace.metadata.uid },
|
|
},
|
|
);
|
|
task.output = adopted.resourcesAdopted
|
|
? `Adopted ${adopted.resourcesAdopted} existing resources`
|
|
: "Workspace ready";
|
|
},
|
|
},
|
|
{
|
|
title: "Reconcile databases",
|
|
enabled: async () =>
|
|
getComposePostgresClaims(await compose()).length > 0,
|
|
task: async (taskCtx, task) => {
|
|
const response = await managementRequest<Record<string, unknown>>(
|
|
project,
|
|
request,
|
|
`${workspacePath}/databases`,
|
|
{ method: "POST", json: { compose: taskCtx.compose } },
|
|
);
|
|
taskCtx.serviceEnv = mergeServiceEnv(
|
|
taskCtx.serviceEnv,
|
|
operationEnvironment(response),
|
|
);
|
|
task.output = `${Object.keys(taskCtx.serviceEnv).length} services`;
|
|
},
|
|
},
|
|
{
|
|
title: "Reconcile S3 storage",
|
|
enabled: async () => getComposeS3Claims(await compose()).length > 0,
|
|
task: async (taskCtx, task) => {
|
|
const response = await managementRequest<Record<string, unknown>>(
|
|
project,
|
|
request,
|
|
`${workspacePath}/storage`,
|
|
{ method: "POST", json: { compose: taskCtx.compose } },
|
|
);
|
|
taskCtx.serviceEnv = mergeServiceEnv(
|
|
taskCtx.serviceEnv,
|
|
operationEnvironment(response),
|
|
);
|
|
task.output = `${Object.keys(taskCtx.serviceEnv).length} services`;
|
|
},
|
|
},
|
|
{
|
|
title: "Render manifests",
|
|
task: async (taskCtx, task) => {
|
|
taskCtx.resources = await composeToKubernetes(
|
|
project,
|
|
taskCtx.compose!,
|
|
cwd,
|
|
taskCtx.serviceEnv,
|
|
taskCtx.buildImages,
|
|
);
|
|
await config.postRender?.(
|
|
taskCtx.resources as KuberResource[],
|
|
await getHookContext(),
|
|
);
|
|
task.output = `${taskCtx.resources.length} resources`;
|
|
},
|
|
},
|
|
{
|
|
title: "Reconcile resources",
|
|
task: async (taskCtx, task) => {
|
|
taskCtx.plan = await reconcileResources(
|
|
project,
|
|
taskCtx.resources!,
|
|
config.rolloutTimeoutMs,
|
|
config.postApply
|
|
? async (resources) =>
|
|
config.postApply?.(
|
|
resources as KuberResource[],
|
|
await getHookContext(),
|
|
)
|
|
: undefined,
|
|
request,
|
|
);
|
|
task.output = `${taskCtx.plan.desired.length} applied, ${taskCtx.plan.stale.length} stale deleted`;
|
|
},
|
|
},
|
|
],
|
|
{ rendererOptions: { collapseErrors: false } },
|
|
).run();
|
|
|
|
return taskCtx;
|
|
}
|
|
|
|
export const up = defineCommand({
|
|
meta: { name: "up", description: "Create and start deployments" },
|
|
args: {
|
|
build: {
|
|
type: "boolean",
|
|
default: true,
|
|
alias: "b",
|
|
description: "Build images before starting deployments",
|
|
negativeDescription: "Don't build an image, even if it's policy",
|
|
},
|
|
},
|
|
async run({ args }) {
|
|
await runUp(args.build);
|
|
},
|
|
});
|