402 lines
11 KiB
TypeScript
402 lines
11 KiB
TypeScript
import { randomUUID } from "node:crypto";
|
|
import { execFile } from "node:child_process";
|
|
import { isAbsolute, relative, resolve, sep } from "node:path";
|
|
import { promisify } from "node:util";
|
|
import type { Writable } from "node:stream";
|
|
import type { ComposeSpecification, Service } from "../schema/docker.d";
|
|
import {
|
|
BUILD_PROTOCOL_VERSION,
|
|
type BuildEvent,
|
|
type BuildRequest,
|
|
type BuildStatus,
|
|
type Sha256Digest,
|
|
} from "../shared/build-protocol";
|
|
import { resolveComposeArch } from "./arch";
|
|
import { apiRequest, type ApiRequestInit } from "./api";
|
|
import { DEFAULT_REGISTRY } from "./config";
|
|
import {
|
|
enumerateWorkspace,
|
|
serializeWorkspaceManifest,
|
|
type WorkspaceSnapshot,
|
|
} from "./workspace";
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
const UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024;
|
|
const DEFAULT_POLL_INTERVAL_MS = 1_000;
|
|
|
|
export type ApiRequester = <T>(
|
|
path: string,
|
|
init?: ApiRequestInit,
|
|
) => Promise<T>;
|
|
|
|
export type BuildOptions = {
|
|
registry?: string;
|
|
request?: ApiRequester;
|
|
pollIntervalMs?: number;
|
|
sleep?: (milliseconds: number) => Promise<void>;
|
|
snapshot?: WorkspaceSnapshot;
|
|
};
|
|
|
|
type BuildPlan = {
|
|
name: string;
|
|
image: string;
|
|
context: string;
|
|
dockerfile?: string;
|
|
target?: string;
|
|
buildArgs: string[];
|
|
};
|
|
|
|
type ProgressReporter = (message: string) => void | Promise<void>;
|
|
type BuildReporter = {
|
|
progress?: ProgressReporter;
|
|
stream?: Writable;
|
|
};
|
|
|
|
export type BuildResult = {
|
|
built: string[];
|
|
changed: string[];
|
|
images: Record<string, string>;
|
|
};
|
|
|
|
type SnapshotNegotiation = {
|
|
workspace: Sha256Digest;
|
|
missing: Sha256Digest[];
|
|
ready: boolean;
|
|
};
|
|
|
|
type ImageResult = {
|
|
image: string;
|
|
digest: Sha256Digest;
|
|
reference: string;
|
|
};
|
|
|
|
function posixRelative(root: string, path: string): string {
|
|
return relative(root, path).split(sep).join("/") || ".";
|
|
}
|
|
|
|
function assertInsideRepo(
|
|
repoRoot: string,
|
|
path: string,
|
|
description: string,
|
|
service: string,
|
|
): string {
|
|
const value = relative(repoRoot, path);
|
|
if (value.startsWith(`..${sep}`) || value === ".." || isAbsolute(value)) {
|
|
throw new Error(
|
|
`${description} must stay inside the git repo for service ${service}`,
|
|
);
|
|
}
|
|
return posixRelative(repoRoot, path);
|
|
}
|
|
|
|
function resolveBuildArgs(service: Service): string[] {
|
|
if (
|
|
!service.build ||
|
|
typeof service.build === "string" ||
|
|
!service.build.args
|
|
)
|
|
return [];
|
|
if (Array.isArray(service.build.args)) return [...service.build.args];
|
|
return Object.entries(service.build.args)
|
|
.filter(([, value]) => value !== null)
|
|
.map(([key, value]) => `${key}=${String(value)}`);
|
|
}
|
|
|
|
function resolveBuildPlan(
|
|
project: string,
|
|
name: string,
|
|
service: Service,
|
|
cwd: string,
|
|
repoRoot: string,
|
|
registry: string,
|
|
): BuildPlan | undefined {
|
|
if (!service.build) return;
|
|
const build = service.build;
|
|
const contextInput =
|
|
typeof build === "string" ? build : (build.context ?? ".");
|
|
if (contextInput.includes("://"))
|
|
throw new Error(
|
|
`Remote build context is not supported for service ${name}`,
|
|
);
|
|
if (typeof build !== "string" && build.dockerfile_inline)
|
|
throw new Error(`dockerfile_inline is not supported for service ${name}`);
|
|
|
|
const contextPath = resolve(cwd, contextInput);
|
|
const context = assertInsideRepo(
|
|
repoRoot,
|
|
contextPath,
|
|
"Build context",
|
|
name,
|
|
);
|
|
const dockerfilePath =
|
|
typeof build === "string" || !build.dockerfile
|
|
? undefined
|
|
: resolve(contextPath, build.dockerfile);
|
|
|
|
return {
|
|
name,
|
|
// The server replaces this requested name with its configured imageName.
|
|
image: getBuildImageName(project, name, registry),
|
|
context,
|
|
dockerfile: dockerfilePath
|
|
? assertInsideRepo(repoRoot, dockerfilePath, "Dockerfile", name)
|
|
: undefined,
|
|
target: typeof build === "string" ? undefined : build.target,
|
|
buildArgs: resolveBuildArgs(service),
|
|
};
|
|
}
|
|
|
|
export function parseImageManifestDigest(output: string): string {
|
|
const manifest = JSON.parse(output) as { digest?: unknown };
|
|
if (
|
|
typeof manifest.digest !== "string" ||
|
|
!/^sha256:[a-f0-9]{64}$/.test(manifest.digest)
|
|
)
|
|
throw new Error("Registry response did not contain a valid image digest");
|
|
return manifest.digest;
|
|
}
|
|
|
|
export function toPinnedImage(image: string, digest: string): string {
|
|
if (!/^sha256:[a-f0-9]{64}$/.test(digest))
|
|
throw new Error(`Invalid image digest ${digest}`);
|
|
return `${image}@${digest}`;
|
|
}
|
|
|
|
export function getBuildImageName(
|
|
project: string,
|
|
service: string,
|
|
registry = DEFAULT_REGISTRY,
|
|
): string {
|
|
return `${registry.replace(/\/+$/, "")}/kuber/${project}-${service}:latest`;
|
|
}
|
|
|
|
export function imageDigestChanged(
|
|
before: string | undefined,
|
|
after: string | undefined,
|
|
): boolean {
|
|
return !before || !after || before !== after;
|
|
}
|
|
|
|
export async function getRepoRoot(cwd: string): Promise<string> {
|
|
const { stdout } = await execFileAsync("git", [
|
|
"-C",
|
|
cwd,
|
|
"rev-parse",
|
|
"--show-toplevel",
|
|
]);
|
|
return stdout.trim();
|
|
}
|
|
|
|
async function uploadBlob(
|
|
digest: Sha256Digest,
|
|
data: Uint8Array,
|
|
request: ApiRequester,
|
|
): Promise<void> {
|
|
const path = `/blobs/${encodeURIComponent(digest)}/uploads`;
|
|
const progress = await request<{ offset: number; complete: boolean }>(path, {
|
|
method: "POST",
|
|
json: { size: data.byteLength },
|
|
});
|
|
let offset = progress.offset;
|
|
while (!progress.complete && offset < data.byteLength) {
|
|
const chunk = data.subarray(offset, offset + UPLOAD_CHUNK_BYTES);
|
|
const uploaded = await request<{ offset: number }>(path, {
|
|
method: "PATCH",
|
|
headers: {
|
|
"content-type": "application/octet-stream",
|
|
"upload-offset": String(offset),
|
|
},
|
|
body: chunk,
|
|
});
|
|
if (uploaded.offset <= offset)
|
|
throw new Error(`Blob upload for ${digest} made no progress`);
|
|
offset = uploaded.offset;
|
|
}
|
|
if (!progress.complete) {
|
|
await request(`${path}/complete`, { method: "POST", json: {} });
|
|
}
|
|
}
|
|
|
|
export async function uploadWorkspaceSnapshot(
|
|
snapshot: WorkspaceSnapshot,
|
|
request: ApiRequester = apiRequest,
|
|
reporter?: BuildReporter,
|
|
): Promise<void> {
|
|
const blobs = new Map(snapshot.blobs.map((blob) => [blob.digest, blob.data]));
|
|
blobs.set(snapshot.digest, serializeWorkspaceManifest(snapshot.manifest));
|
|
|
|
for (;;) {
|
|
const negotiation = await request<SnapshotNegotiation>(
|
|
"/snapshots/negotiate",
|
|
{
|
|
method: "POST",
|
|
json: { workspace: snapshot.digest },
|
|
},
|
|
);
|
|
if (negotiation.ready) return;
|
|
if (negotiation.missing.length === 0)
|
|
throw new Error(
|
|
"Snapshot negotiation is incomplete but reported no missing blobs",
|
|
);
|
|
for (const digest of negotiation.missing) {
|
|
const data = blobs.get(digest);
|
|
if (!data)
|
|
throw new Error(`Server requested unknown workspace blob ${digest}`);
|
|
await reporter?.progress?.(`Uploading ${digest}`);
|
|
await uploadBlob(digest, data, request);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function reportBuildEvent(
|
|
event: BuildEvent,
|
|
reporter?: BuildReporter,
|
|
reportedStates?: Set<BuildStatus["state"]>,
|
|
): Promise<number> {
|
|
if (event.type === "status") {
|
|
if (!reportedStates?.has(event.status.state)) {
|
|
reportedStates?.add(event.status.state);
|
|
await reporter?.progress?.(`Build ${event.status.state}`);
|
|
}
|
|
return 0;
|
|
}
|
|
if (reporter?.stream) reporter.stream.write(event.message);
|
|
else await reporter?.progress?.(event.message.trimEnd());
|
|
return event.sequence;
|
|
}
|
|
|
|
async function waitForBuild(
|
|
id: string,
|
|
request: ApiRequester,
|
|
reporter: BuildReporter | undefined,
|
|
pollIntervalMs: number,
|
|
sleep: (milliseconds: number) => Promise<void>,
|
|
initial: BuildStatus,
|
|
): Promise<BuildStatus> {
|
|
let status = initial;
|
|
let sequence = 0;
|
|
const reportedStates = new Set<BuildStatus["state"]>();
|
|
for (;;) {
|
|
const events = await request<BuildEvent[]>(
|
|
`/builds/${encodeURIComponent(id)}/events?after=${sequence}`,
|
|
);
|
|
for (const event of events)
|
|
sequence = Math.max(
|
|
sequence,
|
|
await reportBuildEvent(event, reporter, reportedStates),
|
|
);
|
|
if (status.state === "succeeded" || status.state === "failed")
|
|
return status;
|
|
status = await request<BuildStatus>(
|
|
`/builds/${encodeURIComponent(id)}/reconcile`,
|
|
{
|
|
method: "POST",
|
|
json: {},
|
|
},
|
|
);
|
|
if (status.state !== "succeeded" && status.state !== "failed")
|
|
await sleep(pollIntervalMs);
|
|
}
|
|
}
|
|
|
|
export async function resolveBuildImages(
|
|
project: string,
|
|
compose: ComposeSpecification,
|
|
options: BuildOptions = {},
|
|
): Promise<Record<string, string>> {
|
|
const request = options.request ?? apiRequest;
|
|
const images: Record<string, string> = {};
|
|
for (const [service, definition] of Object.entries(compose.services ?? {})) {
|
|
if (!definition.build) continue;
|
|
try {
|
|
images[service] = (
|
|
await request<ImageResult>("/images/resolve", {
|
|
method: "POST",
|
|
json: { project, service },
|
|
})
|
|
).reference;
|
|
} catch (error) {
|
|
throw new Error(
|
|
`Cannot resolve a published image for service ${service}. Run kuber up to build it.`,
|
|
{ cause: error },
|
|
);
|
|
}
|
|
}
|
|
return images;
|
|
}
|
|
|
|
export async function buildServices(
|
|
project: string,
|
|
compose: ComposeSpecification,
|
|
cwd = process.cwd(),
|
|
reporter?: BuildReporter,
|
|
options: BuildOptions = {},
|
|
): Promise<BuildResult> {
|
|
if (!Object.values(compose.services ?? {}).some((service) => service.build))
|
|
return { built: [], changed: [], images: {} };
|
|
|
|
const request = options.request ?? apiRequest;
|
|
const repoRoot = await getRepoRoot(cwd);
|
|
const snapshot = options.snapshot ?? (await enumerateWorkspace(repoRoot));
|
|
const plans = Object.entries(compose.services ?? {}).flatMap(
|
|
([name, service]) => {
|
|
const plan = resolveBuildPlan(
|
|
project,
|
|
name,
|
|
service,
|
|
cwd,
|
|
repoRoot,
|
|
options.registry ?? DEFAULT_REGISTRY,
|
|
);
|
|
return plan ? [plan] : [];
|
|
},
|
|
);
|
|
await uploadWorkspaceSnapshot(snapshot, request, reporter);
|
|
|
|
const images: Record<string, string> = {};
|
|
for (const plan of plans) {
|
|
await reporter?.progress?.(`Building ${plan.name}`);
|
|
const id = randomUUID();
|
|
const buildRequest: BuildRequest = {
|
|
version: BUILD_PROTOCOL_VERSION,
|
|
id,
|
|
project,
|
|
service: plan.name,
|
|
spec: {
|
|
architecture: resolveComposeArch(compose),
|
|
image: plan.image,
|
|
context: plan.context,
|
|
dockerfile: plan.dockerfile,
|
|
target: plan.target,
|
|
buildArgs: plan.buildArgs,
|
|
workspace: snapshot.digest,
|
|
},
|
|
};
|
|
const initial = await request<BuildStatus>("/builds", {
|
|
method: "POST",
|
|
json: buildRequest,
|
|
});
|
|
const status = await waitForBuild(
|
|
id,
|
|
request,
|
|
reporter,
|
|
options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS,
|
|
options.sleep ?? ((milliseconds) => Bun.sleep(milliseconds)),
|
|
initial,
|
|
);
|
|
if (status.state !== "succeeded")
|
|
throw new Error(
|
|
`Build failed for service ${plan.name}: ${status.error ?? "unknown error"}`,
|
|
);
|
|
images[plan.name] = (
|
|
await request<ImageResult>(`/builds/${encodeURIComponent(id)}/result`)
|
|
).reference;
|
|
}
|
|
|
|
return {
|
|
built: plans.map((plan) => plan.name),
|
|
changed: plans.map((plan) => plan.name),
|
|
images,
|
|
};
|
|
}
|