import { randomUUID } from "node:crypto"; import { execFile } from "node:child_process"; import { isAbsolute, relative, resolve, sep } from "node:path"; import { promisify } from "node:util"; import type { Writable } from "node:stream"; import type { ComposeSpecification, Service } from "../schema/docker.d"; import { BUILD_PROTOCOL_VERSION, type BuildEvent, type BuildRequest, type BuildStatus, type Sha256Digest, } from "../shared/build-protocol"; import { resolveComposeArch } from "./arch"; import { apiRequest, type ApiRequestInit, type ApiRequestOptions } from "./api"; import { DEFAULT_REGISTRY } from "./config"; import { enumerateWorkspace, serializeWorkspaceManifest, type WorkspaceSnapshot, } from "./workspace"; const execFileAsync = promisify(execFile); const UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024; const DEFAULT_POLL_INTERVAL_MS = 1_000; const BUILD_POLL_REQUEST_TIMEOUT_MS = 300_000; const MAX_BUILD_POLL_ATTEMPTS = 3; export const MAX_CONCURRENT_REQUESTS = 20; export const MAX_REQUESTS_PER_SECOND = 40; export type SchedulerClock = { now(): number; }; export type SchedulerSleep = (ms: number) => Promise; export class TaskScheduler { private inflight = 0; private maxInflight: number; private maxPerSecond: number; private requestStarts: number[] = []; private waiting: Array<() => void> = []; private rateGate: Promise = Promise.resolve(); private clock: SchedulerClock; private sleep: SchedulerSleep; constructor(options?: { maxInflight?: number; maxPerSecond?: number; clock?: SchedulerClock; sleep?: SchedulerSleep; }) { this.maxInflight = options?.maxInflight ?? MAX_CONCURRENT_REQUESTS; this.maxPerSecond = options?.maxPerSecond ?? MAX_REQUESTS_PER_SECOND; if (!Number.isSafeInteger(this.maxInflight) || this.maxInflight < 1) throw new RangeError("maxInflight must be a positive integer"); if (!Number.isSafeInteger(this.maxPerSecond) || this.maxPerSecond < 1) throw new RangeError("maxPerSecond must be a positive integer"); this.clock = options?.clock ?? { now: () => Date.now() }; this.sleep = options?.sleep ?? ((ms) => Bun.sleep(ms)); } get currentInflight(): number { return this.inflight; } get maxConcurrent(): number { return this.maxInflight; } get currentRequestStarts(): number { this.pruneOldStarts(); return this.requestStarts.length; } private pruneOldStarts(): void { const cutoff = this.clock.now() - 1000; while (this.requestStarts.length > 0 && this.requestStarts[0]! <= cutoff) { this.requestStarts.shift(); } } private async acquire(): Promise { if (this.inflight < this.maxInflight) { this.inflight++; return; } await new Promise((resolve) => { this.waiting.push(resolve); }); } private release(): void { if (this.waiting.length > 0) { const next = this.waiting.shift()!; next(); } else { this.inflight--; } } private waitForRateLimit(): Promise { const reservation = this.rateGate.then(async () => { for (;;) { this.pruneOldStarts(); if (this.requestStarts.length < this.maxPerSecond) { this.requestStarts.push(this.clock.now()); return; } const oldest = this.requestStarts[0]!; await this.sleep(Math.max(1, oldest + 1000 - this.clock.now())); } }); this.rateGate = reservation.catch(() => {}); return reservation; } async run(fn: () => Promise): Promise { await this.acquire(); try { await this.waitForRateLimit(); return await fn(); } finally { this.release(); } } } async function runConcurrent( values: T[], concurrency: number, run: (value: T) => Promise, ): Promise { let index = 0; const worker = async () => { for (;;) { const current = index++; if (current >= values.length) return; await run(values[current]!); } }; await Promise.all( Array.from({ length: Math.min(concurrency, values.length) }, worker), ); } export type ApiRequester = ( path: string, init?: ApiRequestInit, options?: ApiRequestOptions, ) => Promise; export type BuildOptions = { registry?: string; request?: ApiRequester; pollIntervalMs?: number; sleep?: (milliseconds: number) => Promise; snapshot?: WorkspaceSnapshot; scheduler?: TaskScheduler; }; type BuildPlan = { name: string; image: string; context: string; dockerfile?: string; target?: string; buildArgs: string[]; }; type ProgressReporter = (message: string) => void | Promise; type BuildReporter = { progress?: ProgressReporter; stream?: Writable; }; export type BuildResult = { built: string[]; changed: string[]; images: Record; }; type SnapshotNegotiation = { workspace: Sha256Digest; missing: Sha256Digest[]; ready: boolean; }; type ImageResult = { image: string; digest: Sha256Digest; reference: string; }; function posixRelative(root: string, path: string): string { return relative(root, path).split(sep).join("/") || "."; } function assertInsideRepo( repoRoot: string, path: string, description: string, service: string, ): string { const value = relative(repoRoot, path); if (value.startsWith(`..${sep}`) || value === ".." || isAbsolute(value)) { throw new Error( `${description} must stay inside the git repo for service ${service}`, ); } return posixRelative(repoRoot, path); } function resolveBuildArgs(service: Service): string[] { if ( !service.build || typeof service.build === "string" || !service.build.args ) return []; if (Array.isArray(service.build.args)) return [...service.build.args]; return Object.entries(service.build.args) .filter(([, value]) => value !== null) .map(([key, value]) => `${key}=${String(value)}`); } function resolveBuildPlan( project: string, name: string, service: Service, cwd: string, repoRoot: string, registry: string, ): BuildPlan | undefined { if (!service.build) return; const build = service.build; const contextInput = typeof build === "string" ? build : (build.context ?? "."); if (contextInput.includes("://")) throw new Error( `Remote build context is not supported for service ${name}`, ); if (typeof build !== "string" && build.dockerfile_inline) throw new Error(`dockerfile_inline is not supported for service ${name}`); const contextPath = resolve(cwd, contextInput); const context = assertInsideRepo( repoRoot, contextPath, "Build context", name, ); const dockerfilePath = typeof build === "string" || !build.dockerfile ? undefined : resolve(contextPath, build.dockerfile); return { name, // The server replaces this requested name with its configured imageName. image: getBuildImageName(project, name, registry), context, dockerfile: dockerfilePath ? assertInsideRepo(repoRoot, dockerfilePath, "Dockerfile", name) : undefined, target: typeof build === "string" ? undefined : build.target, buildArgs: resolveBuildArgs(service), }; } export function parseImageManifestDigest(output: string): string { const manifest = JSON.parse(output) as { digest?: unknown }; if ( typeof manifest.digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(manifest.digest) ) throw new Error("Registry response did not contain a valid image digest"); return manifest.digest; } export function toPinnedImage(image: string, digest: string): string { if (!/^sha256:[a-f0-9]{64}$/.test(digest)) throw new Error(`Invalid image digest ${digest}`); return `${image}@${digest}`; } export function getBuildImageName( project: string, service: string, registry = DEFAULT_REGISTRY, ): string { return `${registry.replace(/\/+$/, "")}/kuber/${project}-${service}:latest`; } export function imageDigestChanged( before: string | undefined, after: string | undefined, ): boolean { return !before || !after || before !== after; } export async function getRepoRoot(cwd: string): Promise { const { stdout } = await execFileAsync("git", [ "-C", cwd, "rev-parse", "--show-toplevel", ]); return stdout.trim(); } async function uploadBlob( digest: Sha256Digest, data: Uint8Array, request: ApiRequester, scheduler: TaskScheduler, project?: string, ): Promise { const uploadPath = `/blobs/${encodeURIComponent(digest)}/uploads`; const projectQuery = project ? `?project=${encodeURIComponent(project)}` : ""; const path = `${uploadPath}${projectQuery}`; const progress = await scheduler.run(() => request<{ offset: number; complete: boolean }>(path, { method: "POST", json: { size: data.byteLength }, }), ); let offset = progress.offset; while (!progress.complete && offset < data.byteLength) { const chunk = data.subarray(offset, offset + UPLOAD_CHUNK_BYTES); const uploaded = await scheduler.run(() => request<{ offset: number }>(path, { method: "PATCH", headers: { "content-type": "application/octet-stream", "upload-offset": String(offset), }, body: chunk, }), ); if (uploaded.offset <= offset) throw new Error(`Blob upload for ${digest} made no progress`); offset = uploaded.offset; } if (!progress.complete) { await scheduler.run(() => request(`${uploadPath}/complete${projectQuery}`, { method: "POST", json: {}, }), ); } } export async function uploadWorkspaceSnapshot( snapshot: WorkspaceSnapshot, request: ApiRequester = apiRequest, reporter?: BuildReporter, scheduler?: TaskScheduler, project?: string, ): Promise { const blobs = new Map(snapshot.blobs.map((blob) => [blob.digest, blob.data])); blobs.set(snapshot.digest, serializeWorkspaceManifest(snapshot.manifest)); const requestScheduler = scheduler ?? new TaskScheduler(); for (;;) { const negotiation = await requestScheduler.run(() => request("/snapshots/negotiate", { method: "POST", json: { workspace: snapshot.digest, ...(project && { project }) }, }), ); if (negotiation.ready) return; if (negotiation.missing.length === 0) throw new Error( "Snapshot negotiation is incomplete but reported no missing blobs", ); await runConcurrent( negotiation.missing, requestScheduler.maxConcurrent, async (digest) => { const data = blobs.get(digest); if (!data) throw new Error(`Server requested unknown workspace blob ${digest}`); await reporter?.progress?.(`Uploading ${digest}`); await uploadBlob(digest, data, request, requestScheduler, project); }, ); } } async function reportBuildEvent( event: BuildEvent, reporter?: BuildReporter, reportedStates?: Set, ): Promise { if (event.type === "status") { if (!reportedStates?.has(event.status.state)) { reportedStates?.add(event.status.state); await reporter?.progress?.(`Build ${event.status.state}`); } return 0; } if (reporter?.stream) reporter.stream.write(event.message); else await reporter?.progress?.(event.message.trimEnd()); return event.sequence; } function isTransientBuildPollError(error: unknown): boolean { if (!(error instanceof Error) || error.name === "AbortError") return false; if (error.name === "TimeoutError" || error instanceof TypeError) return true; const code = "code" in error && typeof error.code === "string" ? error.code : error.cause && typeof error.cause === "object" && "code" in error.cause && typeof error.cause.code === "string" ? error.cause.code : undefined; return ( code === "ECONNABORTED" || code === "ECONNRESET" || code === "ECONNREFUSED" || code === "EAI_AGAIN" || code === "ETIMEDOUT" ); } async function requestBuildPoll( request: ApiRequester, path: string, init: ApiRequestInit | undefined, pollIntervalMs: number, sleep: (milliseconds: number) => Promise, ): Promise { for (let attempt = 1; attempt <= MAX_BUILD_POLL_ATTEMPTS; attempt++) { try { return await request(path, init, { timeoutMs: BUILD_POLL_REQUEST_TIMEOUT_MS, }); } catch (error) { if ( !isTransientBuildPollError(error) || attempt === MAX_BUILD_POLL_ATTEMPTS ) { throw error; } await sleep(pollIntervalMs); } } throw new Error("Build poll retries exhausted"); } async function waitForBuild( id: string, request: ApiRequester, reporter: BuildReporter | undefined, pollIntervalMs: number, sleep: (milliseconds: number) => Promise, initial: BuildStatus, ): Promise { let status = initial; let sequence = 0; const reportedStates = new Set(); for (;;) { const events = await requestBuildPoll( request, `/builds/${encodeURIComponent(id)}/events?after=${sequence}`, undefined, pollIntervalMs, sleep, ); for (const event of events) sequence = Math.max( sequence, await reportBuildEvent(event, reporter, reportedStates), ); if (status.state === "succeeded" || status.state === "failed") return status; status = await requestBuildPoll( request, `/builds/${encodeURIComponent(id)}/reconcile`, { method: "POST", json: {}, }, pollIntervalMs, sleep, ); if (status.state !== "succeeded" && status.state !== "failed") await sleep(pollIntervalMs); } } export async function resolveBuildImages( project: string, compose: ComposeSpecification, options: BuildOptions = {}, ): Promise> { const request = options.request ?? apiRequest; const images: Record = {}; for (const [service, definition] of Object.entries(compose.services ?? {})) { if (!definition.build) continue; try { images[service] = ( await request("/images/resolve", { method: "POST", json: { project, service }, }) ).reference; } catch (error) { throw new Error( `Cannot resolve a published image for service ${service}. Run kuber up to build it.`, { cause: error }, ); } } return images; } export async function buildServices( project: string, compose: ComposeSpecification, cwd = process.cwd(), reporter?: BuildReporter, options: BuildOptions = {}, ): Promise { if (!Object.values(compose.services ?? {}).some((service) => service.build)) return { built: [], changed: [], images: {} }; const request = options.request ?? apiRequest; const repoRoot = await getRepoRoot(cwd); const snapshot = options.snapshot ?? (await enumerateWorkspace(repoRoot)); const plans = Object.entries(compose.services ?? {}).flatMap( ([name, service]) => { const plan = resolveBuildPlan( project, name, service, cwd, repoRoot, options.registry ?? DEFAULT_REGISTRY, ); return plan ? [plan] : []; }, ); await uploadWorkspaceSnapshot( snapshot, request, reporter, options.scheduler, project, ); const images: Record = {}; for (const plan of plans) { await reporter?.progress?.(`Building ${plan.name}`); const id = randomUUID(); const buildRequest: BuildRequest = { version: BUILD_PROTOCOL_VERSION, id, project, service: plan.name, spec: { architecture: resolveComposeArch(compose), image: plan.image, context: plan.context, dockerfile: plan.dockerfile, target: plan.target, buildArgs: plan.buildArgs, workspace: snapshot.digest, }, }; const initial = await request( "/builds", { method: "POST", json: buildRequest, }, { timeoutMs: 300_000 }, ); const status = await waitForBuild( id, request, reporter, options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS, options.sleep ?? ((milliseconds) => Bun.sleep(milliseconds)), initial, ); if (status.state !== "succeeded") throw new Error( `Build failed for service ${plan.name}: ${status.error ?? "unknown error"}`, ); images[plan.name] = ( await request(`/builds/${encodeURIComponent(id)}/result`) ).reference; } return { built: plans.map((plan) => plan.name), changed: plans.map((plan) => plan.name), images, }; }