Files
kuber/server/build-controller.ts
T
2026-09-03 11:28:30 +07:00

662 lines
21 KiB
TypeScript

import { createHash } from "node:crypto";
import { rm } from "node:fs/promises";
import { join } from "node:path";
import {
BUILD_PROTOCOL_VERSION,
assertSha256Digest,
type BuildEvent,
type BuildRequest,
type BuildStatus,
type Sha256Digest,
} from "../shared/build-protocol";
import { createBuildJob, type KubernetesJob } from "./build-job";
import {
BUILD_RECORD_API_VERSION,
BuildStoreConflictError,
type BuildRecord,
type BuildStore,
type UploadRecord,
} from "./build-store";
import {
materializeWorkspace,
parseWorkspaceManifest,
type MaterializeCas,
} from "./materialize";
import { parseImageReference, resolveRegistryDigest } from "./registry";
export const DEFAULT_MAX_BLOB_BYTES = 1024 * 1024 * 1024;
export const DEFAULT_MAX_UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024;
export const DEFAULT_MAX_LOG_BYTES = 1024 * 1024;
export interface BuildCas extends MaterializeCas {
put(data: Uint8Array, expected?: Sha256Digest): Promise<Sha256Digest>;
}
export type JobPhase = "queued" | "running" | "succeeded" | "failed";
export interface BuildJobObservation {
phase: JobPhase;
startedAt?: string;
finishedAt?: string;
error?: string;
}
export interface BuildKubernetesOperations {
createJob(job: KubernetesJob): Promise<void>;
getJob(
namespace: string,
name: string,
): Promise<BuildJobObservation | undefined>;
getJobLogs(namespace: string, name: string): Promise<string | Uint8Array>;
deleteJob(namespace: string, name: string): Promise<void>;
}
export interface BuildControllerOptions {
cas: BuildCas;
store: BuildStore;
kubernetes: BuildKubernetesOperations;
namespace: string;
workspaceRoot: string;
workspaceClaimName: string;
cacheImage: string | ((request: BuildRequest) => string);
imageName?: (request: BuildRequest) => string;
pushImage?: (request: BuildRequest) => string;
pushRegistryInsecure?: boolean;
cacheRegistryInsecure?: boolean;
buildkitImage?: string;
serviceAccountName?: string;
registrySecretName?: string;
nodeSelector?: Record<string, string>;
tolerations?: Array<Record<string, unknown>>;
maxBlobBytes?: number;
maxUploadChunkBytes?: number;
maxLogBytes?: number;
now?: () => Date;
materialize?: typeof materializeWorkspace;
resolveDigest?: typeof resolveRegistryDigest;
}
export class BuildValidationError extends Error {
readonly code = "BUILD_INVALID";
}
export class BuildNotFoundError extends Error {
readonly code = "BUILD_NOT_FOUND";
}
export class BuildConflictError extends Error {
readonly code = "BUILD_CONFLICT";
}
export type SnapshotNegotiation = {
workspace: Sha256Digest;
missing: Sha256Digest[];
ready: boolean;
};
export type UploadProgress = {
digest: Sha256Digest;
size: number;
offset: number;
complete: boolean;
};
export function buildImageName(
registry: string,
project: string,
service: string,
): string {
const valid = (value: string) =>
value.length <= 63 && /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/.test(value);
if (!valid(project) || !valid(service))
throw new BuildValidationError(
"Build project and service must be valid Kubernetes names",
);
const owner = registry.replace(/\/+$/, "");
if (!owner) throw new BuildValidationError("Build registry is required");
return `${owner}/kuber/${project}-${service}:latest`;
}
function clone<T>(value: T): T {
return structuredClone(value);
}
function requestFingerprint(request: BuildRequest): string {
return createHash("sha256").update(JSON.stringify(request)).digest("hex");
}
function jobWorkspaceSubPath(workspaceRoot: string, subPath: string): string {
const segments = workspaceRoot.split("/").filter(Boolean);
const prefix = segments.at(-1);
if (!prefix || prefix === ".") return subPath;
return subPath ? `${prefix}/${subPath}` : prefix;
}
function validateRequest(request: BuildRequest): void {
if (request.version !== BUILD_PROTOCOL_VERSION)
throw new BuildValidationError("Unsupported build protocol version");
if (!request.id || !request.project || !request.service)
throw new BuildValidationError(
"Build ID, project, and service are required",
);
if (Buffer.byteLength(request.id) > 256)
throw new BuildValidationError("Build ID is too long");
if (!request.spec || !["amd64", "arm64"].includes(request.spec.architecture))
throw new BuildValidationError("Invalid build architecture");
if (
!Array.isArray(request.spec.buildArgs) ||
!request.spec.buildArgs.every((value) => typeof value === "string")
) {
throw new BuildValidationError("Build arguments must be strings");
}
assertSha256Digest(request.spec.workspace);
parseImageReference(request.spec.image);
}
function recordStatus(record: BuildRecord): BuildStatus {
const {
version,
id,
state,
createdAt,
startedAt,
finishedAt,
digest,
error,
} = record.status;
return {
version,
id,
state,
createdAt,
...(startedAt && { startedAt }),
...(finishedAt && { finishedAt }),
...(digest && { digest }),
...(error && { error }),
};
}
export class BuildController {
private readonly now: () => Date;
private readonly maxBlobBytes: number;
private readonly maxUploadChunkBytes: number;
private readonly maxLogBytes: number;
private readonly materializer: typeof materializeWorkspace;
private readonly digestResolver: typeof resolveRegistryDigest;
constructor(private readonly options: BuildControllerOptions) {
this.now = options.now ?? (() => new Date());
this.maxBlobBytes = options.maxBlobBytes ?? DEFAULT_MAX_BLOB_BYTES;
this.maxUploadChunkBytes =
options.maxUploadChunkBytes ?? DEFAULT_MAX_UPLOAD_CHUNK_BYTES;
this.maxLogBytes = options.maxLogBytes ?? DEFAULT_MAX_LOG_BYTES;
this.materializer = options.materialize ?? materializeWorkspace;
this.digestResolver = options.resolveDigest ?? resolveRegistryDigest;
}
async negotiateSnapshot(
workspace: Sha256Digest,
): Promise<SnapshotNegotiation> {
assertSha256Digest(workspace);
if (!(await this.options.cas.has(workspace)))
return { workspace, missing: [workspace], ready: false };
const manifest = parseWorkspaceManifest(
await this.options.cas.get(workspace),
);
const missing: Sha256Digest[] = [];
const seen = new Set<Sha256Digest>();
for (const file of manifest.files) {
if (!seen.has(file.digest) && !(await this.options.cas.has(file.digest)))
missing.push(file.digest);
seen.add(file.digest);
}
return { workspace, missing, ready: missing.length === 0 };
}
async beginBlobUpload(
digest: Sha256Digest,
size: number,
): Promise<UploadProgress> {
assertSha256Digest(digest);
if (!Number.isSafeInteger(size) || size < 0 || size > this.maxBlobBytes)
throw new BuildValidationError(
`Blob size must be between 0 and ${this.maxBlobBytes}`,
);
if (await this.options.cas.has(digest)) {
const actualSize = (await this.options.cas.get(digest)).byteLength;
if (actualSize !== size)
throw new BuildConflictError(
"Blob size does not match content already in CAS",
);
return { digest, size, offset: size, complete: true };
}
const current = await this.options.store.getUpload(digest);
if (current) {
if (current.spec.size !== size)
throw new BuildConflictError(
"Upload size does not match the existing upload",
);
return { digest, size, offset: current.status.offset, complete: false };
}
const timestamp = this.now().toISOString();
const upload: UploadRecord = {
apiVersion: BUILD_RECORD_API_VERSION,
kind: "BuildUpload",
metadata: {
name: digest.replace(":", "-"),
resourceVersion: "1",
creationTimestamp: timestamp,
},
spec: { digest, size },
status: { offset: 0, data: new Uint8Array() },
};
const stored = await this.options.store.createUpload(upload);
if (stored.spec.size !== size)
throw new BuildConflictError(
"Upload size does not match the existing upload",
);
return { digest, size, offset: stored.status.offset, complete: false };
}
async uploadBlobChunk(
digest: Sha256Digest,
offset: number,
chunk: Uint8Array,
): Promise<UploadProgress> {
assertSha256Digest(digest);
if (
!(chunk instanceof Uint8Array) ||
chunk.byteLength > this.maxUploadChunkBytes
)
throw new BuildValidationError(
`Upload chunks may not exceed ${this.maxUploadChunkBytes} bytes`,
);
const current = await this.options.store.getUpload(digest);
if (!current) {
if (await this.options.cas.has(digest)) {
const size = (await this.options.cas.get(digest)).byteLength;
return { digest, size, offset: size, complete: true };
}
throw new BuildNotFoundError("Blob upload was not initialized");
}
if (offset !== current.status.offset)
throw new BuildConflictError(
`Upload offset mismatch; expected ${current.status.offset}`,
);
const nextOffset = offset + chunk.byteLength;
if (nextOffset > current.spec.size)
throw new BuildValidationError("Upload exceeds the declared blob size");
const data = new Uint8Array(nextOffset);
data.set(current.status.data);
data.set(chunk, offset);
const next = clone(current);
next.metadata.resourceVersion = String(
Number(current.metadata.resourceVersion) + 1,
);
next.status = { offset: nextOffset, data };
await this.options.store.replaceUpload(
next,
current.metadata.resourceVersion,
);
return {
digest,
size: current.spec.size,
offset: nextOffset,
complete: false,
};
}
async completeBlobUpload(digest: Sha256Digest): Promise<UploadProgress> {
assertSha256Digest(digest);
const current = await this.options.store.getUpload(digest);
if (!current) {
if (await this.options.cas.has(digest)) {
const size = (await this.options.cas.get(digest)).byteLength;
return { digest, size, offset: size, complete: true };
}
throw new BuildNotFoundError("Blob upload was not initialized");
}
if (
current.status.offset !== current.spec.size ||
current.status.data.byteLength !== current.spec.size
) {
throw new BuildConflictError(
`Blob upload is incomplete at offset ${current.status.offset}`,
);
}
try {
await this.options.cas.put(current.status.data, digest);
} catch (error) {
throw new BuildValidationError(
error instanceof Error ? error.message : String(error),
);
}
await this.options.store.deleteUpload(digest);
return {
digest,
size: current.spec.size,
offset: current.spec.size,
complete: true,
};
}
async submitBuild(request: BuildRequest): Promise<BuildStatus> {
request = clone(request);
if (this.options.imageName)
request.spec.image = this.options.imageName(request);
validateRequest(request);
const existing = await this.options.store.getBuild(request.id);
if (existing) {
if (
requestFingerprint(existing.spec.request) !==
requestFingerprint(request)
)
throw new BuildConflictError(
"Build ID was already used for a different request",
);
return recordStatus(existing);
}
const snapshot = await this.negotiateSnapshot(request.spec.workspace);
if (!snapshot.ready)
throw new BuildConflictError(
`Workspace snapshot is incomplete: ${snapshot.missing.join(", ")}`,
);
const createdAt = this.now().toISOString();
const hash = createHash("sha256")
.update(request.id)
.digest("hex")
.slice(0, 24);
const jobName = `kuber-build-${hash}`;
const imageKey = `${request.project}\0${request.service}\0${request.spec.image}`;
const initial: BuildStatus = {
version: BUILD_PROTOCOL_VERSION,
id: request.id,
state: "queued",
createdAt,
};
const record: BuildRecord = {
apiVersion: BUILD_RECORD_API_VERSION,
kind: "BuildRecord",
metadata: {
name: request.id,
resourceVersion: "1",
creationTimestamp: createdAt,
labels: { project: request.project, service: request.service },
},
spec: {
request: clone(request),
imageKey,
jobName,
workspaceSubPath: jobName,
},
status: {
...initial,
logBytes: 0,
logOffset: 0,
nextSequence: 1,
events: [{ type: "status", status: initial }],
},
};
let stored: BuildRecord;
try {
const result = await this.options.store.createBuild(record);
stored = result.record;
if (!result.created) {
if (
requestFingerprint(stored.spec.request) !==
requestFingerprint(request)
)
throw new BuildConflictError(
"Build ID was already used for a different request",
);
return recordStatus(stored);
}
} catch (error) {
if (error instanceof BuildStoreConflictError)
throw new BuildConflictError(error.message);
throw error;
}
try {
await this.materializer(
this.options.cas,
request.spec.workspace,
join(this.options.workspaceRoot, stored.spec.workspaceSubPath),
);
const cacheImage =
typeof this.options.cacheImage === "function"
? this.options.cacheImage(request)
: this.options.cacheImage;
const pushImage = this.options.pushImage
? this.options.pushImage(request)
: request.spec.image;
const job = createBuildJob({
name: jobName,
namespace: this.options.namespace,
spec: request.spec,
workspaceClaimName: this.options.workspaceClaimName,
workspaceSubPath: jobWorkspaceSubPath(
this.options.workspaceRoot,
stored.spec.workspaceSubPath,
),
cacheImage,
pushImage,
pushRegistryInsecure: this.options.pushRegistryInsecure,
cacheRegistryInsecure:
this.options.cacheRegistryInsecure ??
this.options.pushRegistryInsecure,
buildkitImage: this.options.buildkitImage,
serviceAccountName: this.options.serviceAccountName,
registrySecretName: this.options.registrySecretName,
nodeSelector: this.options.nodeSelector,
tolerations: this.options.tolerations,
});
await this.options.kubernetes.createJob(job);
await this.updateBuild(stored, (next) => {
next.status.jobCreated = true;
});
return initial;
} catch (error) {
await this.failBuild(
stored.metadata.name,
error instanceof Error ? error.message : String(error),
);
throw error;
}
}
async getBuildStatus(id: string): Promise<BuildStatus> {
const record = await this.requireBuild(id);
return recordStatus(record);
}
async getBuildEvents(id: string, afterSequence = 0): Promise<BuildEvent[]> {
if (!Number.isSafeInteger(afterSequence) || afterSequence < 0)
throw new BuildValidationError(
"Event sequence must be a non-negative integer",
);
const record = await this.requireBuild(id);
return clone(
record.status.events.filter(
(event) => event.type === "status" || event.sequence > afterSequence,
),
);
}
async reconcileBuild(id: string): Promise<BuildStatus> {
let record = await this.requireBuild(id);
if (record.status.state === "succeeded" || record.status.state === "failed")
return recordStatus(record);
if (record.status.jobCreated) record = await this.captureLogs(record);
const observation = await this.options.kubernetes.getJob(
this.options.namespace,
record.spec.jobName,
);
if (!observation) return recordStatus(record);
if (observation.phase === "running" && record.status.state === "queued") {
record = await this.setState(record, "running", {
startedAt: observation.startedAt ?? this.now().toISOString(),
});
} else if (observation.phase === "failed") {
record = await this.setState(record, "failed", {
startedAt: record.status.startedAt ?? observation.startedAt,
finishedAt: observation.finishedAt ?? this.now().toISOString(),
error: observation.error ?? "BuildKit Job failed",
});
} else if (observation.phase === "succeeded") {
try {
const digest = await this.digestResolver(
record.spec.request.spec.image,
);
assertSha256Digest(digest);
record = await this.setState(record, "succeeded", {
startedAt: record.status.startedAt ?? observation.startedAt,
finishedAt: observation.finishedAt ?? this.now().toISOString(),
digest,
});
} catch (error) {
record = await this.setState(record, "failed", {
finishedAt: this.now().toISOString(),
error: `Unable to resolve pushed image digest: ${error instanceof Error ? error.message : String(error)}`,
});
}
}
return recordStatus(record);
}
async cancelBuild(id: string): Promise<BuildStatus> {
const record = await this.requireBuild(id);
if (record.status.state === "succeeded" || record.status.state === "failed")
return recordStatus(record);
if (record.status.jobCreated)
await this.options.kubernetes.deleteJob(
this.options.namespace,
record.spec.jobName,
);
const next = await this.setState(record, "failed", {
finishedAt: this.now().toISOString(),
error: "Build cancelled",
cancelled: true,
});
return recordStatus(next);
}
async cleanupBuild(id: string): Promise<void> {
const record = await this.requireBuild(id);
if (record.status.state !== "succeeded" && record.status.state !== "failed")
throw new BuildConflictError("An active build cannot be cleaned up");
if (record.status.jobCreated)
await this.options.kubernetes.deleteJob(
this.options.namespace,
record.spec.jobName,
);
await rm(join(this.options.workspaceRoot, record.spec.workspaceSubPath), {
recursive: true,
force: true,
});
}
async getBuildResult(
id: string,
): Promise<{ image: string; digest: Sha256Digest; reference: string }> {
const record = await this.requireBuild(id);
if (record.status.state !== "succeeded" || !record.status.digest)
throw new BuildConflictError("Build has no immutable image result");
const parsed = parseImageReference(record.spec.request.spec.image);
const image = `${parsed.registry}/${parsed.repository}`;
return {
image,
digest: record.status.digest,
reference: `${image}@${record.status.digest}`,
};
}
private async requireBuild(id: string): Promise<BuildRecord> {
const record = await this.options.store.getBuild(id);
if (!record) throw new BuildNotFoundError(`Build '${id}' not found`);
return record;
}
private async updateBuild(
record: BuildRecord,
change: (next: BuildRecord) => void,
): Promise<BuildRecord> {
const next = clone(record);
next.metadata.resourceVersion = String(
Number(record.metadata.resourceVersion) + 1,
);
change(next);
await this.options.store.replaceBuild(
next,
record.metadata.resourceVersion,
);
return this.requireBuild(record.metadata.name);
}
private async setState(
record: BuildRecord,
state: BuildStatus["state"],
values: Partial<BuildRecord["status"]>,
): Promise<BuildRecord> {
if (record.status.state === state && state === "running") return record;
return this.updateBuild(record, (next) => {
Object.assign(next.status, values, { state });
next.status.events.push({ type: "status", status: recordStatus(next) });
});
}
private async failBuild(id: string, message: string): Promise<void> {
const current = await this.requireBuild(id);
if (
current.status.state === "succeeded" ||
current.status.state === "failed"
)
return;
await this.setState(current, "failed", {
finishedAt: this.now().toISOString(),
error: message,
});
}
private async captureLogs(record: BuildRecord): Promise<BuildRecord> {
let raw: string | Uint8Array;
try {
raw = await this.options.kubernetes.getJobLogs(
this.options.namespace,
record.spec.jobName,
);
} catch {
return record;
}
const bytes = typeof raw === "string" ? Buffer.from(raw) : Buffer.from(raw);
const offset =
bytes.byteLength < record.status.logOffset ? 0 : record.status.logOffset;
if (bytes.byteLength === offset) return record;
const delta = bytes.subarray(offset);
return this.updateBuild(record, (next) => {
next.status.logOffset = bytes.byteLength;
for (const message of delta
.toString("utf8")
.split(/(?<=\n)/)
.filter(Boolean)) {
const event: BuildEvent = {
type: "log",
id: record.metadata.name,
sequence: next.status.nextSequence++,
message,
};
next.status.events.push(event);
next.status.logBytes += Buffer.byteLength(message);
}
while (next.status.logBytes > this.maxLogBytes) {
const index = next.status.events.findIndex(
(event) => event.type === "log",
);
if (index === -1) break;
const [removed] = next.status.events.splice(index, 1);
if (removed?.type === "log")
next.status.logBytes -= Buffer.byteLength(removed.message);
}
});
}
}