astral/kuber
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden self-managed reconciliation

Browse Source
This commit is contained in:
dmgnr
2026-09-05 10:17:55 +00:00 Unverified
parent 8e9d207915
commit 321f4e807a
40 changed files with 4977 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/build-controller.ts
+99 -41
View File
@@ -26,9 +26,11 @@ import { parseImageReference, resolveRegistryDigest } from "./registry";
export const DEFAULT_MAX_BLOB_BYTES = 1024 * 1024 * 1024;
export const DEFAULT_MAX_UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024;
const SNAPSHOT_CHECK_CONCURRENCY = 20;
export const DEFAULT_MAX_LOG_BYTES = 1024 * 1024;
export interface BuildCas extends MaterializeCas {
has(digest: Sha256Digest): Promise<boolean>;
put(data: Uint8Array, expected?: Sha256Digest): Promise<Sha256Digest>;
}
@@ -101,6 +103,28 @@ export type UploadProgress = {
complete: boolean;
};
async function mapConcurrent<T, R>(
values: T[],
run: (value: T) => Promise<R>,
): Promise<R[]> {
const results = new Array<R>(values.length);
let index = 0;
const worker = async () => {
for (;;) {
const current = index++;
if (current >= values.length) return;
results[current] = await run(values[current]!);
}
};
await Promise.all(
Array.from(
{ length: Math.min(SNAPSHOT_CHECK_CONCURRENCY, values.length) },
worker,
),
);
return results;
}
export function buildImageName(
registry: string,
project: string,
@@ -121,10 +145,6 @@ function clone<T>(value: T): T {
return structuredClone(value);
}
function requestFingerprint(request: BuildRequest): string {
return createHash("sha256").update(JSON.stringify(request)).digest("hex");
}
function jobWorkspaceSubPath(workspaceRoot: string, subPath: string): string {
const segments = workspaceRoot.split("/").filter(Boolean);
const prefix = segments.at(-1);
@@ -183,6 +203,7 @@ export class BuildController {
private readonly maxLogBytes: number;
private readonly materializer: typeof materializeWorkspace;
private readonly digestResolver: typeof resolveRegistryDigest;
private readonly imageSubmissionLocks = new Map<string, Promise<unknown>>();
constructor(private readonly options: BuildControllerOptions) {
this.now = options.now ?? (() => new Date());
@@ -203,13 +224,11 @@ export class BuildController {
const manifest = parseWorkspaceManifest(
await this.options.cas.get(workspace),
);
const missing: Sha256Digest[] = [];
const seen = new Set<Sha256Digest>();
for (const file of manifest.files) {
if (!seen.has(file.digest) && !(await this.options.cas.has(file.digest)))
missing.push(file.digest);
seen.add(file.digest);
}
const digests = [...new Set(manifest.files.map((file) => file.digest))];
const available = await mapConcurrent(digests, (digest) =>
this.options.cas.has(digest),
);
const missing = digests.filter((_digest, index) => !available[index]);
return { workspace, missing, ready: missing.length === 0 };
}
@@ -345,17 +364,24 @@ export class BuildController {
if (this.options.imageName)
request.spec.image = this.options.imageName(request);
validateRequest(request);
const existing = await this.options.store.getBuild(request.id);
if (existing) {
if (
requestFingerprint(existing.spec.request) !==
requestFingerprint(request)
)
throw new BuildConflictError(
"Build ID was already used for a different request",
);
return recordStatus(existing);
const imageKey = `${request.project}\0${request.service}\0${request.spec.image}`;
const previous =
this.imageSubmissionLocks.get(imageKey) ?? Promise.resolve();
const current = previous.then(() => this.submitBuildInternal(request));
const entry = current.catch(() => undefined);
this.imageSubmissionLocks.set(imageKey, entry);
try {
return await current;
} finally {
if (this.imageSubmissionLocks.get(imageKey) === entry)
this.imageSubmissionLocks.delete(imageKey);
}
}
private async submitBuildInternal(
request: BuildRequest,
): Promise<BuildStatus> {
const imageKey = `${request.project}\0${request.service}\0${request.spec.image}`;
const snapshot = await this.negotiateSnapshot(request.spec.workspace);
if (!snapshot.ready)
throw new BuildConflictError(
@@ -367,7 +393,6 @@ export class BuildController {
.digest("hex")
.slice(0, 24);
const jobName = `kuber-build-${hash}`;
const imageKey = `${request.project}\0${request.service}\0${request.spec.image}`;
const initial: BuildStatus = {
version: BUILD_PROTOCOL_VERSION,
id: request.id,
@@ -401,16 +426,27 @@ export class BuildController {
try {
const result = await this.options.store.createBuild(record);
stored = result.record;
if (!result.created) {
if (
requestFingerprint(stored.spec.request) !==
requestFingerprint(request)
)
throw new BuildConflictError(
"Build ID was already used for a different request",
);
return recordStatus(stored);
for (const superseded of result.superseded ?? []) {
// Deletion is best effort and does not wait for the Job or its pods.
await this.options.kubernetes
.deleteJob(this.options.namespace, superseded.spec.jobName)
.catch(() => undefined);
await rm(
join(this.options.workspaceRoot, superseded.spec.workspaceSubPath),
{ recursive: true, force: true },
).catch(() => undefined);
}
if (!result.created) return recordStatus(stored);
const current = await this.options.store.getBuild(request.id);
if (
!current ||
current.status.state !== "queued" ||
(this.options.store.ownsBuild &&
!(await this.options.store.ownsBuild(imageKey, request.id)))
)
throw new BuildConflictError(
`Build '${request.id}' was superseded before Job creation`,
);
} catch (error) {
if (error instanceof BuildStoreConflictError)
throw new BuildConflictError(error.message);
@@ -422,6 +458,13 @@ export class BuildController {
request.spec.workspace,
join(this.options.workspaceRoot, stored.spec.workspaceSubPath),
);
if (
this.options.store.ownsBuild &&
!(await this.options.store.ownsBuild(imageKey, request.id))
)
throw new BuildConflictError(
`Build '${request.id}' lost image lock before Job creation`,
);
const cacheImage =
typeof this.options.cacheImage === "function"
? this.options.cacheImage(request)
@@ -456,6 +499,11 @@ export class BuildController {
});
return initial;
} catch (error) {
try {
await this.terminateJob(stored.spec.jobName);
} catch {
// Job cleanup is best effort; the record still releases its lock.
}
await this.failBuild(
stored.metadata.name,
error instanceof Error ? error.message : String(error),
@@ -469,6 +517,10 @@ export class BuildController {
return recordStatus(record);
}
async getBuildProject(id: string): Promise<string> {
return (await this.requireBuild(id)).spec.request.project;
}
async getBuildEvents(id: string, afterSequence = 0): Promise<BuildEvent[]> {
if (!Number.isSafeInteger(afterSequence) || afterSequence < 0)
throw new BuildValidationError(
@@ -527,11 +579,17 @@ export class BuildController {
const record = await this.requireBuild(id);
if (record.status.state === "succeeded" || record.status.state === "failed")
return recordStatus(record);
if (record.status.jobCreated)
await this.options.kubernetes.deleteJob(
this.options.namespace,
record.spec.jobName,
);
if (record.status.jobCreated) {
try {
await this.terminateJob(record.spec.jobName);
} catch (error) {
await this.failBuild(
record.metadata.name,
`Unable to cancel build safely: ${error instanceof Error ? error.message : String(error)}`,
);
throw error;
}
}
const next = await this.setState(record, "failed", {
finishedAt: this.now().toISOString(),
error: "Build cancelled",
@@ -544,11 +602,7 @@ export class BuildController {
const record = await this.requireBuild(id);
if (record.status.state !== "succeeded" && record.status.state !== "failed")
throw new BuildConflictError("An active build cannot be cleaned up");
if (record.status.jobCreated)
await this.options.kubernetes.deleteJob(
this.options.namespace,
record.spec.jobName,
);
if (record.status.jobCreated) await this.terminateJob(record.spec.jobName);
await rm(join(this.options.workspaceRoot, record.spec.workspaceSubPath), {
recursive: true,
force: true,
@@ -576,6 +630,10 @@ export class BuildController {
return record;
}
private async terminateJob(name: string): Promise<void> {
await this.options.kubernetes.deleteJob(this.options.namespace, name);
}
private async updateBuild(
record: BuildRecord,
change: (next: BuildRecord) => void,