feat: improve API keys and build workflows
This commit is contained in:
+131
-48
@@ -179,6 +179,9 @@ type ProgressReporter = (message: string) => void | Promise<void>;
|
||||
type BuildReporter = {
|
||||
progress?: ProgressReporter;
|
||||
stream?: Writable;
|
||||
/** Per-image output and lifecycle hooks, including for queued images. */
|
||||
service?: (name: string) => BuildReporter | undefined;
|
||||
settled?: (name: string, error?: unknown) => void;
|
||||
};
|
||||
|
||||
export type BuildResult = {
|
||||
@@ -197,6 +200,7 @@ type ImageResult = {
|
||||
image: string;
|
||||
digest: Sha256Digest;
|
||||
reference: string;
|
||||
references?: Record<string, string>;
|
||||
};
|
||||
|
||||
function posixRelative(root: string, path: string): string {
|
||||
@@ -405,17 +409,22 @@ export async function uploadWorkspaceSnapshot(
|
||||
async function reportBuildEvent(
|
||||
event: BuildEvent,
|
||||
reporter?: BuildReporter,
|
||||
reportedStates?: Set<BuildStatus["state"]>,
|
||||
reportedStates?: Set<string>,
|
||||
service?: string,
|
||||
): Promise<number> {
|
||||
): Promise<void> {
|
||||
if (event.type === "status") {
|
||||
if (!reportedStates?.has(event.status.state)) {
|
||||
reportedStates?.add(event.status.state);
|
||||
const phase =
|
||||
event.status.phase ??
|
||||
(event.status.state === "succeeded" || event.status.state === "failed"
|
||||
? "done"
|
||||
: event.status.state);
|
||||
if (!reportedStates?.has(phase)) {
|
||||
reportedStates?.add(phase);
|
||||
await reporter?.progress?.(
|
||||
`${service ? `${service}: ` : ""}Build ${event.status.state}`,
|
||||
`${service ? `${service}: ` : ""}Build ${phase}${event.status.state === "failed" ? `: ${event.status.error ?? "unknown error"}` : ""}`,
|
||||
);
|
||||
}
|
||||
return 0;
|
||||
return;
|
||||
}
|
||||
if (reporter?.stream)
|
||||
reporter.stream.write(
|
||||
@@ -425,7 +434,6 @@ async function reportBuildEvent(
|
||||
await reporter?.progress?.(
|
||||
`${service ? `${service}: ` : ""}${event.message.trimEnd()}`,
|
||||
);
|
||||
return event.sequence;
|
||||
}
|
||||
|
||||
function isTransientBuildPollError(error: unknown): boolean {
|
||||
@@ -482,16 +490,26 @@ async function requestBuildPoll<T>(
|
||||
async function waitForBuild(
|
||||
id: string,
|
||||
request: ApiRequester,
|
||||
reporter: BuildReporter | undefined,
|
||||
reporters: Array<{ name: string; reporter?: BuildReporter }>,
|
||||
pollIntervalMs: number,
|
||||
sleep: (milliseconds: number) => Promise<void>,
|
||||
initial: BuildStatus,
|
||||
signal?: AbortSignal,
|
||||
service?: string,
|
||||
prefixService = false,
|
||||
): Promise<BuildStatus> {
|
||||
let status = initial;
|
||||
let sequence = 0;
|
||||
const reportedStates = new Set<BuildStatus["state"]>();
|
||||
let failureLog = "";
|
||||
// Each physical build belongs only to its destination services. A shared
|
||||
// reporter (the caller's global sink) receives each event just once.
|
||||
const owners = new Map<BuildReporter | undefined, {
|
||||
name: string;
|
||||
reporter?: BuildReporter;
|
||||
states: Set<string>;
|
||||
}>();
|
||||
for (const { name, reporter } of reporters)
|
||||
if (!owners.has(reporter))
|
||||
owners.set(reporter, { name, reporter, states: new Set<string>() });
|
||||
for (;;) {
|
||||
const events = await requestBuildPoll<BuildEvent[]>(
|
||||
request,
|
||||
@@ -501,13 +519,32 @@ async function waitForBuild(
|
||||
sleep,
|
||||
signal,
|
||||
);
|
||||
for (const event of events)
|
||||
sequence = Math.max(
|
||||
sequence,
|
||||
await reportBuildEvent(event, reporter, reportedStates, service),
|
||||
);
|
||||
if (status.state === "succeeded" || status.state === "failed")
|
||||
for (const event of events) {
|
||||
if (event.type === "log" && event.sequence <= sequence) continue;
|
||||
if (event.type === "log")
|
||||
failureLog = (failureLog + event.message).slice(-8_192);
|
||||
for (const { name, reporter, states } of owners.values())
|
||||
await reportBuildEvent(
|
||||
event,
|
||||
reporter,
|
||||
states,
|
||||
prefixService ? name : undefined,
|
||||
);
|
||||
if (event.type === "log") sequence = event.sequence;
|
||||
}
|
||||
if (status.state === "succeeded" || status.state === "failed") {
|
||||
const details = failureLog.trim();
|
||||
if (
|
||||
status.state === "failed" &&
|
||||
details &&
|
||||
!status.error?.includes(details)
|
||||
)
|
||||
return {
|
||||
...status,
|
||||
error: `${status.error ?? "BuildKit Job failed"}\n${details}`,
|
||||
};
|
||||
return status;
|
||||
}
|
||||
status = await requestBuildPoll<BuildStatus>(
|
||||
request,
|
||||
`/builds/${encodeURIComponent(id)}/reconcile`,
|
||||
@@ -545,24 +582,21 @@ export async function resolveBuildImages(
|
||||
const index = next++;
|
||||
if (index >= services.length) return;
|
||||
const service = services[index]!;
|
||||
options.signal?.throwIfAborted();
|
||||
try {
|
||||
references[index] = (
|
||||
await request<ImageResult>(
|
||||
"/images/resolve",
|
||||
{
|
||||
method: "POST",
|
||||
json: { project, service },
|
||||
signal: options.signal,
|
||||
},
|
||||
)
|
||||
).reference;
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Cannot resolve a published image for service ${service}. Run kuber up to build it.`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
options.signal?.throwIfAborted();
|
||||
try {
|
||||
references[index] = (
|
||||
await request<ImageResult>("/images/resolve", {
|
||||
method: "POST",
|
||||
json: { project, service },
|
||||
signal: options.signal,
|
||||
})
|
||||
).reference;
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`Cannot resolve a published image for service ${service}. Run kuber up to build it.`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
),
|
||||
@@ -619,32 +653,80 @@ export async function buildServices(
|
||||
);
|
||||
|
||||
const references: string[] = new Array(plans.length);
|
||||
const reporters = new Map(
|
||||
plans.map((plan) => [
|
||||
plan.name,
|
||||
reporter?.service?.(plan.name) ?? reporter,
|
||||
]),
|
||||
);
|
||||
const architecture = resolveComposeArch(compose);
|
||||
const groups = new Map<string, number[]>();
|
||||
plans.forEach((plan, index) => {
|
||||
const key = JSON.stringify({
|
||||
workspace: snapshot.digest,
|
||||
architecture,
|
||||
context: plan.context,
|
||||
dockerfile: plan.dockerfile,
|
||||
target: plan.target,
|
||||
buildArgs: plan.buildArgs,
|
||||
});
|
||||
const group = groups.get(key) ?? [];
|
||||
group.push(index);
|
||||
groups.set(key, group);
|
||||
});
|
||||
const batches = [...groups.values()];
|
||||
let next = 0;
|
||||
let failure: unknown;
|
||||
let failed = false;
|
||||
const worker = async () => {
|
||||
while (!failed && next < plans.length) {
|
||||
const index = next++;
|
||||
const plan = plans[index]!;
|
||||
while (!failed && next < batches.length) {
|
||||
const batch = batches[next++]!;
|
||||
const groupPlans = batch.map((index) => plans[index]!);
|
||||
try {
|
||||
options.signal?.throwIfAborted();
|
||||
await reporter?.progress?.(`Building ${plan.name}`);
|
||||
references[index] = await buildPlan(plan);
|
||||
for (const plan of groupPlans) {
|
||||
await reporters
|
||||
.get(plan.name)
|
||||
?.progress?.(
|
||||
`${reporter?.service ? "" : `${plan.name}: `}Build queued`,
|
||||
);
|
||||
}
|
||||
const result = await buildPlan(groupPlans);
|
||||
batch.forEach((index) => {
|
||||
const plan = plans[index]!;
|
||||
const reference =
|
||||
index === batch[0]
|
||||
? result.reference
|
||||
: result.references?.[plan.name];
|
||||
if (!reference)
|
||||
throw new Error(
|
||||
`Build result missing image for service ${plan.name}`,
|
||||
);
|
||||
references[index] = reference;
|
||||
});
|
||||
for (const plan of groupPlans) reporter?.settled?.(plan.name);
|
||||
} catch (error) {
|
||||
if (!failed) failure = error;
|
||||
failed = true;
|
||||
for (const plan of groupPlans) reporter?.settled?.(plan.name, error);
|
||||
}
|
||||
}
|
||||
};
|
||||
const buildPlan = async (plan: BuildPlan): Promise<string> => {
|
||||
const buildPlan = async (groupPlans: BuildPlan[]): Promise<ImageResult> => {
|
||||
const plan = groupPlans[0]!;
|
||||
const id = randomUUID();
|
||||
const buildRequest: BuildRequest = {
|
||||
version: BUILD_PROTOCOL_VERSION,
|
||||
id,
|
||||
project,
|
||||
service: plan.name,
|
||||
...(groupPlans.length > 1 && {
|
||||
destinations: groupPlans
|
||||
.slice(1)
|
||||
.map(({ name, image }) => ({ service: name, image })),
|
||||
}),
|
||||
spec: {
|
||||
architecture: resolveComposeArch(compose),
|
||||
architecture,
|
||||
image: plan.image,
|
||||
context: plan.context,
|
||||
dockerfile: plan.dockerfile,
|
||||
@@ -665,26 +747,27 @@ export async function buildServices(
|
||||
const status = await waitForBuild(
|
||||
id,
|
||||
request,
|
||||
reporter,
|
||||
groupPlans.map(({ name }) => ({ name, reporter: reporters.get(name) })),
|
||||
options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS,
|
||||
options.sleep ?? ((milliseconds) => Bun.sleep(milliseconds)),
|
||||
initial,
|
||||
options.signal,
|
||||
plan.name,
|
||||
!reporter?.service,
|
||||
);
|
||||
if (status.state !== "succeeded")
|
||||
throw new Error(
|
||||
`Build failed for service ${plan.name}: ${status.error ?? "unknown error"}`,
|
||||
);
|
||||
return (
|
||||
await request<ImageResult>(`/builds/${encodeURIComponent(id)}/result`, {
|
||||
return await request<ImageResult>(
|
||||
`/builds/${encodeURIComponent(id)}/result`,
|
||||
{
|
||||
signal: options.signal,
|
||||
})
|
||||
).reference;
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(concurrency, plans.length) }, worker),
|
||||
Array.from({ length: Math.min(concurrency, batches.length) }, worker),
|
||||
);
|
||||
if (failed) throw failure;
|
||||
const images = Object.fromEntries(
|
||||
|
||||
Reference in New Issue
Block a user