feat: v2
This commit is contained in:
@@ -0,0 +1,429 @@
|
||||
import {
|
||||
BatchV1Api,
|
||||
CoreV1Api,
|
||||
KubernetesObjectApi,
|
||||
type V1Job,
|
||||
} from "@kubernetes/client-node";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, open, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { Sha256Digest } from "../shared/build-protocol";
|
||||
import type {
|
||||
BuildJobObservation,
|
||||
BuildKubernetesOperations,
|
||||
} from "./build-controller";
|
||||
import type { KubernetesJob } from "./build-job";
|
||||
import {
|
||||
BuildStoreConflictError,
|
||||
type BuildRecord,
|
||||
type BuildStore,
|
||||
type CreateBuildResult,
|
||||
type UploadRecord,
|
||||
} from "./build-store";
|
||||
|
||||
const TYPE_LABEL = "kuber.astrxl.dev/type";
|
||||
|
||||
type ConfigMap = {
|
||||
apiVersion: "v1";
|
||||
kind: "ConfigMap";
|
||||
metadata: {
|
||||
name: string;
|
||||
namespace: string;
|
||||
resourceVersion?: string;
|
||||
labels?: Record<string, string>;
|
||||
};
|
||||
data?: Record<string, string>;
|
||||
};
|
||||
|
||||
export interface BuildObjectApi {
|
||||
create(value: ConfigMap): Promise<unknown>;
|
||||
read(value: ConfigMap): Promise<unknown>;
|
||||
replace(value: ConfigMap): Promise<unknown>;
|
||||
delete(value: ConfigMap): Promise<unknown>;
|
||||
list(
|
||||
apiVersion: string,
|
||||
kind: string,
|
||||
namespace?: string,
|
||||
pretty?: string,
|
||||
exact?: boolean,
|
||||
exportValue?: boolean,
|
||||
fieldSelector?: string,
|
||||
labelSelector?: string,
|
||||
): Promise<{ items: unknown[] }>;
|
||||
}
|
||||
|
||||
function hashName(prefix: string, value: string): string {
|
||||
return `${prefix}-${createHash("sha256").update(value).digest("hex").slice(0, 48)}`;
|
||||
}
|
||||
|
||||
function statusCode(error: unknown): number | undefined {
|
||||
if (!error || typeof error !== "object") return;
|
||||
if ("code" in error && typeof error.code === "number") return error.code;
|
||||
if ("statusCode" in error && typeof error.statusCode === "number")
|
||||
return error.statusCode;
|
||||
}
|
||||
|
||||
function payload<T>(value: unknown): T | undefined {
|
||||
const object = value as ConfigMap;
|
||||
const raw = object.data?.payload;
|
||||
if (!raw) return;
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as T & {
|
||||
metadata?: { resourceVersion?: string };
|
||||
};
|
||||
if (parsed.metadata && object.metadata.resourceVersion)
|
||||
parsed.metadata.resourceVersion = object.metadata.resourceVersion;
|
||||
return parsed;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
function map(
|
||||
namespace: string,
|
||||
name: string,
|
||||
type: "build" | "build-upload" | "build-lock",
|
||||
value?: unknown,
|
||||
resourceVersion?: string,
|
||||
): ConfigMap {
|
||||
return {
|
||||
apiVersion: "v1",
|
||||
kind: "ConfigMap",
|
||||
metadata: {
|
||||
name,
|
||||
namespace,
|
||||
...(resourceVersion && { resourceVersion }),
|
||||
labels: { [TYPE_LABEL]: type },
|
||||
},
|
||||
...(value !== undefined && { data: { payload: JSON.stringify(value) } }),
|
||||
};
|
||||
}
|
||||
|
||||
function terminal(record: BuildRecord): boolean {
|
||||
return record.status.state === "succeeded" || record.status.state === "failed";
|
||||
}
|
||||
|
||||
function sameSpec(left: BuildRecord, right: BuildRecord): boolean {
|
||||
return JSON.stringify(left.spec) === JSON.stringify(right.spec);
|
||||
}
|
||||
|
||||
/** Build metadata lives in ConfigMaps; resumable upload bytes live only on the RWX volume. */
|
||||
export class KubernetesBuildStore implements BuildStore {
|
||||
constructor(
|
||||
private readonly objects: BuildObjectApi,
|
||||
private readonly namespace: string,
|
||||
private readonly uploadRoot: string,
|
||||
) {}
|
||||
|
||||
private buildName(id: string): string {
|
||||
return hashName("build", id);
|
||||
}
|
||||
|
||||
private uploadName(digest: Sha256Digest): string {
|
||||
return hashName("upload", digest);
|
||||
}
|
||||
|
||||
private uploadPath(digest: Sha256Digest): string {
|
||||
return join(this.uploadRoot, digest.slice("sha256:".length));
|
||||
}
|
||||
|
||||
private lockName(imageKey: string): string {
|
||||
return hashName("build-lock", imageKey);
|
||||
}
|
||||
|
||||
private async read<T>(value: ConfigMap): Promise<T | undefined> {
|
||||
try {
|
||||
return payload<T>(await this.objects.read(value));
|
||||
} catch (error) {
|
||||
if (statusCode(error) === 404) return;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async delete(value: ConfigMap): Promise<void> {
|
||||
try {
|
||||
await this.objects.delete(value);
|
||||
} catch (error) {
|
||||
if (statusCode(error) !== 404) throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async createBuild(record: BuildRecord): Promise<CreateBuildResult> {
|
||||
const existing = await this.getBuild(record.metadata.name);
|
||||
if (existing) {
|
||||
if (!sameSpec(existing, record))
|
||||
throw new BuildStoreConflictError(
|
||||
"Build ID was already used for a different request",
|
||||
);
|
||||
return { record: existing, created: false };
|
||||
}
|
||||
|
||||
const lockName = this.lockName(record.spec.imageKey);
|
||||
try {
|
||||
await this.objects.create(
|
||||
map(this.namespace, lockName, "build-lock", {
|
||||
buildId: record.metadata.name,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
if (statusCode(error) !== 409) throw error;
|
||||
const lock = await this.read<{ buildId: string }>(
|
||||
map(this.namespace, lockName, "build-lock"),
|
||||
);
|
||||
const active = lock && (await this.getBuild(lock.buildId));
|
||||
if (!active || terminal(active)) {
|
||||
await this.delete(map(this.namespace, lockName, "build-lock"));
|
||||
return this.createBuild(record);
|
||||
}
|
||||
throw new BuildStoreConflictError(
|
||||
`Build '${active.metadata.name}' is already active for ${record.spec.imageKey}`,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const created = (await this.objects.create(
|
||||
map(this.namespace, this.buildName(record.metadata.name), "build", record),
|
||||
)) as ConfigMap;
|
||||
return { record: payload<BuildRecord>(created) ?? record, created: true };
|
||||
} catch (error) {
|
||||
await this.delete(map(this.namespace, lockName, "build-lock"));
|
||||
if (statusCode(error) === 409) {
|
||||
const concurrent = await this.getBuild(record.metadata.name);
|
||||
if (concurrent && sameSpec(concurrent, record))
|
||||
return { record: concurrent, created: false };
|
||||
throw new BuildStoreConflictError(
|
||||
"Build ID was already used for a different request",
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getBuild(id: string): Promise<BuildRecord | undefined> {
|
||||
const record = await this.read<BuildRecord>(
|
||||
map(this.namespace, this.buildName(id), "build"),
|
||||
);
|
||||
return record?.metadata.name === id ? record : undefined;
|
||||
}
|
||||
|
||||
async listBuilds(): Promise<BuildRecord[]> {
|
||||
const result = await this.objects.list(
|
||||
"v1",
|
||||
"ConfigMap",
|
||||
this.namespace,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
`${TYPE_LABEL}=build`,
|
||||
);
|
||||
return result.items
|
||||
.map((item) => payload<BuildRecord>(item))
|
||||
.filter((item): item is BuildRecord => item?.kind === "BuildRecord")
|
||||
.sort((a, b) =>
|
||||
a.metadata.creationTimestamp.localeCompare(b.metadata.creationTimestamp),
|
||||
);
|
||||
}
|
||||
|
||||
async replaceBuild(
|
||||
record: BuildRecord,
|
||||
expectedResourceVersion: string,
|
||||
): Promise<void> {
|
||||
const current = await this.getBuild(record.metadata.name);
|
||||
if (!current || current.metadata.resourceVersion !== expectedResourceVersion)
|
||||
throw new BuildStoreConflictError("Build record was concurrently modified");
|
||||
if (!sameSpec(current, record))
|
||||
throw new BuildStoreConflictError("Build specification is immutable");
|
||||
if (
|
||||
current.status.state === "succeeded" &&
|
||||
(record.status.state !== "succeeded" ||
|
||||
current.status.digest !== record.status.digest)
|
||||
)
|
||||
throw new BuildStoreConflictError(
|
||||
"A successful image digest is immutable",
|
||||
);
|
||||
|
||||
try {
|
||||
await this.objects.replace(
|
||||
map(
|
||||
this.namespace,
|
||||
this.buildName(record.metadata.name),
|
||||
"build",
|
||||
record,
|
||||
expectedResourceVersion,
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
if (statusCode(error) === 409)
|
||||
throw new BuildStoreConflictError("Build record was concurrently modified");
|
||||
throw error;
|
||||
}
|
||||
if (terminal(record))
|
||||
await this.delete(
|
||||
map(this.namespace, this.lockName(record.spec.imageKey), "build-lock"),
|
||||
);
|
||||
}
|
||||
|
||||
async getUpload(digest: Sha256Digest): Promise<UploadRecord | undefined> {
|
||||
const record = await this.read<UploadRecord>(
|
||||
map(this.namespace, this.uploadName(digest), "build-upload"),
|
||||
);
|
||||
if (!record || record.spec.digest !== digest) return;
|
||||
try {
|
||||
record.status.data = new Uint8Array(await readFile(this.uploadPath(digest)));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
|
||||
record.status.data = new Uint8Array();
|
||||
}
|
||||
if (record.status.data.byteLength !== record.status.offset)
|
||||
throw new Error(`Upload file for ${digest} does not match its record`);
|
||||
return record;
|
||||
}
|
||||
|
||||
async createUpload(record: UploadRecord): Promise<UploadRecord> {
|
||||
const existing = await this.getUpload(record.spec.digest);
|
||||
if (existing) return existing;
|
||||
await mkdir(this.uploadRoot, { recursive: true, mode: 0o700 });
|
||||
const path = this.uploadPath(record.spec.digest);
|
||||
const handle = await open(path, "wx", 0o600).catch((error) => {
|
||||
if ((error as NodeJS.ErrnoException).code === "EEXIST") return;
|
||||
throw error;
|
||||
});
|
||||
await handle?.close();
|
||||
const metadata = structuredClone(record);
|
||||
metadata.status.data = new Uint8Array();
|
||||
try {
|
||||
const created = (await this.objects.create(
|
||||
map(
|
||||
this.namespace,
|
||||
this.uploadName(record.spec.digest),
|
||||
"build-upload",
|
||||
metadata,
|
||||
),
|
||||
)) as ConfigMap;
|
||||
const result = payload<UploadRecord>(created) ?? metadata;
|
||||
result.status.data = new Uint8Array();
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (statusCode(error) === 409) return (await this.getUpload(record.spec.digest))!;
|
||||
await rm(path, { force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async replaceUpload(
|
||||
record: UploadRecord,
|
||||
expectedResourceVersion: string,
|
||||
): Promise<void> {
|
||||
const metadata = structuredClone(record);
|
||||
metadata.status.data = new Uint8Array();
|
||||
try {
|
||||
await this.objects.replace(
|
||||
map(
|
||||
this.namespace,
|
||||
this.uploadName(record.spec.digest),
|
||||
"build-upload",
|
||||
metadata,
|
||||
expectedResourceVersion,
|
||||
),
|
||||
);
|
||||
await writeFile(this.uploadPath(record.spec.digest), record.status.data, {
|
||||
mode: 0o600,
|
||||
});
|
||||
} catch (error) {
|
||||
if (statusCode(error) === 409)
|
||||
throw new BuildStoreConflictError("Upload record was concurrently modified");
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async deleteUpload(digest: Sha256Digest): Promise<void> {
|
||||
await this.delete(
|
||||
map(this.namespace, this.uploadName(digest), "build-upload"),
|
||||
);
|
||||
await rm(this.uploadPath(digest), { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export class KubernetesBuildOperations implements BuildKubernetesOperations {
|
||||
constructor(
|
||||
private readonly batch: BatchV1Api,
|
||||
private readonly core: CoreV1Api,
|
||||
) {}
|
||||
|
||||
async createJob(job: KubernetesJob): Promise<void> {
|
||||
await this.batch.createNamespacedJob({
|
||||
namespace: job.metadata.namespace,
|
||||
body: job as unknown as V1Job,
|
||||
fieldManager: "kuber-server",
|
||||
fieldValidation: "Strict",
|
||||
});
|
||||
}
|
||||
|
||||
async getJob(
|
||||
namespace: string,
|
||||
name: string,
|
||||
): Promise<BuildJobObservation | undefined> {
|
||||
let job: V1Job;
|
||||
try {
|
||||
job = await this.batch.readNamespacedJob({ namespace, name });
|
||||
} catch (error) {
|
||||
if (statusCode(error) === 404) return;
|
||||
throw error;
|
||||
}
|
||||
const failed = job.status?.conditions?.find(
|
||||
(condition) => condition.type === "Failed" && condition.status === "True",
|
||||
);
|
||||
const complete = job.status?.conditions?.find(
|
||||
(condition) => condition.type === "Complete" && condition.status === "True",
|
||||
);
|
||||
const phase = failed
|
||||
? "failed"
|
||||
: complete
|
||||
? "succeeded"
|
||||
: (job.status?.active ?? 0) > 0
|
||||
? "running"
|
||||
: "queued";
|
||||
return {
|
||||
phase,
|
||||
startedAt: job.status?.startTime?.toISOString(),
|
||||
finishedAt: job.status?.completionTime?.toISOString(),
|
||||
...(failed?.message && { error: failed.message }),
|
||||
};
|
||||
}
|
||||
|
||||
async getJobLogs(namespace: string, name: string): Promise<string> {
|
||||
const pods = await this.core.listNamespacedPod({
|
||||
namespace,
|
||||
labelSelector: `job-name=${name}`,
|
||||
});
|
||||
const pod = pods.items
|
||||
.sort((a, b) =>
|
||||
(a.metadata?.creationTimestamp?.getTime() ?? 0) -
|
||||
(b.metadata?.creationTimestamp?.getTime() ?? 0),
|
||||
)
|
||||
.at(-1);
|
||||
if (!pod?.metadata?.name) return "";
|
||||
return this.core.readNamespacedPodLog({
|
||||
namespace,
|
||||
name: pod.metadata.name,
|
||||
container: "buildkit",
|
||||
});
|
||||
}
|
||||
|
||||
async deleteJob(namespace: string, name: string): Promise<void> {
|
||||
try {
|
||||
await this.batch.deleteNamespacedJob({
|
||||
namespace,
|
||||
name,
|
||||
propagationPolicy: "Background",
|
||||
});
|
||||
} catch (error) {
|
||||
if (statusCode(error) !== 404) throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildObjectApi(objects: KubernetesObjectApi): BuildObjectApi {
|
||||
return objects as unknown as BuildObjectApi;
|
||||
}
|
||||
Reference in New Issue
Block a user