import { BatchV1Api, CoreV1Api, KubernetesObjectApi, type V1Job, } from "@kubernetes/client-node"; import { createHash } from "node:crypto"; import { mkdir, open, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { Sha256Digest } from "../shared/build-protocol"; import type { BuildJobObservation, BuildKubernetesOperations, } from "./build-controller"; import type { KubernetesJob } from "./build-job"; import { BuildStoreConflictError, type BuildRecord, type BuildStore, type CreateBuildResult, type UploadRecord, } from "./build-store"; const TYPE_LABEL = "kuber.astrxl.dev/type"; type ConfigMap = { apiVersion: "v1"; kind: "ConfigMap"; metadata: { name: string; namespace: string; resourceVersion?: string; labels?: Record; }; data?: Record; }; export interface BuildObjectApi { create(value: ConfigMap): Promise; read(value: ConfigMap): Promise; replace(value: ConfigMap): Promise; delete(value: ConfigMap): Promise; list( apiVersion: string, kind: string, namespace?: string, pretty?: string, exact?: boolean, exportValue?: boolean, fieldSelector?: string, labelSelector?: string, ): Promise<{ items: unknown[] }>; } function hashName(prefix: string, value: string): string { return `${prefix}-${createHash("sha256").update(value).digest("hex").slice(0, 48)}`; } function statusCode(error: unknown): number | undefined { if (!error || typeof error !== "object") return; if ("code" in error && typeof error.code === "number") return error.code; if ("statusCode" in error && typeof error.statusCode === "number") return error.statusCode; } function payload(value: unknown): T | undefined { const object = value as ConfigMap; const raw = object.data?.payload; if (!raw) return; try { const parsed = JSON.parse(raw) as T & { metadata?: { resourceVersion?: string }; }; if (parsed.metadata && object.metadata.resourceVersion) parsed.metadata.resourceVersion = object.metadata.resourceVersion; return parsed; } catch { return; } } function map( namespace: string, name: string, type: "build" | "build-upload" | "build-lock", value?: unknown, resourceVersion?: string, ): ConfigMap { return { apiVersion: "v1", kind: "ConfigMap", metadata: { name, namespace, ...(resourceVersion && { resourceVersion }), labels: { [TYPE_LABEL]: type }, }, ...(value !== undefined && { data: { payload: JSON.stringify(value) } }), }; } function terminal(record: BuildRecord): boolean { return record.status.state === "succeeded" || record.status.state === "failed"; } function sameSpec(left: BuildRecord, right: BuildRecord): boolean { return JSON.stringify(left.spec) === JSON.stringify(right.spec); } /** Build metadata lives in ConfigMaps; resumable upload bytes live only on the RWX volume. */ export class KubernetesBuildStore implements BuildStore { constructor( private readonly objects: BuildObjectApi, private readonly namespace: string, private readonly uploadRoot: string, ) {} private buildName(id: string): string { return hashName("build", id); } private uploadName(digest: Sha256Digest): string { return hashName("upload", digest); } private uploadPath(digest: Sha256Digest): string { return join(this.uploadRoot, digest.slice("sha256:".length)); } private lockName(imageKey: string): string { return hashName("build-lock", imageKey); } private async read(value: ConfigMap): Promise { try { return payload(await this.objects.read(value)); } catch (error) { if (statusCode(error) === 404) return; throw error; } } private async delete(value: ConfigMap): Promise { try { await this.objects.delete(value); } catch (error) { if (statusCode(error) !== 404) throw error; } } async createBuild(record: BuildRecord): Promise { const existing = await this.getBuild(record.metadata.name); if (existing) { if (!sameSpec(existing, record)) throw new BuildStoreConflictError( "Build ID was already used for a different request", ); return { record: existing, created: false }; } const lockName = this.lockName(record.spec.imageKey); try { await this.objects.create( map(this.namespace, lockName, "build-lock", { buildId: record.metadata.name, }), ); } catch (error) { if (statusCode(error) !== 409) throw error; const lock = await this.read<{ buildId: string }>( map(this.namespace, lockName, "build-lock"), ); const active = lock && (await this.getBuild(lock.buildId)); if (!active || terminal(active)) { await this.delete(map(this.namespace, lockName, "build-lock")); return this.createBuild(record); } throw new BuildStoreConflictError( `Build '${active.metadata.name}' is already active for ${record.spec.imageKey}`, ); } try { const created = (await this.objects.create( map(this.namespace, this.buildName(record.metadata.name), "build", record), )) as ConfigMap; return { record: payload(created) ?? record, created: true }; } catch (error) { await this.delete(map(this.namespace, lockName, "build-lock")); if (statusCode(error) === 409) { const concurrent = await this.getBuild(record.metadata.name); if (concurrent && sameSpec(concurrent, record)) return { record: concurrent, created: false }; throw new BuildStoreConflictError( "Build ID was already used for a different request", ); } throw error; } } async getBuild(id: string): Promise { const record = await this.read( map(this.namespace, this.buildName(id), "build"), ); return record?.metadata.name === id ? record : undefined; } async listBuilds(): Promise { const result = await this.objects.list( "v1", "ConfigMap", this.namespace, undefined, undefined, undefined, undefined, `${TYPE_LABEL}=build`, ); return result.items .map((item) => payload(item)) .filter((item): item is BuildRecord => item?.kind === "BuildRecord") .sort((a, b) => a.metadata.creationTimestamp.localeCompare(b.metadata.creationTimestamp), ); } async replaceBuild( record: BuildRecord, expectedResourceVersion: string, ): Promise { const current = await this.getBuild(record.metadata.name); if (!current || current.metadata.resourceVersion !== expectedResourceVersion) throw new BuildStoreConflictError("Build record was concurrently modified"); if (!sameSpec(current, record)) throw new BuildStoreConflictError("Build specification is immutable"); if ( current.status.state === "succeeded" && (record.status.state !== "succeeded" || current.status.digest !== record.status.digest) ) throw new BuildStoreConflictError( "A successful image digest is immutable", ); try { await this.objects.replace( map( this.namespace, this.buildName(record.metadata.name), "build", record, expectedResourceVersion, ), ); } catch (error) { if (statusCode(error) === 409) throw new BuildStoreConflictError("Build record was concurrently modified"); throw error; } if (terminal(record)) await this.delete( map(this.namespace, this.lockName(record.spec.imageKey), "build-lock"), ); } async getUpload(digest: Sha256Digest): Promise { const record = await this.read( map(this.namespace, this.uploadName(digest), "build-upload"), ); if (!record || record.spec.digest !== digest) return; try { record.status.data = new Uint8Array(await readFile(this.uploadPath(digest))); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; record.status.data = new Uint8Array(); } if (record.status.data.byteLength !== record.status.offset) throw new Error(`Upload file for ${digest} does not match its record`); return record; } async createUpload(record: UploadRecord): Promise { const existing = await this.getUpload(record.spec.digest); if (existing) return existing; await mkdir(this.uploadRoot, { recursive: true, mode: 0o700 }); const path = this.uploadPath(record.spec.digest); const handle = await open(path, "wx", 0o600).catch((error) => { if ((error as NodeJS.ErrnoException).code === "EEXIST") return; throw error; }); await handle?.close(); const metadata = structuredClone(record); metadata.status.data = new Uint8Array(); try { const created = (await this.objects.create( map( this.namespace, this.uploadName(record.spec.digest), "build-upload", metadata, ), )) as ConfigMap; const result = payload(created) ?? metadata; result.status.data = new Uint8Array(); return result; } catch (error) { if (statusCode(error) === 409) return (await this.getUpload(record.spec.digest))!; await rm(path, { force: true }); throw error; } } async replaceUpload( record: UploadRecord, expectedResourceVersion: string, ): Promise { const metadata = structuredClone(record); metadata.status.data = new Uint8Array(); try { await this.objects.replace( map( this.namespace, this.uploadName(record.spec.digest), "build-upload", metadata, expectedResourceVersion, ), ); await writeFile(this.uploadPath(record.spec.digest), record.status.data, { mode: 0o600, }); } catch (error) { if (statusCode(error) === 409) throw new BuildStoreConflictError("Upload record was concurrently modified"); throw error; } } async deleteUpload(digest: Sha256Digest): Promise { await this.delete( map(this.namespace, this.uploadName(digest), "build-upload"), ); await rm(this.uploadPath(digest), { force: true }); } } export class KubernetesBuildOperations implements BuildKubernetesOperations { constructor( private readonly batch: BatchV1Api, private readonly core: CoreV1Api, ) {} async createJob(job: KubernetesJob): Promise { await this.batch.createNamespacedJob({ namespace: job.metadata.namespace, body: job as unknown as V1Job, fieldManager: "kuber-server", fieldValidation: "Strict", }); } async getJob( namespace: string, name: string, ): Promise { let job: V1Job; try { job = await this.batch.readNamespacedJob({ namespace, name }); } catch (error) { if (statusCode(error) === 404) return; throw error; } const failed = job.status?.conditions?.find( (condition) => condition.type === "Failed" && condition.status === "True", ); const complete = job.status?.conditions?.find( (condition) => condition.type === "Complete" && condition.status === "True", ); const phase = failed ? "failed" : complete ? "succeeded" : (job.status?.active ?? 0) > 0 ? "running" : "queued"; return { phase, startedAt: job.status?.startTime?.toISOString(), finishedAt: job.status?.completionTime?.toISOString(), ...(failed?.message && { error: failed.message }), }; } async getJobLogs(namespace: string, name: string): Promise { const pods = await this.core.listNamespacedPod({ namespace, labelSelector: `job-name=${name}`, }); const pod = pods.items .sort((a, b) => (a.metadata?.creationTimestamp?.getTime() ?? 0) - (b.metadata?.creationTimestamp?.getTime() ?? 0), ) .at(-1); if (!pod?.metadata?.name) return ""; return this.core.readNamespacedPodLog({ namespace, name: pod.metadata.name, container: "buildkit", }); } async deleteJob(namespace: string, name: string): Promise { try { await this.batch.deleteNamespacedJob({ namespace, name, propagationPolicy: "Background", }); } catch (error) { if (statusCode(error) !== 404) throw error; } } } export function buildObjectApi(objects: KubernetesObjectApi): BuildObjectApi { return objects as unknown as BuildObjectApi; }