From 321f4e807a7982fbc09f94abaed2d70c9c8162d6 Mon Sep 17 00:00:00 2001 From: dmgnr Date: Sat, 5 Sep 2026 10:17:55 +0000 Subject: [PATCH] feat: harden self-managed reconciliation --- .kuberrc.ts | 150 +++++- README.md | 14 + command/main.ts | 4 +- command/trust.ts | 111 ++++ command/up.ts | 207 +++++++- lib/build.ts | 273 ++++++++-- lib/database.ts | 25 +- lib/exec-api.ts | 3 +- lib/storage.ts | 42 +- lib/trust.ts | 105 ++++ package.json | 2 +- server/app.ts | 377 +++++++++++--- server/build-controller.ts | 140 +++-- server/build-kubernetes.ts | 466 +++++++++++++++-- server/build-store.ts | 149 +++++- server/index.ts | 69 ++- server/kubernetes-state.ts | 151 +++++- server/management.ts | 94 +++- server/materialize.ts | 34 +- server/operation-store.ts | 63 ++- server/redact.ts | 14 + server/trust-store.ts | 39 ++ server/workspace-store.ts | 1 - shared/api.ts | 3 + tests/command/trust.test.ts | 97 ++++ tests/command/up-api.test.ts | 388 +++++++++++++- tests/lib/build-api.test.ts | 503 +++++++++++++++++- tests/lib/config.test.ts | 37 +- tests/lib/trust.test.ts | 50 ++ tests/server/app.test.ts | 717 +++++++++++++++++++++++++- tests/server/build-controller.test.ts | 180 ++++++- tests/server/build-kubernetes.test.ts | 349 +++++++++++++ tests/server/build-store.test.ts | 138 ++++- tests/server/exec-websocket.test.ts | 12 +- tests/server/kubernetes-state.test.ts | 98 +++- tests/server/management.test.ts | 54 +- tests/server/materialize.test.ts | 38 ++ tests/server/operation-store.test.ts | 32 +- tests/server/trust-store.test.ts | 145 ++++++ tests/server/workspace-store.test.ts | 7 +- 40 files changed, 4977 insertions(+), 404 deletions(-) create mode 100644 command/trust.ts create mode 100644 lib/trust.ts create mode 100644 server/trust-store.ts create mode 100644 tests/command/trust.test.ts create mode 100644 tests/lib/trust.test.ts create mode 100644 tests/server/build-kubernetes.test.ts create mode 100644 tests/server/trust-store.test.ts diff --git a/.kuberrc.ts b/.kuberrc.ts index cbbd160..de78100 100644 --- a/.kuberrc.ts +++ b/.kuberrc.ts @@ -24,7 +24,35 @@ export default { { apiGroups: [""], resources: ["secrets", "configmaps", "pods", "pods/log"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], + }, + { + apiGroups: [""], + resources: ["serviceaccounts"], + resourceNames: ["kuber-server"], + verbs: ["get", "update", "patch"], + }, + { + apiGroups: ["rbac.authorization.k8s.io"], + resources: ["roles", "rolebindings"], + resourceNames: ["kuber-server-auth"], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], }, { apiGroups: ["batch"], @@ -34,7 +62,15 @@ export default { { apiGroups: ["coordination.k8s.io"], resources: ["leases"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], }, ], }, @@ -67,8 +103,21 @@ export default { }, { apiGroups: [""], - resources: ["services", "configmaps", "secrets", "persistentvolumeclaims"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + resources: [ + "services", + "configmaps", + "secrets", + "persistentvolumeclaims", + ], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], }, { apiGroups: [""], @@ -82,28 +131,73 @@ export default { }, { apiGroups: ["apps"], - resources: ["deployments", "replicasets", "statefulsets", "daemonsets"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + resources: [ + "deployments", + "replicasets", + "statefulsets", + "daemonsets", + ], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], }, { apiGroups: ["batch"], resources: ["jobs", "cronjobs"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], }, { apiGroups: ["networking.k8s.io"], resources: ["ingresses", "networkpolicies"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], }, { apiGroups: ["autoscaling"], resources: ["horizontalpodautoscalers"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], }, { apiGroups: ["policy"], resources: ["poddisruptionbudgets"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], }, { apiGroups: ["storage.k8s.io"], @@ -113,7 +207,15 @@ export default { { apiGroups: ["traefik.io"], resources: ["ingressroutes"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], }, { apiGroups: ["postgresql.cnpg.io"], @@ -123,12 +225,34 @@ export default { { apiGroups: ["postgresql.cnpg.io"], resources: ["databases"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], }, { apiGroups: ["garage.rajsingh.info"], resources: ["garagebuckets", "garagekeys"], - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + verbs: [ + "get", + "list", + "watch", + "create", + "update", + "patch", + "delete", + ], + }, + { + apiGroups: ["rbac.authorization.k8s.io"], + resources: ["clusterroles", "clusterrolebindings"], + resourceNames: ["kuber-server-manager"], + verbs: ["get", "update", "patch"], }, ], }, diff --git a/README.md b/README.md index a8b359b..316f32f 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,20 @@ a per-project namespace, reconciles managed Postgres and S3 claims, rolls deployments back, streams logs, and exposes interactive `exec` sessions over a WebSocket. +## Directory Trust + +Before `kuber up`, run `kuber trust` from the configured project directory. +Trust is exactly the configured namespace plus a SHA-256 fingerprint of the +resolved current working directory. The local mode-0600 store lets `up` fail +before builds from an untrusted directory. The server stores only namespace and +fingerprint registrations in labelled ConfigMaps in its `kuber-system` control +plane namespace, then checks the pair when resource reconciliation begins. + +This is an accidental-targeting safeguard, not a security boundary: a client +that intentionally forges a registered fingerprint can pass it. `kuber trust +status` shows local/server awareness without printing paths; `kuber trust revoke` +removes the current directory registration. + ## Environment Assumptions kuber targets a specific self-hosted cluster and workstation setup. It is not diff --git a/command/main.ts b/command/main.ts index 0ea7f86..330fb6c 100644 --- a/command/main.ts +++ b/command/main.ts @@ -16,11 +16,12 @@ import { start } from "./start"; import { stop } from "./stop"; import { up } from "./up"; import { users } from "./users"; +import { trust } from "./trust"; export const main = defineCommand({ meta: { name: "kuber", - version: "2.1.0", + version: "2.2.0", description: "Docker Compose -> K8s translation layer", }, args: { @@ -47,6 +48,7 @@ export const main = defineCommand({ stop, up, users, + trust, whoami, }, }); diff --git a/command/trust.ts b/command/trust.ts new file mode 100644 index 0000000..ad7c053 --- /dev/null +++ b/command/trust.ts @@ -0,0 +1,111 @@ +import { defineCommand } from "citty"; +import { apiRequest, KuberApiError, type ApiRequestInit } from "../lib/api"; +import { ctx } from "../lib/context"; +import { + readTrust, + resolveTrustIdentity, + updateTrust, + type LocalTrustRecord, +} from "../lib/trust"; +import type { WorkspaceTrustResponse } from "../shared/api"; + +function route(project: string) { + return `/workspaces/${encodeURIComponent(project)}/trust`; +} + +type TrustApiRequest = (path: string, init?: ApiRequestInit) => Promise; + +async function current() { + const { project, cwd } = ctx(); + return resolveTrustIdentity(project, cwd); +} + +export async function grantTrust( + identity: LocalTrustRecord, + request: TrustApiRequest = apiRequest, +): Promise { + await request(route(identity.project), { + method: "POST", + json: { fingerprint: identity.fingerprint }, + }); + await updateTrust((records) => [ + ...records.filter( + (record) => + record.project !== identity.project || + record.fingerprint !== identity.fingerprint, + ), + identity, + ]); + console.log(`Trusted this directory for namespace ${identity.project}`); +} + +export async function statusTrust( + identity: LocalTrustRecord, + request: TrustApiRequest = apiRequest, +): Promise { + const local = (await readTrust()).some( + (record) => + record.project === identity.project && + record.fingerprint === identity.fingerprint, + ); + const remote = await request(route(identity.project)); + const registered = remote.fingerprints.includes(identity.fingerprint); + console.log(`Namespace: ${identity.project}`); + console.log(`Local: ${local ? "trusted" : "untrusted"}`); + console.log(`Server: ${registered ? "registered" : "not registered"}`); +} + +export async function revokeTrust( + identity: LocalTrustRecord, + request: TrustApiRequest = apiRequest, +): Promise { + let remoteError: unknown; + try { + await request( + `${route(identity.project)}?fingerprint=${encodeURIComponent(identity.fingerprint)}`, + { method: "DELETE" }, + ); + } catch (error) { + // A missing server record is already revoked; all other failures are + // reported after the local registration is removed. + if (!(error instanceof KuberApiError && error.status === 404)) + remoteError = error; + } + await updateTrust((records) => + records.filter( + (record) => + record.project !== identity.project || + record.fingerprint !== identity.fingerprint, + ), + ); + if (remoteError) { + const detail = + remoteError instanceof Error ? remoteError.message : String(remoteError); + throw new Error( + `Removed local trust for namespace ${identity.project}, but failed to revoke the server registration: ${detail}`, + { cause: remoteError }, + ); + } + console.log(`Revoked trust for namespace ${identity.project}`); +} + +export const trust = defineCommand({ + meta: { name: "trust", description: "Trust this directory for kuber up" }, + subCommands: { + status: defineCommand({ + meta: { name: "status", description: "Show local and server trust" }, + async run() { + await statusTrust(await current()); + }, + }), + revoke: defineCommand({ + meta: { name: "revoke", description: "Revoke this directory trust" }, + async run() { + await revokeTrust(await current()); + }, + }), + }, + async run() { + await grantTrust(await current()); + }, +}); diff --git a/command/up.ts b/command/up.ts index 5941d72..13f62f1 100644 --- a/command/up.ts +++ b/command/up.ts @@ -1,8 +1,14 @@ import { defineCommand } from "citty"; import { Listr } from "listr2"; +import { randomUUID } from "node:crypto"; import type { ComposeSpecification } from "../schema/docker.d"; import type { KuberResource } from "../types"; -import { apiRequest, KuberApiError, type ApiRequestInit } from "../lib/api"; +import { + apiRequest, + KuberApiError, + type ApiRequestInit, + type ApiRequestOptions, +} from "../lib/api"; import { buildServices, getRepoRoot, @@ -14,6 +20,11 @@ import { ctx } from "../lib/context"; import { getComposePostgresClaims } from "../lib/database"; import { getComposeS3Claims } from "../lib/storage"; import { enumerateWorkspace, type WorkspaceSnapshot } from "../lib/workspace"; +import { + requireLocalTrust, + resolveTrustIdentity, + trustHeaders, +} from "../lib/trust"; type Workspace = { metadata: { name: string; uid: string; resourceVersion: string }; @@ -33,6 +44,21 @@ type ResourcePlan = { stale: ResourceIdentity[]; }; +type OperationStatus = { + state?: unknown; + error?: { code?: unknown; message?: unknown }; +}; + +type OperationResponse = { + operationId?: unknown; + operation?: { status?: OperationStatus }; +}; + +export type OperationResumeOptions = { + now?: () => number; + sleep?: (milliseconds: number) => Promise; +}; + type UpContext = { compose?: ComposeSpecification; snapshot?: WorkspaceSnapshot; @@ -43,6 +69,14 @@ type UpContext = { }; export const WORKSPACE_ADOPTION_METHOD = "POST"; +const OPERATION_RESUME_INITIAL_BACKOFF_MS = 250; +const OPERATION_RESUME_MAX_BACKOFF_MS = 5_000; +const OPERATION_RESUME_GRACE_MS = 60_000; +const RECOVERABLE_API_ERROR_CODES = new Set([ + "HTTP_502", + "HTTP_503", + "HTTP_504", +]); export function workspaceAdoptionRoute(project: string): string { return `/workspaces/${encodeURIComponent(project)}/adopt`; @@ -141,6 +175,150 @@ async function managementRequest( } } +function operationIdFromResponse(response: unknown): string | undefined { + if (!response || typeof response !== "object") return; + const operationId = (response as OperationResponse).operationId; + return typeof operationId === "string" && operationId + ? operationId + : undefined; +} + +function operationStatusFromResponse( + response: unknown, +): OperationStatus | undefined { + if (!response || typeof response !== "object") return; + const status = (response as OperationResponse).operation?.status; + return status && typeof status === "object" ? status : undefined; +} + +function operationFailure( + operationId: string, + status: OperationStatus, +): KuberApiError { + const error = status.error; + const cancelled = status.state === "cancelled"; + const message = + typeof error?.message === "string" + ? error.message + : cancelled + ? "The operation was cancelled" + : "The operation failed"; + const code = + typeof error?.code === "string" + ? error.code + : cancelled + ? "OPERATION_CANCELLED" + : "OPERATION_FAILED"; + return new KuberApiError(message, cancelled ? 409 : 500, { + title: cancelled ? "Operation cancelled" : "Operation failed", + status: cancelled ? 409 : 500, + code, + operationId, + }); +} + +function isRecoverableConnectionInterruption(error: unknown): boolean { + if (error instanceof KuberApiError) + return RECOVERABLE_API_ERROR_CODES.has(error.code); + if (error instanceof DOMException && error.name === "AbortError") + return false; + if (error instanceof Error && error.name === "AbortError") return false; + return ( + error instanceof TypeError || + (error instanceof Error && error.name === "TimeoutError") + ); +} + +function isInterruptedOperation(status: OperationStatus): boolean { + return ( + status.state === "failed" && status.error?.code === "OPERATION_INTERRUPTED" + ); +} + +function operationResumeDeadline( + rolloutTimeoutMs: number, + now: number, +): number { + return now + Math.max(rolloutTimeoutMs, 0) + OPERATION_RESUME_GRACE_MS; +} + +async function resumeManagedOperation( + project: string, + request: ApiRequester, + path: string, + init: ApiRequestInit, + rolloutTimeoutMs: number, + options: OperationResumeOptions, +): Promise { + const now = options.now ?? Date.now; + const sleep = options.sleep ?? ((milliseconds) => Bun.sleep(milliseconds)); + const deadline = operationResumeDeadline(rolloutTimeoutMs, now()); + const headers = new Headers(init.headers); + headers.set("idempotency-key", randomUUID()); + let operationInit = { ...init, headers }; + let operationId: string | undefined; + let backoffMs = OPERATION_RESUME_INITIAL_BACKOFF_MS; + let restartRetryPending = false; + + for (;;) { + if (restartRetryPending && now() >= deadline) + throw new Error("Timed out while reconnecting to resume the operation"); + try { + let status: OperationStatus | undefined; + if (operationId) { + const operation = await managementRequest<{ status: OperationStatus }>( + project, + request, + `/operations/${encodeURIComponent(operationId)}`, + {}, + ); + status = operation.status; + } else { + restartRetryPending = false; + const response = await managementRequest( + project, + request, + path, + operationInit, + ); + operationId = operationIdFromResponse(response); + status = operationStatusFromResponse(response); + } + + if (!operationId || !status || status.state === "succeeded") return; + if (isInterruptedOperation(status)) { + if (now() >= deadline) throw operationFailure(operationId, status); + operationId = undefined; + restartRetryPending = true; + headers.set("idempotency-key", randomUUID()); + operationInit = { ...init, headers }; + } else if (status.state === "failed" || status.state === "cancelled") + throw operationFailure(operationId, status); + } catch (error) { + if ( + error instanceof KuberApiError && + error.code === "OPERATION_INTERRUPTED" + ) { + if (now() >= deadline) throw adoptionHint(project, error); + operationId = undefined; + restartRetryPending = true; + headers.set("idempotency-key", randomUUID()); + operationInit = { ...init, headers }; + } else { + if (!isRecoverableConnectionInterruption(error)) + throw adoptionHint(project, error); + if (now() >= deadline) throw adoptionHint(project, error); + } + } + + const remainingMs = deadline - now(); + if (remainingMs <= 0) + throw new Error("Timed out while reconnecting to resume the operation"); + await sleep(Math.min(backoffMs, remainingMs)); + backoffMs = Math.min(backoffMs * 2, OPERATION_RESUME_MAX_BACKOFF_MS); + } +} + export async function reconcileResources( project: string, resources: KubernetesResource[], @@ -149,6 +327,7 @@ export async function reconcileResources( | ((resources: KubernetesResource[]) => void | Promise) | undefined, request: ApiRequester = apiRequest, + resumeOptions: OperationResumeOptions = {}, ): Promise { const workspacePath = `/workspaces/${encodeURIComponent(project)}`; const plan = await managementRequest( @@ -157,7 +336,7 @@ export async function reconcileResources( `${workspacePath}/resources/plan`, { method: "POST", json: { resources } }, ); - await managementRequest( + await resumeManagedOperation( project, request, `${workspacePath}/resources/apply`, @@ -165,12 +344,14 @@ export async function reconcileResources( method: "POST", json: { resources: plan.desired }, }, + rolloutTimeoutMs, + resumeOptions, ); await postApply?.(plan.desired); const deployments = getDeploymentNames(plan.desired); if (deployments.length > 0) { - await managementRequest( + await resumeManagedOperation( project, request, `${workspacePath}/resources/wait`, @@ -178,10 +359,12 @@ export async function reconcileResources( method: "POST", json: { deployments, timeoutMs: rolloutTimeoutMs }, }, + rolloutTimeoutMs, + resumeOptions, ); } if (plan.stale.length > 0) { - await managementRequest( + await resumeManagedOperation( project, request, `${workspacePath}/resources/delete`, @@ -189,6 +372,8 @@ export async function reconcileResources( method: "POST", json: { resources: plan.stale }, }, + rolloutTimeoutMs, + resumeOptions, ); } return plan; @@ -199,6 +384,20 @@ export async function runUp( request: ApiRequester = apiRequest, ) { const { project, compose, cwd, config, hookContext: getHookContext } = ctx(); + const trusted = await requireLocalTrust( + await resolveTrustIdentity(project, cwd), + ); + const baseRequest = request; + request = async ( + path: string, + init: ApiRequestInit = {}, + options?: ApiRequestOptions, + ) => { + const headers = new Headers(init.headers); + for (const [key, value] of Object.entries(trustHeaders(trusted))) + headers.set(key, value); + return baseRequest(path, { ...init, headers }, options); + }; const workspacePath = `/workspaces/${encodeURIComponent(project)}`; const taskCtx = await new Listr( diff --git a/lib/build.ts b/lib/build.ts index 0f60d6a..7ef5489 100644 --- a/lib/build.ts +++ b/lib/build.ts @@ -12,7 +12,7 @@ import { type Sha256Digest, } from "../shared/build-protocol"; import { resolveComposeArch } from "./arch"; -import { apiRequest, type ApiRequestInit } from "./api"; +import { apiRequest, type ApiRequestInit, type ApiRequestOptions } from "./api"; import { DEFAULT_REGISTRY } from "./config"; import { enumerateWorkspace, @@ -23,10 +23,132 @@ import { const execFileAsync = promisify(execFile); const UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024; const DEFAULT_POLL_INTERVAL_MS = 1_000; +const BUILD_POLL_REQUEST_TIMEOUT_MS = 300_000; +const MAX_BUILD_POLL_ATTEMPTS = 3; + +export const MAX_CONCURRENT_REQUESTS = 20; +export const MAX_REQUESTS_PER_SECOND = 40; + +export type SchedulerClock = { + now(): number; +}; + +export type SchedulerSleep = (ms: number) => Promise; + +export class TaskScheduler { + private inflight = 0; + private maxInflight: number; + private maxPerSecond: number; + private requestStarts: number[] = []; + private waiting: Array<() => void> = []; + private rateGate: Promise = Promise.resolve(); + private clock: SchedulerClock; + private sleep: SchedulerSleep; + + constructor(options?: { + maxInflight?: number; + maxPerSecond?: number; + clock?: SchedulerClock; + sleep?: SchedulerSleep; + }) { + this.maxInflight = options?.maxInflight ?? MAX_CONCURRENT_REQUESTS; + this.maxPerSecond = options?.maxPerSecond ?? MAX_REQUESTS_PER_SECOND; + if (!Number.isSafeInteger(this.maxInflight) || this.maxInflight < 1) + throw new RangeError("maxInflight must be a positive integer"); + if (!Number.isSafeInteger(this.maxPerSecond) || this.maxPerSecond < 1) + throw new RangeError("maxPerSecond must be a positive integer"); + this.clock = options?.clock ?? { now: () => Date.now() }; + this.sleep = options?.sleep ?? ((ms) => Bun.sleep(ms)); + } + + get currentInflight(): number { + return this.inflight; + } + + get maxConcurrent(): number { + return this.maxInflight; + } + + get currentRequestStarts(): number { + this.pruneOldStarts(); + return this.requestStarts.length; + } + + private pruneOldStarts(): void { + const cutoff = this.clock.now() - 1000; + while (this.requestStarts.length > 0 && this.requestStarts[0]! <= cutoff) { + this.requestStarts.shift(); + } + } + + private async acquire(): Promise { + if (this.inflight < this.maxInflight) { + this.inflight++; + return; + } + await new Promise((resolve) => { + this.waiting.push(resolve); + }); + } + + private release(): void { + if (this.waiting.length > 0) { + const next = this.waiting.shift()!; + next(); + } else { + this.inflight--; + } + } + + private waitForRateLimit(): Promise { + const reservation = this.rateGate.then(async () => { + for (;;) { + this.pruneOldStarts(); + if (this.requestStarts.length < this.maxPerSecond) { + this.requestStarts.push(this.clock.now()); + return; + } + const oldest = this.requestStarts[0]!; + await this.sleep(Math.max(1, oldest + 1000 - this.clock.now())); + } + }); + this.rateGate = reservation.catch(() => {}); + return reservation; + } + + async run(fn: () => Promise): Promise { + await this.acquire(); + try { + await this.waitForRateLimit(); + return await fn(); + } finally { + this.release(); + } + } +} + +async function runConcurrent( + values: T[], + concurrency: number, + run: (value: T) => Promise, +): Promise { + let index = 0; + const worker = async () => { + for (;;) { + const current = index++; + if (current >= values.length) return; + await run(values[current]!); + } + }; + await Promise.all( + Array.from({ length: Math.min(concurrency, values.length) }, worker), + ); +} export type ApiRequester = ( path: string, init?: ApiRequestInit, + options?: ApiRequestOptions, ) => Promise; export type BuildOptions = { @@ -35,6 +157,7 @@ export type BuildOptions = { pollIntervalMs?: number; sleep?: (milliseconds: number) => Promise; snapshot?: WorkspaceSnapshot; + scheduler?: TaskScheduler; }; type BuildPlan = { @@ -191,29 +314,42 @@ async function uploadBlob( digest: Sha256Digest, data: Uint8Array, request: ApiRequester, + scheduler: TaskScheduler, + project?: string, ): Promise { - const path = `/blobs/${encodeURIComponent(digest)}/uploads`; - const progress = await request<{ offset: number; complete: boolean }>(path, { - method: "POST", - json: { size: data.byteLength }, - }); + const uploadPath = `/blobs/${encodeURIComponent(digest)}/uploads`; + const projectQuery = project ? `?project=${encodeURIComponent(project)}` : ""; + const path = `${uploadPath}${projectQuery}`; + const progress = await scheduler.run(() => + request<{ offset: number; complete: boolean }>(path, { + method: "POST", + json: { size: data.byteLength }, + }), + ); let offset = progress.offset; while (!progress.complete && offset < data.byteLength) { const chunk = data.subarray(offset, offset + UPLOAD_CHUNK_BYTES); - const uploaded = await request<{ offset: number }>(path, { - method: "PATCH", - headers: { - "content-type": "application/octet-stream", - "upload-offset": String(offset), - }, - body: chunk, - }); + const uploaded = await scheduler.run(() => + request<{ offset: number }>(path, { + method: "PATCH", + headers: { + "content-type": "application/octet-stream", + "upload-offset": String(offset), + }, + body: chunk, + }), + ); if (uploaded.offset <= offset) throw new Error(`Blob upload for ${digest} made no progress`); offset = uploaded.offset; } if (!progress.complete) { - await request(`${path}/complete`, { method: "POST", json: {} }); + await scheduler.run(() => + request(`${uploadPath}/complete${projectQuery}`, { + method: "POST", + json: {}, + }), + ); } } @@ -221,30 +357,36 @@ export async function uploadWorkspaceSnapshot( snapshot: WorkspaceSnapshot, request: ApiRequester = apiRequest, reporter?: BuildReporter, + scheduler?: TaskScheduler, + project?: string, ): Promise { const blobs = new Map(snapshot.blobs.map((blob) => [blob.digest, blob.data])); blobs.set(snapshot.digest, serializeWorkspaceManifest(snapshot.manifest)); + const requestScheduler = scheduler ?? new TaskScheduler(); for (;;) { - const negotiation = await request( - "/snapshots/negotiate", - { + const negotiation = await requestScheduler.run(() => + request("/snapshots/negotiate", { method: "POST", - json: { workspace: snapshot.digest }, - }, + json: { workspace: snapshot.digest, ...(project && { project }) }, + }), ); if (negotiation.ready) return; if (negotiation.missing.length === 0) throw new Error( "Snapshot negotiation is incomplete but reported no missing blobs", ); - for (const digest of negotiation.missing) { - const data = blobs.get(digest); - if (!data) - throw new Error(`Server requested unknown workspace blob ${digest}`); - await reporter?.progress?.(`Uploading ${digest}`); - await uploadBlob(digest, data, request); - } + await runConcurrent( + negotiation.missing, + requestScheduler.maxConcurrent, + async (digest) => { + const data = blobs.get(digest); + if (!data) + throw new Error(`Server requested unknown workspace blob ${digest}`); + await reporter?.progress?.(`Uploading ${digest}`); + await uploadBlob(digest, data, request, requestScheduler, project); + }, + ); } } @@ -265,6 +407,52 @@ async function reportBuildEvent( return event.sequence; } +function isTransientBuildPollError(error: unknown): boolean { + if (!(error instanceof Error) || error.name === "AbortError") return false; + if (error.name === "TimeoutError" || error instanceof TypeError) return true; + const code = + "code" in error && typeof error.code === "string" + ? error.code + : error.cause && + typeof error.cause === "object" && + "code" in error.cause && + typeof error.cause.code === "string" + ? error.cause.code + : undefined; + return ( + code === "ECONNABORTED" || + code === "ECONNRESET" || + code === "ECONNREFUSED" || + code === "EAI_AGAIN" || + code === "ETIMEDOUT" + ); +} + +async function requestBuildPoll( + request: ApiRequester, + path: string, + init: ApiRequestInit | undefined, + pollIntervalMs: number, + sleep: (milliseconds: number) => Promise, +): Promise { + for (let attempt = 1; attempt <= MAX_BUILD_POLL_ATTEMPTS; attempt++) { + try { + return await request(path, init, { + timeoutMs: BUILD_POLL_REQUEST_TIMEOUT_MS, + }); + } catch (error) { + if ( + !isTransientBuildPollError(error) || + attempt === MAX_BUILD_POLL_ATTEMPTS + ) { + throw error; + } + await sleep(pollIntervalMs); + } + } + throw new Error("Build poll retries exhausted"); +} + async function waitForBuild( id: string, request: ApiRequester, @@ -277,8 +465,12 @@ async function waitForBuild( let sequence = 0; const reportedStates = new Set(); for (;;) { - const events = await request( + const events = await requestBuildPoll( + request, `/builds/${encodeURIComponent(id)}/events?after=${sequence}`, + undefined, + pollIntervalMs, + sleep, ); for (const event of events) sequence = Math.max( @@ -287,12 +479,15 @@ async function waitForBuild( ); if (status.state === "succeeded" || status.state === "failed") return status; - status = await request( + status = await requestBuildPoll( + request, `/builds/${encodeURIComponent(id)}/reconcile`, { method: "POST", json: {}, }, + pollIntervalMs, + sleep, ); if (status.state !== "succeeded" && status.state !== "failed") await sleep(pollIntervalMs); @@ -351,7 +546,13 @@ export async function buildServices( return plan ? [plan] : []; }, ); - await uploadWorkspaceSnapshot(snapshot, request, reporter); + await uploadWorkspaceSnapshot( + snapshot, + request, + reporter, + options.scheduler, + project, + ); const images: Record = {}; for (const plan of plans) { @@ -372,10 +573,14 @@ export async function buildServices( workspace: snapshot.digest, }, }; - const initial = await request("/builds", { - method: "POST", - json: buildRequest, - }); + const initial = await request( + "/builds", + { + method: "POST", + json: buildRequest, + }, + { timeoutMs: 300_000 }, + ); const status = await waitForBuild( id, request, diff --git a/lib/database.ts b/lib/database.ts index 85972fc..1965df1 100644 --- a/lib/database.ts +++ b/lib/database.ts @@ -236,7 +236,15 @@ function toManagedRole(claim: PostgresClaim): ManagedRole { }; } -async function reconcileManagedRoles(claims: PostgresClaim[]): Promise { +function throwIfAborted(signal?: AbortSignal): void { + if (!signal?.aborted) return; + throw new Error("Workspace operation execution was cancelled"); +} + +async function reconcileManagedRoles( + claims: PostgresClaim[], + signal?: AbortSignal, +): Promise { if (claims.length === 0) return; const cluster = await readObject< @@ -264,6 +272,7 @@ async function reconcileManagedRoles(claims: PostgresClaim[]): Promise { } try { + throwIfAborted(signal); await applyResource({ apiVersion: "postgresql.cnpg.io/v1", kind: "Cluster", @@ -287,6 +296,7 @@ async function reconcileManagedRoles(claims: PostgresClaim[]): Promise { async function reconcileDatabases( project: string, claims: PostgresClaim[], + signal?: AbortSignal, ): Promise { const uniqueDatabases = new Map(); for (const claim of claims) { @@ -295,6 +305,7 @@ async function reconcileDatabases( for (const claim of uniqueDatabases.values()) { try { + throwIfAborted(signal); await applyResource({ apiVersion: "postgresql.cnpg.io/v1", kind: "Database", @@ -328,16 +339,19 @@ async function reconcileDatabases( export async function reconcilePostgresClaim( project: string, claim: PostgresClaim, + signal?: AbortSignal, ): Promise { + throwIfAborted(signal); const credentials = await ensureRoleSecret(claim); - await reconcileManagedRoles([claim]); - await reconcileDatabases(project, [claim]); + await reconcileManagedRoles([claim], signal); + await reconcileDatabases(project, [claim], signal); return credentials; } export async function reconcilePostgresClaims( project: string, compose: ComposeSpecification, + signal?: AbortSignal, ): Promise>> { const claims = getComposePostgresClaims(compose); if (claims.length === 0) return {}; @@ -345,11 +359,12 @@ export async function reconcilePostgresClaims( const credentialsBySecret = new Map(); for (const claim of claims) { if (credentialsBySecret.has(claim.secretName)) continue; + throwIfAborted(signal); credentialsBySecret.set(claim.secretName, await ensureRoleSecret(claim)); } - await reconcileManagedRoles(claims); - await reconcileDatabases(project, claims); + await reconcileManagedRoles(claims, signal); + await reconcileDatabases(project, claims, signal); return Object.fromEntries( claims.map((claim) => { diff --git a/lib/exec-api.ts b/lib/exec-api.ts index 4e8e3d8..792a9ed 100644 --- a/lib/exec-api.ts +++ b/lib/exec-api.ts @@ -161,7 +161,8 @@ export async function openExecSession( try { if (!live) { live = true; - for (const frame of pending.splice(0)) socket.send(JSON.stringify(frame)); + for (const frame of pending.splice(0)) + socket.send(JSON.stringify(frame)); } const frame = parseFrame(event.data); const reader = readers.shift(); diff --git a/lib/storage.ts b/lib/storage.ts index 58b7725..89a2809 100644 --- a/lib/storage.ts +++ b/lib/storage.ts @@ -138,15 +138,41 @@ export function getComposeS3Claims(compose: ComposeSpecification): S3Claim[] { return claims; } +function throwIfAborted(signal?: AbortSignal): void { + if (!signal?.aborted) return; + throw new Error("Workspace operation execution was cancelled"); +} + +async function waitForInterval( + delayMs: number, + signal?: AbortSignal, +): Promise { + if (!signal) return new Promise((resolve) => setTimeout(resolve, delayMs)); + throwIfAborted(signal); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal.removeEventListener("abort", cancelWait); + resolve(); + }, delayMs); + const cancelWait = () => { + clearTimeout(timer); + reject(new Error("Workspace operation execution was cancelled")); + }; + signal.addEventListener("abort", cancelWait, { once: true }); + }); +} + async function reconcileBuckets( project: string, claims: S3Claim[], + signal?: AbortSignal, ): Promise { const buckets = new Map(); for (const claim of claims) buckets.set(claim.bucket, claim); for (const claim of buckets.values()) { try { + throwIfAborted(signal); await applyResource({ apiVersion: GARAGE_API_VERSION, kind: "GarageBucket", @@ -177,12 +203,14 @@ async function reconcileBuckets( async function reconcileKeys( project: string, claims: S3Claim[], + signal?: AbortSignal, ): Promise { const keys = new Map(); for (const claim of claims) keys.set(claim.key, claim); for (const claim of keys.values()) { try { + throwIfAborted(signal); await applyResource({ apiVersion: GARAGE_API_VERSION, kind: "GarageKey", @@ -227,11 +255,13 @@ async function reconcileKeys( export async function getS3Credentials( claim: S3Claim, + signal?: AbortSignal, ): Promise> { const deadline = Date.now() + SECRET_WAIT_TIMEOUT_MS; let lastPhase: string | undefined; while (Date.now() < deadline) { + throwIfAborted(signal); const key = await readObject({ apiVersion: GARAGE_API_VERSION, kind: "GarageKey", @@ -274,9 +304,7 @@ export async function getS3Credentials( } } - await new Promise((resolve) => - setTimeout(resolve, SECRET_WAIT_INTERVAL_MS), - ); + await waitForInterval(SECRET_WAIT_INTERVAL_MS, signal); } throw new Error( @@ -287,17 +315,19 @@ export async function getS3Credentials( export async function reconcileS3Claims( project: string, compose: ComposeSpecification, + signal?: AbortSignal, ): Promise>> { const claims = getComposeS3Claims(compose); if (claims.length === 0) return {}; - await reconcileBuckets(project, claims); - await reconcileKeys(project, claims); + await reconcileBuckets(project, claims, signal); + await reconcileKeys(project, claims, signal); const environmentsByKey = new Map>(); for (const claim of claims) { if (environmentsByKey.has(claim.key)) continue; - environmentsByKey.set(claim.key, await getS3Credentials(claim)); + throwIfAborted(signal); + environmentsByKey.set(claim.key, await getS3Credentials(claim, signal)); } return Object.fromEntries( diff --git a/lib/trust.ts b/lib/trust.ts new file mode 100644 index 0000000..d45c2e7 --- /dev/null +++ b/lib/trust.ts @@ -0,0 +1,105 @@ +import { createHash, randomUUID } from "node:crypto"; +import { + chmod, + mkdir, + readFile, + realpath, + rename, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; + +export const TRUST_SCHEMA_VERSION = 1; + +export type TrustIdentity = { project: string; fingerprint: string }; +export type LocalTrustRecord = TrustIdentity; +type TrustFile = { version: number; records: LocalTrustRecord[] }; + +export function getTrustPath(): string { + return join( + process.env.XDG_CONFIG_HOME ?? join(process.env.HOME ?? "/tmp", ".config"), + "kuber", + "trust.json", + ); +} + +export async function resolveTrustIdentity( + project: string, + cwd = process.cwd(), +): Promise { + const path = await realpath(cwd); + return { + project, + fingerprint: createHash("sha256").update(path).digest("hex"), + }; +} + +function validRecord(value: unknown): value is LocalTrustRecord { + return ( + !!value && + typeof value === "object" && + typeof (value as Record).project === "string" && + typeof (value as Record).fingerprint === "string" && + /^[a-f0-9]{64}$/.test( + (value as Record).fingerprint as string, + ) + ); +} + +export async function readTrust(): Promise { + try { + const value: unknown = JSON.parse(await readFile(getTrustPath(), "utf8")); + if ( + !value || + typeof value !== "object" || + (value as TrustFile).version !== TRUST_SCHEMA_VERSION || + !Array.isArray((value as TrustFile).records) + ) + return []; + return (value as TrustFile).records.filter(validRecord); + } catch { + return []; + } +} + +export async function writeTrust(records: LocalTrustRecord[]): Promise { + const path = getTrustPath(); + const directory = path.slice(0, path.lastIndexOf("/")); + await mkdir(directory, { recursive: true, mode: 0o700 }); + await chmod(directory, 0o700); + const temporary = `${path}.${randomUUID()}.tmp`; + await writeFile( + temporary, + JSON.stringify({ version: TRUST_SCHEMA_VERSION, records }, null, 2) + "\n", + { flag: "wx", mode: 0o600 }, + ); + await rename(temporary, path); + await chmod(path, 0o600); +} + +export async function updateTrust( + update: (records: LocalTrustRecord[]) => LocalTrustRecord[], +): Promise { + await writeTrust(update(await readTrust())); +} + +export function trustHeaders(identity: TrustIdentity): Record { + return { + "x-kuber-trust-project": identity.project, + "x-kuber-trust-fingerprint": identity.fingerprint, + }; +} + +export async function requireLocalTrust( + identity: TrustIdentity, +): Promise { + const record = (await readTrust()).find( + (candidate) => + candidate.project === identity.project && + candidate.fingerprint === identity.fingerprint, + ); + if (record) return record; + throw new Error( + "TRUST_REQUIRED: this directory is not trusted for this namespace. Run kuber trust before kuber up.", + ); +} diff --git a/package.json b/package.json index 9ded584..18aae4b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dmgnr/kuber", - "version": "2.1.0", + "version": "2.2.0", "description": "Docker Compose to Kubernetes translation layer", "bin": { "kuber": "dist/index.js" diff --git a/server/app.ts b/server/app.ts index f623ab0..6894363 100644 --- a/server/app.ts +++ b/server/app.ts @@ -18,6 +18,7 @@ import { type Role, } from "./authorization"; import type { AuditStore } from "./audit-store"; +import { validateTrust, type TrustStore } from "./trust-store"; import { BuildConflictError, BuildNotFoundError, @@ -40,6 +41,7 @@ import { OperationConflictError, OperationNotFoundError, OperationValidationError, + sanitizeOperationError, sanitizeOperationResult, type Operation, type OperationStore, @@ -55,6 +57,7 @@ import { type Workspace, type WorkspaceStore, } from "./workspace-store"; +import { redactString } from "./redact"; const API_PREFIX = "/api/v2"; const RUNTIME_SESSION_MS = 24 * 60 * 60 * 1000; @@ -62,6 +65,8 @@ const PERSISTENT_SESSION_MS = 30 * 24 * 60 * 60 * 1000; const LOGIN_WINDOW_MS = 5 * 60 * 1000; const MAX_LOGIN_FAILURES = 5; const DEFAULT_JSON_LIMIT = 1024 * 1024; +const WORKSPACE_LEASE_TTL_MS = 30_000; +const WORKSPACE_LEASE_RENEW_INTERVAL_MS = WORKSPACE_LEASE_TTL_MS / 3; export interface ApiWorkspaceStore extends WorkspaceStore { delete?(id: string): Promise; @@ -77,6 +82,7 @@ export type AppOptions = { workspaceStore?: ApiWorkspaceStore; operationStore?: OperationStore; auditStore?: AuditStore; + trustStore?: TrustStore; management?: ManagementService; builds?: BuildController; logs?: LogService; @@ -240,7 +246,7 @@ export function createApp( type: `https://kuber.astrxl.dev/problems/${error.code.toLowerCase()}`, title: error.title, status: error.status, - detail: error.message, + detail: redactString(error.message), code: error.code, requestId, ...(error.operationId && { operationId: error.operationId }), @@ -508,6 +514,39 @@ export function createApp( return options.management; } + function requireTrustStore(): TrustStore { + if (!options.trustStore) + throw new HttpError( + 503, + "Service unavailable", + "TRUST_STORE_UNAVAILABLE", + "Namespace trust service is unavailable. Contact your Kuber administrator.", + ); + return options.trustStore; + } + + async function requireTrust( + request: Request, + project: string, + ): Promise { + const fingerprint = request.headers.get("x-kuber-trust-fingerprint"); + const suppliedProject = request.headers.get("x-kuber-trust-project"); + if (!fingerprint || suppliedProject !== project) + throw new HttpError( + 428, + "Trusted workspace required", + "TRUST_REQUIRED", + "This workspace reconciliation requires trust. Run kuber trust and retry.", + ); + if (!(await requireTrustStore().has(project, fingerprint))) + throw new HttpError( + 403, + "Trusted workspace rejected", + "TRUST_REQUIRED", + "This directory is not registered for this namespace.", + ); + } + function requireAdoption(): WorkspaceAdoptionService { const workspaceStore = options.workspaceStore; const adoption = @@ -556,6 +595,12 @@ export function createApp( }, status: { ...operation.status, + ...(operation.status.error && { + error: sanitizeOperationError( + operation.status.error, + operation.spec.action, + ), + }), ...(operation.status.result !== undefined && { result: sanitizeOperationResult( operation.status.result, @@ -568,15 +613,58 @@ export function createApp( function operationBody(operation: Operation, result: unknown) { const visible = publicOperation(operation); - return isRecord(result) - ? { ...result, operationId: operation.metadata.name, operation: visible } - : Array.isArray(result) + const safeResult = sanitizeOperationResult(result, operation.spec.action); + return isRecord(safeResult) + ? { + ...safeResult, + operationId: operation.metadata.name, + operation: visible, + } + : Array.isArray(safeResult) ? { - deployments: result, + deployments: safeResult, operationId: operation.metadata.name, operation: visible, } - : { result, operationId: operation.metadata.name, operation: visible }; + : { + result: safeResult, + operationId: operation.metadata.name, + operation: visible, + }; + } + + async function transitionOperationToFailure( + operationId: string, + error: NonNullable, + ): Promise { + try { + return await options.operationStore!.transition(operationId, "failed", { + error, + }); + } catch (transitionError) { + if (!(transitionError instanceof OperationConflictError)) + throw transitionError; + const latest = await options.operationStore!.get(operationId); + if (latest?.status.state === "failed") return latest; + throw transitionError; + } + } + + function operationFailureError(operation: Operation): HttpError { + const failure = operation.status.error + ? sanitizeOperationError(operation.status.error, operation.spec.action) + : { code: "OPERATION_FAILED", message: "Operation failed" }; + return new HttpError( + failure.code === "WORKSPACE_BUSY" || + failure.code === "WORKSPACE_LEASE_LOST" + ? 409 + : 500, + "Operation failed", + failure.code, + failure.message, + undefined, + operation.metadata.name, + ); } async function runOperation( @@ -585,7 +673,7 @@ export function createApp( workspace: Workspace, action: string, input: unknown, - execute: () => Promise, + execute: (signal: AbortSignal) => Promise, ): Promise { if (!options.operationStore) throw new HttpError( @@ -594,7 +682,7 @@ export function createApp( "OPERATION_STORE_UNAVAILABLE", "Operation storage is not configured", ); - const operation = await options.operationStore.create({ + const { operation } = await options.operationStore.createOrReuse({ workspaceId: workspace.metadata.name, workspaceUid: workspace.metadata.uid, action, @@ -602,14 +690,7 @@ export function createApp( request: input, }); if (operation.status.state === "failed") - throw new HttpError( - operation.status.error?.code === "WORKSPACE_BUSY" ? 409 : 500, - "Operation failed", - operation.status.error?.code ?? "OPERATION_FAILED", - operation.status.error?.message ?? "Operation failed", - undefined, - operation.metadata.name, - ); + throw operationFailureError(operation); if (operation.status.state === "cancelled") throw new HttpError( 409, @@ -640,16 +721,6 @@ export function createApp( ) : undefined; if (options.leases && !lease) { - await options.operationStore.transition( - operation.metadata.name, - "failed", - { - error: { - code: "WORKSPACE_BUSY", - message: "Another workspace operation is running", - }, - }, - ); throw new HttpError( 409, "Conflict", @@ -659,9 +730,76 @@ export function createApp( operation.metadata.name, ); } - await options.operationStore.transition(operation.metadata.name, "running"); + if (!lease && options.leases) throw new Error("Unreachable lease state"); + + let operationStarted = false; + let leaseRenewalTimer: ReturnType | undefined; + let leaseOwnershipLost = false; + let renewalInFlight: Promise | undefined; + const executionController = new AbortController(); + const renewLease = async (): Promise => { + if (!lease || leaseOwnershipLost) return false; + if (renewalInFlight) return renewalInFlight; + renewalInFlight = lease + .renew(WORKSPACE_LEASE_TTL_MS) + .catch(() => false) + .then((renewed) => { + if (!renewed) { + leaseOwnershipLost = true; + executionController.abort("Workspace lease ownership was lost"); + } + return renewed; + }) + .finally(() => { + renewalInFlight = undefined; + }); + return renewalInFlight; + }; + const scheduleLeaseRenewal = () => { + if (!lease || leaseOwnershipLost) return; + leaseRenewalTimer = setTimeout(() => { + void renewLease().finally(scheduleLeaseRenewal); + }, WORKSPACE_LEASE_RENEW_INTERVAL_MS); + }; + const requireLeaseOwnership = async () => { + if (lease && !(await renewLease())) + throw new HttpError( + 409, + "Conflict", + "WORKSPACE_LEASE_LOST", + "Workspace operation lease ownership was lost", + undefined, + operation.metadata.name, + ); + }; try { - const result = await execute(); + const claimed = await options.operationStore.claimExecution( + operation.metadata.name, + ); + if (!claimed) { + const latest = await options.operationStore.get( + operation.metadata.name, + ); + if (latest?.status.state === "succeeded") + return response(operationBody(latest, latest.status.result), 200, { + location: `${API_PREFIX}/operations/${operation.metadata.name}`, + }); + if (latest?.status.state === "failed") + throw operationFailureError(latest); + return response( + { + operationId: operation.metadata.name, + operation: publicOperation(latest ?? operation), + }, + 202, + { location: `${API_PREFIX}/operations/${operation.metadata.name}` }, + ); + } + operationStarted = true; + scheduleLeaseRenewal(); + await requireLeaseOwnership(); + const result = await execute(executionController.signal); + await requireLeaseOwnership(); const completed = await options.operationStore.transition( operation.metadata.name, "succeeded", @@ -687,21 +825,30 @@ export function createApp( location: `${API_PREFIX}/operations/${operation.metadata.name}`, }); } catch (error) { + if (!operationStarted) throw error; const message = error instanceof Error ? error.message : String(error); - await options.operationStore.transition( + const leaseLost = + leaseOwnershipLost || + (error instanceof HttpError && error.code === "WORKSPACE_LEASE_LOST"); + const failed = await transitionOperationToFailure( operation.metadata.name, - "failed", { - error: { code: "OPERATION_FAILED", message }, + code: leaseLost ? "WORKSPACE_LEASE_LOST" : "OPERATION_FAILED", + message: leaseLost + ? "Workspace operation lease ownership was lost" + : message, }, ); + const failure = failed.status.error; + const failureCode = failure?.code ?? "OPERATION_FAILED"; + const failureMessage = failure?.message ?? message; try { await audit( identity, request, action, "failure", - { error: message }, + { error: failureMessage }, workspace.metadata.name, operation.metadata.name, ); @@ -712,14 +859,17 @@ export function createApp( ); } throw new HttpError( - 500, - "Operation failed", - "OPERATION_FAILED", - message, + failureCode === "WORKSPACE_LEASE_LOST" ? 409 : 500, + failureCode === "WORKSPACE_LEASE_LOST" + ? "Conflict" + : "Operation failed", + failureCode, + failureMessage, undefined, operation.metadata.name, ); } finally { + if (leaseRenewalTimer !== undefined) clearTimeout(leaseRenewalTimer); try { await lease?.release(); } catch (error) { @@ -810,12 +960,89 @@ export function createApp( return new Response(null, { status: 204 }); } + const trustMatch = new RegExp( + `^${API_PREFIX}/workspaces/([^/]+)/trust$`, + ).exec(path); + if (trustMatch && options.trustStore) { + const project = pathPart(trustMatch[1]!); + if (request.method === "GET") { + await requireCapability(identity, request, "kubernetes:read"); + return response({ + fingerprints: await options.trustStore.list(project), + }); + } + if (request.method === "POST") { + await requireCapability(identity, request, "kubernetes:write"); + const body = await readJson(request); + if (typeof body.fingerprint !== "string") + throw new HttpError( + 400, + "Invalid trust request", + "TRUST_INVALID", + "fingerprint is required", + ); + try { + validateTrust(project, body.fingerprint); + } catch (error) { + throw new HttpError( + 400, + "Invalid trust request", + "TRUST_INVALID", + error instanceof Error ? error.message : "Invalid trust request", + ); + } + await options.trustStore.grant(project, body.fingerprint); + await audit( + identity, + request, + "trust.grant", + "success", + undefined, + project, + ); + return new Response(null, { status: 204 }); + } + if (request.method === "DELETE") { + await requireCapability(identity, request, "kubernetes:write"); + const fingerprint = url.searchParams.get("fingerprint"); + if (!fingerprint) + throw new HttpError( + 400, + "Invalid trust request", + "TRUST_INVALID", + "fingerprint is required", + ); + if (!(await options.trustStore.revoke(project, fingerprint))) + throw new HttpError( + 404, + "Not found", + "TRUST_REGISTRATION_MISSING", + "Trust registration was not found", + ); + await audit( + identity, + request, + "trust.revoke", + "success", + undefined, + project, + ); + return new Response(null, { status: 204 }); + } + } + if (path === `${API_PREFIX}/builds` && request.method === "POST") { await requireCapability(identity, request, "kubernetes:write"); + const body = await readJson(request); + if (typeof body.project !== "string" || !body.project) + throw new HttpError( + 400, + "Invalid build request", + "BUILD_INVALID", + "project is required", + ); return response( - await requireBuilds().submitBuild( - (await readJson(request)) as unknown as BuildRequest, - ), + await requireBuilds().submitBuild(body as unknown as BuildRequest), 202, ); } @@ -826,6 +1053,13 @@ export function createApp( ) { await requireCapability(identity, request, "kubernetes:write"); const body = await readJson(request); + if (typeof body.project !== "string" || !body.project) + throw new HttpError( + 400, + "Invalid snapshot request", + "BUILD_INVALID", + "project is required", + ); return response( await requireBuilds().negotiateSnapshot(body.workspace as Sha256Digest), ); @@ -886,6 +1120,14 @@ export function createApp( ).exec(path); if (blobMatch) { await requireCapability(identity, request, "kubernetes:write"); + const project = url.searchParams.get("project"); + if (!project) + throw new HttpError( + 400, + "Invalid upload request", + "BUILD_INVALID", + "project is required", + ); const digest = `sha256:${blobMatch[2]!.toLowerCase()}` as Sha256Digest; if (blobMatch[3] === "complete" && request.method === "POST") return response(await requireBuilds().completeBlobUpload(digest)); @@ -1151,13 +1393,16 @@ export function createApp( workspace, "workspace.delete", { full: true }, - async () => { + async (signal) => { const result = options.management ? await options.management.down( workspaceIdentity(workspace), true, + { signal }, ) : undefined; + if (signal.aborted) + throw new Error("Workspace operation execution was cancelled"); if (!requireWorkspaceStore().delete) throw new Error("Workspace store does not support deletion"); await requireWorkspaceStore().delete!(id); @@ -1274,24 +1519,30 @@ export function createApp( workspace, `workspace.${lifecycleAction}`, body, - async () => { + async (signal) => { const management = requireManagement(); const names = Array.isArray(body.services) ? (body.services as string[]) : undefined; if (lifecycleAction === "stop") - return management.stop(workspaceIdentity(workspace), names); + return management.stop(workspaceIdentity(workspace), names, { + signal, + }); if (lifecycleAction === "restart") - return management.restart(workspaceIdentity(workspace), names); + return management.restart(workspaceIdentity(workspace), names, { + signal, + }); if (lifecycleAction === "rollback") return management.rollback( workspaceIdentity(workspace), names, typeof body.timeoutMs === "number" ? body.timeoutMs : undefined, + { signal }, ); return management.down( workspaceIdentity(workspace), body.full === true, + { signal }, ); }, ); @@ -1322,13 +1573,14 @@ export function createApp( ) { await requireCapability(identity, request, "kubernetes:write"); const body = await readJson(request); + if (subpath === "resources/apply") await requireTrust(request, id); return runOperation( identity, request, workspace, subpath!, body, - async () => { + async (signal) => { const management = requireManagement(); if (subpath === "resources/apply") { if (!Array.isArray(body.resources)) @@ -1336,6 +1588,7 @@ export function createApp( return management.applyResources( workspaceIdentity(workspace), body.resources as KubernetesObject[], + { signal }, ); } if (subpath === "resources/wait") { @@ -1345,6 +1598,7 @@ export function createApp( workspaceIdentity(workspace), body.deployments as string[], typeof body.timeoutMs === "number" ? body.timeoutMs : undefined, + { signal }, ); return { ready: true }; } @@ -1353,6 +1607,7 @@ export function createApp( await management.deleteResources( workspaceIdentity(workspace), body.resources as unknown as ResourceIdentity[], + { signal }, ); return { deleted: body.resources.length }; }, @@ -1369,10 +1624,11 @@ export function createApp( workspace, "databases.reconcile", body, - () => + (signal) => requireManagement().reconcileDatabases( workspaceIdentity(workspace), compose, + { signal }, ), ); } @@ -1429,10 +1685,11 @@ export function createApp( workspace, "storage.reconcile", body, - () => + (signal) => requireManagement().reconcileStorage( workspaceIdentity(workspace), compose, + { signal }, ), ); } @@ -1634,9 +1891,7 @@ export function createApp( }; } -const EXEC_UPGRADE_PATH = new RegExp( - `^${API_PREFIX}/workspaces/([^/]+)/exec$`, -); +const EXEC_UPGRADE_PATH = new RegExp(`^${API_PREFIX}/workspaces/([^/]+)/exec$`); export function execUpgradeMatch(url: URL): string | undefined { const match = EXEC_UPGRADE_PATH.exec(url.pathname); @@ -1709,10 +1964,7 @@ export async function authorizeExecConnection( return { identity, workspace }; } -export function execProblem( - error: unknown, - requestId = "", -): Response { +export function execProblem(error: unknown, requestId = ""): Response { let httpError: HttpError; if (error instanceof HttpError) httpError = error; else { @@ -1782,9 +2034,7 @@ const DEFAULT_EXEC_MAX_FRAME_BYTES = 64 * 1024; export class WireExecSession { private readonly controller = new AbortController(); - private session?: Awaited< - ReturnType - >; + private session?: Awaited>; private readonly maxFrameBytes: number; private started = false; private done: Promise = Promise.resolve(); @@ -1796,8 +2046,7 @@ export class WireExecSession { private readonly connection: ExecConnection, options: ExecLinkOptions = {}, ) { - this.maxFrameBytes = - options.maxFrameBytes ?? DEFAULT_EXEC_MAX_FRAME_BYTES; + this.maxFrameBytes = options.maxFrameBytes ?? DEFAULT_EXEC_MAX_FRAME_BYTES; } private send(frame: ExecServerWireFrame) { @@ -1851,14 +2100,10 @@ export class WireExecSession { private toClientFrame(frame: ExecClientWireFrame): ExecClientFrame { switch (frame.type) { case "stdin": { - if ( - frame.encoding !== "base64" || - typeof frame.data !== "string" - ) + if (frame.encoding !== "base64" || typeof frame.data !== "string") throw new Error("Invalid stdin frame"); const data = base64ToBytes(frame.data); - if (!data) - throw new Error("Invalid stdin frame"); + if (!data) throw new Error("Invalid stdin frame"); return { type: "stdin", data, @@ -1946,7 +2191,7 @@ export class WireExecSession { error instanceof Error && "code" in error && typeof (error as { code?: unknown }).code === "string" - ? ((error as { code: string }).code) + ? (error as { code: string }).code : "EXEC_FAILED", message: error instanceof Error ? error.message : "Exec failed to start", @@ -1985,7 +2230,7 @@ export class WireExecSession { error instanceof Error && "code" in error && typeof (error as { code?: unknown }).code === "string" - ? ((error as { code: string }).code) + ? (error as { code: string }).code : "EXEC_FAILED"; this.send({ type: "error", diff --git a/server/build-controller.ts b/server/build-controller.ts index b93c7b6..45e3a9c 100644 --- a/server/build-controller.ts +++ b/server/build-controller.ts @@ -26,9 +26,11 @@ import { parseImageReference, resolveRegistryDigest } from "./registry"; export const DEFAULT_MAX_BLOB_BYTES = 1024 * 1024 * 1024; export const DEFAULT_MAX_UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024; +const SNAPSHOT_CHECK_CONCURRENCY = 20; export const DEFAULT_MAX_LOG_BYTES = 1024 * 1024; export interface BuildCas extends MaterializeCas { + has(digest: Sha256Digest): Promise; put(data: Uint8Array, expected?: Sha256Digest): Promise; } @@ -101,6 +103,28 @@ export type UploadProgress = { complete: boolean; }; +async function mapConcurrent( + values: T[], + run: (value: T) => Promise, +): Promise { + const results = new Array(values.length); + let index = 0; + const worker = async () => { + for (;;) { + const current = index++; + if (current >= values.length) return; + results[current] = await run(values[current]!); + } + }; + await Promise.all( + Array.from( + { length: Math.min(SNAPSHOT_CHECK_CONCURRENCY, values.length) }, + worker, + ), + ); + return results; +} + export function buildImageName( registry: string, project: string, @@ -121,10 +145,6 @@ function clone(value: T): T { return structuredClone(value); } -function requestFingerprint(request: BuildRequest): string { - return createHash("sha256").update(JSON.stringify(request)).digest("hex"); -} - function jobWorkspaceSubPath(workspaceRoot: string, subPath: string): string { const segments = workspaceRoot.split("/").filter(Boolean); const prefix = segments.at(-1); @@ -183,6 +203,7 @@ export class BuildController { private readonly maxLogBytes: number; private readonly materializer: typeof materializeWorkspace; private readonly digestResolver: typeof resolveRegistryDigest; + private readonly imageSubmissionLocks = new Map>(); constructor(private readonly options: BuildControllerOptions) { this.now = options.now ?? (() => new Date()); @@ -203,13 +224,11 @@ export class BuildController { const manifest = parseWorkspaceManifest( await this.options.cas.get(workspace), ); - const missing: Sha256Digest[] = []; - const seen = new Set(); - for (const file of manifest.files) { - if (!seen.has(file.digest) && !(await this.options.cas.has(file.digest))) - missing.push(file.digest); - seen.add(file.digest); - } + const digests = [...new Set(manifest.files.map((file) => file.digest))]; + const available = await mapConcurrent(digests, (digest) => + this.options.cas.has(digest), + ); + const missing = digests.filter((_digest, index) => !available[index]); return { workspace, missing, ready: missing.length === 0 }; } @@ -345,17 +364,24 @@ export class BuildController { if (this.options.imageName) request.spec.image = this.options.imageName(request); validateRequest(request); - const existing = await this.options.store.getBuild(request.id); - if (existing) { - if ( - requestFingerprint(existing.spec.request) !== - requestFingerprint(request) - ) - throw new BuildConflictError( - "Build ID was already used for a different request", - ); - return recordStatus(existing); + const imageKey = `${request.project}\0${request.service}\0${request.spec.image}`; + const previous = + this.imageSubmissionLocks.get(imageKey) ?? Promise.resolve(); + const current = previous.then(() => this.submitBuildInternal(request)); + const entry = current.catch(() => undefined); + this.imageSubmissionLocks.set(imageKey, entry); + try { + return await current; + } finally { + if (this.imageSubmissionLocks.get(imageKey) === entry) + this.imageSubmissionLocks.delete(imageKey); } + } + + private async submitBuildInternal( + request: BuildRequest, + ): Promise { + const imageKey = `${request.project}\0${request.service}\0${request.spec.image}`; const snapshot = await this.negotiateSnapshot(request.spec.workspace); if (!snapshot.ready) throw new BuildConflictError( @@ -367,7 +393,6 @@ export class BuildController { .digest("hex") .slice(0, 24); const jobName = `kuber-build-${hash}`; - const imageKey = `${request.project}\0${request.service}\0${request.spec.image}`; const initial: BuildStatus = { version: BUILD_PROTOCOL_VERSION, id: request.id, @@ -401,16 +426,27 @@ export class BuildController { try { const result = await this.options.store.createBuild(record); stored = result.record; - if (!result.created) { - if ( - requestFingerprint(stored.spec.request) !== - requestFingerprint(request) - ) - throw new BuildConflictError( - "Build ID was already used for a different request", - ); - return recordStatus(stored); + for (const superseded of result.superseded ?? []) { + // Deletion is best effort and does not wait for the Job or its pods. + await this.options.kubernetes + .deleteJob(this.options.namespace, superseded.spec.jobName) + .catch(() => undefined); + await rm( + join(this.options.workspaceRoot, superseded.spec.workspaceSubPath), + { recursive: true, force: true }, + ).catch(() => undefined); } + if (!result.created) return recordStatus(stored); + const current = await this.options.store.getBuild(request.id); + if ( + !current || + current.status.state !== "queued" || + (this.options.store.ownsBuild && + !(await this.options.store.ownsBuild(imageKey, request.id))) + ) + throw new BuildConflictError( + `Build '${request.id}' was superseded before Job creation`, + ); } catch (error) { if (error instanceof BuildStoreConflictError) throw new BuildConflictError(error.message); @@ -422,6 +458,13 @@ export class BuildController { request.spec.workspace, join(this.options.workspaceRoot, stored.spec.workspaceSubPath), ); + if ( + this.options.store.ownsBuild && + !(await this.options.store.ownsBuild(imageKey, request.id)) + ) + throw new BuildConflictError( + `Build '${request.id}' lost image lock before Job creation`, + ); const cacheImage = typeof this.options.cacheImage === "function" ? this.options.cacheImage(request) @@ -456,6 +499,11 @@ export class BuildController { }); return initial; } catch (error) { + try { + await this.terminateJob(stored.spec.jobName); + } catch { + // Job cleanup is best effort; the record still releases its lock. + } await this.failBuild( stored.metadata.name, error instanceof Error ? error.message : String(error), @@ -469,6 +517,10 @@ export class BuildController { return recordStatus(record); } + async getBuildProject(id: string): Promise { + return (await this.requireBuild(id)).spec.request.project; + } + async getBuildEvents(id: string, afterSequence = 0): Promise { if (!Number.isSafeInteger(afterSequence) || afterSequence < 0) throw new BuildValidationError( @@ -527,11 +579,17 @@ export class BuildController { const record = await this.requireBuild(id); if (record.status.state === "succeeded" || record.status.state === "failed") return recordStatus(record); - if (record.status.jobCreated) - await this.options.kubernetes.deleteJob( - this.options.namespace, - record.spec.jobName, - ); + if (record.status.jobCreated) { + try { + await this.terminateJob(record.spec.jobName); + } catch (error) { + await this.failBuild( + record.metadata.name, + `Unable to cancel build safely: ${error instanceof Error ? error.message : String(error)}`, + ); + throw error; + } + } const next = await this.setState(record, "failed", { finishedAt: this.now().toISOString(), error: "Build cancelled", @@ -544,11 +602,7 @@ export class BuildController { const record = await this.requireBuild(id); if (record.status.state !== "succeeded" && record.status.state !== "failed") throw new BuildConflictError("An active build cannot be cleaned up"); - if (record.status.jobCreated) - await this.options.kubernetes.deleteJob( - this.options.namespace, - record.spec.jobName, - ); + if (record.status.jobCreated) await this.terminateJob(record.spec.jobName); await rm(join(this.options.workspaceRoot, record.spec.workspaceSubPath), { recursive: true, force: true, @@ -576,6 +630,10 @@ export class BuildController { return record; } + private async terminateJob(name: string): Promise { + await this.options.kubernetes.deleteJob(this.options.namespace, name); + } + private async updateBuild( record: BuildRecord, change: (next: BuildRecord) => void, diff --git a/server/build-kubernetes.ts b/server/build-kubernetes.ts index bce9942..84ec4e0 100644 --- a/server/build-kubernetes.ts +++ b/server/build-kubernetes.ts @@ -2,6 +2,7 @@ import { BatchV1Api, CoreV1Api, KubernetesObjectApi, + type V1DeleteOptions, type V1Job, } from "@kubernetes/client-node"; import { createHash } from "node:crypto"; @@ -15,6 +16,8 @@ import type { import type { KubernetesJob } from "./build-job"; import { BuildStoreConflictError, + buildTimestamp, + compareBuildRecords, type BuildRecord, type BuildStore, type CreateBuildResult, @@ -22,6 +25,7 @@ import { } from "./build-store"; const TYPE_LABEL = "kuber.astrxl.dev/type"; +const IMAGE_LABEL = "kuber.astrxl.dev/image"; type ConfigMap = { apiVersion: "v1"; @@ -30,6 +34,7 @@ type ConfigMap = { name: string; namespace: string; resourceVersion?: string; + creationTimestamp?: string; labels?: Record; }; data?: Record; @@ -39,7 +44,7 @@ export interface BuildObjectApi { create(value: ConfigMap): Promise; read(value: ConfigMap): Promise; replace(value: ConfigMap): Promise; - delete(value: ConfigMap): Promise; + delete(value: ConfigMap, options?: V1DeleteOptions): Promise; list( apiVersion: string, kind: string, @@ -69,10 +74,12 @@ function payload(value: unknown): T | undefined { if (!raw) return; try { const parsed = JSON.parse(raw) as T & { - metadata?: { resourceVersion?: string }; + metadata?: { resourceVersion?: string; creationTimestamp?: string }; }; if (parsed.metadata && object.metadata.resourceVersion) parsed.metadata.resourceVersion = object.metadata.resourceVersion; + if (parsed.metadata && object.metadata.creationTimestamp) + parsed.metadata.creationTimestamp = object.metadata.creationTimestamp; return parsed; } catch { return; @@ -85,6 +92,7 @@ function map( type: "build" | "build-upload" | "build-lock", value?: unknown, resourceVersion?: string, + labels?: Record, ): ConfigMap { return { apiVersion: "v1", @@ -93,20 +101,59 @@ function map( name, namespace, ...(resourceVersion && { resourceVersion }), - labels: { [TYPE_LABEL]: type }, + labels: { [TYPE_LABEL]: type, ...labels }, }, ...(value !== undefined && { data: { payload: JSON.stringify(value) } }), }; } function terminal(record: BuildRecord): boolean { - return record.status.state === "succeeded" || record.status.state === "failed"; + return ( + record.status.state === "succeeded" || record.status.state === "failed" + ); } function sameSpec(left: BuildRecord, right: BuildRecord): boolean { return JSON.stringify(left.spec) === JSON.stringify(right.spec); } +function newer(left: BuildRecord, right: BuildRecord): boolean { + return compareBuildRecords(left, right) > 0; +} + +function supersede(record: BuildRecord, finishedAt: string): BuildRecord { + const reason = "Superseded by newer build"; + const next = structuredClone(record); + next.metadata.resourceVersion = String( + Number(record.metadata.resourceVersion) + 1, + ); + next.status = { + ...record.status, + state: "failed", + finishedAt, + error: reason, + cancelled: true, + events: [ + ...record.status.events, + { + type: "status", + status: { + version: record.status.version, + id: record.status.id, + state: "failed", + createdAt: record.status.createdAt, + ...(record.status.startedAt && { + startedAt: record.status.startedAt, + }), + finishedAt, + error: reason, + }, + }, + ], + }; + return next; +} + /** Build metadata lives in ConfigMaps; resumable upload bytes live only on the RWX volume. */ export class KubernetesBuildStore implements BuildStore { constructor( @@ -131,6 +178,68 @@ export class KubernetesBuildStore implements BuildStore { return hashName("build-lock", imageKey); } + private imageLabel(imageKey: string): string { + return createHash("sha256").update(imageKey).digest("hex").slice(0, 63); + } + + private buildLabels(record: BuildRecord): Record { + return { + ...record.metadata.labels, + [IMAGE_LABEL]: this.imageLabel(record.spec.imageKey), + }; + } + + private async listImageBuilds(imageKey: string): Promise { + const result = await this.objects.list( + "v1", + "ConfigMap", + this.namespace, + undefined, + undefined, + undefined, + undefined, + `${TYPE_LABEL}=build,${IMAGE_LABEL}=${this.imageLabel(imageKey)}`, + ); + const records = result.items + .map((item) => payload(item)) + .filter( + (item): item is BuildRecord => + item?.kind === "BuildRecord" && item.spec?.imageKey === imageKey, + ); + // Legacy records lack the image index label. Keep this compatibility scan + // bounded to build records, then match all identifying payload fields. + const [project, service] = imageKey.split("\0"); + if (!project || !service) return records; + const legacy = await this.objects.list( + "v1", + "ConfigMap", + this.namespace, + undefined, + undefined, + undefined, + undefined, + `${TYPE_LABEL}=build`, + ); + const legacyRecords = legacy.items + .map((item) => payload(item)) + .filter( + (item): item is BuildRecord => + item?.kind === "BuildRecord" && + item.spec?.request?.project === project && + item.spec?.request?.service === service && + item.spec?.imageKey === imageKey, + ); + return [ + ...records, + ...legacyRecords.filter( + (legacyRecord) => + !records.some( + (record) => record.metadata.name === legacyRecord.metadata.name, + ), + ), + ]; + } + private async read(value: ConfigMap): Promise { try { return payload(await this.objects.read(value)); @@ -140,14 +249,73 @@ export class KubernetesBuildStore implements BuildStore { } } - private async delete(value: ConfigMap): Promise { + private async readConfigMap( + value: ConfigMap, + ): Promise { try { - await this.objects.delete(value); + return (await this.objects.read(value)) as ConfigMap; } catch (error) { - if (statusCode(error) !== 404) throw error; + if (statusCode(error) === 404) return; + throw error; } } + private async delete( + value: ConfigMap, + expectedResourceVersion?: string, + ): Promise { + try { + await this.objects.delete( + value, + expectedResourceVersion + ? { preconditions: { resourceVersion: expectedResourceVersion } } + : undefined, + ); + } catch (error) { + if (statusCode(error) !== 404 && statusCode(error) !== 409) throw error; + } + } + + private async supersedeBuild( + record: BuildRecord, + finishedAt: string, + releaseLock = false, + ): Promise { + const next = supersede(record, finishedAt); + await this.replaceBuildRecord( + next, + record.metadata.resourceVersion, + releaseLock, + ); + return next; + } + + /** + * Build records are indexed by image label, rather than scanning all records. + * This repairs record-first crashes and makes the record ordering authoritative + * even when one of the records never acquired a lock. + */ + private async reconcileOlderBuilds( + candidate: BuildRecord, + ): Promise { + const records = await this.listImageBuilds(candidate.spec.imageKey); + const superseded: BuildRecord[] = []; + for (const other of records) { + if (other.metadata.name === candidate.metadata.name || terminal(other)) + continue; + if (newer(other, candidate)) { + superseded.push( + await this.supersedeBuild(candidate, buildTimestamp(candidate)), + ); + return superseded; + } + superseded.push( + await this.supersedeBuild(other, buildTimestamp(candidate)), + ); + } + return superseded; + } + async createBuild(record: BuildRecord): Promise { const existing = await this.getBuild(record.metadata.name); if (existing) { @@ -155,48 +323,195 @@ export class KubernetesBuildStore implements BuildStore { throw new BuildStoreConflictError( "Build ID was already used for a different request", ); - return { record: existing, created: false }; + if (terminal(existing)) return { record: existing, created: false }; + record = existing; } - const lockName = this.lockName(record.spec.imageKey); - try { - await this.objects.create( - map(this.namespace, lockName, "build-lock", { - buildId: record.metadata.name, - }), - ); - } catch (error) { - if (statusCode(error) !== 409) throw error; - const lock = await this.read<{ buildId: string }>( - map(this.namespace, lockName, "build-lock"), - ); - const active = lock && (await this.getBuild(lock.buildId)); - if (!active || terminal(active)) { - await this.delete(map(this.namespace, lockName, "build-lock")); - return this.createBuild(record); - } - throw new BuildStoreConflictError( - `Build '${active.metadata.name}' is already active for ${record.spec.imageKey}`, - ); - } - - try { - const created = (await this.objects.create( - map(this.namespace, this.buildName(record.metadata.name), "build", record), - )) as ConfigMap; - return { record: payload(created) ?? record, created: true }; - } catch (error) { - await this.delete(map(this.namespace, lockName, "build-lock")); - if (statusCode(error) === 409) { + if (!existing) { + // Persist before locking so contenders can find and supersede queued records. + try { + await this.objects.create( + map( + this.namespace, + this.buildName(record.metadata.name), + "build", + record, + undefined, + this.buildLabels(record), + ), + ); + } catch (error) { + if (statusCode(error) !== 409) throw error; const concurrent = await this.getBuild(record.metadata.name); if (concurrent && sameSpec(concurrent, record)) - return { record: concurrent, created: false }; + return this.createBuild(concurrent); throw new BuildStoreConflictError( "Build ID was already used for a different request", ); } + const stored = await this.getBuild(record.metadata.name); + if (!stored) + throw new BuildStoreConflictError("Build record disappeared"); + record = stored; + } + + const reconciled = await this.reconcileOlderBuilds(record); + if ( + reconciled.some((value) => value.metadata.name === record.metadata.name) + ) + return { + record: (await this.getBuild(record.metadata.name))!, + created: false, + superseded: reconciled, + }; + + const lockName = this.lockName(record.spec.imageKey); + for (let attempt = 0; attempt < 8; attempt++) { + try { + await this.objects.create( + map(this.namespace, lockName, "build-lock", { + buildId: record.metadata.name, + }), + ); + return { + record: (await this.getBuild(record.metadata.name))!, + created: true, + ...(reconciled.length && { superseded: reconciled }), + }; + } catch (error) { + if (statusCode(error) !== 409) { + await this.failCreatedBuild(record, error); + throw error; + } + const lockObject = await this.readConfigMap( + map(this.namespace, lockName, "build-lock"), + ); + if (!lockObject) continue; + const lock = payload<{ buildId: string }>(lockObject); + if (!lock?.buildId) + throw new BuildStoreConflictError("Invalid build lock"); + if (lock.buildId === record.metadata.name) + return { + record: (await this.getBuild(record.metadata.name))!, + created: false, + ...(reconciled.length && { superseded: reconciled }), + }; + const active = await this.getBuild(lock.buildId); + if (!active) { + await this.delete(lockObject, lockObject.metadata.resourceVersion); + continue; + } + if (terminal(active)) { + await this.delete(lockObject, lockObject.metadata.resourceVersion); + continue; + } + if (newer(active, record)) { + const superseded = await this.supersedeBuild( + record, + buildTimestamp(record), + ); + return { record: superseded, created: false }; + } + try { + const nextOld = await this.supersedeBuild( + active, + buildTimestamp(record), + ); + await this.objects.replace( + map( + this.namespace, + lockName, + "build-lock", + { buildId: record.metadata.name }, + lockObject.metadata.resourceVersion, + ), + ); + return { + record: (await this.getBuild(record.metadata.name))!, + created: true, + superseded: [...reconciled, nextOld], + }; + } catch (takeoverError) { + if (statusCode(takeoverError) === 409) continue; + await this.failCreatedBuild(record, takeoverError); + throw takeoverError; + } + } + } + await this.failCreatedBuild( + record, + new BuildStoreConflictError("Build lock acquisition timed out"), + ); + throw new BuildStoreConflictError("Build lock acquisition timed out"); + } + + private async failCreatedBuild( + record: BuildRecord, + error: unknown, + ): Promise { + const current = await this.getBuild(record.metadata.name); + if (!current || terminal(current)) return; + const next = structuredClone(current); + next.metadata.resourceVersion = String( + Number(current.metadata.resourceVersion) + 1, + ); + next.status = { + ...current.status, + state: "failed", + finishedAt: new Date().toISOString(), + error: error instanceof Error ? error.message : String(error), + events: [ + ...current.status.events, + { + type: "status", + status: { + version: current.status.version, + id: current.status.id, + state: "failed", + createdAt: current.status.createdAt, + finishedAt: next.status.finishedAt, + error: next.status.error, + }, + }, + ], + }; + await this.replaceBuildRecord(next, current.metadata.resourceVersion); + } + + private async replaceBuildRecord( + record: BuildRecord, + expectedResourceVersion: string, + releaseLock = true, + ): Promise { + const current = await this.getBuild(record.metadata.name); + if ( + !current || + current.metadata.resourceVersion !== expectedResourceVersion || + !sameSpec(current, record) + ) + throw new BuildStoreConflictError( + "Build record was concurrently modified", + ); + try { + await this.objects.replace( + map( + this.namespace, + this.buildName(record.metadata.name), + "build", + record, + expectedResourceVersion, + this.buildLabels(record), + ), + ); + } catch (error) { + if (statusCode(error) === 409) + throw new BuildStoreConflictError( + "Build record was concurrently modified", + ); throw error; } + if (terminal(record) && releaseLock) + await this.releaseLock(record.spec.imageKey, record.metadata.name); } async getBuild(id: string): Promise { @@ -220,9 +535,7 @@ export class KubernetesBuildStore implements BuildStore { return result.items .map((item) => payload(item)) .filter((item): item is BuildRecord => item?.kind === "BuildRecord") - .sort((a, b) => - a.metadata.creationTimestamp.localeCompare(b.metadata.creationTimestamp), - ); + .sort(compareBuildRecords); } async replaceBuild( @@ -230,8 +543,13 @@ export class KubernetesBuildStore implements BuildStore { expectedResourceVersion: string, ): Promise { const current = await this.getBuild(record.metadata.name); - if (!current || current.metadata.resourceVersion !== expectedResourceVersion) - throw new BuildStoreConflictError("Build record was concurrently modified"); + if ( + !current || + current.metadata.resourceVersion !== expectedResourceVersion + ) + throw new BuildStoreConflictError( + "Build record was concurrently modified", + ); if (!sameSpec(current, record)) throw new BuildStoreConflictError("Build specification is immutable"); if ( @@ -251,17 +569,46 @@ export class KubernetesBuildStore implements BuildStore { "build", record, expectedResourceVersion, + this.buildLabels(record), ), ); } catch (error) { if (statusCode(error) === 409) - throw new BuildStoreConflictError("Build record was concurrently modified"); + throw new BuildStoreConflictError( + "Build record was concurrently modified", + ); throw error; } - if (terminal(record)) - await this.delete( - map(this.namespace, this.lockName(record.spec.imageKey), "build-lock"), - ); + if (terminal(record)) { + await this.releaseLock(record.spec.imageKey, record.metadata.name); + } + } + + async ownsBuild(imageKey: string, buildId: string): Promise { + const lock = await this.read<{ buildId: string }>( + map(this.namespace, this.lockName(imageKey), "build-lock"), + ); + return lock?.buildId === buildId; + } + + private async releaseLock(imageKey: string, buildId: string): Promise { + const name = this.lockName(imageKey); + const object = await this.readConfigMap( + map(this.namespace, name, "build-lock"), + ); + if (!object || payload<{ buildId: string }>(object)?.buildId !== buildId) + return; + if (!object.metadata.resourceVersion) return; + await this.delete( + map( + this.namespace, + name, + "build-lock", + undefined, + object.metadata.resourceVersion, + ), + object.metadata.resourceVersion, + ); } async getUpload(digest: Sha256Digest): Promise { @@ -270,7 +617,9 @@ export class KubernetesBuildStore implements BuildStore { ); if (!record || record.spec.digest !== digest) return; try { - record.status.data = new Uint8Array(await readFile(this.uploadPath(digest))); + record.status.data = new Uint8Array( + await readFile(this.uploadPath(digest)), + ); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; record.status.data = new Uint8Array(); @@ -305,7 +654,8 @@ export class KubernetesBuildStore implements BuildStore { result.status.data = new Uint8Array(); return result; } catch (error) { - if (statusCode(error) === 409) return (await this.getUpload(record.spec.digest))!; + if (statusCode(error) === 409) + return (await this.getUpload(record.spec.digest))!; await rm(path, { force: true }); throw error; } @@ -332,7 +682,9 @@ export class KubernetesBuildStore implements BuildStore { }); } catch (error) { if (statusCode(error) === 409) - throw new BuildStoreConflictError("Upload record was concurrently modified"); + throw new BuildStoreConflictError( + "Upload record was concurrently modified", + ); throw error; } } @@ -375,7 +727,8 @@ export class KubernetesBuildOperations implements BuildKubernetesOperations { (condition) => condition.type === "Failed" && condition.status === "True", ); const complete = job.status?.conditions?.find( - (condition) => condition.type === "Complete" && condition.status === "True", + (condition) => + condition.type === "Complete" && condition.status === "True", ); const phase = failed ? "failed" @@ -398,9 +751,10 @@ export class KubernetesBuildOperations implements BuildKubernetesOperations { labelSelector: `job-name=${name}`, }); const pod = pods.items - .sort((a, b) => - (a.metadata?.creationTimestamp?.getTime() ?? 0) - - (b.metadata?.creationTimestamp?.getTime() ?? 0), + .sort( + (a, b) => + (a.metadata?.creationTimestamp?.getTime() ?? 0) - + (b.metadata?.creationTimestamp?.getTime() ?? 0), ) .at(-1); if (!pod?.metadata?.name) return ""; diff --git a/server/build-store.ts b/server/build-store.ts index 27c3c80..5d5dd18 100644 --- a/server/build-store.ts +++ b/server/build-store.ts @@ -44,7 +44,11 @@ export interface UploadRecord { status: { offset: number; data: Uint8Array }; } -export type CreateBuildResult = { record: BuildRecord; created: boolean }; +export type CreateBuildResult = { + record: BuildRecord; + created: boolean; + superseded?: BuildRecord[]; +}; /** Implementations must make createBuild and replace operations atomic. */ export interface BuildStore { @@ -55,6 +59,7 @@ export interface BuildStore { record: BuildRecord, expectedResourceVersion: string, ): Promise; + ownsBuild?(imageKey: string, buildId: string): Promise; getUpload(digest: Sha256Digest): Promise; createUpload(record: UploadRecord): Promise; replaceUpload( @@ -82,10 +87,63 @@ function sameSpec(left: BuildRecord, right: BuildRecord): boolean { return JSON.stringify(left.spec) === JSON.stringify(right.spec); } +export function buildTimestamp(record: BuildRecord): string { + const creationTimestamp = record.metadata?.creationTimestamp; + if (typeof creationTimestamp === "string") return creationTimestamp; + const createdAt = record.status?.createdAt; + return typeof createdAt === "string" ? createdAt : ""; +} + +export function compareBuildRecords( + left: BuildRecord, + right: BuildRecord, +): number { + const timestamp = buildTimestamp(left).localeCompare(buildTimestamp(right)); + if (timestamp !== 0) return timestamp; + return left.metadata.name.localeCompare(right.metadata.name); +} + +function newer(left: BuildRecord, right: BuildRecord): boolean { + return compareBuildRecords(left, right) > 0; +} + +function supersede(record: BuildRecord, finishedAt: string): BuildRecord { + const reason = "Superseded by newer build"; + record.status = { + ...record.status, + state: "failed", + finishedAt, + error: reason, + cancelled: true, + events: [ + ...record.status.events, + { + type: "status", + status: { + version: record.status.version, + id: record.status.id, + state: "failed", + createdAt: record.status.createdAt, + ...(record.status.startedAt && { + startedAt: record.status.startedAt, + }), + finishedAt, + error: reason, + }, + }, + ], + }; + record.metadata.resourceVersion = String( + Number(record.metadata.resourceVersion) + 1, + ); + return record; +} + export class MemoryBuildStore implements BuildStore { private readonly builds = new Map(); private readonly uploads = new Map(); private readonly active = new Map(); + private readonly byImage = new Map>(); async createBuild(record: BuildRecord): Promise { const existing = this.builds.get(record.metadata.name); @@ -95,20 +153,79 @@ export class MemoryBuildStore implements BuildStore { "Build ID was already used for a different request", ); } - return { record: clone(existing), created: false }; - } - const activeId = this.active.get(record.spec.imageKey); - if (activeId) { - const active = this.builds.get(activeId); - if (active && !terminal(active)) { - throw new BuildStoreConflictError( - `Build '${activeId}' is already active for ${record.spec.imageKey}`, + if (terminal(existing)) + return { record: clone(existing), created: false }; + const candidates = [...(this.byImage.get(existing.spec.imageKey) ?? [])] + .map((id) => this.builds.get(id)) + .filter( + (value): value is BuildRecord => + !!value && + !terminal(value) && + value.metadata.name !== existing.metadata.name, ); + const newerCandidate = candidates.find((candidate) => + newer(candidate, existing), + ); + if (newerCandidate) { + const superseded = supersede(existing, buildTimestamp(existing)); + this.builds.set(existing.metadata.name, clone(superseded)); + if (this.active.get(existing.spec.imageKey) === existing.metadata.name) + this.active.delete(existing.spec.imageKey); + return { + record: clone(superseded), + created: false, + superseded: [clone(superseded)], + }; } + const superseded: BuildRecord[] = []; + for (const old of candidates) { + const next = supersede(old, buildTimestamp(existing)); + this.builds.set(old.metadata.name, clone(next)); + superseded.push(clone(next)); + } + this.active.set(existing.spec.imageKey, existing.metadata.name); + return { + record: clone(existing), + created: false, + ...(superseded.length && { superseded }), + }; + } + const candidates = [...(this.byImage.get(record.spec.imageKey) ?? [])] + .map((id) => this.builds.get(id)) + .filter((value): value is BuildRecord => !!value && !terminal(value)); + const active = candidates.sort(compareBuildRecords).at(-1); + if (active && newer(active, record)) { + const superseded = supersede(record, record.status.createdAt); + this.builds.set(record.metadata.name, clone(superseded)); + this.byImage.get(record.spec.imageKey)?.add(record.metadata.name); + return { + record: clone(superseded), + created: false, + superseded: [clone(superseded)], + }; + } + if (active) { + const old = supersede(active, record.status.createdAt); + this.builds.set(active.metadata.name, clone(old)); + if (this.active.get(record.spec.imageKey) === active.metadata.name) + this.active.delete(record.spec.imageKey); } this.builds.set(record.metadata.name, clone(record)); + this.byImage.set( + record.spec.imageKey, + new Set([ + ...(this.byImage.get(record.spec.imageKey) ?? []), + record.metadata.name, + ]), + ); this.active.set(record.spec.imageKey, record.metadata.name); - return { record: clone(record), created: true }; + return { + record: clone(record), + created: true, + superseded: active + ? [clone(this.builds.get(active.metadata.name)!)] + : undefined, + }; } async getBuild(id: string) { @@ -117,13 +234,7 @@ export class MemoryBuildStore implements BuildStore { } async listBuilds() { - return [...this.builds.values()] - .sort((a, b) => - a.metadata.creationTimestamp.localeCompare( - b.metadata.creationTimestamp, - ), - ) - .map(clone); + return [...this.builds.values()].sort(compareBuildRecords).map(clone); } async replaceBuild(record: BuildRecord, expectedResourceVersion: string) { @@ -157,6 +268,10 @@ export class MemoryBuildStore implements BuildStore { } } + async ownsBuild(imageKey: string, buildId: string): Promise { + return this.active.get(imageKey) === buildId; + } + async getUpload(digest: Sha256Digest) { const value = this.uploads.get(digest); return value && clone(value); diff --git a/server/index.ts b/server/index.ts index db21711..bd0e855 100644 --- a/server/index.ts +++ b/server/index.ts @@ -2,10 +2,7 @@ import { createApp, cleanupExpiredSessions } from "./app"; import { RedactingAuditStore } from "./audit-store"; import { readFile } from "node:fs/promises"; import { IMAGE_REGISTRY } from "../const"; -import { - BuildController, - buildImageName, -} from "./build-controller"; +import { BuildController, buildImageName } from "./build-controller"; import { KubernetesBuildOperations, KubernetesBuildStore, @@ -22,6 +19,7 @@ import { KubernetesWorkspaceLeaseProvider, KubernetesWorkspacePersistence, KubernetesWorkspaceStore, + KubernetesTrustStore, } from "./kubernetes-state"; import { execUpgradeMatch, @@ -51,23 +49,23 @@ const operationStore = new PersistentOperationStore( const auditStore = new RedactingAuditStore( new KubernetesAuditPersistence(clients.objects), ); +const trustStore = new KubernetesTrustStore(clients.objects); const management = createManagementService( createKubernetesManagementDependencies(clients), ); const namespace = process.env.KUBER_SYSTEM_NAMESPACE?.trim() || "kuber-system"; const dataRoot = process.env.KUBER_DATA_ROOT?.trim() || "/data"; -const registry = (process.env.KUBER_BUILD_REGISTRY?.trim() || IMAGE_REGISTRY).replace( - /\/+$/, - "", -); +const registry = ( + process.env.KUBER_BUILD_REGISTRY?.trim() || IMAGE_REGISTRY +).replace(/\/+$/, ""); const registryConfigPath = - process.env.KUBER_REGISTRY_CONFIG?.trim() || "/etc/kuber/registry/config.json"; + process.env.KUBER_REGISTRY_CONFIG?.trim() || + "/etc/kuber/registry/config.json"; const registryResolveOrigin = process.env.KUBER_REGISTRY_RESOLVE_ORIGIN?.trim(); const internalRegistryHost = process.env.KUBER_INTERNAL_REGISTRY_HOST?.trim(); const internalRegistryInsecure = process.env.KUBER_INTERNAL_REGISTRY_INSECURE?.trim() === "true"; -const pushImagePrefix = - process.env.KUBER_PUSH_IMAGE_PREFIX?.trim() || "kuber/"; +const pushImagePrefix = process.env.KUBER_PUSH_IMAGE_PREFIX?.trim() || "kuber/"; if (!process.env.KUBER_REGISTRY_SECRET?.trim()) { console.warn( "KUBER_REGISTRY_SECRET is unset; BuildKit will use anonymous registry access", @@ -88,10 +86,15 @@ async function registryCredentials(): Promise { config.auths?.[`https://${host}`] ?? config.auths?.[`https://${host}/v1/`]; if (!entry?.auth) return; - const separator = Buffer.from(entry.auth, "base64").toString("utf8").indexOf(":"); + const separator = Buffer.from(entry.auth, "base64") + .toString("utf8") + .indexOf(":"); if (separator < 0) return; const value = Buffer.from(entry.auth, "base64").toString("utf8"); - return { username: value.slice(0, separator), password: value.slice(separator + 1) }; + return { + username: value.slice(0, separator), + password: value.slice(separator + 1), + }; } const buildStore = new KubernetesBuildStore( @@ -105,10 +108,12 @@ const builds = new BuildController({ kubernetes: new KubernetesBuildOperations(clients.batch, clients.core), namespace, workspaceRoot: `${dataRoot}/workspaces`, - workspaceClaimName: process.env.KUBER_BUILD_DATA_CLAIM?.trim() || "kuber-build-data", + workspaceClaimName: + process.env.KUBER_BUILD_DATA_CLAIM?.trim() || "kuber-build-data", cacheImage: (request) => - `${(internalRegistryHost ?? registry)}/kuber/cache-${request.project}-${request.service}`, - imageName: (request) => buildImageName(registry, request.project, request.service), + `${internalRegistryHost ?? registry}/kuber/cache-${request.project}-${request.service}`, + imageName: (request) => + buildImageName(registry, request.project, request.service), pushImage: internalRegistryHost ? (request) => `${internalRegistryHost}/${pushImagePrefix}${request.project}-${request.service}:latest` @@ -162,15 +167,18 @@ try { const sessionCleanupIntervalMs = Number( process.env.KUBER_SESSION_CLEANUP_MS ?? 10 * 60 * 1000, ); -const sessionCleanupTimer = setInterval(async () => { - try { - await cleanupExpiredSessions(store); - } catch (error) { - console.error("Expired session cleanup failed", error); - } -}, Number.isFinite(sessionCleanupIntervalMs) && sessionCleanupIntervalMs > 0 - ? sessionCleanupIntervalMs - : 10 * 60 * 1000); +const sessionCleanupTimer = setInterval( + async () => { + try { + await cleanupExpiredSessions(store); + } catch (error) { + console.error("Expired session cleanup failed", error); + } + }, + Number.isFinite(sessionCleanupIntervalMs) && sessionCleanupIntervalMs > 0 + ? sessionCleanupIntervalMs + : 10 * 60 * 1000, +); sessionCleanupTimer.unref?.(); const app = createApp({ @@ -178,6 +186,7 @@ const app = createApp({ workspaceStore, operationStore, auditStore, + trustStore, management, builds, logs, @@ -190,7 +199,11 @@ const app = createApp({ origin: registryResolveOrigin, insecure: registryResolveOrigin?.startsWith("http://"), }); - return { image: image.replace(/:latest$/, ""), digest, reference: `${image.replace(/:latest$/, "")}@${digest}` }; + return { + image: image.replace(/:latest$/, ""), + digest, + reference: `${image.replace(/:latest$/, "")}@${digest}`, + }; }, allowedOrigins: ( process.env.KUBER_ALLOWED_ORIGINS ?? "https://kuber.astrxl.dev" @@ -242,7 +255,9 @@ const server = Bun.serve({ session?: WireExecSession; }; state.session?.receive( - typeof message === "string" ? message : new TextDecoder().decode(message), + typeof message === "string" + ? message + : new TextDecoder().decode(message), ); }, close(ws) { diff --git a/server/kubernetes-state.ts b/server/kubernetes-state.ts index ec3c682..6d33915 100644 --- a/server/kubernetes-state.ts +++ b/server/kubernetes-state.ts @@ -18,12 +18,14 @@ import { type AuditEvent, type AuditPersistence } from "./audit-store"; import { managementDependencies, type ManagementDependencies, + type OperationExecution, type ResourceIdentity, } from "./management"; import type { RollbackCandidate } from "../lib/rollback"; import { LABELS } from "../const"; import type { WorkspaceAdoptionResult, WorkspaceAdoptionService } from "./app"; import { WorkspaceAdoptionError } from "./app"; +import { validateTrust, type TrustStore } from "./trust-store"; import { RESERVED_NAMESPACES, WORKSPACE_PROJECT_LABEL, @@ -164,7 +166,11 @@ export interface LeaseObjects { create(value: V1Lease): Promise; read(name: string, namespace: string): Promise; replace(value: V1Lease): Promise; - delete(name: string, namespace: string): Promise; + delete( + name: string, + namespace: string, + expectedResourceVersion?: string, + ): Promise; } /** Wraps a CoordinationV1Api client in the LeaseObjects adapter. */ @@ -193,9 +199,17 @@ export function createKubernetesLeaseObjects( body: value, }); }, - async delete(name, namespace) { + async delete(name, namespace, expectedResourceVersion) { try { - await coordination.deleteNamespacedLease({ name, namespace }); + await coordination.deleteNamespacedLease({ + name, + namespace, + ...(expectedResourceVersion && { + body: { + preconditions: { resourceVersion: expectedResourceVersion }, + }, + }), + }); } catch (error) { if (isNotFound(error)) return; throw error; @@ -217,8 +231,9 @@ const MAX_LEASE_ACQUIRE_RETRIES = 5; */ function microTimeString(ms: number): string { const iso = new Date(ms).toISOString(); - const match = - /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d+))?Z$/.exec(iso); + const match = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d+))?Z$/.exec( + iso, + ); if (!match) return iso; const fraction = (match[2] ?? "").padEnd(6, "0"); return `${match[1]}.${fraction}Z`; @@ -254,7 +269,11 @@ export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider return digestName("lease", workspaceId); } - private leaseSpec(workspaceId: string, holder: string, ttlMs: number): V1Lease { + private leaseSpec( + workspaceId: string, + holder: string, + ttlMs: number, + ): V1Lease { return { apiVersion: KUBER_LEASE_API_VERSION, kind: "Lease", @@ -268,7 +287,9 @@ export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider acquireTime: microTimeString( this.now(), ) as unknown as V1LeaseSpec["acquireTime"], - renewTime: microTimeString(this.now()) as unknown as V1LeaseSpec["renewTime"], + renewTime: microTimeString( + this.now(), + ) as unknown as V1LeaseSpec["renewTime"], leaseTransitions: 0, }, }; @@ -328,8 +349,7 @@ export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider workspaceId: string, holder: string, ): WorkspaceLease { - const leaseDurationMs = - (lease.spec?.leaseDurationSeconds ?? 30) * 1000; + const leaseDurationMs = (lease.spec?.leaseDurationSeconds ?? 30) * 1000; const expiresAt = ( Date.parse(String(lease.spec?.renewTime)) + leaseDurationMs ).toString(); @@ -370,7 +390,16 @@ export class KubernetesWorkspaceLeaseProvider implements WorkspaceLeaseProvider if (!name) return; const current = await this.objects.read(name, this.namespace); if (current?.spec?.holderIdentity !== holder) return; - await this.objects.delete(name, this.namespace); + const resourceVersion = current.metadata?.resourceVersion; + if (!resourceVersion) return; + try { + await this.objects.delete(name, this.namespace, resourceVersion); + } catch (error) { + // A replace by a successor between read and delete invalidates the + // resourceVersion precondition. Its lease must remain intact. + if (isConflict(error) || isNotFound(error)) return; + throw error; + } }; return { workspaceId, holder, expiresAt, renew, release }; @@ -821,7 +850,7 @@ export class KubernetesWorkspaceAdoptionService implements WorkspaceAdoptionServ export class KubernetesOperationPersistence implements OperationPersistence { constructor(private readonly objects = createKubernetesClients().objects) {} - async createIdempotent(operation: Operation): Promise { + async createIdempotent(operation: Operation) { const operationName = digestName( "operation", `${operation.spec.workspaceId}\0${operation.spec.idempotencyKey}`, @@ -841,7 +870,7 @@ export class KubernetesOperationPersistence implements OperationPersistence { deterministic.spec.workspaceId, ), ); - return deterministic; + return { operation: deterministic, created: true }; } catch (error) { if (!isConflict(error)) throw error; const existing = await this.get(operationName); @@ -849,7 +878,7 @@ export class KubernetesOperationPersistence implements OperationPersistence { throw new OperationConflictError( "Idempotent operation could not be recovered", ); - return existing; + return { operation: existing, created: false }; } } @@ -931,6 +960,44 @@ export class KubernetesAuditPersistence implements AuditPersistence { } } +export class KubernetesTrustStore implements TrustStore { + constructor(private readonly objects = createKubernetesClients().objects) {} + private name(project: string, fingerprint: string) { + return digestName("trust", `${project}\0${fingerprint}`); + } + async grant(project: string, fingerprint: string): Promise { + validateTrust(project, fingerprint); + if (await this.has(project, fingerprint)) return; + await createObject( + this.objects, + stateObject( + "ConfigMap", + this.name(project, fingerprint), + "trust", + { project, fingerprint }, + project, + ), + ); + } + async list(project: string): Promise { + return (await list(this.objects, "ConfigMap", "trust", project)) + .map((item) => parsePayload<{ project: string; fingerprint: string }>(item)) + .filter((record): record is { project: string; fingerprint: string } => + Boolean(record && record.project === project && /^[a-f0-9]{64}$/.test(record.fingerprint)), + ) + .map((record) => record.fingerprint); + } + async has(project: string, fingerprint: string): Promise { + const item = await read(this.objects, "ConfigMap", this.name(project, fingerprint)); + const record = item && parsePayload<{ project: string; fingerprint: string }>(item); + return record?.project === project && record.fingerprint === fingerprint; + } + async revoke(project: string, fingerprint: string): Promise { + if (!(await this.has(project, fingerprint))) return false; + return deleteObject(this.objects, "ConfigMap", this.name(project, fingerprint)); + } +} + function resource(identity: ResourceIdentity): KubernetesObject { return { apiVersion: identity.apiVersion, @@ -943,6 +1010,30 @@ function resource(identity: ResourceIdentity): KubernetesObject { }; } +async function sleepUntilExecutionCancelled( + delayMs: number, + execution?: OperationExecution, +): Promise { + const signal = execution?.signal; + if (!signal) { + await Bun.sleep(delayMs); + return; + } + if (signal.aborted) + throw new Error("Workspace operation execution was cancelled"); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal.removeEventListener("abort", cancelSleep); + resolve(); + }, delayMs); + const cancelSleep = () => { + clearTimeout(timer); + reject(new Error("Workspace operation execution was cancelled")); + }; + signal.addEventListener("abort", cancelSleep, { once: true }); + }); +} + export function createKubernetesManagementDependencies( clients = createKubernetesClients(), ): ManagementDependencies { @@ -965,17 +1056,16 @@ export function createKubernetesManagementDependencies( }; }; const replicaSets = async (project: string, deploymentUid?: string) => - ( - // Deployment-created ReplicaSets inherit only the pod-template labels - // (e.g. app, pod-template-hash), never the Deployment's metadata - // managed-by label, so a managed selector here excludes every revision - // and rollback reports "no previous release". List namespace-wide and - // restrict by ownerReference instead. - await apps.listNamespacedReplicaSet({ namespace: project }) - ).items.filter((item) => - item.metadata?.ownerReferences?.some( - (owner) => owner.kind === "Deployment" && owner.uid === deploymentUid, - ), + // Deployment-created ReplicaSets inherit only the pod-template labels + // (e.g. app, pod-template-hash), never the Deployment's metadata + // managed-by label, so a managed selector here excludes every revision + // and rollback reports "no previous release". List namespace-wide and + // restrict by ownerReference instead. + (await apps.listNamespacedReplicaSet({ namespace: project })).items.filter( + (item) => + item.metadata?.ownerReferences?.some( + (owner) => owner.kind === "Deployment" && owner.uid === deploymentUid, + ), ); const overrides: Partial = { listDeployments: async (project) => @@ -1007,9 +1097,16 @@ export function createKubernetesManagementDependencies( }, }, }), - waitForDeployment: async (project, name, timeoutMs = 300_000) => { + waitForDeployment: async ( + project, + name, + timeoutMs = 300_000, + execution?: OperationExecution, + ) => { const started = Date.now(); while (Date.now() - started < timeoutMs) { + if (execution?.signal?.aborted) + throw new Error("Workspace operation execution was cancelled"); const deployment = await apps.readNamespacedDeployment({ namespace: project, name, @@ -1023,7 +1120,9 @@ export function createKubernetesManagementDependencies( (deployment.status?.unavailableReplicas ?? 0) === 0 ) return; - await Bun.sleep(2_000); + if (execution?.signal?.aborted) + throw new Error("Workspace operation execution was cancelled"); + await sleepUntilExecutionCancelled(2_000, execution); } throw new Error(`Timed out waiting for deployment ${name} rollout`); }, diff --git a/server/management.ts b/server/management.ts index 8676edc..8535a9b 100644 --- a/server/management.ts +++ b/server/management.ts @@ -49,7 +49,6 @@ export const RESERVED_NAMESPACES = new Set([ "kube-system", "kube-public", "kube-node-lease", - "kuber-system", DATABASE_NAMESPACE, STORAGE_NAMESPACE, ]); @@ -99,6 +98,8 @@ export type CredentialMetadata = { secretName?: string; }; +export type OperationExecution = { signal?: AbortSignal }; + export type ManagementDependencies = { readNamespace(project: string): Promise; listDeployments(project: string): Promise; @@ -106,18 +107,25 @@ export type ManagementDependencies = { project: string, name: string, replicas: number, + execution?: OperationExecution, + ): Promise; + restartDeployment( + project: string, + name: string, + execution?: OperationExecution, ): Promise; - restartDeployment(project: string, name: string): Promise; waitForDeployment( project: string, name: string, timeoutMs?: number, + execution?: OperationExecution, ): Promise; fetchGraphObjects(project: string): Promise; planRollback(project: string, names?: string[]): Promise; rollbackDeployment( project: string, candidate: RollbackCandidate, + execution?: OperationExecution, ): Promise; listProjectResources(project: string): Promise; listDatabaseResources(project: string): Promise; @@ -126,16 +134,24 @@ export type ManagementDependencies = { project: string, desired: KubernetesObject[], ): Promise; - applyResource(resource: KubernetesObject): Promise; - deleteResource(identity: ResourceIdentity): Promise; + applyResource( + resource: KubernetesObject, + execution?: OperationExecution, + ): Promise; + deleteResource( + identity: ResourceIdentity, + execution?: OperationExecution, + ): Promise; reconcileDatabases( project: string, compose: ComposeSpecification, + execution?: OperationExecution, ): Promise>>; getDatabaseCredentials(username: string): Promise; reconcileStorage( project: string, compose: ComposeSpecification, + execution?: OperationExecution, ): Promise>>; getStorageCredentials(claim: S3Claim): Promise>; }; @@ -161,9 +177,11 @@ const defaultOperations: Omit< listStorageResources: listManagedStorageResources, findStaleResources: getStaleResources, applyResource, - reconcileDatabases: reconcilePostgresClaims, + reconcileDatabases: async (project, compose, execution) => + reconcilePostgresClaims(project, compose, execution?.signal), getDatabaseCredentials: getRoleCredentials, - reconcileStorage: reconcileS3Claims, + reconcileStorage: async (project, compose, execution) => + reconcileS3Claims(project, compose, execution?.signal), getStorageCredentials: getS3Credentials, }; @@ -191,6 +209,11 @@ function validateWorkspace(workspace: Workspace): void { } } +function throwIfExecutionAborted(execution?: OperationExecution): void { + if (!execution?.signal?.aborted) return; + throw new Error("Workspace operation execution was cancelled"); +} + function resourceName(resource: KubernetesObject): string { const name = resource.metadata?.name; if (!resource.apiVersion || !resource.kind || !name) { @@ -378,10 +401,13 @@ export function createManagementService(dependencies: ManagementDependencies) { workspace: Workspace, namespace: string, resources: KubernetesObject[], + execution?: OperationExecution, ): Promise { for (const resource of resources) { + throwIfExecutionAborted(execution); await dependencies.applyResource( labelExternal(workspace, resource, namespace), + execution, ); } } @@ -389,6 +415,7 @@ export function createManagementService(dependencies: ManagementDependencies) { async function deleteResources( workspace: Workspace, resources: ResourceIdentity[], + execution?: OperationExecution, ): Promise { await assertSafe(workspace); for (const resource of resources) { @@ -399,7 +426,8 @@ export function createManagementService(dependencies: ManagementDependencies) { } } for (const resource of resources) { - await dependencies.deleteResource(resource); + throwIfExecutionAborted(execution); + await dependencies.deleteResource(resource, execution); } } @@ -472,7 +500,12 @@ export function createManagementService(dependencies: ManagementDependencies) { ); }, - async stop(workspace: Workspace, names?: string[]): Promise { + async stop( + workspace: Workspace, + names?: string[], + execution?: OperationExecution, + ): Promise { + throwIfExecutionAborted(execution); const selected = await targets(workspace, names); const selectedSet = new Set(selected); const hpas = (await dependencies.listProjectResources(workspace.project)) @@ -485,18 +518,24 @@ export function createManagementService(dependencies: ManagementDependencies) { ) .map((resource) => identity(workspace, resource)); if (hpas.length > 0) { - await deleteResources(workspace, hpas); + await deleteResources(workspace, hpas, execution); } for (const name of selected) { - await dependencies.scaleDeployment(workspace.project, name, 0); + throwIfExecutionAborted(execution); + await dependencies.scaleDeployment(workspace.project, name, 0, execution); } return selected; }, - async restart(workspace: Workspace, names?: string[]): Promise { + async restart( + workspace: Workspace, + names?: string[], + execution?: OperationExecution, + ): Promise { const selected = await targets(workspace, names); for (const name of selected) { - await dependencies.restartDeployment(workspace.project, name); + throwIfExecutionAborted(execution); + await dependencies.restartDeployment(workspace.project, name, execution); } return selected; }, @@ -505,20 +544,25 @@ export function createManagementService(dependencies: ManagementDependencies) { workspace: Workspace, names?: string[], timeoutMs?: number, + execution?: OperationExecution, ): Promise { + throwIfExecutionAborted(execution); await assertSafe(workspace); const candidates = await dependencies.planRollback( workspace.project, names, ); for (const candidate of candidates) { - await dependencies.rollbackDeployment(workspace.project, candidate); + throwIfExecutionAborted(execution); + await dependencies.rollbackDeployment(workspace.project, candidate, execution); } for (const candidate of candidates) { + throwIfExecutionAborted(execution); await dependencies.waitForDeployment( workspace.project, candidate.name, timeoutMs, + execution, ); } return candidates; @@ -540,16 +584,20 @@ export function createManagementService(dependencies: ManagementDependencies) { async reconcileDatabases( workspace: Workspace, compose: ComposeSpecification, + execution?: OperationExecution, ) { + throwIfExecutionAborted(execution); await assertSafe(workspace, true); const environment = await dependencies.reconcileDatabases( workspace.project, compose, + execution, ); await ownExternalResources( workspace, DATABASE_NAMESPACE, await dependencies.listDatabaseResources(workspace.project), + execution, ); return environment; }, @@ -589,16 +637,20 @@ export function createManagementService(dependencies: ManagementDependencies) { async reconcileStorage( workspace: Workspace, compose: ComposeSpecification, + execution?: OperationExecution, ) { + throwIfExecutionAborted(execution); await assertSafe(workspace, true); const environment = await dependencies.reconcileStorage( workspace.project, compose, + execution, ); await ownExternalResources( workspace, STORAGE_NAMESPACE, await dependencies.listStorageResources(workspace.project), + execution, ); return environment; }, @@ -651,13 +703,15 @@ export function createManagementService(dependencies: ManagementDependencies) { async applyResources( workspace: Workspace, resources: KubernetesObject[], + execution?: OperationExecution, ): Promise { await assertSafe(workspace, true); const applied: KubernetesObject[] = []; for (const resource of sortResources( resources.map((item) => labelDesired(workspace, item)), )) { - applied.push(await dependencies.applyResource(resource)); + throwIfExecutionAborted(execution); + applied.push(await dependencies.applyResource(resource, execution)); } return applied; }, @@ -666,13 +720,16 @@ export function createManagementService(dependencies: ManagementDependencies) { workspace: Workspace, deploymentTargets: string[], timeoutMs?: number, + execution?: OperationExecution, ): Promise { const selected = await targets(workspace, deploymentTargets); for (const name of selected) { + throwIfExecutionAborted(execution); await dependencies.waitForDeployment( workspace.project, name, timeoutMs, + execution, ); } }, @@ -681,9 +738,14 @@ export function createManagementService(dependencies: ManagementDependencies) { planDown, - async down(workspace: Workspace, full = false): Promise { + async down( + workspace: Workspace, + full = false, + execution?: OperationExecution, + ): Promise { + throwIfExecutionAborted(execution); const plan = await planDown(workspace, full); - await deleteResources(workspace, plan.delete); + await deleteResources(workspace, plan.delete, execution); return plan; }, }; diff --git a/server/materialize.ts b/server/materialize.ts index f2ec638..ca5700b 100644 --- a/server/materialize.ts +++ b/server/materialize.ts @@ -27,7 +27,30 @@ import { export interface MaterializeCas { get(digest: Sha256Digest): Promise; - has(digest: Sha256Digest): Promise; +} + +const MATERIALIZE_CONCURRENCY = 20; + +async function mapConcurrent( + values: T[], + run: (value: T) => Promise, +): Promise { + const results = new Array(values.length); + let index = 0; + const worker = async () => { + for (;;) { + const current = index++; + if (current >= values.length) return; + results[current] = await run(values[current]!); + } + }; + await Promise.all( + Array.from( + { length: Math.min(MATERIALIZE_CONCURRENCY, values.length) }, + worker, + ), + ); + return results; } function safePath(path: string): boolean { @@ -105,11 +128,6 @@ export async function materializeWorkspace( ): Promise { assertSha256Digest(manifestDigest); const manifest = parseWorkspaceManifest(await cas.get(manifestDigest)); - const missing: Sha256Digest[] = []; - for (const file of manifest.files) - if (!(await cas.has(file.digest))) missing.push(file.digest); - if (missing.length) - throw new Error(`Workspace blobs are missing: ${missing.join(", ")}`); await mkdir(dirname(destination), { recursive: true }); try { @@ -124,7 +142,7 @@ export async function materializeWorkspace( ); await mkdir(temporary, { mode: 0o755 }); try { - for (const file of manifest.files) { + await mapConcurrent(manifest.files, async (file) => { const target = join(temporary, file.path); if (relative(temporary, target).startsWith("..")) throw new Error("Unsafe workspace path"); @@ -151,7 +169,7 @@ export async function materializeWorkspace( } await chmod(target, file.mode); } - } + }); await rename(temporary, destination); } catch (error) { await rm(temporary, { recursive: true, force: true }); diff --git a/server/operation-store.ts b/server/operation-store.ts index d407c3a..488e9a7 100644 --- a/server/operation-store.ts +++ b/server/operation-store.ts @@ -48,14 +48,26 @@ export class OperationNotFoundError extends Error { /** createIdempotent must atomically index workspaceId + idempotencyKey. */ export interface OperationPersistence { - createIdempotent(operation: Operation): Promise; + createIdempotent(operation: Operation): Promise; get(id: string): Promise; list(workspaceId?: string): Promise; replace(operation: Operation, expectedResourceVersion: string): Promise; } +/** The idempotent operation and whether this caller created its record. */ +export interface CreatedOperation { + operation: Operation; + created: boolean; +} + export interface OperationStore { create(input: CreateOperationInput): Promise; + createOrReuse(input: CreateOperationInput): Promise; + /** + * Atomically claims a pending operation for execution. A false result means + * another request has already moved it out of pending. + */ + claimExecution(id: string): Promise; get(id: string): Promise; list(workspaceId?: string): Promise; transition( @@ -193,6 +205,20 @@ export function sanitizeOperationResult( ); } +/** Keep operation error codes stable while preventing provider details from escaping. */ +export function sanitizeOperationError( + error: OperationError, + action?: string, +): OperationError { + const message = + action === "databases.reconcile" + ? "Database reconciliation failed" + : action === "storage.reconcile" + ? "Storage reconciliation failed" + : redactString(error.message); + return { code: error.code, message }; +} + export class PersistentOperationStore implements OperationStore { constructor( private readonly persistence: OperationPersistence, @@ -201,6 +227,10 @@ export class PersistentOperationStore implements OperationStore { ) {} async create(input: CreateOperationInput): Promise { + return (await this.createOrReuse(input)).operation; + } + + async createOrReuse(input: CreateOperationInput): Promise { if (!input.workspaceId || !input.action.trim()) { throw new OperationValidationError( "Workspace ID and action are required", @@ -233,12 +263,12 @@ export class PersistentOperationStore implements OperationStore { status: { state: "pending" }, }; const stored = await this.persistence.createIdempotent(operation); - if (stored.spec.requestHash !== operation.spec.requestHash) { + if (stored.operation.spec.requestHash !== operation.spec.requestHash) { throw new OperationConflictError( "Idempotency key was already used for a different request", ); } - return clone(stored); + return { operation: clone(stored.operation), created: stored.created }; } async get(id: string) { @@ -246,6 +276,15 @@ export class PersistentOperationStore implements OperationStore { return operation && clone(operation); } + async claimExecution(id: string): Promise { + try { + return await this.transition(id, "running"); + } catch (error) { + if (!(error instanceof OperationConflictError)) throw error; + return; + } + } + async list(workspaceId?: string) { return clone(await this.persistence.list(workspaceId)); } @@ -288,7 +327,9 @@ export class PersistentOperationStore implements OperationStore { sanitizeOperationResult(options.result, current.spec.action), ), }), - ...(options.error && { error: clone(options.error) }), + ...(options.error && { + error: sanitizeOperationError(options.error, current.spec.action), + }), }; await this.persistence.replace(operation, current.metadata.resourceVersion); return clone(operation); @@ -302,10 +343,14 @@ export class MemoryOperationPersistence implements OperationPersistence { async createIdempotent(operation: Operation) { const key = `${operation.spec.workspaceId}\0${operation.spec.idempotencyKey}`; const existingId = this.idempotency.get(key); - if (existingId) return clone(this.operations.get(existingId)!); + if (existingId) + return { + operation: clone(this.operations.get(existingId)!), + created: false, + }; this.operations.set(operation.metadata.name, clone(operation)); this.idempotency.set(key, operation.metadata.name); - return clone(operation); + return { operation: clone(operation), created: true }; } async get(id: string) { @@ -355,11 +400,7 @@ export class MemoryWorkspaceLeaseProvider implements WorkspaceLeaseProvider { throw new OperationValidationError("Invalid workspace lease request"); } const current = this.leases.get(workspaceId); - if ( - current && - current.expiresAt > this.now() && - current.holder !== holder - ) { + if (current && current.expiresAt > this.now()) { return undefined; } const token = randomUUID(); diff --git a/server/redact.ts b/server/redact.ts index 25cd0c1..9910e64 100644 --- a/server/redact.ts +++ b/server/redact.ts @@ -10,10 +10,24 @@ const AWS_ACCESS_KEY_ID = /\bAKIA[0-9A-Z]{16}\b/g; // OpenSSH and other PEM private key headers embedded anywhere in a string. const PRIVATE_KEY_HEADER = /-----BEGIN [A-Z ]*PRIVATE KEY-----/g; +// Secret-bearing environment assignments are commonly included in provider +// and command error messages (for example, DB_PASSWORD=...). +const SECRET_ASSIGNMENT = + /\b([A-Z][A-Z0-9_]*(?:PASSWORD|PASSWD|TOKEN|SECRET|CREDENTIAL|PRIVATE_KEY|ACCESS_KEY|CONNECTION_STRING|DATABASE_URL)[A-Z0-9_]*)\s*[=:]\s*([^\s,;\]}]+)/gi; +const SECRET_FIELD = + /(["']?(?:password|passwd|token|secret|credential|privateKey|accessKey|connectionString|databaseUrl)["']?\s*:\s*["']?)([^"'\s,}\]]+)/gi; +const SECRET_DATA = + /((?:["'](?:kind|type)["']\s*:\s*["'](?:Secret|secret)["'][\s\S]{0,1000}?["'](?:data|stringData)["']\s*:\s*)\{[^{}]*\})/gi; + /** Redact common credential-bearing substrings while leaving everything else intact. */ export function redactString(value: string): string { return value .replace(AWS_ACCESS_KEY_ID, REDACTED) .replace(PRIVATE_KEY_HEADER, REDACTED) + .replace(SECRET_ASSIGNMENT, (_match, key) => `${key}=${REDACTED}`) + .replace(SECRET_FIELD, (_match, prefix) => `${prefix}${REDACTED}`) + .replace(SECRET_DATA, (_match, prefix) => + prefix.replace(/\{[^{}]*\}$/, `{${REDACTED}}`), + ) .replace(URL_CREDENTIALS, (_match, scheme) => `${scheme}${REDACTED}@`); } diff --git a/server/trust-store.ts b/server/trust-store.ts new file mode 100644 index 0000000..e74b704 --- /dev/null +++ b/server/trust-store.ts @@ -0,0 +1,39 @@ +export interface TrustStore { + grant(project: string, fingerprint: string): Promise; + list(project: string): Promise; + has(project: string, fingerprint: string): Promise; + revoke(project: string, fingerprint: string): Promise; +} + +export function validateTrust(project: string, fingerprint: string): void { + if (project.length > 63 || !/^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/.test(project)) + throw new Error("project must be a Kubernetes name"); + if (!/^[a-f0-9]{64}$/.test(fingerprint)) + throw new Error("fingerprint is invalid"); +} + +export class MemoryTrustStore implements TrustStore { + private readonly fingerprints = new Map>(); + + async grant(project: string, fingerprint: string): Promise { + validateTrust(project, fingerprint); + const values = this.fingerprints.get(project) ?? new Set(); + values.add(fingerprint); + this.fingerprints.set(project, values); + } + + async list(project: string): Promise { + return [...(this.fingerprints.get(project) ?? [])]; + } + + async has(project: string, fingerprint: string): Promise { + return (await this.list(project)).includes(fingerprint); + } + + async revoke(project: string, fingerprint: string): Promise { + const values = this.fingerprints.get(project); + if (!values?.delete(fingerprint)) return false; + if (!values.size) this.fingerprints.delete(project); + return true; + } +} diff --git a/server/workspace-store.ts b/server/workspace-store.ts index b519719..6fdd93f 100644 --- a/server/workspace-store.ts +++ b/server/workspace-store.ts @@ -7,7 +7,6 @@ export const MAX_REVISION_PAYLOAD_BYTES = 900 * 1024; export const WORKSPACE_ID_PATTERN = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/; export const RESERVED_WORKSPACE_IDS: ReadonlySet = new Set([ - "kuber-system", "database", "garage-system", "routing", diff --git a/shared/api.ts b/shared/api.ts index 2ac7f2b..15ee09f 100644 --- a/shared/api.ts +++ b/shared/api.ts @@ -63,6 +63,9 @@ export type UpdateWorkspaceRequest = Omit; export type CreateWorkspaceResponse = Workspace; export type ListWorkspacesResponse = Page; export type GetWorkspaceResponse = Workspace; +export type WorkspaceTrustResponse = { + fingerprints: string[]; +}; export type DeleteWorkspaceResponse = OperationAcceptedResponse; export type AdoptWorkspaceResponse = { workspaceId: string; diff --git a/tests/command/trust.test.ts b/tests/command/trust.test.ts new file mode 100644 index 0000000..00e11e9 --- /dev/null +++ b/tests/command/trust.test.ts @@ -0,0 +1,97 @@ +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import { rm } from "node:fs/promises"; +import { KuberApiError, type ApiRequestInit } from "../../lib/api"; +import { getTrustPath, readTrust, updateTrust } from "../../lib/trust"; +import { grantTrust, revokeTrust, statusTrust } from "../../command/trust"; + +const originalConfig = process.env.XDG_CONFIG_HOME; +const identity = { project: "demo", fingerprint: "a".repeat(64) }; + +afterEach(async () => { + const path = getTrustPath(); + if (originalConfig === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = originalConfig; + await rm(path.slice(0, path.lastIndexOf("/")), { + recursive: true, + force: true, + }); +}); + +function requester( + calls: Array<{ path: string; init?: ApiRequestInit }>, + response: unknown = undefined, +) { + return async (path: string, init?: ApiRequestInit): Promise => { + calls.push({ path, init }); + return response as T; + }; +} + +describe("trust command", () => { + test("grants, reports, and revokes the current namespace fingerprint", async () => { + process.env.XDG_CONFIG_HOME = `/tmp/kuber-trust-command-${crypto.randomUUID()}`; + const calls: Array<{ path: string; init?: ApiRequestInit }> = []; + const output = spyOn(console, "log").mockImplementation(() => {}); + + await grantTrust(identity, requester(calls)); + expect(calls).toEqual([ + { + path: "/workspaces/demo/trust", + init: { method: "POST", json: { fingerprint: identity.fingerprint } }, + }, + ]); + expect(await readTrust()).toEqual([identity]); + + calls.length = 0; + await statusTrust( + identity, + requester(calls, { fingerprints: [identity.fingerprint] }), + ); + expect(calls).toEqual([ + { path: "/workspaces/demo/trust", init: undefined }, + ]); + expect(output.mock.calls.map(([line]) => line)).toEqual([ + "Trusted this directory for namespace demo", + "Namespace: demo", + "Local: trusted", + "Server: registered", + ]); + + calls.length = 0; + await revokeTrust(identity, requester(calls)); + expect(calls).toEqual([ + { + path: `/workspaces/demo/trust?fingerprint=${identity.fingerprint}`, + init: { method: "DELETE" }, + }, + ]); + expect(await readTrust()).toEqual([]); + output.mockRestore(); + }); + + test("removes local trust when the server registration is already absent", async () => { + process.env.XDG_CONFIG_HOME = `/tmp/kuber-trust-command-${crypto.randomUUID()}`; + await updateTrust(() => [identity]); + const output = spyOn(console, "log").mockImplementation(() => {}); + + await revokeTrust(identity, async () => { + throw new KuberApiError("not found", 404); + }); + + expect(await readTrust()).toEqual([]); + expect(output).toHaveBeenCalledWith("Revoked trust for namespace demo"); + output.mockRestore(); + }); + + test("removes local trust before reporting a remote revoke failure", async () => { + process.env.XDG_CONFIG_HOME = `/tmp/kuber-trust-command-${crypto.randomUUID()}`; + await updateTrust(() => [identity]); + + await expect( + revokeTrust(identity, async () => { + throw new KuberApiError("unavailable", 503); + }), + ).rejects.toThrow("Removed local trust for namespace demo"); + expect(await readTrust()).toEqual([]); + }); +}); diff --git a/tests/command/up-api.test.ts b/tests/command/up-api.test.ts index cb0e092..1ea6d80 100644 --- a/tests/command/up-api.test.ts +++ b/tests/command/up-api.test.ts @@ -1,12 +1,18 @@ import { describe, expect, test } from "bun:test"; -import type { ApiRequestInit } from "../../lib/api"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ApiRequestInit, ApiRequestOptions } from "../../lib/api"; import { KuberApiError } from "../../lib/api"; import type { ApiRequester } from "../../lib/build"; import { ensureWorkspace, reconcileResources, + runUp, workspaceAdoptionRoute, } from "../../command/up"; +import { provideContext } from "../../lib/context"; +import { resolveTrustIdentity, updateTrust } from "../../lib/trust"; import { workspaceManifestDigest } from "../../lib/workspace"; const manifest = { version: 1 as const, files: [] }; @@ -17,6 +23,87 @@ const snapshot = { }; describe("up API pipeline", () => { + test("forwards the build timeout through the trusted requester", async () => { + const root = await mkdtemp(join(tmpdir(), "kuber-up-api-")); + const previousCwd = process.cwd(); + const previousConfigHome = process.env.XDG_CONFIG_HOME; + const configHome = join(root, "config"); + const calls: Array<{ + path: string; + init?: ApiRequestInit; + options?: ApiRequestOptions; + }> = []; + + try { + await writeFile( + join(root, "compose.yml"), + "services:\n web:\n build: .\n", + ); + await writeFile( + join(root, ".kuberrc.ts"), + 'export default { project: "shop" };\n', + ); + const git = Bun.spawn(["git", "init", "-q", root]); + expect(await git.exited).toBe(0); + + process.chdir(root); + process.env.XDG_CONFIG_HOME = configHome; + const identity = await resolveTrustIdentity("shop", root); + await updateTrust((records) => [...records, identity]); + + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + options?: ApiRequestOptions, + ) => { + calls.push({ path, init, options }); + if (path === "/snapshots/negotiate") + return { workspace: snapshot.digest, missing: [], ready: true } as T; + if (path === "/builds") { + const buildRequest = init?.json as { id?: unknown } | undefined; + if (typeof buildRequest?.id !== "string") + throw new Error("Expected build request ID"); + return { + version: 1, + id: buildRequest.id, + state: "succeeded", + createdAt: "2026-01-01T00:00:00Z", + } as T; + } + if (path.includes("/events")) return [] as T; + if (path.endsWith("/result")) + return { + image: "registry.server/kuber/shop-web", + digest: `sha256:${"a".repeat(64)}`, + reference: `registry.server/kuber/shop-web@sha256:${"a".repeat(64)}`, + } as T; + if (path === "/workspaces/shop") + throw new KuberApiError("missing", 404); + if (path === "/workspaces") + return { + metadata: { name: "shop", uid: "workspace", resourceVersion: "1" }, + } as T; + if (path.endsWith("/adopt")) return { resourcesAdopted: 0 } as T; + if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; + return {} as T; + }; + + await provideContext(() => runUp(true, request)); + + const build = calls.find(({ path }) => path === "/builds"); + if (!build) throw new Error("Expected build submission"); + expect(build.options).toEqual({ timeoutMs: 300_000 }); + expect( + new Headers(build.init?.headers).get("x-kuber-trust-project"), + ).toBe("shop"); + } finally { + process.chdir(previousCwd); + if (previousConfigHome === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = previousConfigHome; + await rm(root, { recursive: true, force: true }); + } + }); + test("creates workspace metadata without embedding source blobs", async () => { const calls: Array<{ path: string; init?: ApiRequestInit }> = []; const request: ApiRequester = async ( @@ -101,6 +188,305 @@ describe("up API pipeline", () => { expect(order).toEqual(["plan", "apply", "hook", "wait", "delete"]); }); + test("resumes an interrupted reconcile operation by its persisted ID", async () => { + const calls: string[] = []; + const applyIdempotencyKeys: Array = []; + let applyAttempts = 0; + let operationPolls = 0; + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + calls.push(path); + if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; + if (path.endsWith("/apply")) { + applyIdempotencyKeys.push( + new Headers(init?.headers).get("idempotency-key"), + ); + applyAttempts++; + if (applyAttempts === 1) throw new TypeError("fetch failed"); + return { + operationId: "operation-apply", + operation: { status: { state: "running" } }, + } as T; + } + if (path === "/operations/operation-apply") { + operationPolls++; + if (operationPolls === 1) throw new TypeError("connection reset"); + return { status: { state: "succeeded" } } as T; + } + throw new Error(`Unexpected request: ${path}`); + }; + + await reconcileResources("shop", [], 1, undefined, request, { + sleep: async () => {}, + }); + + expect(calls).toEqual([ + "/workspaces/shop/resources/plan", + "/workspaces/shop/resources/apply", + "/workspaces/shop/resources/apply", + "/operations/operation-apply", + "/operations/operation-apply", + ]); + expect(applyIdempotencyKeys[0]).toBeTruthy(); + expect(applyIdempotencyKeys[1]).toBe(applyIdempotencyKeys[0]); + }); + + test("restarts after an interrupted apply is persisted before its ID is received", async () => { + const applyRequests: Array<{ key: string | null; json: unknown }> = []; + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; + if (path.endsWith("/apply")) { + applyRequests.push({ + key: new Headers(init?.headers).get("idempotency-key"), + json: init?.json, + }); + if (applyRequests.length === 1) throw new TypeError("fetch failed"); + if (applyRequests.length === 2) + throw new KuberApiError("operation interrupted", 500, { + title: "Operation interrupted", + status: 500, + code: "OPERATION_INTERRUPTED", + }); + return { + operationId: "operation-apply", + operation: { status: { state: "succeeded" } }, + } as T; + } + throw new Error(`Unexpected request: ${path}`); + }; + + await reconcileResources("shop", [], 1, undefined, request, { + sleep: async () => {}, + }); + + expect(applyRequests).toHaveLength(3); + expect(applyRequests[0]?.key).toBeTruthy(); + expect(applyRequests[1]?.key).toBe(applyRequests[0]?.key); + expect(applyRequests[2]?.key).toBeTruthy(); + expect(applyRequests[2]?.key).not.toBe(applyRequests[0]?.key); + expect(applyRequests[2]?.json).toEqual(applyRequests[0]?.json); + }); + + test("resubmits with a fresh key after server restart interruption", async () => { + const applyRequests: Array<{ key: string | null; json: unknown }> = []; + let operationPolls = 0; + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; + if (path.endsWith("/apply")) { + applyRequests.push({ + key: new Headers(init?.headers).get("idempotency-key"), + json: init?.json, + }); + return { + operationId: `operation-apply-${applyRequests.length}`, + operation: { status: { state: "running" } }, + } as T; + } + if (path === "/operations/operation-apply-1") { + operationPolls++; + if (operationPolls === 1) + return { + status: { + state: "failed", + error: { + code: "OPERATION_INTERRUPTED", + message: "server restarted", + }, + }, + } as T; + return { status: { state: "succeeded" } } as T; + } + if (path === "/operations/operation-apply-2") + return { status: { state: "succeeded" } } as T; + throw new Error(`Unexpected request: ${path}`); + }; + + await reconcileResources("shop", [], 1, undefined, request, { + sleep: async () => {}, + }); + + expect(applyRequests).toHaveLength(2); + expect(applyRequests[0]?.key).toBeTruthy(); + expect(applyRequests[1]?.key).toBeTruthy(); + expect(applyRequests[1]?.key).not.toBe(applyRequests[0]?.key); + expect(applyRequests[1]?.json).toEqual(applyRequests[0]?.json); + }); + + test.each([ + ["failed", "OPERATION_FAILED"], + ["cancelled", "OPERATION_CANCELLED"], + ])("does not retry arbitrary %s operations", async (state, code) => { + let applyAttempts = 0; + let operationPolls = 0; + const request: ApiRequester = async (path: string) => { + if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; + if (path.endsWith("/apply")) { + applyAttempts++; + return { + operationId: "operation-apply", + operation: { status: { state: "running" } }, + } as T; + } + if (path === "/operations/operation-apply") { + operationPolls++; + return { + status: { state, error: { code, message: "terminal" } }, + } as T; + } + throw new Error(`Unexpected request: ${path}`); + }; + + await expect( + reconcileResources("shop", [], 1, undefined, request, { + sleep: async () => {}, + }), + ).rejects.toMatchObject({ code }); + expect(applyAttempts).toBe(1); + expect(operationPolls).toBe(1); + }); + + test("bounds restart recovery sleeps by the resume deadline", async () => { + let currentTime = 0; + const sleeps: Array<{ startedAt: number; milliseconds: number }> = []; + let operationNumber = 0; + const request: ApiRequester = async (path: string) => { + if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; + if (path.endsWith("/apply")) { + operationNumber++; + return { + operationId: `operation-${operationNumber}`, + operation: { status: { state: "running" } }, + } as T; + } + return { + status: { + state: "failed", + error: { code: "OPERATION_INTERRUPTED", message: "restarted" }, + }, + } as T; + }; + + await expect( + reconcileResources("shop", [], 0, undefined, request, { + now: () => currentTime, + sleep: async (milliseconds) => { + sleeps.push({ startedAt: currentTime, milliseconds }); + currentTime += milliseconds; + }, + }), + ).rejects.toThrow("Timed out while reconnecting to resume the operation"); + + expect(sleeps).not.toHaveLength(0); + expect( + sleeps.every( + ({ startedAt, milliseconds }) => startedAt + milliseconds <= 60_000, + ), + ).toBe(true); + expect(currentTime).toBe(60_000); + }); + + test("does not restart after an apply interruption reaches the resume deadline", async () => { + let currentTime = 0; + let applyAttempts = 0; + const request: ApiRequester = async (path: string) => { + if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; + applyAttempts++; + if (applyAttempts === 1) throw new TypeError("connection reset"); + throw new KuberApiError("operation interrupted", 500, { + title: "Operation interrupted", + status: 500, + code: "OPERATION_INTERRUPTED", + }); + }; + + await expect( + reconcileResources("shop", [], 0, undefined, request, { + now: () => currentTime, + sleep: async (milliseconds) => { + currentTime += milliseconds; + if (currentTime < 60_000) currentTime = 60_000; + }, + }), + ).rejects.toMatchObject({ + code: "OPERATION_INTERRUPTED", + status: 500, + }); + expect(applyAttempts).toBe(2); + expect(currentTime).toBe(60_000); + }); + + test("fails immediately for a typed 503 operation-store error", async () => { + const unavailable = new KuberApiError( + "Operation storage is not configured", + 503, + { + title: "Service unavailable", + status: 503, + code: "OPERATION_STORE_UNAVAILABLE", + }, + ); + const calls: string[] = []; + const sleeps: number[] = []; + const request: ApiRequester = async (path: string) => { + calls.push(path); + if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; + throw unavailable; + }; + + await expect( + reconcileResources("shop", [], 1, undefined, request, { + sleep: async (milliseconds) => { + sleeps.push(milliseconds); + }, + }), + ).rejects.toBe(unavailable); + + expect(calls).toEqual([ + "/workspaces/shop/resources/plan", + "/workspaces/shop/resources/apply", + ]); + expect(sleeps).toEqual([]); + }); + + test("never sleeps past the operation resume deadline", async () => { + let currentTime = 0; + const sleeps: Array<{ startedAt: number; milliseconds: number }> = []; + const request: ApiRequester = async (path: string) => { + if (path.endsWith("/plan")) return { desired: [], stale: [] } as T; + throw new TypeError("connection reset"); + }; + + await expect( + reconcileResources("shop", [], 0, undefined, request, { + now: () => currentTime, + sleep: async (milliseconds) => { + sleeps.push({ startedAt: currentTime, milliseconds }); + currentTime += milliseconds; + }, + }), + ).rejects.toThrow("connection reset"); + + expect(sleeps).not.toHaveLength(0); + expect( + sleeps.every( + ({ startedAt, milliseconds }) => startedAt + milliseconds <= 60_000, + ), + ).toBe(true); + expect(sleeps.at(-1)).toEqual({ + startedAt: 57_750, + milliseconds: 2_250, + }); + expect(currentTime).toBe(60_000); + }); + test("fails closed with the precise missing adoption route", async () => { const request: ApiRequester = async (path: string) => { if (path.endsWith("/plan")) diff --git a/tests/lib/build-api.test.ts b/tests/lib/build-api.test.ts index d82034e..d491b42 100644 --- a/tests/lib/build-api.test.ts +++ b/tests/lib/build-api.test.ts @@ -2,12 +2,15 @@ import { afterEach, describe, expect, test } from "bun:test"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import type { ApiRequestInit } from "../../lib/api"; +import type { ApiRequestInit, ApiRequestOptions } from "../../lib/api"; import { buildServices, resolveBuildImages, uploadWorkspaceSnapshot, type ApiRequester, + TaskScheduler, + MAX_CONCURRENT_REQUESTS, + MAX_REQUESTS_PER_SECOND, } from "../../lib/build"; import { workspaceManifestDigest, @@ -22,6 +25,42 @@ function emptySnapshot(): WorkspaceSnapshot { return { manifest, digest: workspaceManifestDigest(manifest), blobs: [] }; } +function createSnapshotWithBlobs( + count: number, + blobSize: number, +): WorkspaceSnapshot { + const blobs = Array.from({ length: count }, (_, i) => { + const data = new Uint8Array(blobSize); + data.fill(i + 1); + const digest = `sha256:${"0".repeat(62)}${String(i + 1).padStart(2, "0")}`; + return { + digest: digest as import("../../shared/build-protocol").Sha256Digest, + data, + }; + }); + + const manifest = { + version: 1 as const, + files: blobs + .map((blob, i) => ({ + path: `file${i}.txt`, + type: "file" as const, + digest: blob.digest, + size: blob.data.byteLength, + mode: 0o644 as const, + })) + .sort((left, right) => + Buffer.from(left.path).compare(Buffer.from(right.path)), + ), + }; + + return { + manifest, + digest: workspaceManifestDigest(manifest), + blobs, + }; +} + afterEach(async () => { await Promise.all( directories @@ -33,13 +72,18 @@ afterEach(async () => { describe("authenticated build API pipeline", () => { test("negotiates and uploads the manifest through resumable blob routes", async () => { const snapshot = emptySnapshot(); - const calls: Array<{ path: string; init?: ApiRequestInit }> = []; + const calls: Array<{ + path: string; + init?: ApiRequestInit; + options?: ApiRequestOptions; + }> = []; let negotiations = 0; const request: ApiRequester = async ( path: string, init?: ApiRequestInit, + options?: ApiRequestOptions, ) => { - calls.push({ path, init }); + calls.push({ path, init, options }); if (path === "/snapshots/negotiate") { negotiations += 1; return ( @@ -76,20 +120,385 @@ describe("authenticated build API pipeline", () => { expect(new Headers(calls[2]!.init?.headers).get("upload-offset")).toBe("0"); }); + test("uses project-scoped URLs for every resumable blob upload endpoint", async () => { + const snapshot = emptySnapshot(); + const project = "shop & staging"; + const uploadPath = `/blobs/${encodeURIComponent(snapshot.digest)}/uploads`; + const calls: Array<{ method: string; path: string }> = []; + let negotiations = 0; + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + const url = new URL(path, "https://kuber.test"); + const method = init?.method ?? "GET"; + calls.push({ method, path }); + + if (url.pathname === "/snapshots/negotiate") { + negotiations += 1; + return ( + negotiations === 1 + ? { + workspace: snapshot.digest, + missing: [snapshot.digest], + ready: false, + } + : { workspace: snapshot.digest, missing: [], ready: true } + ) as T; + } + if (url.pathname === uploadPath && method === "POST") + return { offset: 0, complete: false } as T; + if (url.pathname === uploadPath && method === "PATCH") + return { offset: (init!.body as Uint8Array).byteLength } as T; + if (url.pathname === `${uploadPath}/complete` && method === "POST") + return { complete: true } as T; + throw new Error(`Unexpected request ${method} ${path}`); + }; + + await uploadWorkspaceSnapshot( + snapshot, + request, + undefined, + undefined, + project, + ); + + const projectQuery = `?project=${encodeURIComponent(project)}`; + expect(calls).toEqual([ + { method: "POST", path: "/snapshots/negotiate" }, + { method: "POST", path: `${uploadPath}${projectQuery}` }, + { method: "PATCH", path: `${uploadPath}${projectQuery}` }, + { method: "POST", path: `${uploadPath}/complete${projectQuery}` }, + { method: "POST", path: "/snapshots/negotiate" }, + ]); + }); + + describe("TaskScheduler", () => { + test("uses the production request limits", () => { + expect(MAX_CONCURRENT_REQUESTS).toBe(20); + expect(MAX_REQUESTS_PER_SECOND).toBe(40); + }); + + test("limits concurrent in-flight operations", async () => { + let maxInflight = 0; + let currentInflight = 0; + const scheduler = new TaskScheduler({ + maxInflight: 5, + maxPerSecond: 100, + }); + + const tasks = Array.from({ length: 10 }, () => + scheduler.run(async () => { + currentInflight++; + maxInflight = Math.max(maxInflight, currentInflight); + await Bun.sleep(10); + currentInflight--; + }), + ); + + await Promise.all(tasks); + expect(maxInflight).toBeLessThanOrEqual(5); + }); + + test("rate limits request starts to maxPerSecond", async () => { + const requestStarts: number[] = []; + let currentTime = 0; + const clock = { now: () => currentTime }; + const sleep = async (ms: number) => { + currentTime += ms; + }; + const scheduler = new TaskScheduler({ + maxInflight: 100, + maxPerSecond: 4, + clock, + sleep, + }); + + const tasks = Array.from({ length: 8 }, () => + scheduler.run(async () => { + requestStarts.push(currentTime); + }), + ); + + await Promise.all(tasks); + + for (const start of requestStarts) { + expect( + requestStarts.filter( + (candidate) => candidate >= start && candidate < start + 1000, + ).length, + ).toBeLessThanOrEqual(4); + } + }); + }); + + describe("concurrent blob uploads", () => { + test("uploads multiple blobs concurrently", async () => { + const snapshot = createSnapshotWithBlobs(5, 1024); + let negotiations = 0; + let inflight = 0; + let maxInflight = 0; + + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + if (path === "/snapshots/negotiate") { + negotiations += 1; + return ( + negotiations === 1 + ? { + workspace: snapshot.digest, + missing: snapshot.blobs.map((blob) => blob.digest), + ready: false, + } + : { workspace: snapshot.digest, missing: [], ready: true } + ) as T; + } + inflight += 1; + maxInflight = Math.max(maxInflight, inflight); + await Bun.sleep(5); + inflight -= 1; + if (init?.method === "POST" && !path.endsWith("/complete")) + return { offset: 0, complete: false } as T; + if (init?.method === "PATCH") { + return { offset: (init.body as Uint8Array).byteLength } as T; + } + return { complete: true } as T; + }; + + await uploadWorkspaceSnapshot( + snapshot, + request, + undefined, + new TaskScheduler({ maxInflight: 5, maxPerSecond: 100 }), + ); + + expect(maxInflight).toBeGreaterThan(1); + expect(negotiations).toBe(2); + }); + + test("completes active blobs before initializing the full backlog", async () => { + const snapshot = createSnapshotWithBlobs(12, 1024); + const methods: string[] = []; + let negotiations = 0; + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + if (path === "/snapshots/negotiate") { + negotiations += 1; + return ( + negotiations === 1 + ? { + workspace: snapshot.digest, + missing: snapshot.blobs.map((blob) => blob.digest), + ready: false, + } + : { workspace: snapshot.digest, missing: [], ready: true } + ) as T; + } + methods.push( + path.endsWith("/complete") ? "complete" : (init?.method ?? "GET"), + ); + if (init?.method === "POST" && !path.endsWith("/complete")) + return { offset: 0, complete: false } as T; + if (init?.method === "PATCH") + return { offset: (init.body as Uint8Array).byteLength } as T; + return { complete: true } as T; + }; + + await uploadWorkspaceSnapshot( + snapshot, + request, + undefined, + new TaskScheduler({ maxInflight: 4, maxPerSecond: 100 }), + ); + + expect(methods.indexOf("complete")).toBeLessThan( + methods.lastIndexOf("POST"), + ); + }); + + test("inflight never exceeds configured maximum", async () => { + const snapshot = createSnapshotWithBlobs(30, 1024); + let currentInflight = 0; + let maxObservedInflight = 0; + const maxInflight = 10; + let negotiations = 0; + + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + if (path === "/snapshots/negotiate") { + negotiations += 1; + return ( + negotiations === 1 + ? { + workspace: snapshot.digest, + missing: snapshot.blobs.map((blob) => blob.digest), + ready: false, + } + : { workspace: snapshot.digest, missing: [], ready: true } + ) as T; + } + currentInflight += 1; + maxObservedInflight = Math.max(maxObservedInflight, currentInflight); + await Bun.sleep(2); + currentInflight -= 1; + if (init?.method === "POST" && !path.endsWith("/complete")) + return { offset: 0, complete: false } as T; + if (init?.method === "PATCH") { + return { offset: (init.body as Uint8Array).byteLength } as T; + } + return { complete: true } as T; + }; + + await uploadWorkspaceSnapshot( + snapshot, + request, + undefined, + new TaskScheduler({ maxInflight, maxPerSecond: 1000 }), + ); + + expect(maxObservedInflight).toBeLessThanOrEqual(maxInflight); + }); + + test("request starts are rate-limited to maxPerSecond", async () => { + const snapshot = createSnapshotWithBlobs(8, 1024); + const requestStarts: Array<{ path: string; time: number }> = []; + let currentTime = 0; + const clock = { now: () => currentTime }; + const sleep = async (ms: number) => { + currentTime += ms; + }; + let negotiations = 0; + + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + requestStarts.push({ path, time: currentTime }); + if (path === "/snapshots/negotiate") { + negotiations += 1; + return ( + negotiations === 1 + ? { + workspace: snapshot.digest, + missing: snapshot.blobs.map((blob) => blob.digest), + ready: false, + } + : { workspace: snapshot.digest, missing: [], ready: true } + ) as T; + } + if (init?.method === "POST" && !path.endsWith("/complete")) { + return { offset: 0, complete: false } as T; + } + if (init?.method === "PATCH") { + return { offset: (init.body as Uint8Array).byteLength } as T; + } + if (path.endsWith("/complete")) { + return { complete: true } as T; + } + return { complete: true } as T; + }; + + const maxPerSecond = 4; + await uploadWorkspaceSnapshot( + snapshot, + request, + undefined, + new TaskScheduler({ + maxInflight: 100, + maxPerSecond, + clock, + sleep, + }), + ); + + for (const { time } of requestStarts) { + expect( + requestStarts.filter( + ({ time: candidate }) => + candidate >= time && candidate < time + 1000, + ).length, + ).toBeLessThanOrEqual(maxPerSecond); + } + }); + + test("chunks of one blob remain ordered during concurrent uploads", async () => { + const blobSize = 25 * 1024 * 1024; + const snapshot = createSnapshotWithBlobs(3, blobSize); + const chunkOrders = new Map(); + let negotiations = 0; + + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + ) => { + if (path === "/snapshots/negotiate") { + negotiations += 1; + return ( + negotiations === 1 + ? { + workspace: snapshot.digest, + missing: snapshot.blobs.map((blob) => blob.digest), + ready: false, + } + : { workspace: snapshot.digest, missing: [], ready: true } + ) as T; + } + if (init?.method === "POST" && !path.endsWith("/complete")) { + return { offset: 0, complete: false } as T; + } + if (init?.method === "PATCH") { + const digest = decodeURIComponent(path.split("/")[2]!); + const offset = Number( + new Headers(init.headers).get("upload-offset") ?? "0", + ); + const chunkIndex = Math.floor(offset / (8 * 1024 * 1024)); + if (!chunkOrders.has(digest)) chunkOrders.set(digest, []); + chunkOrders.get(digest)!.push(chunkIndex); + return { offset: offset + (init.body as Uint8Array).byteLength } as T; + } + return { complete: true } as T; + }; + + await uploadWorkspaceSnapshot( + snapshot, + request, + undefined, + new TaskScheduler({ maxInflight: 3, maxPerSecond: 100 }), + ); + + for (const [, chunks] of chunkOrders) { + for (let i = 1; i < chunks.length; i++) { + expect(chunks[i]!).toBeGreaterThan(chunks[i - 1]!); + } + } + }); + }); + test("submits, reconciles, reports logs, and returns the server image reference", async () => { const root = await mkdtemp(join(tmpdir(), "kuber-build-api-")); directories.push(root); const git = Bun.spawn(["git", "init", "-q", root]); expect(await git.exited).toBe(0); const snapshot = emptySnapshot(); - const calls: Array<{ path: string; init?: ApiRequestInit }> = []; + const calls: Array<{ + path: string; + init?: ApiRequestInit; + options?: ApiRequestOptions; + }> = []; let submitted: BuildRequest | undefined; const output: string[] = []; const request: ApiRequester = async ( path: string, init?: ApiRequestInit, + options?: ApiRequestOptions, ) => { - calls.push({ path, init }); + calls.push({ path, init, options }); if (path === "/snapshots/negotiate") return { workspace: snapshot.digest, missing: [], ready: true } as T; if (path === "/builds") { @@ -155,9 +564,93 @@ describe("authenticated build API pipeline", () => { }, }); expect(output).toContain("build log"); + expect(calls.find(({ path }) => path === "/builds")?.options).toEqual({ + timeoutMs: 300_000, + }); + expect( + calls + .filter( + ({ path }) => path.includes("/events") || path.endsWith("/reconcile"), + ) + .map(({ options }) => options), + ).toEqual([ + { timeoutMs: 300_000 }, + { timeoutMs: 300_000 }, + { timeoutMs: 300_000 }, + ]); expect(calls.some(({ path }) => path.endsWith("/result"))).toBe(true); }); + test("retries a timed out poll request with the build request timeout", async () => { + const root = await mkdtemp(join(tmpdir(), "kuber-build-api-")); + directories.push(root); + const git = Bun.spawn(["git", "init", "-q", root]); + expect(await git.exited).toBe(0); + const snapshot = emptySnapshot(); + const pollCalls: Array<{ path: string; options?: ApiRequestOptions }> = []; + let buildId = ""; + let eventRequests = 0; + const request: ApiRequester = async ( + path: string, + init?: ApiRequestInit, + options?: ApiRequestOptions, + ) => { + if (path === "/snapshots/negotiate") + return { workspace: snapshot.digest, missing: [], ready: true } as T; + if (path === "/builds") { + const buildRequest = init?.json as BuildRequest | undefined; + if (!buildRequest) throw new Error("Expected build request"); + buildId = buildRequest.id; + return { + version: 1, + id: buildId, + state: "queued", + createdAt: "2026-01-01T00:00:00Z", + } as T; + } + if (path.includes("/events")) { + pollCalls.push({ path, options }); + eventRequests += 1; + if (eventRequests === 1) + throw new DOMException("request timed out", "TimeoutError"); + return [] as T; + } + if (path.endsWith("/reconcile")) { + pollCalls.push({ path, options }); + return { + version: 1, + id: buildId, + state: "succeeded", + createdAt: "2026-01-01T00:00:00Z", + digest: `sha256:${"a".repeat(64)}`, + } as T; + } + if (path.endsWith("/result")) + return { + image: "registry.server/kuber/shop-web", + digest: `sha256:${"a".repeat(64)}`, + reference: `registry.server/kuber/shop-web@sha256:${"a".repeat(64)}`, + } as T; + throw new Error(`Unexpected request ${path}`); + }; + + await buildServices( + "shop", + { services: { web: { build: "." } } }, + root, + undefined, + { request, snapshot, sleep: async () => {}, pollIntervalMs: 0 }, + ); + + expect(eventRequests).toBe(3); + expect(pollCalls).toEqual([ + expect.objectContaining({ options: { timeoutMs: 300_000 } }), + expect.objectContaining({ options: { timeoutMs: 300_000 } }), + expect.objectContaining({ options: { timeoutMs: 300_000 } }), + expect.objectContaining({ options: { timeoutMs: 300_000 } }), + ]); + }); + test("no-build image resolution uses only the resolve route", async () => { const calls: string[] = []; const images = await resolveBuildImages( diff --git a/tests/lib/config.test.ts b/tests/lib/config.test.ts index 7bd3317..bc41e0a 100644 --- a/tests/lib/config.test.ts +++ b/tests/lib/config.test.ts @@ -94,7 +94,9 @@ describe("kuber config", () => { { services: { app: { build: "." } } }, process.cwd(), {}, - { app: `registry.example.com/team/kuber/project-app:latest@sha256:${"a".repeat(64)}` }, + { + app: `registry.example.com/team/kuber/project-app:latest@sha256:${"a".repeat(64)}`, + }, ); const deployment = resources.find( (resource) => resource.kind === "Deployment", @@ -103,6 +105,39 @@ describe("kuber config", () => { `registry.example.com/team/kuber/project-app:latest@sha256:${"a".repeat(64)}`, ); }); + + test("scopes the server RBAC self-management rules", async () => { + const config = await loadConfig(process.cwd()); + const resources: any[] = []; + await config.postRender?.(resources, { + cwd: process.cwd(), + project: "kuber-system", + composeFile: "compose.yml", + }); + + const role = resources.find((resource) => resource.kind === "Role"); + const manager = resources.find( + (resource) => resource.kind === "ClusterRole", + ); + expect(role.rules).toContainEqual({ + apiGroups: [""], + resources: ["serviceaccounts"], + resourceNames: ["kuber-server"], + verbs: ["get", "update", "patch"], + }); + expect(role.rules).toContainEqual({ + apiGroups: ["rbac.authorization.k8s.io"], + resources: ["roles", "rolebindings"], + resourceNames: ["kuber-server-auth"], + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"], + }); + expect(manager.rules).toContainEqual({ + apiGroups: ["rbac.authorization.k8s.io"], + resources: ["clusterroles", "clusterrolebindings"], + resourceNames: ["kuber-server-manager"], + verbs: ["get", "update", "patch"], + }); + }); }); describe("global config argument", () => { diff --git a/tests/lib/trust.test.ts b/tests/lib/trust.test.ts new file mode 100644 index 0000000..9280b77 --- /dev/null +++ b/tests/lib/trust.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { realpath, rm, stat } from "node:fs/promises"; +import { + getTrustPath, + readTrust, + requireLocalTrust, + resolveTrustIdentity, + updateTrust, +} from "../../lib/trust"; + +const originalConfig = process.env.XDG_CONFIG_HOME; + +afterEach(async () => { + const path = getTrustPath(); + if (originalConfig === undefined) delete process.env.XDG_CONFIG_HOME; + else process.env.XDG_CONFIG_HOME = originalConfig; + await rm(path, { force: true }); + await rm(path.slice(0, path.lastIndexOf("/")), { + recursive: true, + force: true, + }); +}); + +describe("local namespace trust", () => { + test("grants and revokes a resolved CWD fingerprint in a mode-0600 store", async () => { + process.env.XDG_CONFIG_HOME = `/tmp/kuber-trust-${crypto.randomUUID()}`; + const cwd = await realpath("."); + const identity = await resolveTrustIdentity("demo", cwd); + await updateTrust((records) => [...records, identity]); + expect(await requireLocalTrust(identity)).toEqual(identity); + expect((await stat(getTrustPath())).mode & 0o777).toBe(0o600); + await updateTrust((records) => + records.filter( + (record) => + record.project !== identity.project || + record.fingerprint !== identity.fingerprint, + ), + ); + await expect(requireLocalTrust(identity)).rejects.toThrow("TRUST_REQUIRED"); + }); + + test("rejects an unregistered directory in the same namespace", async () => { + process.env.XDG_CONFIG_HOME = `/tmp/kuber-trust-${crypto.randomUUID()}`; + const first = { project: "demo", fingerprint: "a".repeat(64) }; + const second = { project: "demo", fingerprint: "b".repeat(64) }; + await updateTrust(() => [first]); + await expect(requireLocalTrust(second)).rejects.toThrow("TRUST_REQUIRED"); + expect(await readTrust()).toEqual([first]); + }); +}); diff --git a/tests/server/app.test.ts b/tests/server/app.test.ts index 59c6491..47dccff 100644 --- a/tests/server/app.test.ts +++ b/tests/server/app.test.ts @@ -8,6 +8,7 @@ import { MemoryWorkspaceLeaseProvider, } from "../../server/operation-store"; import { MemoryWorkspaceStore } from "../../server/workspace-store"; +import { MemoryTrustStore } from "../../server/trust-store"; function request( path: string, @@ -137,6 +138,151 @@ describe("kuber API authentication", () => { }); }); +describe("operation response safety", () => { + test("redacts database and storage reconciliation results immediately", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const seen: unknown[] = []; + const databaseResult = { + credentials: { password: "database-password" }, + env: [{ name: "DB_PASSWORD", value: "database-password" }], + }; + const storageResult = { + credentials: { secretKey: "storage-secret" }, + env: [{ name: "STORAGE_SECRET", value: "storage-secret" }], + }; + const management = { + reconcileDatabases: async (_workspace: unknown, compose: unknown) => { + seen.push(compose); + return databaseResult; + }, + reconcileStorage: async (_workspace: unknown, compose: unknown) => { + seen.push(compose); + return storageResult; + }, + } as unknown as ManagementService; + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore: new MemoryOperationStore(undefined, () => + crypto.randomUUID(), + ), + management, + }); + const compose = { + services: {}, + secret: "internal-compose-secret", + }; + for (const [path, key, secret] of [ + ["databases", "db-once", "database-password"], + ["storage", "storage-once", "storage-secret"], + ] as const) { + const result = await app( + request( + `/api/v2/workspaces/demo/${path}`, + { + method: "POST", + headers: { "idempotency-key": key }, + body: JSON.stringify({ compose }), + }, + "token", + ), + ); + expect(result.status).toBe(200); + const body = JSON.stringify(await result.json()); + expect(body).not.toContain(secret); + expect(body).not.toContain("internal-compose-secret"); + expect(body).toContain('"redacted":true'); + } + expect(seen).toHaveLength(2); + expect(JSON.stringify(seen[0])).toContain("internal-compose-secret"); + expect(databaseResult.credentials.password).toBe("database-password"); + expect(storageResult.credentials.secretKey).toBe("storage-secret"); + }); + + test("redacts failed reconciliation errors immediately and when retrieved", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const operationStore = new MemoryOperationStore(undefined, () => + crypto.randomUUID(), + ); + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore, + management: { + reconcileDatabases: async () => { + throw new Error( + 'database provider failed DB_PASSWORD=database-password response={"data":{"token":"kube-secret"}}', + ); + }, + reconcileStorage: async () => { + throw new Error( + 'storage provider failed STORAGE_SECRET=storage-secret response={"data":{"password":"storage-kube-secret"}}', + ); + }, + } as unknown as ManagementService, + }); + const path = "/api/v2/workspaces/demo/databases"; + const init = { + method: "POST", + headers: { "idempotency-key": "failed-reconcile" }, + body: JSON.stringify({ compose: { services: {} } }), + }; + const immediate = await app(request(path, init, "token")); + const immediateBody = JSON.stringify(await immediate.json()); + expect(immediate.status).toBe(500); + expect(immediateBody).not.toContain("database-password"); + expect(immediateBody).not.toContain("kube-secret"); + expect(immediateBody).toContain("OPERATION_FAILED"); + + const operationId = (await operationStore.list())[0]!.metadata.name; + const retrieved = await app( + request(`/api/v2/operations/${operationId}`, {}, "token"), + ); + const retrievedBody = JSON.stringify(await retrieved.json()); + expect(retrievedBody).not.toContain("database-password"); + expect(retrievedBody).not.toContain("kube-secret"); + + const storageImmediate = await app( + request( + "/api/v2/workspaces/demo/storage", + { + ...init, + headers: { "idempotency-key": "failed-storage" }, + }, + "token", + ), + ); + const storageBody = JSON.stringify(await storageImmediate.json()); + expect(storageImmediate.status).toBe(500); + expect(storageBody).not.toContain("storage-secret"); + expect(storageBody).not.toContain("storage-kube-secret"); + const storageId = (await operationStore.list())[1]!.metadata.name; + const storageRetrieved = await app( + request(`/api/v2/operations/${storageId}`, {}, "token"), + ); + expect(JSON.stringify(await storageRetrieved.json())).not.toContain( + "storage-kube-secret", + ); + + const idempotent = await app(request(path, init, "token")); + const idempotentBody = JSON.stringify(await idempotent.json()); + expect(idempotentBody).not.toContain("database-password"); + expect(idempotentBody).not.toContain("kube-secret"); + }); +}); + async function authenticatedStore(role: "viewer" | "operator" | "admin") { const store = new MemoryAuthStore(); await store.putUser({ username: role, passwordHash: "hash", roles: [role] }); @@ -150,6 +296,101 @@ async function authenticatedStore(role: "viewer" | "operator" | "admin") { } describe("kuber v2 HTTP routes", () => { + test("grants, lists, revokes, and enforces namespace trust for applies", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const trustStore = new MemoryTrustStore(); + const fingerprint = "a".repeat(64); + const headers = { + "x-kuber-trust-project": "demo", + "x-kuber-trust-fingerprint": fingerprint, + }; + const apply = (app: ReturnType) => + app( + request( + "/api/v2/workspaces/demo/resources/apply", + { + method: "POST", + headers, + body: JSON.stringify({ resources: [] }), + }, + "token", + ), + ); + const unavailable = await createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore: new MemoryOperationStore(), + management: { + applyResources: async () => [], + } as unknown as ManagementService, + }); + const unavailableApply = await apply(unavailable); + expect(unavailableApply.status).toBe(503); + expect(await unavailableApply.json()).toMatchObject({ + code: "TRUST_STORE_UNAVAILABLE", + }); + + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore: new MemoryOperationStore(), + trustStore, + management: { + applyResources: async () => [], + } as unknown as ManagementService, + }); + const untrustedApply = (headers?: RequestInit["headers"]) => + app( + request( + "/api/v2/workspaces/demo/resources/apply", + { + method: "POST", + headers, + body: JSON.stringify({ resources: [] }), + }, + "token", + ), + ); + + expect((await untrustedApply()).status).toBe(428); + expect((await apply(app)).status).toBe(403); + expect( + ( + await app( + request( + "/api/v2/workspaces/demo/trust", + { method: "POST", body: JSON.stringify({ fingerprint }) }, + "token", + ), + ) + ).status, + ).toBe(204); + const status = await app( + request("/api/v2/workspaces/demo/trust", {}, "token"), + ); + expect(status.status).toBe(200); + expect(await status.json()).toEqual({ fingerprints: [fingerprint] }); + expect((await apply(app)).status).toBe(200); + expect( + ( + await app( + request( + `/api/v2/workspaces/demo/trust?fingerprint=${fingerprint}`, + { method: "DELETE" }, + "token", + ), + ) + ).status, + ).toBe(204); + expect((await apply(app)).status).toBe(403); + }); + test("uses exact origins, request IDs, and problem+json errors", async () => { const app = createApp({ store: new MemoryAuthStore(), @@ -405,10 +646,484 @@ describe("kuber v2 HTTP routes", () => { operationId: "operation-blocked", }); expect((await operationStore.get("operation-blocked"))?.status.state).toBe( - "failed", + "pending", ); }); + test("keeps a concurrent idempotent operation pending when its lease acquisition is denied", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const operationStore = new MemoryOperationStore( + undefined, + () => "same-key", + ); + let acquireCount = 0; + let releaseCount = 0; + let allowFirstAcquire!: () => void; + let signalFirstAcquire!: () => void; + const firstAcquireStarted = new Promise((resolve) => { + signalFirstAcquire = resolve; + }); + const leases = { + acquire: async () => { + acquireCount += 1; + if (acquireCount === 2) return undefined; + signalFirstAcquire(); + await new Promise((resolve) => { + allowFirstAcquire = resolve; + }); + return { + workspaceId: "demo", + holder: "operation-same-key", + expiresAt: new Date().toISOString(), + renew: async () => true, + release: async () => { + releaseCount += 1; + }, + }; + }, + }; + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore, + leases, + management: { stop: async () => ["api"] } as unknown as ManagementService, + }); + const stop = () => + app( + request( + "/api/v2/workspaces/demo/lifecycle", + { + method: "POST", + headers: { "idempotency-key": "same-key" }, + body: JSON.stringify({ action: "stop" }), + }, + "token", + ), + ); + + const owner = stop(); + await firstAcquireStarted; + const duplicate = await stop(); + expect(duplicate.status).toBe(409); + expect((await operationStore.get("operation-same-key"))?.status.state).toBe( + "pending", + ); + + allowFirstAcquire(); + expect((await owner).status).toBe(200); + expect((await operationStore.get("operation-same-key"))?.status.state).toBe( + "succeeded", + ); + expect(acquireCount).toBe(2); + expect(releaseCount).toBe(1); + }); + + test("lets the lease winner claim execution even when a duplicate created the operation", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const operationStore = new MemoryOperationStore( + undefined, + () => "same-key", + ); + let acquireCount = 0; + let releaseFirstAcquire!: () => void; + let firstAcquireStarted!: () => void; + const firstAcquire = new Promise((resolve) => { + firstAcquireStarted = resolve; + }); + const leases = { + acquire: async () => { + acquireCount += 1; + if (acquireCount === 1) { + firstAcquireStarted(); + await new Promise((resolve) => { + releaseFirstAcquire = resolve; + }); + return undefined; + } + return { + workspaceId: "demo", + holder: "operation-same-key", + expiresAt: new Date().toISOString(), + renew: async () => true, + release: async () => {}, + }; + }, + }; + let executions = 0; + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore, + leases, + management: { + stop: async () => { + executions += 1; + return ["api"]; + }, + } as unknown as ManagementService, + }); + const stop = () => + app( + request( + "/api/v2/workspaces/demo/lifecycle", + { + method: "POST", + headers: { "idempotency-key": "same-key" }, + body: JSON.stringify({ action: "stop" }), + }, + "token", + ), + ); + + const creator = stop(); + await firstAcquire; + expect((await stop()).status).toBe(200); + releaseFirstAcquire(); + expect((await creator).status).toBe(409); + expect(executions).toBe(1); + expect((await operationStore.get("operation-same-key"))?.status.state).toBe( + "succeeded", + ); + }); + + test("releases a lease when the initial execution claim fails", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const operationStore = new MemoryOperationStore( + undefined, + () => "claim-fails", + ); + spyOn(operationStore, "claimExecution").mockRejectedValue( + new Error("claim unavailable"), + ); + let releases = 0; + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore, + leases: { + acquire: async () => ({ + workspaceId: "demo", + holder: "operation-claim-fails", + expiresAt: new Date().toISOString(), + renew: async () => true, + release: async () => { + releases += 1; + }, + }), + }, + management: { stop: async () => [] } as unknown as ManagementService, + }); + + expect( + ( + await app( + request( + "/api/v2/workspaces/demo/lifecycle", + { method: "POST", body: JSON.stringify({ action: "stop" }) }, + "token", + ), + ) + ).status, + ).toBe(500); + expect(releases).toBe(1); + }); + + test("fails a claimed operation before execution when lease ownership is lost", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const operationStore = new MemoryOperationStore( + undefined, + () => "lost-lease", + ); + let executions = 0; + let releases = 0; + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore, + leases: { + acquire: async () => ({ + workspaceId: "demo", + holder: "operation-lost-lease", + expiresAt: new Date().toISOString(), + renew: async () => false, + release: async () => { + releases += 1; + }, + }), + }, + management: { + stop: async () => { + executions += 1; + return []; + }, + } as unknown as ManagementService, + }); + + const result = await app( + request( + "/api/v2/workspaces/demo/lifecycle", + { method: "POST", body: JSON.stringify({ action: "stop" }) }, + "token", + ), + ); + expect(result.status).toBe(409); + expect(executions).toBe(0); + expect(releases).toBe(1); + expect(await result.json()).toMatchObject({ + code: "WORKSPACE_LEASE_LOST", + }); + expect( + (await operationStore.get("operation-lost-lease"))?.status, + ).toMatchObject({ + state: "failed", + error: { code: "WORKSPACE_LEASE_LOST" }, + }); + }); + + test("aborts blocked execution and fails the operation when renewal loses the lease", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const operationStore = new MemoryOperationStore( + undefined, + () => "blocked-loss", + ); + const originalSetTimeout = globalThis.setTimeout; + let scheduledRenewal!: () => void; + globalThis.setTimeout = ((callback: Parameters[0]) => { + scheduledRenewal = callback as () => void; + return 0 as unknown as ReturnType; + }) as typeof setTimeout; + try { + let executionStarted!: () => void; + const started = new Promise((resolve) => { + executionStarted = resolve; + }); + let observedAbort = false; + let renewals = 0; + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore, + leases: { + acquire: async () => ({ + workspaceId: "demo", + holder: "operation-blocked-loss", + expiresAt: new Date().toISOString(), + renew: async () => ++renewals === 1, + release: async () => {}, + }), + }, + management: { + stop: async ( + _workspace: { project: string; uid: string }, + _names?: string[], + execution?: { signal?: AbortSignal }, + ) => { + executionStarted(); + await new Promise((resolve) => { + execution?.signal?.addEventListener( + "abort", + () => { + observedAbort = true; + resolve(); + }, + { once: true }, + ); + }); + return []; + }, + } as unknown as ManagementService, + }); + const pending = app( + request( + "/api/v2/workspaces/demo/lifecycle", + { method: "POST", body: JSON.stringify({ action: "stop" }) }, + "token", + ), + ); + await started; + scheduledRenewal(); + const result = await pending; + + expect(observedAbort).toBe(true); + expect(result.status).toBe(409); + expect(await result.json()).toMatchObject({ + code: "WORKSPACE_LEASE_LOST", + }); + expect( + (await operationStore.get("operation-blocked-loss"))?.status, + ).toMatchObject({ + state: "failed", + error: { code: "WORKSPACE_LEASE_LOST" }, + }); + } finally { + globalThis.setTimeout = originalSetTimeout; + } + }); + + test("renews the workspace lease while an operation remains in flight", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + let scheduledRenewal: (() => void) | undefined; + const originalSetTimeout = globalThis.setTimeout; + const originalClearTimeout = globalThis.clearTimeout; + const clearedTimers: unknown[] = []; + globalThis.setTimeout = ((callback: Parameters[0]) => { + scheduledRenewal = callback as () => void; + return 0 as unknown as ReturnType; + }) as typeof setTimeout; + globalThis.clearTimeout = ((timer: ReturnType) => { + clearedTimers.push(timer); + }) as typeof clearTimeout; + try { + let releaseOperation!: () => void; + let operationStarted!: () => void; + const started = new Promise((resolve) => { + operationStarted = resolve; + }); + const completed = new Promise((resolve) => { + releaseOperation = () => resolve(["api"]); + }); + let renewals = 0; + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore: new MemoryOperationStore(undefined, () => "renewing"), + leases: { + acquire: async () => ({ + workspaceId: "demo", + holder: "operation-renewing", + expiresAt: new Date().toISOString(), + renew: async () => { + renewals += 1; + return true; + }, + release: async () => {}, + }), + }, + management: { + stop: async () => { + operationStarted(); + return completed; + }, + } as unknown as ManagementService, + }); + const response = app( + request( + "/api/v2/workspaces/demo/lifecycle", + { method: "POST", body: JSON.stringify({ action: "stop" }) }, + "token", + ), + ); + await started; + expect(renewals).toBe(1); + scheduledRenewal?.(); + await Promise.resolve(); + await Promise.resolve(); + expect(renewals).toBe(2); + releaseOperation(); + expect((await response).status).toBe(200); + expect(renewals).toBe(3); + expect(clearedTimers).toEqual([0]); + } finally { + globalThis.setTimeout = originalSetTimeout; + globalThis.clearTimeout = originalClearTimeout; + } + }); + + test("reuses a failed reconciliation without attempting a second failure transition", async () => { + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-uid", + }); + await workspaceStore.create({ + id: "demo", + source: { uri: "oci://example/demo", digest: "sha256:abc" }, + }); + const operationStore = new MemoryOperationStore( + undefined, + () => "reconcile", + ); + let reconciliations = 0; + const management = { + reconcileDatabases: async () => { + reconciliations += 1; + const [operation] = await operationStore.list("demo"); + await operationStore.transition(operation!.metadata.name, "failed", { + error: { + code: "RECONCILE_FAILED", + message: "Database reconciliation failed", + }, + }); + return {}; + }, + } as unknown as ManagementService; + const app = createApp({ + store: await authenticatedStore("operator"), + workspaceStore, + operationStore, + management, + }); + const reconcile = () => + app( + request( + "/api/v2/workspaces/demo/databases", + { + method: "POST", + headers: { "idempotency-key": "reconcile-once" }, + body: JSON.stringify({ compose: {} }), + }, + "token", + ), + ); + + for (const result of [await reconcile(), await reconcile()]) { + expect(result.status).toBe(500); + expect(await result.json()).toMatchObject({ + code: "RECONCILE_FAILED", + detail: "Database reconciliation failed", + }); + } + expect(reconciliations).toBe(1); + expect( + (await operationStore.get("operation-reconcile"))?.status.state, + ).toBe("failed"); + }); + test("routes workspace adoption and keeps platform adoption admin-only", async () => { const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-uid", diff --git a/tests/server/build-controller.test.ts b/tests/server/build-controller.test.ts index ad55229..4daaf15 100644 --- a/tests/server/build-controller.test.ts +++ b/tests/server/build-controller.test.ts @@ -10,7 +10,9 @@ import { type BuildJobObservation, type BuildKubernetesOperations, } from "../../server/build-controller"; -import { MemoryBuildStore } from "../../server/build-store"; +import { createApp } from "../../server/app"; +import { hashToken, MemoryAuthStore } from "../../server/auth"; +import { MemoryBuildStore, type BuildStore } from "../../server/build-store"; import { FilesystemCas } from "../../server/cas"; import type { KubernetesJob } from "../../server/build-job"; import { @@ -36,7 +38,10 @@ class FakeKubernetes implements BuildKubernetesOperations { deleted: string[] = []; observation: BuildJobObservation | undefined = { phase: "queued" }; logs = ""; + createError?: Error; + preserveAfterDelete = false; async createJob(job: KubernetesJob) { + if (this.createError) throw this.createError; this.jobs.push(job); } async getJob() { @@ -47,14 +52,39 @@ class FakeKubernetes implements BuildKubernetesOperations { } async deleteJob(_namespace: string, name: string) { this.deleted.push(name); + if (!this.preserveAfterDelete) this.observation = undefined; } } -async function fixture(maxLogBytes = 1024) { +class SupersededBeforeJobStore extends MemoryBuildStore { + private superseded = false; + + override async ownsBuild( + imageKey: string, + buildId: string, + ): Promise { + if (!this.superseded) { + this.superseded = true; + const current = await this.getBuild(buildId); + if (current) { + const newer = structuredClone(current); + newer.metadata.name = "newer-build"; + newer.metadata.creationTimestamp = "2026-09-02T00:00:01.000Z"; + newer.spec.request.id = "newer-build"; + await super.createBuild(newer); + } + } + return super.ownsBuild(imageKey, buildId); + } +} + +async function fixture( + maxLogBytes = 1024, + store: BuildStore = new MemoryBuildStore(), +) { const root = await mkdtemp(join(tmpdir(), "kuber-controller-")); roots.push(root); const cas = new FilesystemCas(join(root, "cas")); - const store = new MemoryBuildStore(); const kubernetes = new FakeKubernetes(); const source = Buffer.from("FROM scratch\n"); const sourceDigest = await cas.put(source); @@ -237,7 +267,7 @@ describe("build controller", () => { }); }); - test("submits once, blocks competing image builds, captures bounded logs, and resolves immutable results", async () => { + test("submits once, supersedes competing image builds, captures bounded logs, and resolves immutable results", async () => { const { controller, kubernetes, request } = await fixture(8); expect(await controller.submitBuild(request)).toMatchObject({ state: "queued", @@ -247,18 +277,18 @@ describe("build controller", () => { ).toMatchObject({ state: "queued" }); expect(kubernetes.jobs).toHaveLength(1); const jobSpec = kubernetes.jobs[0]!.spec as any; - expect( - jobSpec.template.spec.containers[0].volumeMounts, - ).toContainEqual( + expect(jobSpec.template.spec.containers[0].volumeMounts).toContainEqual( expect.objectContaining({ name: "workspace", mountPath: "/workspace", subPath: `workspaces/${kubernetes.jobs[0]!.metadata.name}`, }), ); - await expect( - controller.submitBuild({ ...request, id: "request-two" }), - ).rejects.toBeInstanceOf(BuildConflictError); + const replacement = { ...request, id: "request-two" }; + expect(await controller.submitBuild(replacement)).toMatchObject({ + state: "queued", + }); + expect(kubernetes.jobs).toHaveLength(2); await expect( controller.submitBuild({ ...request, project: "changed" }), ).rejects.toBeInstanceOf(BuildConflictError); @@ -268,10 +298,10 @@ describe("build controller", () => { phase: "running", startedAt: "2026-09-02T00:00:03.000Z", }; - expect(await controller.reconcileBuild(request.id)).toMatchObject({ + expect(await controller.reconcileBuild(replacement.id)).toMatchObject({ state: "running", }); - const logs = (await controller.getBuildEvents(request.id)).filter( + const logs = (await controller.getBuildEvents(replacement.id)).filter( (event) => event.type === "log", ); expect( @@ -286,17 +316,137 @@ describe("build controller", () => { phase: "succeeded", finishedAt: "2026-09-02T00:00:04.000Z", }; - const status = await controller.reconcileBuild(request.id); + const status = await controller.reconcileBuild(replacement.id); expect(status).toMatchObject({ state: "succeeded", digest: `sha256:${"f".repeat(64)}`, }); - expect(await controller.getBuildResult(request.id)).toEqual({ + expect(await controller.getBuildResult(replacement.id)).toEqual({ image: "registry.test/demo/web", digest: `sha256:${"f".repeat(64)}`, reference: `registry.test/demo/web@sha256:${"f".repeat(64)}`, }); - expect(await controller.reconcileBuild(request.id)).toEqual(status); + expect(await controller.reconcileBuild(replacement.id)).toEqual(status); + }); + + test("concurrent controllers converge on one same-ID record and Job", async () => { + const first = await fixture(); + const second = new BuildController({ + cas: first.cas, + store: first.store, + kubernetes: first.kubernetes, + namespace: "builds", + workspaceRoot: join(first.root, "workspaces"), + workspaceClaimName: "workspaces", + cacheImage: "registry.test/cache/app", + }); + await Promise.all([ + first.controller.submitBuild(first.request), + second.submitBuild(structuredClone(first.request)), + ]); + expect(first.kubernetes.jobs).toHaveLength(1); + expect(await first.store.getBuild(first.request.id)).toMatchObject({ + spec: { request: { id: first.request.id } }, + }); + }); + + test("starts a replacement without waiting for superseded Job deletion", async () => { + const { controller, kubernetes, request, root, store } = await fixture(); + await controller.submitBuild(request); + const oldJobName = kubernetes.jobs[0]!.metadata.name; + kubernetes.preserveAfterDelete = true; + await expect( + controller.submitBuild({ ...request, id: "replacement" }), + ).resolves.toMatchObject({ + state: "queued", + }); + expect(kubernetes.deleted).toEqual([oldJobName]); + expect(kubernetes.jobs).toHaveLength(2); + await expect( + lstat(join(root, "workspaces", oldJobName)), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect((await store.getBuild(request.id))?.status).toMatchObject({ + state: "failed", + error: "Superseded by newer build", + cancelled: true, + }); + }); + + test("deletes an ambiguous superseded Job even when status was not persisted", async () => { + const { controller, kubernetes, request, store } = await fixture(); + await controller.submitBuild(request); + const oldJobName = kubernetes.jobs[0]!.metadata.name; + const old = (await store.getBuild(request.id))!; + old.status.jobCreated = false; + await store.replaceBuild(old, old.metadata.resourceVersion); + + await expect( + controller.submitBuild({ ...request, id: "replacement" }), + ).resolves.toMatchObject({ state: "queued" }); + expect(kubernetes.deleted).toEqual([oldJobName]); + }); + + test("does not create a Job when the candidate is superseded before Job creation", async () => { + const store = new SupersededBeforeJobStore(); + const { controller, kubernetes, request } = await fixture(1024, store); + + await expect(controller.submitBuild(request)).rejects.toBeInstanceOf( + BuildConflictError, + ); + expect(kubernetes.jobs).toHaveLength(0); + }); + + test("maps supersession before Job creation to HTTP 409", async () => { + const auth = new MemoryAuthStore(); + await auth.putUser({ + username: "operator", + passwordHash: "hash", + roles: ["operator"], + }); + await auth.putSession({ + tokenHash: hashToken("token"), + username: "operator", + authVersion: 1, + expiresAt: "2030-01-01T00:00:00.000Z", + }); + const store = new SupersededBeforeJobStore(); + const { + controller, + kubernetes, + request: buildRequest, + } = await fixture(1024, store); + const app = createApp({ store: auth, builds: controller }); + + const response = await app( + new Request("https://kuber.test/api/v2/builds", { + method: "POST", + headers: { + authorization: "Bearer token", + "content-type": "application/json", + }, + body: JSON.stringify(buildRequest), + }), + ); + + expect(response.status).toBe(409); + expect(await response.json()).toMatchObject({ code: "BUILD_CONFLICT" }); + expect(kubernetes.jobs).toHaveLength(0); + }); + + test("releases the image lock after an ambiguous Job creation failure", async () => { + const { controller, kubernetes, request, store } = await fixture(); + kubernetes.createError = new Error("create response lost"); + kubernetes.preserveAfterDelete = true; + await expect(controller.submitBuild(request)).rejects.toThrow( + "create response lost", + ); + expect((await store.getBuild(request.id))?.status).toMatchObject({ + state: "failed", + }); + kubernetes.createError = undefined; + await expect( + controller.submitBuild({ ...request, id: "replacement" }), + ).resolves.toMatchObject({ state: "queued" }); }); test("cancels idempotently and cleans up only terminal build resources", async () => { diff --git a/tests/server/build-kubernetes.test.ts b/tests/server/build-kubernetes.test.ts new file mode 100644 index 0000000..a3b1048 --- /dev/null +++ b/tests/server/build-kubernetes.test.ts @@ -0,0 +1,349 @@ +import { createHash } from "node:crypto"; +import { describe, expect, test } from "bun:test"; +import type { V1DeleteOptions } from "@kubernetes/client-node"; +import { + BUILD_RECORD_API_VERSION, + type BuildRecord, +} from "../../server/build-store"; +import { + KubernetesBuildStore, + type BuildObjectApi, +} from "../../server/build-kubernetes"; +import { + BUILD_PROTOCOL_VERSION, + type BuildRequest, + type Sha256Digest, +} from "../../shared/build-protocol"; + +const namespace = "kuber-test"; +const imageKey = "project\0service\0registry.test/app:latest"; +const workspace = `sha256:${"a".repeat(64)}` as Sha256Digest; + +function validLabelValue(value: string): boolean { + return ( + value.length <= 63 && + /^[A-Za-z0-9](?:[-_.A-Za-z0-9]*[A-Za-z0-9])?$/.test(value) + ); +} + +function name(prefix: string, value: string): string { + return `${prefix}-${createHash("sha256").update(value).digest("hex").slice(0, 48)}`; +} + +function build(id: string, createdAt: string): BuildRecord { + const request: BuildRequest = { + version: BUILD_PROTOCOL_VERSION, + id, + project: "project", + service: "service", + spec: { + architecture: "amd64", + image: "registry.test/app:latest", + context: ".", + buildArgs: [], + workspace, + }, + }; + const status = { + version: BUILD_PROTOCOL_VERSION, + id, + state: "queued" as const, + createdAt, + }; + return { + apiVersion: BUILD_RECORD_API_VERSION, + kind: "BuildRecord", + metadata: { + name: id, + resourceVersion: "1", + creationTimestamp: createdAt, + labels: { project: "project", service: "service" }, + }, + spec: { request, imageKey, jobName: `job-${id}`, workspaceSubPath: id }, + status: { + ...status, + logBytes: 0, + logOffset: 0, + nextSequence: 1, + events: [{ type: "status", status }], + }, + }; +} + +type ConfigMap = Parameters[0]; +type DeleteOptions = Pick; + +class FakeObjects implements BuildObjectApi { + readonly maps = new Map(); + readonly selectors: string[] = []; + readonly deletes: Array<{ name: string; options?: DeleteOptions }> = []; + onLockRead?: () => void; + private lockRead = false; + + async create(value: ConfigMap): Promise { + if ( + Object.values(value.metadata.labels ?? {}).some( + (label) => !validLabelValue(label), + ) + ) + throw { code: 422 }; + if (this.maps.has(value.metadata.name)) throw { code: 409 }; + this.maps.set( + value.metadata.name, + structuredClone({ + ...value, + metadata: { ...value.metadata, resourceVersion: "1" }, + }), + ); + return value; + } + + async read(value: ConfigMap): Promise { + const found = this.maps.get(value.metadata.name); + if (!found) throw { code: 404 }; + if (value.metadata.name.startsWith("build-lock-") && !this.lockRead) { + this.lockRead = true; + this.onLockRead?.(); + } + return structuredClone(found); + } + + async replace(value: ConfigMap): Promise { + if ( + Object.values(value.metadata.labels ?? {}).some( + (label) => !validLabelValue(label), + ) + ) + throw { code: 422 }; + const current = this.maps.get(value.metadata.name); + if ( + !current || + current.metadata.resourceVersion !== value.metadata.resourceVersion + ) + throw { code: 409 }; + const next = structuredClone({ + ...value, + metadata: { + ...value.metadata, + resourceVersion: String(Number(current.metadata.resourceVersion) + 1), + }, + }); + this.maps.set(value.metadata.name, next); + return next; + } + + async delete(value: ConfigMap, options?: DeleteOptions): Promise { + const current = this.maps.get(value.metadata.name); + this.deletes.push({ name: value.metadata.name, options }); + if (!current) throw { code: 404 }; + if ( + options?.preconditions?.resourceVersion !== + current.metadata.resourceVersion + ) + throw { code: 409 }; + this.maps.delete(value.metadata.name); + return undefined; + } + + async list( + _apiVersion: string, + _kind: string, + _namespace?: string, + _pretty?: string, + _exact?: boolean, + _exportValue?: boolean, + _fieldSelector?: string, + labelSelector?: string, + ): Promise<{ items: unknown[] }> { + this.selectors.push(labelSelector ?? ""); + const labels = Object.fromEntries( + (labelSelector ?? "") + .split(",") + .filter(Boolean) + .map((part) => part.split("=")), + ); + return { + items: [...this.maps.values()].filter((item) => + Object.entries(labels).every( + ([key, value]) => item.metadata.labels?.[key] === value, + ), + ), + }; + } +} + +function store(fake: FakeObjects): KubernetesBuildStore { + return new KubernetesBuildStore( + fake, + namespace, + "/tmp/kuber-build-store-test", + ); +} + +function configMap(record: BuildRecord, imageLabel = true): ConfigMap { + return { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: name("build", record.metadata.name), + namespace, + resourceVersion: record.metadata.resourceVersion, + labels: { + "kuber.astrxl.dev/type": "build", + ...(imageLabel && { + "kuber.astrxl.dev/image": createHash("sha256") + .update(record.spec.imageKey) + .digest("hex") + .slice(0, 63), + }), + }, + }, + data: { payload: JSON.stringify(record) }, + }; +} + +describe("KubernetesBuildStore", () => { + test("creates records with valid, deterministic image index labels", async () => { + const fake = new FakeObjects(); + const record = build("validated", "2026-09-02T00:00:00.000Z"); + const result = await store(fake).createBuild(record); + const labels = fake.maps.get(name("build", record.metadata.name))?.metadata + .labels; + + expect(result.created).toBe(true); + expect(labels).toBeDefined(); + expect(Object.values(labels ?? {}).every(validLabelValue)).toBe(true); + expect(labels?.["kuber.astrxl.dev/image"]).toBe( + createHash("sha256").update(imageKey).digest("hex").slice(0, 63), + ); + }); + + test("uses the image label fast path", async () => { + const fake = new FakeObjects(); + const result = await store(fake).createBuild( + build("old", "2026-09-02T00:00:00.000Z"), + ); + expect(result.created).toBe(true); + expect( + fake.selectors.some((selector) => + selector.includes("kuber.astrxl.dev/image="), + ), + ).toBe(true); + }); + + test("finds legacy records through the bounded project/service fallback", async () => { + const fake = new FakeObjects(); + const legacy = build( + "5ef44ce9-f087-4b65-8ff1-2a60dab9112e", + "2026-09-02T00:00:00.000Z", + ); + fake.maps.set( + name("build", legacy.metadata.name), + configMap(legacy, false), + ); + expect( + fake.maps.get(name("build", legacy.metadata.name))?.metadata.labels, + ).toEqual({ + "kuber.astrxl.dev/type": "build", + }); + const result = await store(fake).createBuild( + build("replacement", "2026-09-02T00:00:01.000Z"), + ); + expect( + (await store(fake).getBuild("5ef44ce9-f087-4b65-8ff1-2a60dab9112e")) + ?.status.state, + ).toBe("failed"); + expect(fake.selectors).toContain("kuber.astrxl.dev/type=build"); + }); + + test("returns every older record superseded during reconciliation", async () => { + const fake = new FakeObjects(); + const kuber = store(fake); + const oldOne = build("old-one", "2026-09-02T00:00:00.000Z"); + const oldTwo = build("old-two", "2026-09-02T00:00:01.000Z"); + fake.maps.set(name("build", oldOne.metadata.name), configMap(oldOne)); + fake.maps.set(name("build", oldTwo.metadata.name), configMap(oldTwo)); + + const result = await kuber.createBuild( + build("replacement", "2026-09-02T00:00:02.000Z"), + ); + + expect(result.created).toBe(true); + expect(result.superseded?.map((record) => record.metadata.name)).toEqual([ + "old-one", + "old-two", + ]); + }); + + test("returns created false when an older candidate is already superseded", async () => { + const fake = new FakeObjects(); + const kuber = store(fake); + await kuber.createBuild(build("newer", "2026-09-02T00:00:01.000Z")); + + const result = await kuber.createBuild( + build("older", "2026-09-02T00:00:00.000Z"), + ); + + expect(result.created).toBe(false); + expect(result.record.status.error).toBe("Superseded by newer build"); + }); + + test("orders and reconciles legacy records with invalid metadata timestamps", async () => { + const fake = new FakeObjects(); + const kuber = store(fake); + const legacy = build("legacy", "2026-09-02T00:00:00.000Z"); + legacy.metadata.creationTimestamp = undefined as unknown as string; + fake.maps.set(name("build", legacy.metadata.name), configMap(legacy)); + + const replacement = await kuber.createBuild( + build("replacement", "2026-09-02T00:00:01.000Z"), + ); + expect(replacement.created).toBe(true); + expect((await kuber.getBuild("legacy"))?.status.state).toBe("failed"); + + const malformed = build("a", "2026-09-02T00:00:00.000Z"); + malformed.metadata.creationTimestamp = 0 as unknown as string; + fake.maps.set(name("build", malformed.metadata.name), configMap(malformed)); + expect( + (await kuber.listBuilds()).map((record) => record.metadata.name), + ).toEqual(["a", "legacy", "replacement"]); + }); + + test("recovers an orphan record and takes over its missing lock", async () => { + const fake = new FakeObjects(); + const kuber = store(fake); + const orphan = build("orphan", "2026-09-02T00:00:00.000Z"); + fake.maps.set( + name("build", orphan.metadata.name), + configMap(orphan, false), + ); + fake.maps.delete(name("build-lock", imageKey)); + const result = await kuber.createBuild( + build("replacement", "2026-09-02T00:00:01.000Z"), + ); + expect((await kuber.getBuild("orphan"))?.status.state).toBe("failed"); + expect(await kuber.ownsBuild(imageKey, "replacement")).toBe(true); + }); + + test("does not delete a successor lock after a read/delete race", async () => { + const fake = new FakeObjects(); + const kuber = store(fake); + await kuber.createBuild(build("old", "2026-09-02T00:00:00.000Z")); + fake.onLockRead = () => { + const lock = fake.maps.get(name("build-lock", imageKey))!; + fake.maps.set(lock.metadata.name, { + ...lock, + data: { payload: JSON.stringify({ buildId: "successor" }) }, + metadata: { ...lock.metadata, resourceVersion: "2" }, + }); + }; + const old = (await kuber.getBuild("old"))!; + old.status.state = "succeeded"; + await kuber.replaceBuild(old, "1"); + expect(await kuber.ownsBuild(imageKey, "successor")).toBe(true); + expect(fake.maps.has(name("build-lock", imageKey))).toBe(true); + expect(fake.deletes.at(-1)?.options?.preconditions?.resourceVersion).toBe( + "1", + ); + }); +}); diff --git a/tests/server/build-store.test.ts b/tests/server/build-store.test.ts index 8f391e6..dfd7d57 100644 --- a/tests/server/build-store.test.ts +++ b/tests/server/build-store.test.ts @@ -13,7 +13,11 @@ import { const workspace = `sha256:${"a".repeat(64)}` as Sha256Digest; -function build(id: string, imageKey = "project\0service\0image"): BuildRecord { +function build( + id: string, + imageKey = "project\0service\0image", + createdAt = "2026-09-02T00:00:00.000Z", +): BuildRecord { const request: BuildRequest = { version: BUILD_PROTOCOL_VERSION, id, @@ -31,7 +35,7 @@ function build(id: string, imageKey = "project\0service\0image"): BuildRecord { version: BUILD_PROTOCOL_VERSION, id, state: "queued" as const, - createdAt: "2026-09-02T00:00:00.000Z", + createdAt, }; return { apiVersion: BUILD_RECORD_API_VERSION, @@ -61,25 +65,28 @@ describe("build store", () => { await expect( store.createBuild(build("one", "different-key")), ).rejects.toBeInstanceOf(BuildStoreConflictError); - await expect(store.createBuild(build("two"))).rejects.toBeInstanceOf( - BuildStoreConflictError, - ); + const replacement = await store.createBuild(build("two")); + expect(replacement.created).toBe(true); + expect(replacement.superseded?.[0]?.metadata.name).toBe("one"); const first = (await store.getBuild("one"))!; const digest = `sha256:${"b".repeat(64)}` as Sha256Digest; - first.metadata.resourceVersion = "2"; + first.metadata.resourceVersion = "3"; Object.assign(first.status, { state: "succeeded", digest, finishedAt: first.status.createdAt, }); - await store.replaceBuild(first, "1"); - expect((await store.createBuild(build("two"))).created).toBe(true); + await store.replaceBuild(first, "2"); + expect( + (await store.createBuild(build("three", "2026-09-02T00:00:02.000Z"))) + .created, + ).toBe(true); const changed = (await store.getBuild("one"))!; changed.metadata.resourceVersion = "3"; changed.status.digest = `sha256:${"c".repeat(64)}`; - await expect(store.replaceBuild(changed, "2")).rejects.toThrow("immutable"); + await expect(store.replaceBuild(changed, "3")).rejects.toThrow("immutable"); }); test("optimistically updates resumable upload records without leaking mutable data", async () => { @@ -107,4 +114,117 @@ describe("build store", () => { BuildStoreConflictError, ); }); + + test("supersedes only the current image and leaves terminal history", async () => { + const store = new MemoryBuildStore(); + await store.createBuild( + build("old", "project\0service\0image", "2026-09-02T00:00:00.000Z"), + ); + const result = await store.createBuild( + build("new", "project\0service\0image", "2026-09-02T00:00:01.000Z"), + ); + expect(result.superseded?.[0]?.status.error).toBe( + "Superseded by newer build", + ); + expect((await store.getBuild("old"))?.status.state).toBe("failed"); + expect((await store.createBuild(build("new"))).created).toBe(false); + expect( + (await store.createBuild(build("other", "project\0service\0other"))) + .created, + ).toBe(true); + }); + + test("concurrent same-image submissions have one active record", async () => { + const store = new MemoryBuildStore(); + const results = await Promise.all([ + store.createBuild(build("first")), + store.createBuild(build("second")), + store.createBuild(build("third")), + ]); + expect(results.filter((result) => result.created)).toHaveLength(3); + const active = (await store.listBuilds()).filter( + (record) => + record.status.state !== "succeeded" && record.status.state !== "failed", + ); + expect(active).toHaveLength(1); + expect(active[0]?.metadata.name).toBe("third"); + }); + + test("recovers an active record whose in-memory lock was lost", async () => { + const store = new MemoryBuildStore(); + await store.createBuild(build("orphan")); + (store as unknown as { active: Map }).active.clear(); + + const replacement = await store.createBuild(build("replacement")); + expect(replacement.superseded?.[0]?.metadata.name).toBe("orphan"); + expect((await store.getBuild("orphan"))?.status.state).toBe("failed"); + expect( + await store.ownsBuild("project\0service\0image", "replacement"), + ).toBe(true); + }); + + test("a delayed older retry cannot reclaim a newer same-image lock", async () => { + const store = new MemoryBuildStore(); + await store.createBuild( + build("a", "project\0service\0image", "2026-09-02T00:00:00.000Z"), + ); + (store as unknown as { active: Map }).active.clear(); + await store.createBuild( + build("b", "project\0service\0image", "2026-09-02T00:00:01.000Z"), + ); + + const delayed = await store.createBuild( + build("a", "project\0service\0image", "2026-09-02T00:00:00.000Z"), + ); + expect(delayed.created).toBe(false); + expect(delayed.record.status.error).toBe("Superseded by newer build"); + expect(await store.ownsBuild("project\0service\0image", "b")).toBe(true); + }); + + test("returns cleanup metadata when a delayed candidate loses to a newer record", async () => { + const store = new MemoryBuildStore(); + await store.createBuild( + build("newer", "project\0service\0image", "2026-09-02T00:00:01.000Z"), + ); + + const result = await store.createBuild( + build("delayed", "project\0service\0image", "2026-09-02T00:00:00.000Z"), + ); + + expect(result.created).toBe(false); + expect(result.superseded?.map((record) => record.metadata.name)).toEqual([ + "delayed", + ]); + expect( + (await store.createBuild(build("delayed"))).superseded, + ).toBeUndefined(); + }); + + test("orders and reconciles legacy records with invalid metadata timestamps", async () => { + const store = new MemoryBuildStore(); + const legacy = build( + "legacy", + "project\0service\0image", + "2026-09-02T00:00:00.000Z", + ); + legacy.metadata.creationTimestamp = undefined as unknown as string; + await store.createBuild(legacy); + + const replacement = await store.createBuild( + build( + "replacement", + "project\0service\0image", + "2026-09-02T00:00:01.000Z", + ), + ); + expect(replacement.created).toBe(true); + expect((await store.getBuild("legacy"))?.status.state).toBe("failed"); + + const malformed = build("a", "project\0service\0other"); + malformed.metadata.creationTimestamp = 0 as unknown as string; + await store.createBuild(malformed); + expect( + (await store.listBuilds()).map((record) => record.metadata.name), + ).toEqual(["a", "legacy", "replacement"]); + }); }); diff --git a/tests/server/exec-websocket.test.ts b/tests/server/exec-websocket.test.ts index 0498af3..b6f231c 100644 --- a/tests/server/exec-websocket.test.ts +++ b/tests/server/exec-websocket.test.ts @@ -155,7 +155,9 @@ function connection(role: "viewer" | "operator" | "admin" = "operator") { } as ExecConnection; } -async function authenticatedStore(role: "viewer" | "operator" | "admin" = "operator") { +async function authenticatedStore( + role: "viewer" | "operator" | "admin" = "operator", +) { const store = new MemoryAuthStore(); await store.putUser({ username: role, passwordHash: "hash", roles: [role] }); await store.putSession({ @@ -176,7 +178,9 @@ function request(path = "/api/v2/workspaces/shop/exec") { describe("authorizeExecConnection", () => { test("resolves the workspace UID server-side for authorized operators", async () => { const store = await authenticatedStore("operator"); - const workspaceStore = new MemoryWorkspaceStore({ uid: () => "workspace-1" }); + const workspaceStore = new MemoryWorkspaceStore({ + uid: () => "workspace-1", + }); await workspaceStore.create({ id: "shop", source: { uri: "oci://example/demo", digest: "sha256:abc" }, @@ -285,7 +289,9 @@ describe("WireExecSession", () => { createExecService(new FakeBackend()), connection(), ); - await session.receive(JSON.stringify({ type: "resize", columns: 1, rows: 1 })); + await session.receive( + JSON.stringify({ type: "resize", columns: 1, rows: 1 }), + ); expect(JSON.parse(socket.sent[0]!)).toMatchObject({ type: "error", code: "EXEC_INVALID", diff --git a/tests/server/kubernetes-state.test.ts b/tests/server/kubernetes-state.test.ts index 996e5b2..02df54d 100644 --- a/tests/server/kubernetes-state.test.ts +++ b/tests/server/kubernetes-state.test.ts @@ -172,9 +172,10 @@ describe("Kubernetes state persistence", () => { ); const first = await persistence.createIdempotent(operation("first")); const second = await persistence.createIdempotent(operation("second")); - expect(second).toEqual(first); + expect(first.created).toBe(true); + expect(second).toEqual({ operation: first.operation, created: false }); expect(fake.objects.size).toBe(1); - expect(first.metadata.name).toBe( + expect(first.operation.metadata.name).toBe( `operation-${createHash("sha256") .update("demo\0same-key") .digest("hex") @@ -239,7 +240,7 @@ describe("Kubernetes state persistence", () => { item.metadata?.labels?.[WORKSPACE_UID_LABEL] === "workspace-uid", ), ).toBe(true); - await expect(adoption.adopt("kuber-system", "uid")).rejects.toThrow( + await expect(adoption.adopt("kube-system", "uid")).rejects.toThrow( "reserved", ); }); @@ -302,7 +303,10 @@ describe("Kubernetes state persistence", () => { }); expect(fake.patches).toHaveLength(2); const [namespacePatch, resourcePatch] = fake.patches; - expect(namespacePatch).toMatchObject({ apiVersion: "v1", kind: "Namespace" }); + expect(namespacePatch).toMatchObject({ + apiVersion: "v1", + kind: "Namespace", + }); expect(resourcePatch).toMatchObject({ apiVersion: "apps/v1", kind: "Deployment", @@ -314,6 +318,7 @@ describe("Kubernetes state persistence", () => { class FakeLeaseStore implements LeaseObjects { readonly leases = new Map(); private rv = 0; + beforeDelete?: () => void; private key(name: string, namespace: string) { return `${namespace}/${name}`; @@ -360,8 +365,18 @@ class FakeLeaseStore implements LeaseObjects { return structuredClone(stored); } - async delete(name: string, namespace: string) { - this.leases.delete(this.key(name, namespace)); + async delete(name: string, namespace: string, expectedResourceVersion?: string) { + this.beforeDelete?.(); + this.beforeDelete = undefined; + const key = this.key(name, namespace); + const current = this.leases.get(key); + if (!current) throw { code: 404 }; + if ( + expectedResourceVersion && + current.metadata?.resourceVersion !== expectedResourceVersion + ) + throw { code: 409 }; + this.leases.delete(key); } } @@ -374,9 +389,7 @@ describe("Kubernetes workspace lease provider", () => { expect(lease!.workspaceId).toBe("demo"); expect(lease!.holder).toBe("worker-a"); expect(fake.leases.size).toBe(1); - expect( - await provider.acquire("demo", "worker-b", 1000), - ).toBeUndefined(); + expect(await provider.acquire("demo", "worker-b", 1000)).toBeUndefined(); }); test("renews optimistically and refuses after expiry or holder change", async () => { @@ -428,6 +441,36 @@ describe("Kubernetes workspace lease provider", () => { expect(await provider.acquire("demo", "worker-b", 1000)).toBeDefined(); }); + test("stale release cannot delete a successor that takes over after expiry", async () => { + const fake = new FakeLeaseStore(); + let now = 0; + const provider = new KubernetesWorkspaceLeaseProvider( + fake, + "kuber-system", + () => now, + ); + const stale = await provider.acquire("demo", "worker-a", 1000); + now = 1500; + fake.beforeDelete = () => { + const current = [...fake.leases.values()][0]!; + fake.leases.set( + `${current.metadata!.namespace}/${current.metadata!.name}`, + { + ...current, + metadata: { + ...current.metadata, + resourceVersion: "successor-version", + }, + spec: { ...current.spec, holderIdentity: "worker-b" }, + }, + ); + }; + + await stale!.release(); + + expect([...fake.leases.values()][0]?.spec?.holderIdentity).toBe("worker-b"); + }); + test("writes acquireTime and renewTime as microsecond MicroTime strings", async () => { const fake = new FakeLeaseStore(); const provider = new KubernetesWorkspaceLeaseProvider( @@ -436,8 +479,7 @@ describe("Kubernetes workspace lease provider", () => { () => Date.parse("2026-09-03T00:23:00.205Z"), ); const lease = await provider.acquire("demo", "worker-a", 1000); - const microRegex = - /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/; + const microRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/; for (const stored of fake.leases.values()) { expect(String(stored.spec?.acquireTime)).toMatch(microRegex); expect(String(stored.spec?.renewTime)).toMatch(microRegex); @@ -505,7 +547,9 @@ class TakeoverContentionStore implements LeaseObjects { spec: { holderIdentity: "contender", leaseDurationSeconds: 1, - renewTime: new Date(this.now - 5000).toISOString() as unknown as V1LeaseSpec["renewTime"], + renewTime: new Date( + this.now - 5000, + ).toISOString() as unknown as V1LeaseSpec["renewTime"], }, }; } @@ -557,7 +601,9 @@ class SustainedContentionStore implements LeaseObjects { spec: { holderIdentity: "contender", leaseDurationSeconds: 1, - renewTime: new Date(this.now - 5000).toISOString() as unknown as V1LeaseSpec["renewTime"], + renewTime: new Date( + this.now - 5000, + ).toISOString() as unknown as V1LeaseSpec["renewTime"], }, }; } @@ -671,7 +717,9 @@ function replicaSet( metadata: { name, namespace, - ...(hasManagedBy && { labels: { ...labels, "app.kubernetes.io/managed-by": "kuber" } }), + ...(hasManagedBy && { + labels: { ...labels, "app.kubernetes.io/managed-by": "kuber" }, + }), ...(!hasManagedBy && Object.keys(labels).length && { labels }), annotations: { "deployment.kubernetes.io/revision": String(revisionNumber), @@ -682,7 +730,9 @@ function replicaSet( }, spec: { template: { - metadata: { labels: { app: name, "pod-template-hash": `hash${revisionNumber}` } }, + metadata: { + labels: { app: name, "pod-template-hash": `hash${revisionNumber}` }, + }, spec: { containers: [{ name: "app", image }] }, }, }, @@ -721,13 +771,17 @@ class FakeApps { async listNamespacedDeployment({ namespace }: { namespace: string }) { return { - items: this.deployments.filter((d) => d.metadata?.namespace === namespace), + items: this.deployments.filter( + (d) => d.metadata?.namespace === namespace, + ), }; } async listNamespacedReplicaSet({ namespace }: { namespace: string }) { return { - items: this.replicaSets.filter((rs) => rs.metadata?.namespace === namespace), + items: this.replicaSets.filter( + (rs) => rs.metadata?.namespace === namespace, + ), }; } } @@ -819,7 +873,15 @@ describe("rollback ReplicaSet discovery", () => { replicaSet("web-2", "demo", depUid, 2, "img:v2", { app: "web" }), // An unrelated RS that happens to carry the managed-by label (e.g. a // standalone non-Deployment object) must still be ignored. - replicaSet("standalone", "demo", "other-uid", 1, "img:standalone", { app: "standalone" }, true), + replicaSet( + "standalone", + "demo", + "other-uid", + 1, + "img:standalone", + { app: "standalone" }, + true, + ), ]; const deps = createKubernetesManagementDependencies( managementClients(deployments, replicaSets), diff --git a/tests/server/management.test.ts b/tests/server/management.test.ts index 1487206..2e31129 100644 --- a/tests/server/management.test.ts +++ b/tests/server/management.test.ts @@ -233,13 +233,22 @@ describe("server management service", () => { dependencies({ listDeployments: async () => ["api", "worker", "batch"].map( - (name) => - object("Deployment", name, `${name}-uid`) as V1Deployment, + (name) => object("Deployment", name, `${name}-uid`) as V1Deployment, ), listProjectResources: async () => [ object("HorizontalPodAutoscaler", "api", "api-hpa-uid", undefined), - object("HorizontalPodAutoscaler", "worker", "worker-hpa-uid", undefined), - object("HorizontalPodAutoscaler", "batch", "batch-hpa-uid", undefined), + object( + "HorizontalPodAutoscaler", + "worker", + "worker-hpa-uid", + undefined, + ), + object( + "HorizontalPodAutoscaler", + "batch", + "batch-hpa-uid", + undefined, + ), ], scaleDeployment: async (_project, name, replicas) => { calls.push(`scale:${name}:${replicas}`); @@ -268,13 +277,22 @@ describe("server management service", () => { dependencies({ listDeployments: async () => ["api", "worker"].map( - (name) => - object("Deployment", name, `${name}-uid`) as V1Deployment, + (name) => object("Deployment", name, `${name}-uid`) as V1Deployment, ), listProjectResources: async () => [ object("HorizontalPodAutoscaler", "api", "api-hpa-uid", undefined), - object("HorizontalPodAutoscaler", "worker", "worker-hpa-uid", undefined), - object("HorizontalPodAutoscaler", "legacy", "legacy-hpa-uid", undefined), + object( + "HorizontalPodAutoscaler", + "worker", + "worker-hpa-uid", + undefined, + ), + object( + "HorizontalPodAutoscaler", + "legacy", + "legacy-hpa-uid", + undefined, + ), ], scaleDeployment: async (_project, name, replicas) => { calls.push(`scale:${name}:${replicas}`); @@ -297,8 +315,9 @@ describe("server management service", () => { test("refuses to delete an HPA not owned by the workspace", async () => { const service = createManagementService( dependencies({ - listDeployments: async () => - [object("Deployment", "api", "api-uid") as V1Deployment], + listDeployments: async () => [ + object("Deployment", "api", "api-uid") as V1Deployment, + ], listProjectResources: async () => [ object("HorizontalPodAutoscaler", "api", "api-hpa-uid", { "app.kubernetes.io/managed-by": "kuber", @@ -319,8 +338,7 @@ describe("server management service", () => { dependencies({ listDeployments: async () => ["api", "worker"].map( - (name) => - object("Deployment", name, `${name}-uid`) as V1Deployment, + (name) => object("Deployment", name, `${name}-uid`) as V1Deployment, ), listProjectResources: async () => [ object("ConfigMap", "env", "env-uid"), @@ -351,4 +369,16 @@ describe("server management service", () => { ).rejects.toThrow("reserved"); expect(read).toBe(false); }); + + test("allows the kuber-system workspace to manage itself", async () => { + const service = createManagementService( + dependencies({ + readNamespace: async () => ({ uid: "namespace-uid" }), + }), + ); + + await expect( + service.namespaceSafety({ project: "kuber-system", uid: "workspace-1" }), + ).resolves.toMatchObject({ project: "kuber-system", status: "external" }); + }); }); diff --git a/tests/server/materialize.test.ts b/tests/server/materialize.test.ts index 5104b0c..4223242 100644 --- a/tests/server/materialize.test.ts +++ b/tests/server/materialize.test.ts @@ -26,6 +26,44 @@ function digest(data: Uint8Array | string): Sha256Digest { } describe("source materialization", () => { + test("bounds concurrent CAS reads while materializing many files", async () => { + const root = await mkdtemp(join(tmpdir(), "kuber-materialize-")); + roots.push(root); + const data = Buffer.from("x"); + const blobDigest = digest(data); + const manifest: WorkspaceManifest = { + version: BUILD_PROTOCOL_VERSION, + files: Array.from({ length: 40 }, (_, index) => ({ + path: `file-${String(index).padStart(2, "0")}`, + type: "file" as const, + digest: blobDigest, + size: data.byteLength, + mode: 0o644 as const, + })), + }; + const manifestBytes = Buffer.from(JSON.stringify(manifest)); + const workspace = digest(manifestBytes); + let inflight = 0; + let maxInflight = 0; + const read = async (result: T): Promise => { + inflight += 1; + maxInflight = Math.max(maxInflight, inflight); + await Bun.sleep(2); + inflight -= 1; + return result; + }; + const cas = { + has: async () => read(true), + get: async (requested: Sha256Digest) => + requested === workspace ? manifestBytes : read(data), + }; + + await materializeWorkspace(cas, workspace, join(root, "workspace")); + + expect(maxInflight).toBeGreaterThan(1); + expect(maxInflight).toBeLessThanOrEqual(20); + }); + test("materializes files and safe symlinks with declared modes", async () => { const root = await mkdtemp(join(tmpdir(), "kuber-materialize-")); roots.push(root); diff --git a/tests/server/operation-store.test.ts b/tests/server/operation-store.test.ts index e22e3fc..dacc710 100644 --- a/tests/server/operation-store.test.ts +++ b/tests/server/operation-store.test.ts @@ -6,6 +6,7 @@ import { OperationValidationError, recoverStaleOperations, sanitizeOperationResult, + sanitizeOperationError, } from "../../server/operation-store"; describe("operation store", () => { @@ -92,10 +93,33 @@ describe("operation store", () => { "checkout https://git@example.com/org/repo.git", "AKIAIOSFODNN7EXAMPLE", ]), - ).toEqual([ - "checkout https://git@example.com/org/repo.git", - "[REDACTED]", - ]); + ).toEqual(["checkout https://git@example.com/org/repo.git", "[REDACTED]"]); + }); + + test("sanitizes operation failures while preserving their codes", () => { + expect( + sanitizeOperationError( + { + code: "PROVIDER_FAILED", + message: + 'database failed: DB_PASSWORD=database-password response={"data":{"token":"kube-secret"}}', + }, + "deploy", + ), + ).toEqual({ + code: "PROVIDER_FAILED", + message: + 'database failed: DB_PASSWORD=[REDACTED] response={"data":{"token":"[REDACTED]"}}', + }); + expect( + sanitizeOperationError( + { code: "RECONCILE_FAILED", message: "secret: db-password" }, + "databases.reconcile", + ), + ).toEqual({ + code: "RECONCILE_FAILED", + message: "Database reconciliation failed", + }); }); test("enforces the operation state machine", async () => { diff --git a/tests/server/trust-store.test.ts b/tests/server/trust-store.test.ts new file mode 100644 index 0000000..6bb40d3 --- /dev/null +++ b/tests/server/trust-store.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, test } from "bun:test"; +import type { KubernetesObjectApi } from "@kubernetes/client-node"; +import { KubernetesTrustStore } from "../../server/kubernetes-state"; +import { MemoryTrustStore } from "../../server/trust-store"; + +type ConfigMap = { + apiVersion: string; + kind: string; + metadata: { + name: string; + namespace?: string; + labels?: Record; + }; + data?: Record; +}; + +class FakeObjects { + readonly maps = new Map(); + readonly selectors: string[] = []; + + async create(value: ConfigMap): Promise { + this.maps.set(value.metadata.name, structuredClone(value)); + } + + async read(value: ConfigMap): Promise { + const found = this.maps.get(value.metadata.name); + if (!found) throw { code: 404 }; + return structuredClone(found); + } + + async delete(value: ConfigMap): Promise { + if (!this.maps.delete(value.metadata.name)) throw { code: 404 }; + } + + async list( + _apiVersion: string, + _kind: string, + _namespace?: string, + _pretty?: string, + _exact?: boolean, + _exportValue?: boolean, + _fieldSelector?: string, + labelSelector?: string, + ): Promise<{ items: ConfigMap[] }> { + this.selectors.push(labelSelector ?? ""); + const labels = Object.fromEntries( + (labelSelector ?? "") + .split(",") + .filter(Boolean) + .map((part) => part.split("=")), + ); + return { + items: [...this.maps.values()].filter((item) => + Object.entries(labels).every( + ([key, value]) => item.metadata.labels?.[key] === value, + ), + ), + }; + } +} + +describe("namespace trust store", () => { + test("rejects invalid namespace and fingerprint", async () => { + const store = new MemoryTrustStore(); + await expect(store.grant("Demo", "a".repeat(64))).rejects.toThrow(); + await expect(store.grant("a".repeat(64), "a".repeat(64))).rejects.toThrow(); + await expect(store.grant("demo", "bad")).rejects.toThrow(); + }); + + test("requires explicit registration for each CWD fingerprint", async () => { + const store = new MemoryTrustStore(); + const first = "a".repeat(64); + const second = "b".repeat(64); + await store.grant("demo", first); + expect(await store.has("demo", first)).toBe(true); + expect(await store.has("demo", second)).toBe(false); + await store.grant("demo", second); + expect(await store.list("demo")).toEqual([first, second]); + expect(await store.revoke("demo", first)).toBe(true); + expect(await store.has("demo", first)).toBe(false); + }); + + test("persists valid ConfigMap records and ignores malformed payloads", async () => { + const objects = new FakeObjects(); + const store = new KubernetesTrustStore( + objects as unknown as KubernetesObjectApi, + ); + const fingerprint = "a".repeat(64); + + await store.grant("demo", fingerprint); + const [record] = [...objects.maps.values()]; + expect(record).toMatchObject({ + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + namespace: "kuber-system", + labels: { + "kuber.astrxl.dev/type": "trust", + "kuber.astrxl.dev/workspace": "demo", + }, + }, + data: { payload: JSON.stringify({ project: "demo", fingerprint }) }, + }); + expect(record?.metadata.name).toMatch(/^trust-[a-f0-9]{48}$/); + expect(await store.list("demo")).toEqual([fingerprint]); + expect(objects.selectors).toContain( + "kuber.astrxl.dev/type=trust,kuber.astrxl.dev/workspace=demo", + ); + + objects.maps.set("trust-malformed", { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: "trust-malformed", + labels: { + "kuber.astrxl.dev/type": "trust", + "kuber.astrxl.dev/workspace": "demo", + }, + }, + data: { payload: "not-json" }, + }); + objects.maps.set("trust-wrong-project", { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: "trust-wrong-project", + labels: { + "kuber.astrxl.dev/type": "trust", + "kuber.astrxl.dev/workspace": "demo", + }, + }, + data: { + payload: JSON.stringify({ + project: "other", + fingerprint: "b".repeat(64), + }), + }, + }); + expect(await store.list("demo")).toEqual([fingerprint]); + + expect(await store.revoke("demo", fingerprint)).toBe(true); + expect(await store.has("demo", fingerprint)).toBe(false); + expect(await store.revoke("demo", fingerprint)).toBe(false); + }); +}); diff --git a/tests/server/workspace-store.test.ts b/tests/server/workspace-store.test.ts index c0e8ac2..72158d2 100644 --- a/tests/server/workspace-store.test.ts +++ b/tests/server/workspace-store.test.ts @@ -16,7 +16,6 @@ describe("workspace store", () => { "Upper", "has_dot", "kube-public", - "kuber-system", "database", "garage-system", "routing", @@ -25,6 +24,12 @@ describe("workspace store", () => { WorkspaceValidationError, ); } + + await expect( + store.create({ id: "kuber-system", source }), + ).resolves.toMatchObject({ + metadata: { name: "kuber-system" }, + }); }); test("creates immutable revisions and uses ETags for replacement", async () => {