astral/kuber
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden self-managed reconciliation

Browse Source
This commit is contained in:
dmgnr
2026-09-05 10:17:55 +00:00 Unverified
parent 8e9d207915
commit 321f4e807a
40 changed files with 4977 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app.ts
+311 -66
View File
@@ -18,6 +18,7 @@ import {
type Role,
} from "./authorization";
import type { AuditStore } from "./audit-store";
import { validateTrust, type TrustStore } from "./trust-store";
import {
BuildConflictError,
BuildNotFoundError,
@@ -40,6 +41,7 @@ import {
OperationConflictError,
OperationNotFoundError,
OperationValidationError,
sanitizeOperationError,
sanitizeOperationResult,
type Operation,
type OperationStore,
@@ -55,6 +57,7 @@ import {
type Workspace,
type WorkspaceStore,
} from "./workspace-store";
import { redactString } from "./redact";
const API_PREFIX = "/api/v2";
const RUNTIME_SESSION_MS = 24 * 60 * 60 * 1000;
@@ -62,6 +65,8 @@ const PERSISTENT_SESSION_MS = 30 * 24 * 60 * 60 * 1000;
const LOGIN_WINDOW_MS = 5 * 60 * 1000;
const MAX_LOGIN_FAILURES = 5;
const DEFAULT_JSON_LIMIT = 1024 * 1024;
const WORKSPACE_LEASE_TTL_MS = 30_000;
const WORKSPACE_LEASE_RENEW_INTERVAL_MS = WORKSPACE_LEASE_TTL_MS / 3;
export interface ApiWorkspaceStore extends WorkspaceStore {
delete?(id: string): Promise<boolean>;
@@ -77,6 +82,7 @@ export type AppOptions = {
workspaceStore?: ApiWorkspaceStore;
operationStore?: OperationStore;
auditStore?: AuditStore;
trustStore?: TrustStore;
management?: ManagementService;
builds?: BuildController;
logs?: LogService;
@@ -240,7 +246,7 @@ export function createApp(
type: `https://kuber.astrxl.dev/problems/${error.code.toLowerCase()}`,
title: error.title,
status: error.status,
detail: error.message,
detail: redactString(error.message),
code: error.code,
requestId,
...(error.operationId && { operationId: error.operationId }),
@@ -508,6 +514,39 @@ export function createApp(
return options.management;
}
function requireTrustStore(): TrustStore {
if (!options.trustStore)
throw new HttpError(
503,
"Service unavailable",
"TRUST_STORE_UNAVAILABLE",
"Namespace trust service is unavailable. Contact your Kuber administrator.",
);
return options.trustStore;
}
async function requireTrust(
request: Request,
project: string,
): Promise<void> {
const fingerprint = request.headers.get("x-kuber-trust-fingerprint");
const suppliedProject = request.headers.get("x-kuber-trust-project");
if (!fingerprint || suppliedProject !== project)
throw new HttpError(
428,
"Trusted workspace required",
"TRUST_REQUIRED",
"This workspace reconciliation requires trust. Run kuber trust and retry.",
);
if (!(await requireTrustStore().has(project, fingerprint)))
throw new HttpError(
403,
"Trusted workspace rejected",
"TRUST_REQUIRED",
"This directory is not registered for this namespace.",
);
}
function requireAdoption(): WorkspaceAdoptionService {
const workspaceStore = options.workspaceStore;
const adoption =
@@ -556,6 +595,12 @@ export function createApp(
},
status: {
...operation.status,
...(operation.status.error && {
error: sanitizeOperationError(
operation.status.error,
operation.spec.action,
),
}),
...(operation.status.result !== undefined && {
result: sanitizeOperationResult(
operation.status.result,
@@ -568,15 +613,58 @@ export function createApp(
function operationBody(operation: Operation, result: unknown) {
const visible = publicOperation(operation);
return isRecord(result)
? { ...result, operationId: operation.metadata.name, operation: visible }
: Array.isArray(result)
const safeResult = sanitizeOperationResult(result, operation.spec.action);
return isRecord(safeResult)
? {
...safeResult,
operationId: operation.metadata.name,
operation: visible,
}
: Array.isArray(safeResult)
? {
deployments: result,
deployments: safeResult,
operationId: operation.metadata.name,
operation: visible,
}
: { result, operationId: operation.metadata.name, operation: visible };
: {
result: safeResult,
operationId: operation.metadata.name,
operation: visible,
};
}
async function transitionOperationToFailure(
operationId: string,
error: NonNullable<Operation["status"]["error"]>,
): Promise<Operation> {
try {
return await options.operationStore!.transition(operationId, "failed", {
error,
});
} catch (transitionError) {
if (!(transitionError instanceof OperationConflictError))
throw transitionError;
const latest = await options.operationStore!.get(operationId);
if (latest?.status.state === "failed") return latest;
throw transitionError;
}
}
function operationFailureError(operation: Operation): HttpError {
const failure = operation.status.error
? sanitizeOperationError(operation.status.error, operation.spec.action)
: { code: "OPERATION_FAILED", message: "Operation failed" };
return new HttpError(
failure.code === "WORKSPACE_BUSY" ||
failure.code === "WORKSPACE_LEASE_LOST"
? 409
: 500,
"Operation failed",
failure.code,
failure.message,
undefined,
operation.metadata.name,
);
}
async function runOperation(
@@ -585,7 +673,7 @@ export function createApp(
workspace: Workspace,
action: string,
input: unknown,
execute: () => Promise<unknown>,
execute: (signal: AbortSignal) => Promise<unknown>,
): Promise<Response> {
if (!options.operationStore)
throw new HttpError(
@@ -594,7 +682,7 @@ export function createApp(
"OPERATION_STORE_UNAVAILABLE",
"Operation storage is not configured",
);
const operation = await options.operationStore.create({
const { operation } = await options.operationStore.createOrReuse({
workspaceId: workspace.metadata.name,
workspaceUid: workspace.metadata.uid,
action,
@@ -602,14 +690,7 @@ export function createApp(
request: input,
});
if (operation.status.state === "failed")
throw new HttpError(
operation.status.error?.code === "WORKSPACE_BUSY" ? 409 : 500,
"Operation failed",
operation.status.error?.code ?? "OPERATION_FAILED",
operation.status.error?.message ?? "Operation failed",
undefined,
operation.metadata.name,
);
throw operationFailureError(operation);
if (operation.status.state === "cancelled")
throw new HttpError(
409,
@@ -640,16 +721,6 @@ export function createApp(
)
: undefined;
if (options.leases && !lease) {
await options.operationStore.transition(
operation.metadata.name,
"failed",
{
error: {
code: "WORKSPACE_BUSY",
message: "Another workspace operation is running",
},
},
);
throw new HttpError(
409,
"Conflict",
@@ -659,9 +730,76 @@ export function createApp(
operation.metadata.name,
);
}
await options.operationStore.transition(operation.metadata.name, "running");
if (!lease && options.leases) throw new Error("Unreachable lease state");
let operationStarted = false;
let leaseRenewalTimer: ReturnType<typeof setTimeout> | undefined;
let leaseOwnershipLost = false;
let renewalInFlight: Promise<boolean> | undefined;
const executionController = new AbortController();
const renewLease = async (): Promise<boolean> => {
if (!lease || leaseOwnershipLost) return false;
if (renewalInFlight) return renewalInFlight;
renewalInFlight = lease
.renew(WORKSPACE_LEASE_TTL_MS)
.catch(() => false)
.then((renewed) => {
if (!renewed) {
leaseOwnershipLost = true;
executionController.abort("Workspace lease ownership was lost");
}
return renewed;
})
.finally(() => {
renewalInFlight = undefined;
});
return renewalInFlight;
};
const scheduleLeaseRenewal = () => {
if (!lease || leaseOwnershipLost) return;
leaseRenewalTimer = setTimeout(() => {
void renewLease().finally(scheduleLeaseRenewal);
}, WORKSPACE_LEASE_RENEW_INTERVAL_MS);
};
const requireLeaseOwnership = async () => {
if (lease && !(await renewLease()))
throw new HttpError(
409,
"Conflict",
"WORKSPACE_LEASE_LOST",
"Workspace operation lease ownership was lost",
undefined,
operation.metadata.name,
);
};
try {
const result = await execute();
const claimed = await options.operationStore.claimExecution(
operation.metadata.name,
);
if (!claimed) {
const latest = await options.operationStore.get(
operation.metadata.name,
);
if (latest?.status.state === "succeeded")
return response(operationBody(latest, latest.status.result), 200, {
location: `${API_PREFIX}/operations/${operation.metadata.name}`,
});
if (latest?.status.state === "failed")
throw operationFailureError(latest);
return response(
{
operationId: operation.metadata.name,
operation: publicOperation(latest ?? operation),
},
202,
{ location: `${API_PREFIX}/operations/${operation.metadata.name}` },
);
}
operationStarted = true;
scheduleLeaseRenewal();
await requireLeaseOwnership();
const result = await execute(executionController.signal);
await requireLeaseOwnership();
const completed = await options.operationStore.transition(
operation.metadata.name,
"succeeded",
@@ -687,21 +825,30 @@ export function createApp(
location: `${API_PREFIX}/operations/${operation.metadata.name}`,
});
} catch (error) {
if (!operationStarted) throw error;
const message = error instanceof Error ? error.message : String(error);
await options.operationStore.transition(
const leaseLost =
leaseOwnershipLost ||
(error instanceof HttpError && error.code === "WORKSPACE_LEASE_LOST");
const failed = await transitionOperationToFailure(
operation.metadata.name,
"failed",
{
error: { code: "OPERATION_FAILED", message },
code: leaseLost ? "WORKSPACE_LEASE_LOST" : "OPERATION_FAILED",
message: leaseLost
? "Workspace operation lease ownership was lost"
: message,
},
);
const failure = failed.status.error;
const failureCode = failure?.code ?? "OPERATION_FAILED";
const failureMessage = failure?.message ?? message;
try {
await audit(
identity,
request,
action,
"failure",
{ error: message },
{ error: failureMessage },
workspace.metadata.name,
operation.metadata.name,
);
@@ -712,14 +859,17 @@ export function createApp(
);
}
throw new HttpError(
500,
"Operation failed",
"OPERATION_FAILED",
message,
failureCode === "WORKSPACE_LEASE_LOST" ? 409 : 500,
failureCode === "WORKSPACE_LEASE_LOST"
? "Conflict"
: "Operation failed",
failureCode,
failureMessage,
undefined,
operation.metadata.name,
);
} finally {
if (leaseRenewalTimer !== undefined) clearTimeout(leaseRenewalTimer);
try {
await lease?.release();
} catch (error) {
@@ -810,12 +960,89 @@ export function createApp(
return new Response(null, { status: 204 });
}
const trustMatch = new RegExp(
`^${API_PREFIX}/workspaces/([^/]+)/trust$`,
).exec(path);
if (trustMatch && options.trustStore) {
const project = pathPart(trustMatch[1]!);
if (request.method === "GET") {
await requireCapability(identity, request, "kubernetes:read");
return response({
fingerprints: await options.trustStore.list(project),
});
}
if (request.method === "POST") {
await requireCapability(identity, request, "kubernetes:write");
const body = await readJson(request);
if (typeof body.fingerprint !== "string")
throw new HttpError(
400,
"Invalid trust request",
"TRUST_INVALID",
"fingerprint is required",
);
try {
validateTrust(project, body.fingerprint);
} catch (error) {
throw new HttpError(
400,
"Invalid trust request",
"TRUST_INVALID",
error instanceof Error ? error.message : "Invalid trust request",
);
}
await options.trustStore.grant(project, body.fingerprint);
await audit(
identity,
request,
"trust.grant",
"success",
undefined,
project,
);
return new Response(null, { status: 204 });
}
if (request.method === "DELETE") {
await requireCapability(identity, request, "kubernetes:write");
const fingerprint = url.searchParams.get("fingerprint");
if (!fingerprint)
throw new HttpError(
400,
"Invalid trust request",
"TRUST_INVALID",
"fingerprint is required",
);
if (!(await options.trustStore.revoke(project, fingerprint)))
throw new HttpError(
404,
"Not found",
"TRUST_REGISTRATION_MISSING",
"Trust registration was not found",
);
await audit(
identity,
request,
"trust.revoke",
"success",
undefined,
project,
);
return new Response(null, { status: 204 });
}
}
if (path === `${API_PREFIX}/builds` && request.method === "POST") {
await requireCapability(identity, request, "kubernetes:write");
const body = await readJson(request);
if (typeof body.project !== "string" || !body.project)
throw new HttpError(
400,
"Invalid build request",
"BUILD_INVALID",
"project is required",
);
return response(
await requireBuilds().submitBuild(
(await readJson(request)) as unknown as BuildRequest,
),
await requireBuilds().submitBuild(body as unknown as BuildRequest),
202,
);
}
@@ -826,6 +1053,13 @@ export function createApp(
) {
await requireCapability(identity, request, "kubernetes:write");
const body = await readJson(request);
if (typeof body.project !== "string" || !body.project)
throw new HttpError(
400,
"Invalid snapshot request",
"BUILD_INVALID",
"project is required",
);
return response(
await requireBuilds().negotiateSnapshot(body.workspace as Sha256Digest),
);
@@ -886,6 +1120,14 @@ export function createApp(
).exec(path);
if (blobMatch) {
await requireCapability(identity, request, "kubernetes:write");
const project = url.searchParams.get("project");
if (!project)
throw new HttpError(
400,
"Invalid upload request",
"BUILD_INVALID",
"project is required",
);
const digest = `sha256:${blobMatch[2]!.toLowerCase()}` as Sha256Digest;
if (blobMatch[3] === "complete" && request.method === "POST")
return response(await requireBuilds().completeBlobUpload(digest));
@@ -1151,13 +1393,16 @@ export function createApp(
workspace,
"workspace.delete",
{ full: true },
async () => {
async (signal) => {
const result = options.management
? await options.management.down(
workspaceIdentity(workspace),
true,
{ signal },
)
: undefined;
if (signal.aborted)
throw new Error("Workspace operation execution was cancelled");
if (!requireWorkspaceStore().delete)
throw new Error("Workspace store does not support deletion");
await requireWorkspaceStore().delete!(id);
@@ -1274,24 +1519,30 @@ export function createApp(
workspace,
`workspace.${lifecycleAction}`,
body,
async () => {
async (signal) => {
const management = requireManagement();
const names = Array.isArray(body.services)
? (body.services as string[])
: undefined;
if (lifecycleAction === "stop")
return management.stop(workspaceIdentity(workspace), names);
return management.stop(workspaceIdentity(workspace), names, {
signal,
});
if (lifecycleAction === "restart")
return management.restart(workspaceIdentity(workspace), names);
return management.restart(workspaceIdentity(workspace), names, {
signal,
});
if (lifecycleAction === "rollback")
return management.rollback(
workspaceIdentity(workspace),
names,
typeof body.timeoutMs === "number" ? body.timeoutMs : undefined,
{ signal },
);
return management.down(
workspaceIdentity(workspace),
body.full === true,
{ signal },
);
},
);
@@ -1322,13 +1573,14 @@ export function createApp(
) {
await requireCapability(identity, request, "kubernetes:write");
const body = await readJson(request);
if (subpath === "resources/apply") await requireTrust(request, id);
return runOperation(
identity,
request,
workspace,
subpath!,
body,
async () => {
async (signal) => {
const management = requireManagement();
if (subpath === "resources/apply") {
if (!Array.isArray(body.resources))
@@ -1336,6 +1588,7 @@ export function createApp(
return management.applyResources(
workspaceIdentity(workspace),
body.resources as KubernetesObject[],
{ signal },
);
}
if (subpath === "resources/wait") {
@@ -1345,6 +1598,7 @@ export function createApp(
workspaceIdentity(workspace),
body.deployments as string[],
typeof body.timeoutMs === "number" ? body.timeoutMs : undefined,
{ signal },
);
return { ready: true };
}
@@ -1353,6 +1607,7 @@ export function createApp(
await management.deleteResources(
workspaceIdentity(workspace),
body.resources as unknown as ResourceIdentity[],
{ signal },
);
return { deleted: body.resources.length };
},
@@ -1369,10 +1624,11 @@ export function createApp(
workspace,
"databases.reconcile",
body,
() =>
(signal) =>
requireManagement().reconcileDatabases(
workspaceIdentity(workspace),
compose,
{ signal },
),
);
}
@@ -1429,10 +1685,11 @@ export function createApp(
workspace,
"storage.reconcile",
body,
() =>
(signal) =>
requireManagement().reconcileStorage(
workspaceIdentity(workspace),
compose,
{ signal },
),
);
}
@@ -1634,9 +1891,7 @@ export function createApp(
};
}
const EXEC_UPGRADE_PATH = new RegExp(
`^${API_PREFIX}/workspaces/([^/]+)/exec$`,
);
const EXEC_UPGRADE_PATH = new RegExp(`^${API_PREFIX}/workspaces/([^/]+)/exec$`);
export function execUpgradeMatch(url: URL): string | undefined {
const match = EXEC_UPGRADE_PATH.exec(url.pathname);
@@ -1709,10 +1964,7 @@ export async function authorizeExecConnection(
return { identity, workspace };
}
export function execProblem(
error: unknown,
requestId = "",
): Response {
export function execProblem(error: unknown, requestId = ""): Response {
let httpError: HttpError;
if (error instanceof HttpError) httpError = error;
else {
@@ -1782,9 +2034,7 @@ const DEFAULT_EXEC_MAX_FRAME_BYTES = 64 * 1024;
export class WireExecSession {
private readonly controller = new AbortController();
private session?: Awaited<
ReturnType<ExecService["openInteractive"]>
>;
private session?: Awaited<ReturnType<ExecService["openInteractive"]>>;
private readonly maxFrameBytes: number;
private started = false;
private done: Promise<void> = Promise.resolve();
@@ -1796,8 +2046,7 @@ export class WireExecSession {
private readonly connection: ExecConnection,
options: ExecLinkOptions = {},
) {
this.maxFrameBytes =
options.maxFrameBytes ?? DEFAULT_EXEC_MAX_FRAME_BYTES;
this.maxFrameBytes = options.maxFrameBytes ?? DEFAULT_EXEC_MAX_FRAME_BYTES;
}
private send(frame: ExecServerWireFrame) {
@@ -1851,14 +2100,10 @@ export class WireExecSession {
private toClientFrame(frame: ExecClientWireFrame): ExecClientFrame {
switch (frame.type) {
case "stdin": {
if (
frame.encoding !== "base64" ||
typeof frame.data !== "string"
)
if (frame.encoding !== "base64" || typeof frame.data !== "string")
throw new Error("Invalid stdin frame");
const data = base64ToBytes(frame.data);
if (!data)
throw new Error("Invalid stdin frame");
if (!data) throw new Error("Invalid stdin frame");
return {
type: "stdin",
data,
@@ -1946,7 +2191,7 @@ export class WireExecSession {
error instanceof Error &&
"code" in error &&
typeof (error as { code?: unknown }).code === "string"
? ((error as { code: string }).code)
? (error as { code: string }).code
: "EXEC_FAILED",
message:
error instanceof Error ? error.message : "Exec failed to start",
@@ -1985,7 +2230,7 @@ export class WireExecSession {
error instanceof Error &&
"code" in error &&
typeof (error as { code?: unknown }).code === "string"
? ((error as { code: string }).code)
? (error as { code: string }).code
: "EXEC_FAILED";
this.send({
type: "error",