astral/kuber
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden self-managed reconciliation

Browse Source
This commit is contained in:
dmgnr
2026-09-05 10:17:55 +00:00 Unverified
parent 8e9d207915
commit 321f4e807a
40 changed files with 4977 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files
command/main.ts
+3 -1
View File
@@ -16,11 +16,12 @@ import { start } from "./start";
import { stop } from "./stop";
import { up } from "./up";
import { users } from "./users";
import { trust } from "./trust";
export const main = defineCommand({
meta: {
name: "kuber",
version: "2.1.0",
version: "2.2.0",
description: "Docker Compose -> K8s translation layer",
},
args: {
@@ -47,6 +48,7 @@ export const main = defineCommand({
stop,
up,
users,
trust,
whoami,
},
});
command/trust.ts
+111
View File
@@ -0,0 +1,111 @@
import { defineCommand } from "citty";
import { apiRequest, KuberApiError, type ApiRequestInit } from "../lib/api";
import { ctx } from "../lib/context";
import {
readTrust,
resolveTrustIdentity,
updateTrust,
type LocalTrustRecord,
} from "../lib/trust";
import type { WorkspaceTrustResponse } from "../shared/api";
function route(project: string) {
return `/workspaces/${encodeURIComponent(project)}/trust`;
}
type TrustApiRequest = <T>(path: string, init?: ApiRequestInit) => Promise<T>;
async function current() {
const { project, cwd } = ctx();
return resolveTrustIdentity(project, cwd);
}
export async function grantTrust(
identity: LocalTrustRecord,
request: TrustApiRequest = apiRequest,
): Promise<void> {
await request(route(identity.project), {
method: "POST",
json: { fingerprint: identity.fingerprint },
});
await updateTrust((records) => [
...records.filter(
(record) =>
record.project !== identity.project ||
record.fingerprint !== identity.fingerprint,
),
identity,
]);
console.log(`Trusted this directory for namespace ${identity.project}`);
}
export async function statusTrust(
identity: LocalTrustRecord,
request: TrustApiRequest = apiRequest,
): Promise<void> {
const local = (await readTrust()).some(
(record) =>
record.project === identity.project &&
record.fingerprint === identity.fingerprint,
);
const remote = await request<WorkspaceTrustResponse>(route(identity.project));
const registered = remote.fingerprints.includes(identity.fingerprint);
console.log(`Namespace: ${identity.project}`);
console.log(`Local: ${local ? "trusted" : "untrusted"}`);
console.log(`Server: ${registered ? "registered" : "not registered"}`);
}
export async function revokeTrust(
identity: LocalTrustRecord,
request: TrustApiRequest = apiRequest,
): Promise<void> {
let remoteError: unknown;
try {
await request<void>(
`${route(identity.project)}?fingerprint=${encodeURIComponent(identity.fingerprint)}`,
{ method: "DELETE" },
);
} catch (error) {
// A missing server record is already revoked; all other failures are
// reported after the local registration is removed.
if (!(error instanceof KuberApiError && error.status === 404))
remoteError = error;
}
await updateTrust((records) =>
records.filter(
(record) =>
record.project !== identity.project ||
record.fingerprint !== identity.fingerprint,
),
);
if (remoteError) {
const detail =
remoteError instanceof Error ? remoteError.message : String(remoteError);
throw new Error(
`Removed local trust for namespace ${identity.project}, but failed to revoke the server registration: ${detail}`,
{ cause: remoteError },
);
}
console.log(`Revoked trust for namespace ${identity.project}`);
}
export const trust = defineCommand({
meta: { name: "trust", description: "Trust this directory for kuber up" },
subCommands: {
status: defineCommand({
meta: { name: "status", description: "Show local and server trust" },
async run() {
await statusTrust(await current());
},
}),
revoke: defineCommand({
meta: { name: "revoke", description: "Revoke this directory trust" },
async run() {
await revokeTrust(await current());
},
}),
},
async run() {
await grantTrust(await current());
},
});
command/up.ts
+203 -4
View File
@@ -1,8 +1,14 @@
import { defineCommand } from "citty";
import { Listr } from "listr2";
import { randomUUID } from "node:crypto";
import type { ComposeSpecification } from "../schema/docker.d";
import type { KuberResource } from "../types";
import { apiRequest, KuberApiError, type ApiRequestInit } from "../lib/api";
import {
apiRequest,
KuberApiError,
type ApiRequestInit,
type ApiRequestOptions,
} from "../lib/api";
import {
buildServices,
getRepoRoot,
@@ -14,6 +20,11 @@ import { ctx } from "../lib/context";
import { getComposePostgresClaims } from "../lib/database";
import { getComposeS3Claims } from "../lib/storage";
import { enumerateWorkspace, type WorkspaceSnapshot } from "../lib/workspace";
import {
requireLocalTrust,
resolveTrustIdentity,
trustHeaders,
} from "../lib/trust";
type Workspace = {
metadata: { name: string; uid: string; resourceVersion: string };
@@ -33,6 +44,21 @@ type ResourcePlan = {
stale: ResourceIdentity[];
};
type OperationStatus = {
state?: unknown;
error?: { code?: unknown; message?: unknown };
};
type OperationResponse = {
operationId?: unknown;
operation?: { status?: OperationStatus };
};
export type OperationResumeOptions = {
now?: () => number;
sleep?: (milliseconds: number) => Promise<void>;
};
type UpContext = {
compose?: ComposeSpecification;
snapshot?: WorkspaceSnapshot;
@@ -43,6 +69,14 @@ type UpContext = {
};
export const WORKSPACE_ADOPTION_METHOD = "POST";
const OPERATION_RESUME_INITIAL_BACKOFF_MS = 250;
const OPERATION_RESUME_MAX_BACKOFF_MS = 5_000;
const OPERATION_RESUME_GRACE_MS = 60_000;
const RECOVERABLE_API_ERROR_CODES = new Set([
"HTTP_502",
"HTTP_503",
"HTTP_504",
]);
export function workspaceAdoptionRoute(project: string): string {
return `/workspaces/${encodeURIComponent(project)}/adopt`;
@@ -141,6 +175,150 @@ async function managementRequest<T>(
}
}
function operationIdFromResponse(response: unknown): string | undefined {
if (!response || typeof response !== "object") return;
const operationId = (response as OperationResponse).operationId;
return typeof operationId === "string" && operationId
? operationId
: undefined;
}
function operationStatusFromResponse(
response: unknown,
): OperationStatus | undefined {
if (!response || typeof response !== "object") return;
const status = (response as OperationResponse).operation?.status;
return status && typeof status === "object" ? status : undefined;
}
function operationFailure(
operationId: string,
status: OperationStatus,
): KuberApiError {
const error = status.error;
const cancelled = status.state === "cancelled";
const message =
typeof error?.message === "string"
? error.message
: cancelled
? "The operation was cancelled"
: "The operation failed";
const code =
typeof error?.code === "string"
? error.code
: cancelled
? "OPERATION_CANCELLED"
: "OPERATION_FAILED";
return new KuberApiError(message, cancelled ? 409 : 500, {
title: cancelled ? "Operation cancelled" : "Operation failed",
status: cancelled ? 409 : 500,
code,
operationId,
});
}
function isRecoverableConnectionInterruption(error: unknown): boolean {
if (error instanceof KuberApiError)
return RECOVERABLE_API_ERROR_CODES.has(error.code);
if (error instanceof DOMException && error.name === "AbortError")
return false;
if (error instanceof Error && error.name === "AbortError") return false;
return (
error instanceof TypeError ||
(error instanceof Error && error.name === "TimeoutError")
);
}
function isInterruptedOperation(status: OperationStatus): boolean {
return (
status.state === "failed" && status.error?.code === "OPERATION_INTERRUPTED"
);
}
function operationResumeDeadline(
rolloutTimeoutMs: number,
now: number,
): number {
return now + Math.max(rolloutTimeoutMs, 0) + OPERATION_RESUME_GRACE_MS;
}
async function resumeManagedOperation(
project: string,
request: ApiRequester,
path: string,
init: ApiRequestInit,
rolloutTimeoutMs: number,
options: OperationResumeOptions,
): Promise<void> {
const now = options.now ?? Date.now;
const sleep = options.sleep ?? ((milliseconds) => Bun.sleep(milliseconds));
const deadline = operationResumeDeadline(rolloutTimeoutMs, now());
const headers = new Headers(init.headers);
headers.set("idempotency-key", randomUUID());
let operationInit = { ...init, headers };
let operationId: string | undefined;
let backoffMs = OPERATION_RESUME_INITIAL_BACKOFF_MS;
let restartRetryPending = false;
for (;;) {
if (restartRetryPending && now() >= deadline)
throw new Error("Timed out while reconnecting to resume the operation");
try {
let status: OperationStatus | undefined;
if (operationId) {
const operation = await managementRequest<{ status: OperationStatus }>(
project,
request,
`/operations/${encodeURIComponent(operationId)}`,
{},
);
status = operation.status;
} else {
restartRetryPending = false;
const response = await managementRequest<OperationResponse>(
project,
request,
path,
operationInit,
);
operationId = operationIdFromResponse(response);
status = operationStatusFromResponse(response);
}
if (!operationId || !status || status.state === "succeeded") return;
if (isInterruptedOperation(status)) {
if (now() >= deadline) throw operationFailure(operationId, status);
operationId = undefined;
restartRetryPending = true;
headers.set("idempotency-key", randomUUID());
operationInit = { ...init, headers };
} else if (status.state === "failed" || status.state === "cancelled")
throw operationFailure(operationId, status);
} catch (error) {
if (
error instanceof KuberApiError &&
error.code === "OPERATION_INTERRUPTED"
) {
if (now() >= deadline) throw adoptionHint(project, error);
operationId = undefined;
restartRetryPending = true;
headers.set("idempotency-key", randomUUID());
operationInit = { ...init, headers };
} else {
if (!isRecoverableConnectionInterruption(error))
throw adoptionHint(project, error);
if (now() >= deadline) throw adoptionHint(project, error);
}
}
const remainingMs = deadline - now();
if (remainingMs <= 0)
throw new Error("Timed out while reconnecting to resume the operation");
await sleep(Math.min(backoffMs, remainingMs));
backoffMs = Math.min(backoffMs * 2, OPERATION_RESUME_MAX_BACKOFF_MS);
}
}
export async function reconcileResources(
project: string,
resources: KubernetesResource[],
@@ -149,6 +327,7 @@ export async function reconcileResources(
| ((resources: KubernetesResource[]) => void | Promise<void>)
| undefined,
request: ApiRequester = apiRequest,
resumeOptions: OperationResumeOptions = {},
): Promise<ResourcePlan> {
const workspacePath = `/workspaces/${encodeURIComponent(project)}`;
const plan = await managementRequest<ResourcePlan>(
@@ -157,7 +336,7 @@ export async function reconcileResources(
`${workspacePath}/resources/plan`,
{ method: "POST", json: { resources } },
);
await managementRequest(
await resumeManagedOperation(
project,
request,
`${workspacePath}/resources/apply`,
@@ -165,12 +344,14 @@ export async function reconcileResources(
method: "POST",
json: { resources: plan.desired },
},
rolloutTimeoutMs,
resumeOptions,
);
await postApply?.(plan.desired);
const deployments = getDeploymentNames(plan.desired);
if (deployments.length > 0) {
await managementRequest(
await resumeManagedOperation(
project,
request,
`${workspacePath}/resources/wait`,
@@ -178,10 +359,12 @@ export async function reconcileResources(
method: "POST",
json: { deployments, timeoutMs: rolloutTimeoutMs },
},
rolloutTimeoutMs,
resumeOptions,
);
}
if (plan.stale.length > 0) {
await managementRequest(
await resumeManagedOperation(
project,
request,
`${workspacePath}/resources/delete`,
@@ -189,6 +372,8 @@ export async function reconcileResources(
method: "POST",
json: { resources: plan.stale },
},
rolloutTimeoutMs,
resumeOptions,
);
}
return plan;
@@ -199,6 +384,20 @@ export async function runUp(
request: ApiRequester = apiRequest,
) {
const { project, compose, cwd, config, hookContext: getHookContext } = ctx();
const trusted = await requireLocalTrust(
await resolveTrustIdentity(project, cwd),
);
const baseRequest = request;
request = async <T>(
path: string,
init: ApiRequestInit = {},
options?: ApiRequestOptions,
) => {
const headers = new Headers(init.headers);
for (const [key, value] of Object.entries(trustHeaders(trusted)))
headers.set(key, value);
return baseRequest<T>(path, { ...init, headers }, options);
};
const workspacePath = `/workspaces/${encodeURIComponent(project)}`;
const taskCtx = await new Listr<UpContext>(