astral/kuber
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add maintenance toggle command

Browse Source
This commit is contained in:
dmgnr
2026-09-18 15:21:28 +00:00 Unverified
parent e3df2f4d76
commit dcdacd5c04
12 changed files with 712 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app.ts
+66 -2
View File
@@ -72,6 +72,11 @@ import {
safeLog,
type ProcessLogEntry,
} from "../lib/request-log";
import {
MaintenanceBusyError,
normalizeMaintenanceHost,
type MaintenanceService,
} from "./maintenance";
const API_PREFIX = "/api/v2";
const RUNTIME_SESSION_MS = 24 * 60 * 60 * 1000;
@@ -103,6 +108,7 @@ export type AppOptions = {
builds?: BuildController;
logs?: LogService;
execService?: ExecService;
maintenance?: MaintenanceService;
resolveImage?: (
project: string,
service: string,
@@ -1792,6 +1798,57 @@ export function createApp(
return response(adopted);
}
match = new RegExp(`^${API_PREFIX}/maintenance/([^/]+)$`).exec(path);
if (match) {
const host = pathPart(match[1]!);
try {
normalizeMaintenanceHost(host);
} catch (error) {
throw new HttpError(
400,
"Invalid hostname",
"MAINTENANCE_HOST_INVALID",
error instanceof Error ? error.message : "Invalid hostname",
);
}
if (!options.maintenance)
throw new HttpError(
503,
"Service unavailable",
"MAINTENANCE_UNAVAILABLE",
"Maintenance service is not configured",
);
if (identity.apiKey?.workspace) {
await audit(identity, request, "authorization.workspace", "denied", {
workspace: identity.apiKey.workspace,
});
throw new HttpError(
403,
"Forbidden",
"MAINTENANCE_GLOBAL_SCOPE_REQUIRED",
"Maintenance requires an unscoped API key or an operator session",
);
}
if (request.method === "GET") {
await requireCapability(identity, request, "kubernetes:write");
return response(await options.maintenance.status(host));
}
if (request.method === "POST") {
await requireCapability(identity, request, "kubernetes:write");
const body = await readJson(request);
if (typeof body.enabled !== "boolean")
throw new HttpError(
400,
"Invalid request",
"MAINTENANCE_ENABLED_REQUIRED",
"enabled must be a boolean",
);
const result = await options.maintenance.set(host, body.enabled);
await audit(identity, request, "maintenance.toggle", "success", result);
return response(result);
}
}
match = new RegExp(`^${API_PREFIX}/workspaces/([^/]+)(?:/(.*))?$`).exec(
path,
);
@@ -2326,9 +2383,16 @@ export function createApp(
error instanceof WorkspaceConflictError ||
error instanceof OperationConflictError ||
error instanceof BuildConflictError ||
error instanceof WorkspaceAdoptionError
error instanceof WorkspaceAdoptionError ||
error instanceof MaintenanceBusyError
)
return new HttpError(409, "Conflict", error.code, error.message);
return new HttpError(
409,
"Conflict",
error.code,
error.message,
error instanceof MaintenanceBusyError ? { "retry-after": "1" } : undefined,
);
if (
error instanceof WorkspaceValidationError ||
error instanceof OperationValidationError ||
server/index.ts
+115
View File
@@ -11,6 +11,13 @@ import {
} from "./build-kubernetes";
import { FilesystemCas } from "./cas";
import { KubernetesAuthStore } from "./kubernetes-store";
import {
MAINTENANCE_ROUTE_NAME,
MAINTENANCE_STATE_NAME,
MAINTENANCE_STATE_NAMESPACE,
MaintenanceService,
} from "./maintenance";
import { PatchStrategy, type KubernetesObject } from "@kubernetes/client-node";
import {
createKubernetesClients,
createKubernetesLeaseObjects,
@@ -112,6 +119,107 @@ const leases = new KubernetesWorkspaceLeaseProvider(
createKubernetesLeaseObjects(clients.coordination),
namespace,
);
const maintenance = new MaintenanceService(
{
async readState() {
try {
const value = (await clients.objects.read({
apiVersion: "v1",
kind: "ConfigMap",
metadata: {
name: MAINTENANCE_STATE_NAME,
namespace: MAINTENANCE_STATE_NAMESPACE,
},
})) as { data?: Record<string, string> };
const hosts = JSON.parse(value.data?.hosts ?? "[]");
return Array.isArray(hosts) &&
hosts.every((host) => typeof host === "string")
? { hosts }
: { hosts: [] };
} catch (error) {
if (
error &&
typeof error === "object" &&
(("code" in error && error.code === 404) ||
("statusCode" in error && error.statusCode === 404))
)
return;
throw error;
}
},
async writeState({ hosts }) {
await clients.objects.patch(
{
apiVersion: "v1",
kind: "ConfigMap",
metadata: {
name: MAINTENANCE_STATE_NAME,
namespace: MAINTENANCE_STATE_NAMESPACE,
labels: { "kuber.astrxl.dev/type": "maintenance" },
},
data: { hosts: JSON.stringify(hosts) },
} as KubernetesObject,
undefined,
undefined,
"kuber-server",
true,
PatchStrategy.ServerSideApply,
);
},
async routeExists() {
try {
await clients.objects.read({
apiVersion: "traefik.io/v1alpha1",
kind: "IngressRoute",
metadata: { name: MAINTENANCE_ROUTE_NAME, namespace: "routing" },
});
return true;
} catch (error) {
if (
error &&
typeof error === "object" &&
(("code" in error && error.code === 404) ||
("statusCode" in error && error.statusCode === 404))
)
return false;
throw error;
}
},
async apply(resource) {
await clients.objects.patch(
resource as KubernetesObject,
undefined,
undefined,
"kuber-server",
true,
PatchStrategy.ServerSideApply,
);
},
async deleteRoute() {
try {
await clients.objects.delete({
apiVersion: "traefik.io/v1alpha1",
kind: "IngressRoute",
metadata: { name: MAINTENANCE_ROUTE_NAME, namespace: "routing" },
});
} catch (error) {
if (
!(
error &&
typeof error === "object" &&
(("code" in error && error.code === 404) ||
("statusCode" in error && error.statusCode === 404))
)
)
throw error;
}
},
},
new KubernetesWorkspaceLeaseProvider(
createKubernetesLeaseObjects(clients.coordination),
MAINTENANCE_STATE_NAMESPACE,
),
);
const builds = new BuildController({
cas: new FilesystemCas(`${dataRoot}/cas`),
store: buildStore,
@@ -177,6 +285,12 @@ try {
console.error("Startup operation recovery failed", error);
}
try {
await maintenance.reconcile();
} catch (error) {
console.error("Startup maintenance reconciliation failed", error);
}
const sessionCleanupIntervalMs = Number(
process.env.KUBER_SESSION_CLEANUP_MS ?? 10 * 60 * 1000,
);
@@ -218,6 +332,7 @@ const app = createApp({
builds,
logs,
execService,
maintenance,
leases,
resolveImage: async (project, service) => {
const image = buildImageName(registry, project, service);
server/maintenance.ts
+157
View File
@@ -0,0 +1,157 @@
import { randomUUID } from "node:crypto";
export const MAINTENANCE_NAMESPACE = "routing";
export const MAINTENANCE_STATE_NAMESPACE = "kuber-system";
export const MAINTENANCE_ROUTE_NAME = "maintenance-override";
export const MAINTENANCE_STATE_NAME = "maintenance-override";
export type MaintenanceStatus = {
host: string;
enabled: boolean;
hosts: string[];
};
export interface MaintenancePersistence {
readState(): Promise<{ hosts: string[] } | undefined>;
writeState(value: { hosts: string[] }): Promise<void>;
routeExists(): Promise<boolean>;
apply(resource: Record<string, unknown>): Promise<void>;
deleteRoute(): Promise<void>;
}
export interface MaintenanceLeaseProvider {
acquire(
name: string,
holder: string,
ttlMs?: number,
): Promise<{ release(): Promise<void> } | undefined>;
}
/** Accept DNS hostnames only: no URL syntax, port, address literals, or wildcards. */
export function normalizeMaintenanceHost(value: unknown): string {
if (typeof value !== "string") throw new Error("host is required");
const host = value.trim().toLowerCase().replace(/\.$/, "");
if (
host.length === 0 ||
host.length > 253 ||
!host.includes(".") ||
!/^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/.test(host)
)
throw new Error("host must be a DNS hostname");
return host;
}
export function maintenanceMiddleware(): Record<string, unknown> {
return {
apiVersion: "traefik.io/v1alpha1",
kind: "Middleware",
metadata: {
name: MAINTENANCE_ROUTE_NAME,
namespace: MAINTENANCE_NAMESPACE,
},
spec: { replacePathRegex: { regex: "^/.*", replacement: "/__error/1001" } },
};
}
export function maintenanceRoute(hosts: string[]): Record<string, unknown> {
return {
apiVersion: "traefik.io/v1alpha1",
kind: "IngressRoute",
metadata: {
name: MAINTENANCE_ROUTE_NAME,
namespace: MAINTENANCE_NAMESPACE,
},
spec: {
routes: [
{
kind: "Rule",
match: hosts.map((host) => `Host(\`${host}\`)`).join(" || "),
// This must override every workspace route for an affected host.
priority: 1_000_000,
middlewares: [
{ name: MAINTENANCE_ROUTE_NAME, namespace: MAINTENANCE_NAMESPACE },
],
services: [
{
name: "error-page",
namespace: MAINTENANCE_NAMESPACE,
port: 3000,
scheme: "http",
},
],
},
],
},
};
}
export class MaintenanceBusyError extends Error {
readonly code = "MAINTENANCE_BUSY";
}
export class MaintenanceService {
constructor(
private readonly persistence: MaintenancePersistence,
private readonly leases: MaintenanceLeaseProvider,
) {}
async status(host: string): Promise<MaintenanceStatus> {
const normalized = normalizeMaintenanceHost(host);
const hosts = this.hosts(await this.persistence.readState());
await this.reconcile(hosts);
return {
host: normalized,
enabled: hosts.includes(normalized) && (await this.persistence.routeExists()),
hosts,
};
}
async toggle(host: string): Promise<MaintenanceStatus> {
return this.update(host);
}
/** Setting the desired state makes a retried mutation safe after a timeout. */
async set(host: string, enabled: boolean): Promise<MaintenanceStatus> {
return this.update(host, enabled);
}
private async update(
host: string,
enabled?: boolean,
): Promise<MaintenanceStatus> {
const normalized = normalizeMaintenanceHost(host);
const lease = await this.leases.acquire(
"maintenance-override",
randomUUID(),
);
if (!lease) throw new MaintenanceBusyError("Maintenance state is busy; retry shortly");
try {
const current = this.hosts(await this.persistence.readState());
const targetEnabled = enabled ?? !current.includes(normalized);
const hosts = targetEnabled
? [...new Set([...current, normalized])].sort()
: current.filter((item) => item !== normalized);
await this.reconcile(hosts);
// Persist only after Traefik has accepted the desired override. If this
// write fails, the next reconciliation restores the prior desired state.
await this.persistence.writeState({ hosts });
return { host: normalized, enabled: targetEnabled, hosts };
} finally {
await lease.release();
}
}
async reconcile(hosts?: string[]): Promise<void> {
const desired = hosts ?? this.hosts(await this.persistence.readState());
await this.persistence.apply(maintenanceMiddleware());
if (desired.length) await this.persistence.apply(maintenanceRoute(desired));
else await this.persistence.deleteRoute();
}
private hosts(value: { hosts: string[] } | undefined): string[] {
if (!value || !Array.isArray(value.hosts)) return [];
return [
...new Set(value.hosts.map((host) => normalizeMaintenanceHost(host))),
].sort();
}
}