feat: add maintenance toggle command

.kuberrc.ts

@@ -206,7 +206,7 @@ export default {
           },
           {
             apiGroups: ["traefik.io"],
-            resources: ["ingressroutes"],
+            resources: ["ingressroutes", "middlewares"],
             verbs: [
               "get",
               "list",

README.md

@@ -83,8 +83,8 @@ days; `logout` revokes the server-side session. `readSession` falls back to the
 persistent file when no runtime session exists.
 
 The default login is scoped to the current user and the authenticated identity
-is available to every v2 command. `login`, `logout`, and `whoami` are the only
-commands that run without a loaded project configuration.
+is available to every v2 command. `login`, `logout`, `whoami`, and global
+`maintenance` are the only commands that run without a loaded project configuration.
 
 ### Roles and Authorization
 

command/main.ts

@@ -18,6 +18,7 @@ import { stop } from "./stop";
 import { up } from "./up";
 import { users } from "./users";
 import { trust } from "./trust";
+import { maintenance } from "./maintenance";
 
 export const main = defineCommand({
   meta: {
@@ -39,6 +40,7 @@ export const main = defineCommand({
     export: exportCommand,
     exec,
     logs,
+    maintenance,
     login,
     logout,
     operations,

command/maintenance.ts

@@ -0,0 +1,75 @@
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { defineCommand } from "citty";
+import { apiRequest, type ApiRequestInit } from "../lib/api";
+import type { MaintenanceStatus } from "../shared/api";
+
+type Request = <T>(path: string, init?: ApiRequestInit) => Promise<T>;
+type Confirmation = (host: string) => Promise<boolean>;
+const confirmationPath = join(
+  process.env.XDG_RUNTIME_DIR ?? "/tmp",
+  "kuber",
+  "maintenance-confirmation",
+);
+const TTL_MS = 60_000;
+
+async function confirmed(host: string): Promise<boolean> {
+  try {
+    const value = JSON.parse(await readFile(confirmationPath, "utf8")) as {
+      host?: string;
+      expiresAt?: number;
+    };
+    if (
+      value.host === host &&
+      typeof value.expiresAt === "number" &&
+      value.expiresAt > Date.now()
+    ) {
+      await rm(confirmationPath, { force: true });
+      return true;
+    }
+  } catch {}
+  await mkdir(join(confirmationPath, ".."), { recursive: true, mode: 0o700 });
+  await writeFile(
+    confirmationPath,
+    JSON.stringify({ host, expiresAt: Date.now() + TTL_MS }),
+    { mode: 0o600 },
+  );
+  return false;
+}
+
+export async function runMaintenance(
+  host: string,
+  request: Request = apiRequest,
+  confirm: Confirmation = confirmed,
+): Promise<MaintenanceStatus> {
+  const status = await request<MaintenanceStatus>(
+    `/maintenance/${encodeURIComponent(host)}`,
+  );
+  if (!(await confirm(status.host))) {
+    console.log(
+      `${status.host} maintenance is currently ${status.enabled ? "\x1b[31mON\x1b[0m" : "\x1b[32mOFF\x1b[0m"}, run this command again to toggle.`,
+    );
+    return status;
+  }
+  const result = await request<MaintenanceStatus>(
+    `/maintenance/${encodeURIComponent(status.host)}`,
+    { method: "POST", json: { enabled: !status.enabled } },
+  );
+  console.log(
+    `${result.host} maintenance is now ${result.enabled ? "\x1b[31mON\x1b[0m" : "\x1b[32mOFF\x1b[0m"}`,
+  );
+  return result;
+}
+
+export const maintenance = defineCommand({
+  meta: {
+    name: "maintenance",
+    description: "Toggle a global hostname maintenance override",
+  },
+  args: {
+    host: { type: "positional", required: true, description: "DNS hostname" },
+  },
+  async run({ args }) {
+    await runMaintenance(args.host);
+  },
+});

index.ts

@@ -11,7 +11,7 @@ async function run() {
   wrapCommandErrors(main);
   const cli = createMain(main);
   const { configPath, rawArgs } = extractConfigArgument(process.argv.slice(2));
-  const contextFree = new Set(["login", "logout", "whoami"]);
+  const contextFree = new Set(["login", "logout", "whoami", "maintenance"]);
   if (rawArgs[0] && contextFree.has(rawArgs[0])) {
     await cli({ rawArgs });
     return;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dmgnr/kuber",
-  "version": "2.4.0",
+  "version": "2.4.1",
   "description": "Docker Compose to Kubernetes translation layer",
   "bin": {
     "kuber": "dist/index.js"

server/app.ts

@@ -72,6 +72,11 @@ import {
   safeLog,
   type ProcessLogEntry,
 } from "../lib/request-log";
+import {
+  MaintenanceBusyError,
+  normalizeMaintenanceHost,
+  type MaintenanceService,
+} from "./maintenance";
 
 const API_PREFIX = "/api/v2";
 const RUNTIME_SESSION_MS = 24 * 60 * 60 * 1000;
@@ -103,6 +108,7 @@ export type AppOptions = {
   builds?: BuildController;
   logs?: LogService;
   execService?: ExecService;
+  maintenance?: MaintenanceService;
   resolveImage?: (
     project: string,
     service: string,
@@ -1792,6 +1798,57 @@ export function createApp(
       return response(adopted);
     }
 
+    match = new RegExp(`^${API_PREFIX}/maintenance/([^/]+)$`).exec(path);
+    if (match) {
+      const host = pathPart(match[1]!);
+      try {
+        normalizeMaintenanceHost(host);
+      } catch (error) {
+        throw new HttpError(
+          400,
+          "Invalid hostname",
+          "MAINTENANCE_HOST_INVALID",
+          error instanceof Error ? error.message : "Invalid hostname",
+        );
+      }
+      if (!options.maintenance)
+        throw new HttpError(
+          503,
+          "Service unavailable",
+          "MAINTENANCE_UNAVAILABLE",
+          "Maintenance service is not configured",
+        );
+      if (identity.apiKey?.workspace) {
+        await audit(identity, request, "authorization.workspace", "denied", {
+          workspace: identity.apiKey.workspace,
+        });
+        throw new HttpError(
+          403,
+          "Forbidden",
+          "MAINTENANCE_GLOBAL_SCOPE_REQUIRED",
+          "Maintenance requires an unscoped API key or an operator session",
+        );
+      }
+      if (request.method === "GET") {
+        await requireCapability(identity, request, "kubernetes:write");
+        return response(await options.maintenance.status(host));
+      }
+      if (request.method === "POST") {
+        await requireCapability(identity, request, "kubernetes:write");
+        const body = await readJson(request);
+        if (typeof body.enabled !== "boolean")
+          throw new HttpError(
+            400,
+            "Invalid request",
+            "MAINTENANCE_ENABLED_REQUIRED",
+            "enabled must be a boolean",
+          );
+        const result = await options.maintenance.set(host, body.enabled);
+        await audit(identity, request, "maintenance.toggle", "success", result);
+        return response(result);
+      }
+    }
+
     match = new RegExp(`^${API_PREFIX}/workspaces/([^/]+)(?:/(.*))?$`).exec(
       path,
     );
@@ -2326,9 +2383,16 @@ export function createApp(
       error instanceof WorkspaceConflictError ||
       error instanceof OperationConflictError ||
       error instanceof BuildConflictError ||
-      error instanceof WorkspaceAdoptionError
+      error instanceof WorkspaceAdoptionError ||
+      error instanceof MaintenanceBusyError
     )
-      return new HttpError(409, "Conflict", error.code, error.message);
+      return new HttpError(
+        409,
+        "Conflict",
+        error.code,
+        error.message,
+        error instanceof MaintenanceBusyError ? { "retry-after": "1" } : undefined,
+      );
     if (
       error instanceof WorkspaceValidationError ||
       error instanceof OperationValidationError ||

server/index.ts

@@ -11,6 +11,13 @@ import {
 } from "./build-kubernetes";
 import { FilesystemCas } from "./cas";
 import { KubernetesAuthStore } from "./kubernetes-store";
+import {
+  MAINTENANCE_ROUTE_NAME,
+  MAINTENANCE_STATE_NAME,
+  MAINTENANCE_STATE_NAMESPACE,
+  MaintenanceService,
+} from "./maintenance";
+import { PatchStrategy, type KubernetesObject } from "@kubernetes/client-node";
 import {
   createKubernetesClients,
   createKubernetesLeaseObjects,
@@ -112,6 +119,107 @@ const leases = new KubernetesWorkspaceLeaseProvider(
   createKubernetesLeaseObjects(clients.coordination),
   namespace,
 );
+const maintenance = new MaintenanceService(
+  {
+    async readState() {
+      try {
+        const value = (await clients.objects.read({
+          apiVersion: "v1",
+          kind: "ConfigMap",
+          metadata: {
+            name: MAINTENANCE_STATE_NAME,
+            namespace: MAINTENANCE_STATE_NAMESPACE,
+          },
+        })) as { data?: Record<string, string> };
+        const hosts = JSON.parse(value.data?.hosts ?? "[]");
+        return Array.isArray(hosts) &&
+          hosts.every((host) => typeof host === "string")
+          ? { hosts }
+          : { hosts: [] };
+      } catch (error) {
+        if (
+          error &&
+          typeof error === "object" &&
+          (("code" in error && error.code === 404) ||
+            ("statusCode" in error && error.statusCode === 404))
+        )
+          return;
+        throw error;
+      }
+    },
+    async writeState({ hosts }) {
+      await clients.objects.patch(
+        {
+          apiVersion: "v1",
+          kind: "ConfigMap",
+          metadata: {
+            name: MAINTENANCE_STATE_NAME,
+            namespace: MAINTENANCE_STATE_NAMESPACE,
+            labels: { "kuber.astrxl.dev/type": "maintenance" },
+          },
+          data: { hosts: JSON.stringify(hosts) },
+        } as KubernetesObject,
+        undefined,
+        undefined,
+        "kuber-server",
+        true,
+        PatchStrategy.ServerSideApply,
+      );
+    },
+    async routeExists() {
+      try {
+        await clients.objects.read({
+          apiVersion: "traefik.io/v1alpha1",
+          kind: "IngressRoute",
+          metadata: { name: MAINTENANCE_ROUTE_NAME, namespace: "routing" },
+        });
+        return true;
+      } catch (error) {
+        if (
+          error &&
+          typeof error === "object" &&
+          (("code" in error && error.code === 404) ||
+            ("statusCode" in error && error.statusCode === 404))
+        )
+          return false;
+        throw error;
+      }
+    },
+    async apply(resource) {
+      await clients.objects.patch(
+        resource as KubernetesObject,
+        undefined,
+        undefined,
+        "kuber-server",
+        true,
+        PatchStrategy.ServerSideApply,
+      );
+    },
+    async deleteRoute() {
+      try {
+        await clients.objects.delete({
+          apiVersion: "traefik.io/v1alpha1",
+          kind: "IngressRoute",
+          metadata: { name: MAINTENANCE_ROUTE_NAME, namespace: "routing" },
+        });
+      } catch (error) {
+        if (
+          !(
+            error &&
+            typeof error === "object" &&
+            (("code" in error && error.code === 404) ||
+              ("statusCode" in error && error.statusCode === 404))
+          )
+        )
+          throw error;
+      }
+    },
+  },
+  new KubernetesWorkspaceLeaseProvider(
+    createKubernetesLeaseObjects(clients.coordination),
+    MAINTENANCE_STATE_NAMESPACE,
+  ),
+);
 const builds = new BuildController({
   cas: new FilesystemCas(`${dataRoot}/cas`),
   store: buildStore,
@@ -177,6 +285,12 @@ try {
   console.error("Startup operation recovery failed", error);
 }
 
+try {
+  await maintenance.reconcile();
+} catch (error) {
+  console.error("Startup maintenance reconciliation failed", error);
+}
+
 const sessionCleanupIntervalMs = Number(
   process.env.KUBER_SESSION_CLEANUP_MS ?? 10 * 60 * 1000,
 );
@@ -218,6 +332,7 @@ const app = createApp({
   builds,
   logs,
   execService,
+  maintenance,
   leases,
   resolveImage: async (project, service) => {
     const image = buildImageName(registry, project, service);

server/maintenance.ts

@@ -0,0 +1,157 @@
+import { randomUUID } from "node:crypto";
+
+export const MAINTENANCE_NAMESPACE = "routing";
+export const MAINTENANCE_STATE_NAMESPACE = "kuber-system";
+export const MAINTENANCE_ROUTE_NAME = "maintenance-override";
+export const MAINTENANCE_STATE_NAME = "maintenance-override";
+
+export type MaintenanceStatus = {
+  host: string;
+  enabled: boolean;
+  hosts: string[];
+};
+
+export interface MaintenancePersistence {
+  readState(): Promise<{ hosts: string[] } | undefined>;
+  writeState(value: { hosts: string[] }): Promise<void>;
+  routeExists(): Promise<boolean>;
+  apply(resource: Record<string, unknown>): Promise<void>;
+  deleteRoute(): Promise<void>;
+}
+
+export interface MaintenanceLeaseProvider {
+  acquire(
+    name: string,
+    holder: string,
+    ttlMs?: number,
+  ): Promise<{ release(): Promise<void> } | undefined>;
+}
+
+/** Accept DNS hostnames only: no URL syntax, port, address literals, or wildcards. */
+export function normalizeMaintenanceHost(value: unknown): string {
+  if (typeof value !== "string") throw new Error("host is required");
+  const host = value.trim().toLowerCase().replace(/\.$/, "");
+  if (
+    host.length === 0 ||
+    host.length > 253 ||
+    !host.includes(".") ||
+    !/^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,63}$/.test(host)
+  )
+    throw new Error("host must be a DNS hostname");
+  return host;
+}
+
+export function maintenanceMiddleware(): Record<string, unknown> {
+  return {
+    apiVersion: "traefik.io/v1alpha1",
+    kind: "Middleware",
+    metadata: {
+      name: MAINTENANCE_ROUTE_NAME,
+      namespace: MAINTENANCE_NAMESPACE,
+    },
+    spec: { replacePathRegex: { regex: "^/.*", replacement: "/__error/1001" } },
+  };
+}
+
+export function maintenanceRoute(hosts: string[]): Record<string, unknown> {
+  return {
+    apiVersion: "traefik.io/v1alpha1",
+    kind: "IngressRoute",
+    metadata: {
+      name: MAINTENANCE_ROUTE_NAME,
+      namespace: MAINTENANCE_NAMESPACE,
+    },
+    spec: {
+      routes: [
+        {
+          kind: "Rule",
+          match: hosts.map((host) => `Host(\`${host}\`)`).join(" || "),
+          // This must override every workspace route for an affected host.
+          priority: 1_000_000,
+          middlewares: [
+            { name: MAINTENANCE_ROUTE_NAME, namespace: MAINTENANCE_NAMESPACE },
+          ],
+          services: [
+            {
+              name: "error-page",
+              namespace: MAINTENANCE_NAMESPACE,
+              port: 3000,
+              scheme: "http",
+            },
+          ],
+        },
+      ],
+    },
+  };
+}
+
+export class MaintenanceBusyError extends Error {
+  readonly code = "MAINTENANCE_BUSY";
+}
+
+export class MaintenanceService {
+  constructor(
+    private readonly persistence: MaintenancePersistence,
+    private readonly leases: MaintenanceLeaseProvider,
+  ) {}
+
+  async status(host: string): Promise<MaintenanceStatus> {
+    const normalized = normalizeMaintenanceHost(host);
+    const hosts = this.hosts(await this.persistence.readState());
+    await this.reconcile(hosts);
+    return {
+      host: normalized,
+      enabled: hosts.includes(normalized) && (await this.persistence.routeExists()),
+      hosts,
+    };
+  }
+
+  async toggle(host: string): Promise<MaintenanceStatus> {
+    return this.update(host);
+  }
+
+  /** Setting the desired state makes a retried mutation safe after a timeout. */
+  async set(host: string, enabled: boolean): Promise<MaintenanceStatus> {
+    return this.update(host, enabled);
+  }
+
+  private async update(
+    host: string,
+    enabled?: boolean,
+  ): Promise<MaintenanceStatus> {
+    const normalized = normalizeMaintenanceHost(host);
+    const lease = await this.leases.acquire(
+      "maintenance-override",
+      randomUUID(),
+    );
+    if (!lease) throw new MaintenanceBusyError("Maintenance state is busy; retry shortly");
+    try {
+      const current = this.hosts(await this.persistence.readState());
+      const targetEnabled = enabled ?? !current.includes(normalized);
+      const hosts = targetEnabled
+        ? [...new Set([...current, normalized])].sort()
+        : current.filter((item) => item !== normalized);
+      await this.reconcile(hosts);
+      // Persist only after Traefik has accepted the desired override. If this
+      // write fails, the next reconciliation restores the prior desired state.
+      await this.persistence.writeState({ hosts });
+      return { host: normalized, enabled: targetEnabled, hosts };
+    } finally {
+      await lease.release();
+    }
+  }
+
+  async reconcile(hosts?: string[]): Promise<void> {
+    const desired = hosts ?? this.hosts(await this.persistence.readState());
+    await this.persistence.apply(maintenanceMiddleware());
+    if (desired.length) await this.persistence.apply(maintenanceRoute(desired));
+    else await this.persistence.deleteRoute();
+  }
+
+  private hosts(value: { hosts: string[] } | undefined): string[] {
+    if (!value || !Array.isArray(value.hosts)) return [];
+    return [
+      ...new Set(value.hosts.map((host) => normalizeMaintenanceHost(host))),
+    ].sort();
+  }
+}

shared/api.ts

@@ -277,3 +277,8 @@ export type ListAuditEventsRequest = {
   until?: string;
 };
 export type ListAuditEventsResponse = Page<AuditEvent>;
+export type MaintenanceStatus = {
+  host: string;
+  enabled: boolean;
+  hosts: string[];
+};

tests/command/maintenance.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, spyOn, test } from "bun:test";
+import { runMaintenance } from "../../command/maintenance";
+import type { ApiRequestInit } from "../../lib/api";
+
+describe("maintenance command", () => {
+  test("first call reads status and asks for confirmation without mutation", async () => {
+    const calls: string[] = [];
+    const output = spyOn(console, "log").mockImplementation(() => {});
+    try {
+      await runMaintenance(
+        "a.example.com",
+        async <T>(path: string) => {
+          calls.push(path);
+          return { host: "a.example.com", enabled: false, hosts: [] } as T;
+        },
+        async () => false,
+      );
+      expect(calls).toEqual(["/maintenance/a.example.com"]);
+      expect(output).toHaveBeenCalledWith(
+        expect.stringContaining("currently \x1b[32mOFF\x1b[0m"),
+      );
+    } finally {
+      output.mockRestore();
+    }
+  });
+
+  test("confirmed call posts an idempotent desired state and prints result", async () => {
+    const calls: Array<{
+      path: string;
+      init?: { method?: string; json?: unknown };
+    }> = [];
+    const output = spyOn(console, "log").mockImplementation(() => {});
+    try {
+      await runMaintenance(
+        "a.example.com",
+        async <T>(path: string, init?: ApiRequestInit) => {
+          calls.push({ path, init });
+          return (
+            calls.length === 1
+              ? { host: "a.example.com", enabled: false, hosts: [] }
+              : {
+                  host: "a.example.com",
+                  enabled: true,
+                  hosts: ["a.example.com"],
+                }
+          ) as T;
+        },
+        async () => true,
+      );
+      expect(calls).toEqual([
+        { path: "/maintenance/a.example.com", init: undefined },
+        {
+          path: "/maintenance/a.example.com",
+          init: { method: "POST", json: { enabled: true } },
+        },
+      ]);
+      expect(output).toHaveBeenCalledWith(
+        "a.example.com maintenance is now \x1b[31mON\x1b[0m",
+      );
+    } finally {
+      output.mockRestore();
+    }
+  });
+});

tests/server/maintenance.test.ts

@@ -0,0 +1,223 @@
+import { describe, expect, test } from "bun:test";
+import {
+  MaintenanceService,
+  maintenanceMiddleware,
+  maintenanceRoute,
+  normalizeMaintenanceHost,
+  type MaintenancePersistence,
+} from "../../server/maintenance";
+import { createApp } from "../../server/app";
+import { hashToken, MemoryAuthStore } from "../../server/auth";
+
+class MemoryPersistence implements MaintenancePersistence {
+  state: { hosts: string[] } | undefined;
+  resources: Record<string, unknown>[] = [];
+  deleted = 0;
+  routePresent = false;
+  failApply = false;
+  failWrite = false;
+  async readState() {
+    return this.state;
+  }
+  async writeState(value: { hosts: string[] }) {
+    if (this.failWrite) throw new Error("state write failed");
+    this.state = value;
+  }
+  async routeExists() {
+    return this.routePresent;
+  }
+  async apply(resource: Record<string, unknown>) {
+    if (this.failApply) throw new Error("route apply failed");
+    this.resources.push(resource);
+    if (resource.kind === "IngressRoute") this.routePresent = true;
+  }
+  async deleteRoute() {
+    this.deleted += 1;
+    this.routePresent = false;
+  }
+}
+
+const lease = { acquire: async () => ({ release: async () => {} }) };
+
+describe("maintenance override", () => {
+  test("normalizes only DNS hostnames", () => {
+    expect(normalizeMaintenanceHost(" Sub.Domain.COM. ")).toBe(
+      "sub.domain.com",
+    );
+    for (const host of [
+      "http://example.com",
+      "example.com:443",
+      "127.0.0.1",
+      "[::1]",
+      "*.example.com",
+      "example",
+      "a..com",
+    ])
+      expect(() => normalizeMaintenanceHost(host)).toThrow();
+  });
+
+  test("enables, deduplicates, and disables the last host", async () => {
+    const persistence = new MemoryPersistence();
+    const service = new MaintenanceService(persistence, lease);
+    expect((await service.status("a.example.com")).enabled).toBe(false);
+    expect((await service.toggle("A.example.com")).hosts).toEqual([
+      "a.example.com",
+    ]);
+    persistence.state = {
+      hosts: ["a.example.com", "b.example.com", "A.example.com"],
+    };
+    expect((await service.toggle("a.example.com")).hosts).toEqual([
+      "b.example.com",
+    ]);
+    expect((await service.toggle("b.example.com")).hosts).toEqual([]);
+    expect(persistence.deleted).toBe(2);
+    expect(persistence.state).toEqual({ hosts: [] });
+  });
+
+  test("renders the single shared error route and rewrite middleware", () => {
+    expect(maintenanceMiddleware()).toMatchObject({
+      metadata: { name: "maintenance-override", namespace: "routing" },
+      spec: {
+        replacePathRegex: { regex: "^/.*", replacement: "/__error/1001" },
+      },
+    });
+    expect(maintenanceRoute(["a.example.com", "b.example.com"])).toMatchObject({
+      metadata: { name: "maintenance-override", namespace: "routing" },
+      spec: {
+        routes: [
+          {
+            match: "Host(`a.example.com`) || Host(`b.example.com`)",
+            priority: 1_000_000,
+            services: [
+              {
+                name: "error-page",
+                namespace: "routing",
+                port: 3000,
+                scheme: "http",
+              },
+            ],
+          },
+        ],
+      },
+    });
+  });
+
+  test("maps an unavailable global lease to a retryable API conflict", async () => {
+    const persistence = new MemoryPersistence();
+    const store = new MemoryAuthStore();
+    await store.putUser({ username: "operator", passwordHash: "hash", roles: ["operator"] });
+    await store.putSession({ tokenHash: hashToken("operator-token"), username: "operator", roles: ["operator"], expiresAt: "2099-01-01T00:00:00.000Z" });
+    const app = createApp({
+      store,
+      maintenance: new MaintenanceService(persistence, { acquire: async () => undefined }),
+    });
+    const result = await app(new Request("https://kuber.astrxl.dev/api/v2/maintenance/a.example.com", {
+      method: "POST",
+      headers: { authorization: "Bearer operator-token", "content-type": "application/json" },
+      body: JSON.stringify({ enabled: true }),
+    }));
+    expect(result.status).toBe(409);
+    expect(result.headers.get("retry-after")).toBe("1");
+    expect(await result.json()).toMatchObject({ code: "MAINTENANCE_BUSY" });
+    expect(persistence.state).toBeUndefined();
+  });
+
+  test("recovers the persisted desired state after route or state failures", async () => {
+    const persistence = new MemoryPersistence();
+    const service = new MaintenanceService(persistence, lease);
+    persistence.failApply = true;
+    await expect(service.set("a.example.com", true)).rejects.toThrow("route apply failed");
+    expect(persistence.state).toBeUndefined();
+    persistence.failApply = false;
+    expect(await service.status("a.example.com")).toMatchObject({ enabled: false });
+
+    persistence.failWrite = true;
+    await expect(service.set("a.example.com", true)).rejects.toThrow("state write failed");
+    expect(persistence.routePresent).toBe(true);
+    persistence.failWrite = false;
+    expect(await service.status("a.example.com")).toMatchObject({ enabled: false });
+    expect(persistence.routePresent).toBe(false);
+
+    expect(await service.set("a.example.com", true)).toMatchObject({ enabled: true });
+    persistence.routePresent = false;
+    expect(await service.status("a.example.com")).toMatchObject({ enabled: true });
+    expect(persistence.routePresent).toBe(true);
+  });
+
+  test("requires kubernetes write without workspace or trust context", async () => {
+    const store = new MemoryAuthStore();
+    await store.putUser({
+      username: "viewer",
+      passwordHash: "hash",
+      roles: ["viewer"],
+    });
+    await store.putUser({
+      username: "operator",
+      passwordHash: "hash",
+      roles: ["operator"],
+    });
+    for (const token of ["viewer-token", "operator-token"])
+      await store.putSession({
+        tokenHash: hashToken(token),
+        username: token.startsWith("viewer") ? "viewer" : "operator",
+        roles: token.startsWith("viewer") ? ["viewer"] : ["operator"],
+        expiresAt: "2099-01-01T00:00:00.000Z",
+      });
+    await store.createApiKey({
+      id: "maintenance-scoped-key",
+      tokenHash: hashToken("scoped-key"),
+      username: "operator",
+      capabilities: ["kubernetes:write"],
+      workspace: "shop",
+      expiresAt: "2099-01-01T00:00:00.000Z",
+    });
+    await store.createApiKey({
+      id: "maintenance-global-key",
+      tokenHash: hashToken("global-key"),
+      username: "operator",
+      capabilities: ["kubernetes:write"],
+      expiresAt: "2099-01-01T00:00:00.000Z",
+    });
+    const persistence = new MemoryPersistence();
+    const app = createApp({
+      store,
+      maintenance: new MaintenanceService(persistence, lease),
+    });
+    const viewer = await app(
+      new Request("https://kuber.astrxl.dev/api/v2/maintenance/a.example.com", {
+        headers: { authorization: "Bearer viewer-token" },
+      }),
+    );
+    expect(viewer.status).toBe(403);
+    const scoped = await app(
+      new Request("https://kuber.astrxl.dev/api/v2/maintenance/a.example.com", {
+        headers: { authorization: "Bearer scoped-key" },
+      }),
+    );
+    expect(scoped.status).toBe(403);
+    expect(await scoped.json()).toMatchObject({ code: "MAINTENANCE_GLOBAL_SCOPE_REQUIRED" });
+    const global = await app(
+      new Request("https://kuber.astrxl.dev/api/v2/maintenance/a.example.com", {
+        headers: { authorization: "Bearer global-key" },
+      }),
+    );
+    expect(global.status).toBe(200);
+    const status = await app(
+      new Request("https://kuber.astrxl.dev/api/v2/maintenance/a.example.com", {
+        headers: { authorization: "Bearer operator-token" },
+      }),
+    );
+    expect(await status.json()).toMatchObject({ enabled: false });
+    const toggle = await app(
+      new Request("https://kuber.astrxl.dev/api/v2/maintenance/a.example.com", {
+        method: "POST",
+        headers: {
+          authorization: "Bearer operator-token",
+          "content-type": "application/json",
+        },
+        body: JSON.stringify({ enabled: true }),
+      }),
+    );
+    expect(await toggle.json()).toMatchObject({ enabled: true });
+  });
+});