fix: stabilize live deployment progress
This commit is contained in:
+1
-1
@@ -22,7 +22,7 @@ import { trust } from "./trust";
|
|||||||
export const main = defineCommand({
|
export const main = defineCommand({
|
||||||
meta: {
|
meta: {
|
||||||
name: "kuber",
|
name: "kuber",
|
||||||
version: "2.3.0",
|
version: "2.3.1",
|
||||||
description: "Docker Compose -> K8s translation layer",
|
description: "Docker Compose -> K8s translation layer",
|
||||||
},
|
},
|
||||||
args: {
|
args: {
|
||||||
|
|||||||
+98
-36
@@ -73,6 +73,7 @@ export type OperationResumeOptions = {
|
|||||||
now?: () => number;
|
now?: () => number;
|
||||||
sleep?: (milliseconds: number) => Promise<void>;
|
sleep?: (milliseconds: number) => Promise<void>;
|
||||||
onEvent?: (event: ResourceOperationEvent) => void;
|
onEvent?: (event: ResourceOperationEvent) => void;
|
||||||
|
signal?: AbortSignal;
|
||||||
};
|
};
|
||||||
|
|
||||||
type OperationEventsResponse = {
|
type OperationEventsResponse = {
|
||||||
@@ -335,6 +336,41 @@ function operationResumeDeadline(
|
|||||||
return now + Math.max(rolloutTimeoutMs, 0) + OPERATION_RESUME_GRACE_MS;
|
return now + Math.max(rolloutTimeoutMs, 0) + OPERATION_RESUME_GRACE_MS;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function abortReason(signal: AbortSignal): unknown {
|
||||||
|
return (
|
||||||
|
signal.reason ?? new DOMException("The operation was aborted", "AbortError")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||||
|
if (signal?.aborted) throw abortReason(signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortableDelay(
|
||||||
|
milliseconds: number,
|
||||||
|
signal: AbortSignal | undefined,
|
||||||
|
): Promise<void> {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
return new Promise<void>((resolve, reject) => {
|
||||||
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
const cleanup = () => {
|
||||||
|
if (timeout !== undefined) clearTimeout(timeout);
|
||||||
|
signal?.removeEventListener("abort", abort);
|
||||||
|
};
|
||||||
|
const complete = () => {
|
||||||
|
cleanup();
|
||||||
|
resolve();
|
||||||
|
};
|
||||||
|
const abort = () => {
|
||||||
|
cleanup();
|
||||||
|
reject(abortReason(signal!));
|
||||||
|
};
|
||||||
|
timeout = setTimeout(complete, milliseconds);
|
||||||
|
signal?.addEventListener("abort", abort, { once: true });
|
||||||
|
if (signal?.aborted) abort();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function resumeManagedOperation(
|
async function resumeManagedOperation(
|
||||||
project: string,
|
project: string,
|
||||||
request: ApiRequester,
|
request: ApiRequester,
|
||||||
@@ -344,18 +380,22 @@ async function resumeManagedOperation(
|
|||||||
options: OperationResumeOptions,
|
options: OperationResumeOptions,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const now = options.now ?? Date.now;
|
const now = options.now ?? Date.now;
|
||||||
const sleep = options.sleep ?? ((milliseconds) => Bun.sleep(milliseconds));
|
|
||||||
const deadline = operationResumeDeadline(rolloutTimeoutMs, now());
|
const deadline = operationResumeDeadline(rolloutTimeoutMs, now());
|
||||||
const headers = new Headers(init.headers);
|
const headers = new Headers(init.headers);
|
||||||
headers.set("idempotency-key", randomUUID());
|
headers.set("idempotency-key", randomUUID());
|
||||||
headers.set("prefer", "respond-async");
|
headers.set("prefer", "respond-async");
|
||||||
let operationInit = { ...init, headers };
|
let operationInit = {
|
||||||
|
...init,
|
||||||
|
headers,
|
||||||
|
signal: options.signal ?? init.signal,
|
||||||
|
};
|
||||||
let operationId: string | undefined;
|
let operationId: string | undefined;
|
||||||
let backoffMs = OPERATION_RESUME_INITIAL_BACKOFF_MS;
|
let backoffMs = OPERATION_RESUME_INITIAL_BACKOFF_MS;
|
||||||
let restartRetryPending = false;
|
let restartRetryPending = false;
|
||||||
let eventCursor = 0;
|
let eventCursor = 0;
|
||||||
|
|
||||||
for (;;) {
|
for (;;) {
|
||||||
|
throwIfAborted(options.signal);
|
||||||
if (restartRetryPending && now() >= deadline)
|
if (restartRetryPending && now() >= deadline)
|
||||||
throw new Error("Timed out while reconnecting to resume the operation");
|
throw new Error("Timed out while reconnecting to resume the operation");
|
||||||
try {
|
try {
|
||||||
@@ -365,7 +405,7 @@ async function resumeManagedOperation(
|
|||||||
project,
|
project,
|
||||||
request,
|
request,
|
||||||
`/operations/${encodeURIComponent(operationId)}`,
|
`/operations/${encodeURIComponent(operationId)}`,
|
||||||
{},
|
{ signal: options.signal },
|
||||||
);
|
);
|
||||||
status = operation.status;
|
status = operation.status;
|
||||||
} else {
|
} else {
|
||||||
@@ -386,7 +426,7 @@ async function resumeManagedOperation(
|
|||||||
project,
|
project,
|
||||||
request,
|
request,
|
||||||
`/operations/${encodeURIComponent(operationId)}/events?after=${eventCursor}`,
|
`/operations/${encodeURIComponent(operationId)}/events?after=${eventCursor}`,
|
||||||
{},
|
{ signal: options.signal },
|
||||||
);
|
);
|
||||||
if (hasOperationEventCursorGap(events, eventCursor))
|
if (hasOperationEventCursorGap(events, eventCursor))
|
||||||
throw new OperationProgressCursorGapError();
|
throw new OperationProgressCursorGapError();
|
||||||
@@ -402,6 +442,7 @@ async function resumeManagedOperation(
|
|||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof OperationProgressCursorGapError) throw error;
|
if (error instanceof OperationProgressCursorGapError) throw error;
|
||||||
|
throwIfAborted(options.signal);
|
||||||
// Event polling is additive; never delay resumable operation status polling.
|
// Event polling is additive; never delay resumable operation status polling.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -413,10 +454,15 @@ async function resumeManagedOperation(
|
|||||||
eventCursor = 0;
|
eventCursor = 0;
|
||||||
restartRetryPending = true;
|
restartRetryPending = true;
|
||||||
headers.set("idempotency-key", randomUUID());
|
headers.set("idempotency-key", randomUUID());
|
||||||
operationInit = { ...init, headers };
|
operationInit = {
|
||||||
|
...init,
|
||||||
|
headers,
|
||||||
|
signal: options.signal ?? init.signal,
|
||||||
|
};
|
||||||
} else if (status.state === "failed" || status.state === "cancelled")
|
} else if (status.state === "failed" || status.state === "cancelled")
|
||||||
throw operationFailure(operationId, status);
|
throw operationFailure(operationId, status);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
throwIfAborted(options.signal);
|
||||||
if (
|
if (
|
||||||
error instanceof KuberApiError &&
|
error instanceof KuberApiError &&
|
||||||
error.code === "OPERATION_INTERRUPTED"
|
error.code === "OPERATION_INTERRUPTED"
|
||||||
@@ -426,7 +472,11 @@ async function resumeManagedOperation(
|
|||||||
eventCursor = 0;
|
eventCursor = 0;
|
||||||
restartRetryPending = true;
|
restartRetryPending = true;
|
||||||
headers.set("idempotency-key", randomUUID());
|
headers.set("idempotency-key", randomUUID());
|
||||||
operationInit = { ...init, headers };
|
operationInit = {
|
||||||
|
...init,
|
||||||
|
headers,
|
||||||
|
signal: options.signal ?? init.signal,
|
||||||
|
};
|
||||||
} else {
|
} else {
|
||||||
if (!isRecoverableConnectionInterruption(error))
|
if (!isRecoverableConnectionInterruption(error))
|
||||||
throw adoptionHint(project, error);
|
throw adoptionHint(project, error);
|
||||||
@@ -437,7 +487,12 @@ async function resumeManagedOperation(
|
|||||||
const remainingMs = deadline - now();
|
const remainingMs = deadline - now();
|
||||||
if (remainingMs <= 0)
|
if (remainingMs <= 0)
|
||||||
throw new Error("Timed out while reconnecting to resume the operation");
|
throw new Error("Timed out while reconnecting to resume the operation");
|
||||||
await sleep(Math.min(backoffMs, remainingMs));
|
const delayMs = Math.min(backoffMs, remainingMs);
|
||||||
|
if (options.sleep) {
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
await options.sleep(delayMs);
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
} else await abortableDelay(delayMs, options.signal);
|
||||||
backoffMs = Math.min(backoffMs * 2, OPERATION_RESUME_MAX_BACKOFF_MS);
|
backoffMs = Math.min(backoffMs * 2, OPERATION_RESUME_MAX_BACKOFF_MS);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -561,52 +616,59 @@ function resourceOperationTaskTitle(
|
|||||||
return `${action} ${resource.kind}/${resource.name}`;
|
return `${action} ${resource.kind}/${resource.name}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runLiveResourceOperation(
|
export function runLiveResourceOperation(
|
||||||
task: ListrTaskWrapper<any, any, any>,
|
task: ListrTaskWrapper<any, any, any>,
|
||||||
phase: ResourceOperationEvent["data"]["phase"],
|
phase: ResourceOperationEvent["data"]["phase"],
|
||||||
targets: ResourceOperationTarget[],
|
targets: ResourceOperationTarget[],
|
||||||
operation: (
|
operation: (
|
||||||
onEvent: (event: ResourceOperationEvent) => void,
|
onEvent: (event: ResourceOperationEvent) => void,
|
||||||
|
signal: AbortSignal,
|
||||||
) => Promise<void>,
|
) => Promise<void>,
|
||||||
options: LiveResourceOperationOptions = {},
|
options: LiveResourceOperationOptions = {},
|
||||||
): Promise<void> {
|
): Listr<ResourceOperationTarget, any, any> | Promise<void> {
|
||||||
if (targets.length === 0) return operation(() => {});
|
if (targets.length === 0) return operation(() => {}, task.signal);
|
||||||
const active = new Map<
|
const active = new Map<
|
||||||
string,
|
string,
|
||||||
ListrTaskWrapper<ResourceOperationTarget, any, any>
|
ListrTaskWrapper<ResourceOperationTarget, any, any>
|
||||||
>();
|
>();
|
||||||
const complete: Array<() => void> = [];
|
const complete = new Set<() => void>();
|
||||||
let started = 0;
|
let started = 0;
|
||||||
let markStarted!: () => void;
|
return task.newListr<ResourceOperationTarget>(
|
||||||
const ready = new Promise<void>((resolve) => {
|
|
||||||
markStarted = resolve;
|
|
||||||
});
|
|
||||||
const children = task.newListr<ResourceOperationTarget>(
|
|
||||||
targets.map((target) => ({
|
targets.map((target) => ({
|
||||||
title: resourceOperationTaskTitle(phase, target),
|
title: resourceOperationTaskTitle(phase, target),
|
||||||
task: (_target, child) => {
|
task: async (_target, child) => {
|
||||||
active.set(resourceOperationKey(target), child);
|
active.set(resourceOperationKey(target), child);
|
||||||
options.onTaskStarted?.(target, child);
|
options.onTaskStarted?.(target, child);
|
||||||
started += 1;
|
started += 1;
|
||||||
if (started === targets.length) markStarted();
|
if (started !== targets.length)
|
||||||
return new Promise<void>((resolve) => complete.push(resolve));
|
return new Promise<void>((resolve) => {
|
||||||
},
|
const completeTask = () => {
|
||||||
})),
|
task.signal.removeEventListener("abort", completeTask);
|
||||||
);
|
complete.delete(completeTask);
|
||||||
const childRun = children.run();
|
resolve();
|
||||||
await ready;
|
};
|
||||||
|
complete.add(completeTask);
|
||||||
|
if (task.signal.aborted) completeTask();
|
||||||
|
else
|
||||||
|
task.signal.addEventListener("abort", completeTask, {
|
||||||
|
once: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
try {
|
try {
|
||||||
await operation((event) => {
|
await operation((event) => {
|
||||||
const resource = event.data.resource;
|
const resource = event.data.resource;
|
||||||
const child = active.get(resourceOperationKey(resource));
|
const activeTask = active.get(resourceOperationKey(resource));
|
||||||
if (!child) return;
|
if (!activeTask) return;
|
||||||
child.output = event.data.state;
|
activeTask.output = event.data.state;
|
||||||
task.output = resourceOperationEventTitle(event);
|
task.output = resourceOperationEventTitle(event);
|
||||||
});
|
}, task.signal);
|
||||||
} finally {
|
} finally {
|
||||||
for (const resolve of complete) resolve();
|
for (const resolve of [...complete]) resolve();
|
||||||
await childRun;
|
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
{ concurrent: true },
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runUp(
|
export async function runUp(
|
||||||
@@ -801,14 +863,14 @@ export async function runUp(
|
|||||||
phaseTask,
|
phaseTask,
|
||||||
"apply",
|
"apply",
|
||||||
desired,
|
desired,
|
||||||
(onEvent) =>
|
(onEvent, signal) =>
|
||||||
resumeManagedOperation(
|
resumeManagedOperation(
|
||||||
project,
|
project,
|
||||||
request,
|
request,
|
||||||
`${resourcePath}/apply`,
|
`${resourcePath}/apply`,
|
||||||
{ method: "POST", json: { resources: plan.desired } },
|
{ method: "POST", json: { resources: plan.desired } },
|
||||||
config.rolloutTimeoutMs,
|
config.rolloutTimeoutMs,
|
||||||
{ onEvent },
|
{ onEvent, signal },
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@@ -829,7 +891,7 @@ export async function runUp(
|
|||||||
phaseTask,
|
phaseTask,
|
||||||
"wait",
|
"wait",
|
||||||
deployments,
|
deployments,
|
||||||
(onEvent) =>
|
(onEvent, signal) =>
|
||||||
resumeManagedOperation(
|
resumeManagedOperation(
|
||||||
project,
|
project,
|
||||||
request,
|
request,
|
||||||
@@ -844,7 +906,7 @@ export async function runUp(
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
config.rolloutTimeoutMs,
|
config.rolloutTimeoutMs,
|
||||||
{ onEvent },
|
{ onEvent, signal },
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@@ -856,14 +918,14 @@ export async function runUp(
|
|||||||
phaseTask,
|
phaseTask,
|
||||||
"delete",
|
"delete",
|
||||||
stale,
|
stale,
|
||||||
(onEvent) =>
|
(onEvent, signal) =>
|
||||||
resumeManagedOperation(
|
resumeManagedOperation(
|
||||||
project,
|
project,
|
||||||
request,
|
request,
|
||||||
`${resourcePath}/delete`,
|
`${resourcePath}/delete`,
|
||||||
{ method: "POST", json: { resources: plan.stale } },
|
{ method: "POST", json: { resources: plan.stale } },
|
||||||
config.rolloutTimeoutMs,
|
config.rolloutTimeoutMs,
|
||||||
{ onEvent },
|
{ onEvent, signal },
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@dmgnr/kuber",
|
"name": "@dmgnr/kuber",
|
||||||
"version": "2.3.0",
|
"version": "2.3.1",
|
||||||
"description": "Docker Compose to Kubernetes translation layer",
|
"description": "Docker Compose to Kubernetes translation layer",
|
||||||
"bin": {
|
"bin": {
|
||||||
"kuber": "dist/index.js"
|
"kuber": "dist/index.js"
|
||||||
|
|||||||
+108
-1
@@ -1,5 +1,11 @@
|
|||||||
import { expect, spyOn, test } from "bun:test";
|
import { expect, spyOn, test } from "bun:test";
|
||||||
import { apiKeyRequest } from "../../command/ci";
|
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { ci, apiKeyRequest, runCi } from "../../command/ci";
|
||||||
|
import { provideContext } from "../../lib/context";
|
||||||
|
import { writeSession } from "../../lib/session";
|
||||||
|
import { resolveTrustIdentity } from "../../lib/trust";
|
||||||
|
|
||||||
test("CI requester supplies an API key without session authentication", async () => {
|
test("CI requester supplies an API key without session authentication", async () => {
|
||||||
const fetch = spyOn(globalThis, "fetch").mockResolvedValue(
|
const fetch = spyOn(globalThis, "fetch").mockResolvedValue(
|
||||||
@@ -17,3 +23,104 @@ test("CI requester supplies an API key without session authentication", async ()
|
|||||||
fetch.mockRestore();
|
fetch.mockRestore();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("CI command requires an API key before loading project context", async () => {
|
||||||
|
const previous = process.env.KUBER_API_KEY;
|
||||||
|
delete process.env.KUBER_API_KEY;
|
||||||
|
try {
|
||||||
|
await expect(
|
||||||
|
ci.run!({
|
||||||
|
args: { apiKey: undefined, build: false, trust: false },
|
||||||
|
} as never),
|
||||||
|
).rejects.toThrow("KUBER_API_KEY or --api-key is required");
|
||||||
|
} finally {
|
||||||
|
if (previous === undefined) delete process.env.KUBER_API_KEY;
|
||||||
|
else process.env.KUBER_API_KEY = previous;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("CI trust grant and deployment pipeline use the API key requester", async () => {
|
||||||
|
const root = await mkdtemp(join(tmpdir(), "kuber-ci-"));
|
||||||
|
const previousCwd = process.cwd();
|
||||||
|
const previousConfigHome = process.env.XDG_CONFIG_HOME;
|
||||||
|
const previousRuntimeDirectory = process.env.XDG_RUNTIME_DIR;
|
||||||
|
const calls: Array<{
|
||||||
|
path: string;
|
||||||
|
method: string | undefined;
|
||||||
|
body: string | undefined;
|
||||||
|
headers: Headers;
|
||||||
|
}> = [];
|
||||||
|
const fetch = spyOn(globalThis, "fetch").mockImplementation((async (
|
||||||
|
input,
|
||||||
|
init,
|
||||||
|
) => {
|
||||||
|
const url = new URL(input.toString());
|
||||||
|
const path = url.pathname.replace("/api/v2", "");
|
||||||
|
calls.push({
|
||||||
|
path,
|
||||||
|
method: init?.method,
|
||||||
|
body: typeof init?.body === "string" ? init.body : undefined,
|
||||||
|
headers: new Headers(init?.headers),
|
||||||
|
});
|
||||||
|
if (path === "/workspaces/shop")
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
metadata: { name: "shop", uid: "workspace", resourceVersion: "1" },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
if (path.endsWith("/plan"))
|
||||||
|
return new Response(JSON.stringify({ desired: [], stale: [] }));
|
||||||
|
if (path === "/snapshots/negotiate")
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({ workspace: "sha256:abc", missing: [], ready: true }),
|
||||||
|
);
|
||||||
|
return new Response(JSON.stringify({ resourcesAdopted: 0 }));
|
||||||
|
}) as typeof globalThis.fetch);
|
||||||
|
|
||||||
|
try {
|
||||||
|
process.env.XDG_CONFIG_HOME = join(root, "config");
|
||||||
|
process.env.XDG_RUNTIME_DIR = join(root, "runtime");
|
||||||
|
await writeSession(
|
||||||
|
{
|
||||||
|
token: "session-secret",
|
||||||
|
expiresAt: "2030-01-01T00:00:00Z",
|
||||||
|
user: { username: "ci", roles: [] },
|
||||||
|
},
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
await writeFile(join(root, "compose.yml"), "services: {}\n");
|
||||||
|
await writeFile(
|
||||||
|
join(root, ".kuberrc.ts"),
|
||||||
|
'export default { project: "shop" };\n',
|
||||||
|
);
|
||||||
|
const git = Bun.spawn(["git", "init", "-q", root]);
|
||||||
|
expect(await git.exited).toBe(0);
|
||||||
|
const fingerprint = (await resolveTrustIdentity("shop", root)).fingerprint;
|
||||||
|
process.chdir(root);
|
||||||
|
await provideContext(() => runCi(false, true, "ci-key"));
|
||||||
|
|
||||||
|
expect(calls[0]?.path).toBe("/workspaces/shop/trust");
|
||||||
|
expect(calls[0]?.method).toBe("POST");
|
||||||
|
expect(JSON.parse(calls[0]?.body ?? "")).toEqual({ fingerprint });
|
||||||
|
expect(calls.some(({ path }) => path.endsWith("/resources/plan"))).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(calls.some(({ path }) => path.endsWith("/resources/apply"))).toBe(
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
calls.every(
|
||||||
|
({ headers }) => headers.get("authorization") === "Bearer ci-key",
|
||||||
|
),
|
||||||
|
).toBe(true);
|
||||||
|
} finally {
|
||||||
|
fetch.mockRestore();
|
||||||
|
process.chdir(previousCwd);
|
||||||
|
if (previousConfigHome === undefined) delete process.env.XDG_CONFIG_HOME;
|
||||||
|
else process.env.XDG_CONFIG_HOME = previousConfigHome;
|
||||||
|
if (previousRuntimeDirectory === undefined)
|
||||||
|
delete process.env.XDG_RUNTIME_DIR;
|
||||||
|
else process.env.XDG_RUNTIME_DIR = previousRuntimeDirectory;
|
||||||
|
await rm(root, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|||||||
+195
-13
@@ -237,6 +237,57 @@ describe("up API pipeline", () => {
|
|||||||
expect(applyIdempotencyKeys[1]).toBe(applyIdempotencyKeys[0]);
|
expect(applyIdempotencyKeys[1]).toBe(applyIdempotencyKeys[0]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("clears the production retry timer and does not retry when cancelled", async () => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const originalSetTimeout = globalThis.setTimeout;
|
||||||
|
const originalClearTimeout = globalThis.clearTimeout;
|
||||||
|
const timer = {} as ReturnType<typeof setTimeout>;
|
||||||
|
let timerCallback: (() => void) | undefined;
|
||||||
|
let timerDelay: number | undefined;
|
||||||
|
let timerCleared = false;
|
||||||
|
let applyAttempts = 0;
|
||||||
|
let startedTimer!: () => void;
|
||||||
|
const timerStarted = new Promise<void>((resolve) => {
|
||||||
|
startedTimer = resolve;
|
||||||
|
});
|
||||||
|
globalThis.setTimeout = ((callback: () => void, milliseconds?: number) => {
|
||||||
|
timerCallback = callback;
|
||||||
|
timerDelay = milliseconds;
|
||||||
|
startedTimer();
|
||||||
|
return timer;
|
||||||
|
}) as typeof setTimeout;
|
||||||
|
globalThis.clearTimeout = ((handle: ReturnType<typeof setTimeout>) => {
|
||||||
|
if (handle === timer) timerCleared = true;
|
||||||
|
}) as typeof clearTimeout;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const request: ApiRequester = async <T>(path: string) => {
|
||||||
|
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
||||||
|
if (path.endsWith("/apply")) {
|
||||||
|
applyAttempts += 1;
|
||||||
|
throw new TypeError("connection reset");
|
||||||
|
}
|
||||||
|
throw new Error(`Unexpected request: ${path}`);
|
||||||
|
};
|
||||||
|
const cancelled = new DOMException("Cancelled", "AbortError");
|
||||||
|
const run = reconcileResources("shop", [], 1, undefined, request, {
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
|
||||||
|
await timerStarted;
|
||||||
|
controller.abort(cancelled);
|
||||||
|
|
||||||
|
await expect(run).rejects.toBe(cancelled);
|
||||||
|
expect(timerDelay).toBe(250);
|
||||||
|
expect(timerCleared).toBe(true);
|
||||||
|
expect(timerCallback).toBeDefined();
|
||||||
|
expect(applyAttempts).toBe(1);
|
||||||
|
} finally {
|
||||||
|
globalThis.setTimeout = originalSetTimeout;
|
||||||
|
globalThis.clearTimeout = originalClearTimeout;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test("restarts after an interrupted apply is persisted before its ID is received", async () => {
|
test("restarts after an interrupted apply is persisted before its ID is received", async () => {
|
||||||
const applyRequests: Array<{ key: string | null; json: unknown }> = [];
|
const applyRequests: Array<{ key: string | null; json: unknown }> = [];
|
||||||
const request: ApiRequester = async <T>(
|
const request: ApiRequester = async <T>(
|
||||||
@@ -385,14 +436,13 @@ describe("up API pipeline", () => {
|
|||||||
expect(operationPolls).toBe(0);
|
expect(operationPolls).toBe(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("starts live resource subtasks before the operation and updates them from progress", async () => {
|
test("attaches concurrent live resource subtasks before driving multi-target operations", async () => {
|
||||||
let finish!: () => void;
|
const children = new Map<string, { title: string; output: string }>();
|
||||||
let child: { title: string; output: string } | undefined;
|
let operationCalls = 0;
|
||||||
const completed = new Promise<void>((resolve) => (finish = resolve));
|
|
||||||
const listr = new Listr([
|
const listr = new Listr([
|
||||||
{
|
{
|
||||||
title: "Apply resources",
|
title: "Apply resources",
|
||||||
task: async (_ctx, task) =>
|
task: (_ctx, task) =>
|
||||||
runLiveResourceOperation(
|
runLiveResourceOperation(
|
||||||
task,
|
task,
|
||||||
"apply",
|
"apply",
|
||||||
@@ -402,9 +452,15 @@ describe("up API pipeline", () => {
|
|||||||
kind: "Service",
|
kind: "Service",
|
||||||
name: "web",
|
name: "web",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
apiVersion: "apps/v1",
|
||||||
|
kind: "Deployment",
|
||||||
|
name: "api",
|
||||||
|
},
|
||||||
],
|
],
|
||||||
async (onEvent) => {
|
async (onEvent) => {
|
||||||
expect(child?.title).toBe("Apply Service/web");
|
operationCalls += 1;
|
||||||
|
expect(children.size).toBe(2);
|
||||||
onEvent({
|
onEvent({
|
||||||
sequence: 1,
|
sequence: 1,
|
||||||
data: {
|
data: {
|
||||||
@@ -417,19 +473,145 @@ describe("up API pipeline", () => {
|
|||||||
state: "started",
|
state: "started",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
expect(child?.output).toBe("started");
|
onEvent({
|
||||||
await completed;
|
sequence: 2,
|
||||||
|
data: {
|
||||||
|
resource: {
|
||||||
|
apiVersion: "apps/v1",
|
||||||
|
kind: "Deployment",
|
||||||
|
name: "api",
|
||||||
|
},
|
||||||
|
phase: "apply",
|
||||||
|
state: "succeeded",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(children.get("web")?.output).toBe("started");
|
||||||
|
expect(children.get("api")?.output).toBe("succeeded");
|
||||||
|
},
|
||||||
|
{
|
||||||
|
onTaskStarted: (target, activeTask) =>
|
||||||
|
children.set(target.name, activeTask),
|
||||||
},
|
},
|
||||||
{ onTaskStarted: (_target, activeTask) => (child = activeTask) },
|
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const run = listr.run();
|
await listr.run();
|
||||||
await Bun.sleep(0);
|
|
||||||
finish();
|
expect(operationCalls).toBe(1);
|
||||||
await run;
|
expect(listr.tasks[0]?.subtasks).toHaveLength(2);
|
||||||
|
expect(listr.tasks[0]?.subtasks.map((task) => task.title)).toEqual([
|
||||||
|
"Apply Service/web",
|
||||||
|
"Apply Deployment/api",
|
||||||
|
]);
|
||||||
|
}, 1_000);
|
||||||
|
|
||||||
|
test("renders a single live resource target and runs zero-target operations directly", async () => {
|
||||||
|
const single = new Listr([
|
||||||
|
{
|
||||||
|
title: "Apply resources",
|
||||||
|
task: (_ctx, task) =>
|
||||||
|
runLiveResourceOperation(
|
||||||
|
task,
|
||||||
|
"apply",
|
||||||
|
[{ apiVersion: "v1", kind: "Service", name: "web" }],
|
||||||
|
async (onEvent) => {
|
||||||
|
onEvent({
|
||||||
|
sequence: 1,
|
||||||
|
data: {
|
||||||
|
resource: { apiVersion: "v1", kind: "Service", name: "web" },
|
||||||
|
phase: "apply",
|
||||||
|
state: "succeeded",
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
},
|
||||||
|
),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
await single.run();
|
||||||
|
expect(single.tasks[0]?.subtasks.map(({ title }) => title)).toEqual([
|
||||||
|
"Apply Service/web",
|
||||||
|
]);
|
||||||
|
|
||||||
|
let called = false;
|
||||||
|
await runLiveResourceOperation(
|
||||||
|
{ signal: new AbortController().signal } as never,
|
||||||
|
"delete",
|
||||||
|
[],
|
||||||
|
async () => {
|
||||||
|
called = true;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
expect(called).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("cancels a multi-resource live operation without leaving child tasks waiting", async () => {
|
||||||
|
const children: Array<{ signal: AbortSignal }> = [];
|
||||||
|
let phaseTask: { cancel: () => void } | undefined;
|
||||||
|
let operationObservedAbort = false;
|
||||||
|
let startOperation: (() => void) | undefined;
|
||||||
|
const operationStarted = new Promise<void>((resolve) => {
|
||||||
|
startOperation = resolve;
|
||||||
|
});
|
||||||
|
const listr = new Listr([
|
||||||
|
{
|
||||||
|
title: "Apply resources",
|
||||||
|
task: (_ctx, task) => {
|
||||||
|
phaseTask = task;
|
||||||
|
return runLiveResourceOperation(
|
||||||
|
task,
|
||||||
|
"apply",
|
||||||
|
[
|
||||||
|
{ apiVersion: "v1", kind: "Service", name: "web" },
|
||||||
|
{ apiVersion: "apps/v1", kind: "Deployment", name: "api" },
|
||||||
|
],
|
||||||
|
async (_onEvent, signal) => {
|
||||||
|
startOperation?.();
|
||||||
|
await new Promise<void>((_resolve, reject) => {
|
||||||
|
signal.addEventListener(
|
||||||
|
"abort",
|
||||||
|
() => {
|
||||||
|
operationObservedAbort = true;
|
||||||
|
reject(signal.reason);
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
{ onTaskStarted: (_target, child) => children.push(child) },
|
||||||
|
);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
const originalExit = process.exit;
|
||||||
|
const exitCodes: Array<number | undefined> = [];
|
||||||
|
process.exit = ((code?: number) => {
|
||||||
|
exitCodes.push(code);
|
||||||
|
return undefined as never;
|
||||||
|
}) as typeof process.exit;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const run = listr.run();
|
||||||
|
await operationStarted;
|
||||||
|
phaseTask?.cancel();
|
||||||
|
await Promise.race([
|
||||||
|
run,
|
||||||
|
Bun.sleep(100).then(() => {
|
||||||
|
throw new Error("Cancelled live operation did not settle promptly");
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
process.exit = originalExit;
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(operationObservedAbort).toBe(true);
|
||||||
|
expect(exitCodes).toEqual([127]);
|
||||||
|
expect(children).toHaveLength(2);
|
||||||
|
expect(children.every(({ signal }) => signal.aborted)).toBe(true);
|
||||||
|
expect(
|
||||||
|
listr.tasks[0]?.subtasks.every(({ state }) => state === "CANCELLED"),
|
||||||
|
).toBe(true);
|
||||||
|
}, 1_000);
|
||||||
|
|
||||||
test("resubmits with a fresh key after server restart interruption", async () => {
|
test("resubmits with a fresh key after server restart interruption", async () => {
|
||||||
const applyRequests: Array<{ key: string | null; json: unknown }> = [];
|
const applyRequests: Array<{ key: string | null; json: unknown }> = [];
|
||||||
|
|||||||
@@ -3,6 +3,9 @@ import { cleanupExpiredSessions, createApp } from "../../server/app";
|
|||||||
import { hashToken, MemoryAuthStore } from "../../server/auth";
|
import { hashToken, MemoryAuthStore } from "../../server/auth";
|
||||||
import { MemoryAuditStore } from "../../server/audit-store";
|
import { MemoryAuditStore } from "../../server/audit-store";
|
||||||
import { MemoryOperationStore } from "../../server/operation-store";
|
import { MemoryOperationStore } from "../../server/operation-store";
|
||||||
|
import type { ManagementService } from "../../server/management";
|
||||||
|
import { MemoryTrustStore } from "../../server/trust-store";
|
||||||
|
import { MemoryWorkspaceStore } from "../../server/workspace-store";
|
||||||
|
|
||||||
const now = Date.parse("2026-09-05T00:00:00.000Z");
|
const now = Date.parse("2026-09-05T00:00:00.000Z");
|
||||||
|
|
||||||
@@ -255,6 +258,67 @@ describe("API keys", () => {
|
|||||||
).toBe(403);
|
).toBe(403);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("allows scoped keys to apply only in their workspace and rejects deleted owners", async () => {
|
||||||
|
const { store } = await setup();
|
||||||
|
const workspaceStore = new MemoryWorkspaceStore({
|
||||||
|
uid: () => "workspace-uid",
|
||||||
|
});
|
||||||
|
for (const id of ["shop", "other"])
|
||||||
|
await workspaceStore.create({
|
||||||
|
id,
|
||||||
|
source: { uri: `oci://example/${id}`, digest: "sha256:abc" },
|
||||||
|
});
|
||||||
|
const trustStore = new MemoryTrustStore();
|
||||||
|
const fingerprint = "a".repeat(64);
|
||||||
|
await trustStore.grant("shop", fingerprint);
|
||||||
|
let applies = 0;
|
||||||
|
const app = createApp({
|
||||||
|
store,
|
||||||
|
workspaceStore,
|
||||||
|
trustStore,
|
||||||
|
operationStore: new MemoryOperationStore(() => new Date(now)),
|
||||||
|
management: {
|
||||||
|
applyResources: async () => {
|
||||||
|
applies += 1;
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
} as unknown as ManagementService,
|
||||||
|
now: () => now,
|
||||||
|
});
|
||||||
|
await store.createApiKey({
|
||||||
|
id: "key_apply_scope_1",
|
||||||
|
tokenHash: hashToken("scoped-apply-key"),
|
||||||
|
username: "ci",
|
||||||
|
capabilities: ["kubernetes:write"],
|
||||||
|
workspace: "shop",
|
||||||
|
expiresAt: "2026-10-05T00:00:00.000Z",
|
||||||
|
});
|
||||||
|
const apply = (workspace: string) =>
|
||||||
|
app(
|
||||||
|
request(
|
||||||
|
`/api/v2/workspaces/${workspace}/resources/apply`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"idempotency-key": `apply-${workspace}`,
|
||||||
|
"x-kuber-trust-project": "shop",
|
||||||
|
"x-kuber-trust-fingerprint": fingerprint,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ resources: [] }),
|
||||||
|
},
|
||||||
|
"scoped-apply-key",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect((await apply("shop")).status).toBe(200);
|
||||||
|
expect((await apply("other")).status).toBe(403);
|
||||||
|
expect(applies).toBe(1);
|
||||||
|
await store.deleteUser("ci");
|
||||||
|
expect(
|
||||||
|
(await app(request("/api/v2/me", {}, "scoped-apply-key"))).status,
|
||||||
|
).toBe(401);
|
||||||
|
});
|
||||||
|
|
||||||
test("limits workspace-scoped keys to their own audit records", async () => {
|
test("limits workspace-scoped keys to their own audit records", async () => {
|
||||||
const { app, store, auditStore } = await setup();
|
const { app, store, auditStore } = await setup();
|
||||||
await store.createApiKey({
|
await store.createApiKey({
|
||||||
|
|||||||
@@ -391,6 +391,51 @@ describe("kuber v2 HTTP routes", () => {
|
|||||||
expect((await apply(app)).status).toBe(403);
|
expect((await apply(app)).status).toBe(403);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("returns OPERATION_CONFLICT when an idempotency key is reused for another body", async () => {
|
||||||
|
const workspaceStore = new MemoryWorkspaceStore({
|
||||||
|
uid: () => "workspace-uid",
|
||||||
|
});
|
||||||
|
await workspaceStore.create({
|
||||||
|
id: "demo",
|
||||||
|
source: { uri: "oci://example/demo", digest: "sha256:abc" },
|
||||||
|
});
|
||||||
|
const trustStore = new MemoryTrustStore();
|
||||||
|
const fingerprint = "c".repeat(64);
|
||||||
|
await trustStore.grant("demo", fingerprint);
|
||||||
|
const app = createApp({
|
||||||
|
store: await authenticatedStore("operator"),
|
||||||
|
workspaceStore,
|
||||||
|
trustStore,
|
||||||
|
operationStore: new MemoryOperationStore(),
|
||||||
|
management: {
|
||||||
|
applyResources: async () => [],
|
||||||
|
} as unknown as ManagementService,
|
||||||
|
});
|
||||||
|
const apply = (resources: unknown[]) =>
|
||||||
|
app(
|
||||||
|
request(
|
||||||
|
"/api/v2/workspaces/demo/resources/apply",
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"idempotency-key": "same-apply",
|
||||||
|
"x-kuber-trust-project": "demo",
|
||||||
|
"x-kuber-trust-fingerprint": fingerprint,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ resources }),
|
||||||
|
},
|
||||||
|
"token",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect((await apply([])).status).toBe(200);
|
||||||
|
const conflict = await apply([
|
||||||
|
{ apiVersion: "v1", kind: "Service", metadata: { name: "web" } },
|
||||||
|
]);
|
||||||
|
expect(conflict.status).toBe(409);
|
||||||
|
expect(await conflict.json()).toMatchObject({ code: "OPERATION_CONFLICT" });
|
||||||
|
});
|
||||||
|
|
||||||
test("starts preferred resource operations before returning so progress can be polled", async () => {
|
test("starts preferred resource operations before returning so progress can be polled", async () => {
|
||||||
const workspaceStore = new MemoryWorkspaceStore({
|
const workspaceStore = new MemoryWorkspaceStore({
|
||||||
uid: () => "workspace-uid",
|
uid: () => "workspace-uid",
|
||||||
|
|||||||
@@ -236,6 +236,35 @@ describe("server management service", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("emits aborted resource progress when an execution signal aborts", async () => {
|
||||||
|
const controller = new AbortController();
|
||||||
|
const events: Array<{ state: string; resource: { name: string } }> = [];
|
||||||
|
const service = createManagementService(
|
||||||
|
dependencies({
|
||||||
|
applyResource: async (_resource, execution) => {
|
||||||
|
controller.abort();
|
||||||
|
if (execution?.signal?.aborted)
|
||||||
|
throw new Error("Workspace operation execution was cancelled");
|
||||||
|
return _resource;
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.applyResources(
|
||||||
|
workspace,
|
||||||
|
[object("Service", "web", "service-uid")],
|
||||||
|
{
|
||||||
|
signal: controller.signal,
|
||||||
|
emit: async (event) => {
|
||||||
|
events.push(event);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
),
|
||||||
|
).rejects.toThrow("cancelled");
|
||||||
|
expect(events.map(({ state }) => state)).toEqual(["started", "aborted"]);
|
||||||
|
});
|
||||||
|
|
||||||
test("rejects namespace and resource ownership mismatches", async () => {
|
test("rejects namespace and resource ownership mismatches", async () => {
|
||||||
const wrongNamespace = createManagementService(
|
const wrongNamespace = createManagementService(
|
||||||
dependencies({
|
dependencies({
|
||||||
|
|||||||
Reference in New Issue
Block a user