fix: stabilize live deployment progress
This commit is contained in:
+195
-13
@@ -237,6 +237,57 @@ describe("up API pipeline", () => {
|
||||
expect(applyIdempotencyKeys[1]).toBe(applyIdempotencyKeys[0]);
|
||||
});
|
||||
|
||||
test("clears the production retry timer and does not retry when cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
const originalSetTimeout = globalThis.setTimeout;
|
||||
const originalClearTimeout = globalThis.clearTimeout;
|
||||
const timer = {} as ReturnType<typeof setTimeout>;
|
||||
let timerCallback: (() => void) | undefined;
|
||||
let timerDelay: number | undefined;
|
||||
let timerCleared = false;
|
||||
let applyAttempts = 0;
|
||||
let startedTimer!: () => void;
|
||||
const timerStarted = new Promise<void>((resolve) => {
|
||||
startedTimer = resolve;
|
||||
});
|
||||
globalThis.setTimeout = ((callback: () => void, milliseconds?: number) => {
|
||||
timerCallback = callback;
|
||||
timerDelay = milliseconds;
|
||||
startedTimer();
|
||||
return timer;
|
||||
}) as typeof setTimeout;
|
||||
globalThis.clearTimeout = ((handle: ReturnType<typeof setTimeout>) => {
|
||||
if (handle === timer) timerCleared = true;
|
||||
}) as typeof clearTimeout;
|
||||
|
||||
try {
|
||||
const request: ApiRequester = async <T>(path: string) => {
|
||||
if (path.endsWith("/plan")) return { desired: [], stale: [] } as T;
|
||||
if (path.endsWith("/apply")) {
|
||||
applyAttempts += 1;
|
||||
throw new TypeError("connection reset");
|
||||
}
|
||||
throw new Error(`Unexpected request: ${path}`);
|
||||
};
|
||||
const cancelled = new DOMException("Cancelled", "AbortError");
|
||||
const run = reconcileResources("shop", [], 1, undefined, request, {
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
await timerStarted;
|
||||
controller.abort(cancelled);
|
||||
|
||||
await expect(run).rejects.toBe(cancelled);
|
||||
expect(timerDelay).toBe(250);
|
||||
expect(timerCleared).toBe(true);
|
||||
expect(timerCallback).toBeDefined();
|
||||
expect(applyAttempts).toBe(1);
|
||||
} finally {
|
||||
globalThis.setTimeout = originalSetTimeout;
|
||||
globalThis.clearTimeout = originalClearTimeout;
|
||||
}
|
||||
});
|
||||
|
||||
test("restarts after an interrupted apply is persisted before its ID is received", async () => {
|
||||
const applyRequests: Array<{ key: string | null; json: unknown }> = [];
|
||||
const request: ApiRequester = async <T>(
|
||||
@@ -385,14 +436,13 @@ describe("up API pipeline", () => {
|
||||
expect(operationPolls).toBe(0);
|
||||
});
|
||||
|
||||
test("starts live resource subtasks before the operation and updates them from progress", async () => {
|
||||
let finish!: () => void;
|
||||
let child: { title: string; output: string } | undefined;
|
||||
const completed = new Promise<void>((resolve) => (finish = resolve));
|
||||
test("attaches concurrent live resource subtasks before driving multi-target operations", async () => {
|
||||
const children = new Map<string, { title: string; output: string }>();
|
||||
let operationCalls = 0;
|
||||
const listr = new Listr([
|
||||
{
|
||||
title: "Apply resources",
|
||||
task: async (_ctx, task) =>
|
||||
task: (_ctx, task) =>
|
||||
runLiveResourceOperation(
|
||||
task,
|
||||
"apply",
|
||||
@@ -402,9 +452,15 @@ describe("up API pipeline", () => {
|
||||
kind: "Service",
|
||||
name: "web",
|
||||
},
|
||||
{
|
||||
apiVersion: "apps/v1",
|
||||
kind: "Deployment",
|
||||
name: "api",
|
||||
},
|
||||
],
|
||||
async (onEvent) => {
|
||||
expect(child?.title).toBe("Apply Service/web");
|
||||
operationCalls += 1;
|
||||
expect(children.size).toBe(2);
|
||||
onEvent({
|
||||
sequence: 1,
|
||||
data: {
|
||||
@@ -417,20 +473,146 @@ describe("up API pipeline", () => {
|
||||
state: "started",
|
||||
},
|
||||
});
|
||||
expect(child?.output).toBe("started");
|
||||
await completed;
|
||||
onEvent({
|
||||
sequence: 2,
|
||||
data: {
|
||||
resource: {
|
||||
apiVersion: "apps/v1",
|
||||
kind: "Deployment",
|
||||
name: "api",
|
||||
},
|
||||
phase: "apply",
|
||||
state: "succeeded",
|
||||
},
|
||||
});
|
||||
expect(children.get("web")?.output).toBe("started");
|
||||
expect(children.get("api")?.output).toBe("succeeded");
|
||||
},
|
||||
{
|
||||
onTaskStarted: (target, activeTask) =>
|
||||
children.set(target.name, activeTask),
|
||||
},
|
||||
{ onTaskStarted: (_target, activeTask) => (child = activeTask) },
|
||||
),
|
||||
},
|
||||
]);
|
||||
|
||||
const run = listr.run();
|
||||
await Bun.sleep(0);
|
||||
finish();
|
||||
await run;
|
||||
await listr.run();
|
||||
|
||||
expect(operationCalls).toBe(1);
|
||||
expect(listr.tasks[0]?.subtasks).toHaveLength(2);
|
||||
expect(listr.tasks[0]?.subtasks.map((task) => task.title)).toEqual([
|
||||
"Apply Service/web",
|
||||
"Apply Deployment/api",
|
||||
]);
|
||||
}, 1_000);
|
||||
|
||||
test("renders a single live resource target and runs zero-target operations directly", async () => {
|
||||
const single = new Listr([
|
||||
{
|
||||
title: "Apply resources",
|
||||
task: (_ctx, task) =>
|
||||
runLiveResourceOperation(
|
||||
task,
|
||||
"apply",
|
||||
[{ apiVersion: "v1", kind: "Service", name: "web" }],
|
||||
async (onEvent) => {
|
||||
onEvent({
|
||||
sequence: 1,
|
||||
data: {
|
||||
resource: { apiVersion: "v1", kind: "Service", name: "web" },
|
||||
phase: "apply",
|
||||
state: "succeeded",
|
||||
},
|
||||
});
|
||||
},
|
||||
),
|
||||
},
|
||||
]);
|
||||
await single.run();
|
||||
expect(single.tasks[0]?.subtasks.map(({ title }) => title)).toEqual([
|
||||
"Apply Service/web",
|
||||
]);
|
||||
|
||||
let called = false;
|
||||
await runLiveResourceOperation(
|
||||
{ signal: new AbortController().signal } as never,
|
||||
"delete",
|
||||
[],
|
||||
async () => {
|
||||
called = true;
|
||||
},
|
||||
);
|
||||
expect(called).toBe(true);
|
||||
});
|
||||
|
||||
test("cancels a multi-resource live operation without leaving child tasks waiting", async () => {
|
||||
const children: Array<{ signal: AbortSignal }> = [];
|
||||
let phaseTask: { cancel: () => void } | undefined;
|
||||
let operationObservedAbort = false;
|
||||
let startOperation: (() => void) | undefined;
|
||||
const operationStarted = new Promise<void>((resolve) => {
|
||||
startOperation = resolve;
|
||||
});
|
||||
const listr = new Listr([
|
||||
{
|
||||
title: "Apply resources",
|
||||
task: (_ctx, task) => {
|
||||
phaseTask = task;
|
||||
return runLiveResourceOperation(
|
||||
task,
|
||||
"apply",
|
||||
[
|
||||
{ apiVersion: "v1", kind: "Service", name: "web" },
|
||||
{ apiVersion: "apps/v1", kind: "Deployment", name: "api" },
|
||||
],
|
||||
async (_onEvent, signal) => {
|
||||
startOperation?.();
|
||||
await new Promise<void>((_resolve, reject) => {
|
||||
signal.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
operationObservedAbort = true;
|
||||
reject(signal.reason);
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
},
|
||||
{ onTaskStarted: (_target, child) => children.push(child) },
|
||||
);
|
||||
},
|
||||
},
|
||||
]);
|
||||
const originalExit = process.exit;
|
||||
const exitCodes: Array<number | undefined> = [];
|
||||
process.exit = ((code?: number) => {
|
||||
exitCodes.push(code);
|
||||
return undefined as never;
|
||||
}) as typeof process.exit;
|
||||
|
||||
try {
|
||||
const run = listr.run();
|
||||
await operationStarted;
|
||||
phaseTask?.cancel();
|
||||
await Promise.race([
|
||||
run,
|
||||
Bun.sleep(100).then(() => {
|
||||
throw new Error("Cancelled live operation did not settle promptly");
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
process.exit = originalExit;
|
||||
}
|
||||
|
||||
expect(operationObservedAbort).toBe(true);
|
||||
expect(exitCodes).toEqual([127]);
|
||||
expect(children).toHaveLength(2);
|
||||
expect(children.every(({ signal }) => signal.aborted)).toBe(true);
|
||||
expect(
|
||||
listr.tasks[0]?.subtasks.every(({ state }) => state === "CANCELLED"),
|
||||
).toBe(true);
|
||||
}, 1_000);
|
||||
|
||||
test("resubmits with a fresh key after server restart interruption", async () => {
|
||||
const applyRequests: Array<{ key: string | null; json: unknown }> = [];
|
||||
let operationPolls = 0;
|
||||
|
||||
Reference in New Issue
Block a user