fix(catalog) : retry transient storage quorum failures

2026-10-12 19:53:19 +07:00
parent 3fb34ffde2
commit fae9e146e1
3 changed files with 73 additions and 16 deletions
@@ -0,0 +1,56 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import { retryTransient } from "./sync-retry";
+
+const label = "Asset catalog/materials/UI_ItemIcon_121767.png";
+const quorumError = new Error('Internal error: Could not reach quorum of 2 (sets=Some(1)). 1 of 2 request succeeded, others returned errors: ["5f70d8ec32eade44: Network error: Not connected: 5f70d8ec32eade44"]');
+
+afterEach(() => vi.useRealTimers());
+
+describe("catalog sync retries", () => {
+  it.each([quorumError, new Error("Network error: Not connected: 5f70d8ec32eade44"), new Error("Request timed out")])("recovers from %s", async (cause) => {
+    vi.useFakeTimers();
+    const task = vi.fn().mockRejectedValueOnce(cause).mockResolvedValue("uploaded");
+    const result = retryTransient(label, task);
+
+    await vi.advanceTimersByTimeAsync(999);
+    expect(task).toHaveBeenCalledTimes(1);
+    await vi.advanceTimersByTimeAsync(1);
+    await expect(result).resolves.toBe("uploaded");
+    expect(task).toHaveBeenCalledTimes(2);
+  });
+
+  it("stops after three failed attempts and preserves the asset and cause", async () => {
+    vi.useFakeTimers();
+    const task = vi.fn().mockRejectedValue(quorumError);
+    const result = expect(retryTransient(label, task)).rejects.toMatchObject({
+      message: `${label}: ${quorumError.message}`,
+      cause: quorumError,
+    });
+
+    await vi.advanceTimersByTimeAsync(1_000);
+    expect(task).toHaveBeenCalledTimes(2);
+    await vi.advanceTimersByTimeAsync(2_000);
+    await result;
+    expect(task).toHaveBeenCalledTimes(3);
+  });
+
+  it("fails immediately on a permanent storage error", async () => {
+    const task = vi.fn().mockRejectedValue(new Error("AccessDenied"));
+    await expect(retryTransient(label, task)).rejects.toThrow(`${label}: AccessDenied`);
+    expect(task).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not retry after cancellation during backoff", async () => {
+    vi.useFakeTimers();
+    const controller = new AbortController();
+    const task = vi.fn().mockRejectedValue(quorumError);
+    const result = expect(retryTransient(label, task, controller.signal)).rejects.toMatchObject({ name: "AbortError" });
+
+    await vi.advanceTimersByTimeAsync(0);
+    controller.abort();
+    await vi.advanceTimersByTimeAsync(1_000);
+    await result;
+    expect(task).toHaveBeenCalledTimes(1);
+  });
+});