fix: recover workspace update races

2026-09-06 09:25:33 +00:00
parent 053f28a367
commit f14e976bcd
6 changed files with 483 additions and 35 deletions
@@ -158,6 +158,163 @@ describe("up API pipeline", () => {
    expect(new Headers(calls[1]?.init?.headers).get("if-match")).toBe('"7"');
  });

+  test("retries one conflict using a fresh ETag", async () => {
+    const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
+    let gets = 0;
+    const request: ApiRequester = async <T>(
+      path: string,
+      init?: ApiRequestInit,
+    ) => {
+      calls.push({ path, init });
+      if (!init) {
+        gets += 1;
+        return {
+          metadata: {
+            name: "shop",
+            uid: "uid",
+            resourceVersion: gets === 1 ? "7" : "8",
+          },
+          spec: { source: { uri: "cas://old", digest: "old" }, config: {} },
+        } as T;
+      }
+      if (calls.length === 2)
+        throw new KuberApiError("conflict", 409, {
+          title: "Conflict",
+          status: 409,
+          code: "WORKSPACE_CONFLICT",
+        });
+      return {
+        metadata: { name: "shop", uid: "uid", resourceVersion: "9" },
+      } as T;
+    };
+
+    await ensureWorkspace("shop", { services: {} }, snapshot, request);
+
+    expect(calls.map(({ path }) => path)).toEqual([
+      "/workspaces/shop",
+      "/workspaces/shop",
+      "/workspaces/shop",
+      "/workspaces/shop",
+    ]);
+    expect(new Headers(calls[3]?.init?.headers).get("if-match")).toBe('"8"');
+  });
+
+  test("accepts a fresh workspace that already matches after a conflict", async () => {
+    let gets = 0;
+    let puts = 0;
+    const desired = {
+      source: { uri: `cas://${snapshot.digest}`, digest: snapshot.digest },
+      config: { compose: { services: {} } },
+    };
+    const request: ApiRequester = async <T>(
+      _path: string,
+      init?: ApiRequestInit,
+    ) => {
+      if (!init) {
+        gets += 1;
+        return {
+          metadata: { name: "shop", uid: "uid", resourceVersion: "8" },
+          spec:
+            gets === 1
+              ? { ...desired, source: { uri: "cas://old", digest: "old" } }
+              : desired,
+        } as T;
+      }
+      puts += 1;
+      throw new KuberApiError("conflict", 409, {
+        title: "Conflict",
+        status: 409,
+        code: "WORKSPACE_CONFLICT",
+      });
+    };
+
+    await ensureWorkspace("shop", { services: {} }, snapshot, request);
+
+    expect({ gets, puts }).toEqual({ gets: 2, puts: 1 });
+  });
+
+  test("recovers an ambiguous transport failure after the update committed", async () => {
+    let gets = 0;
+    let puts = 0;
+    const desired = {
+      source: { uri: `cas://${snapshot.digest}`, digest: snapshot.digest },
+      config: { compose: { services: {} } },
+    };
+    const request: ApiRequester = async <T>(
+      _path: string,
+      init?: ApiRequestInit,
+    ) => {
+      if (!init) {
+        gets += 1;
+        return {
+          metadata: { name: "shop", uid: "uid", resourceVersion: "8" },
+          spec:
+            gets === 1
+              ? { ...desired, source: { uri: "cas://old", digest: "old" } }
+              : desired,
+        } as T;
+      }
+      puts += 1;
+      throw new TypeError("connection reset");
+    };
+
+    await ensureWorkspace("shop", { services: {} }, snapshot, request);
+
+    expect({ gets, puts }).toEqual({ gets: 2, puts: 1 });
+  });
+
+  test("recovers an ambiguous transport failure with one fresh-ETag retry", async () => {
+    const etags: string[] = [];
+    let gets = 0;
+    const request: ApiRequester = async <T>(
+      _path: string,
+      init?: ApiRequestInit,
+    ) => {
+      if (!init) {
+        gets += 1;
+        return {
+          metadata: {
+            name: "shop",
+            uid: "uid",
+            resourceVersion: gets === 1 ? "7" : "8",
+          },
+          spec: { source: { uri: "cas://old", digest: "old" }, config: {} },
+        } as T;
+      }
+      etags.push(new Headers(init.headers).get("if-match")!);
+      if (etags.length === 1) throw new TypeError("connection reset");
+      return {
+        metadata: { name: "shop", uid: "uid", resourceVersion: "9" },
+      } as T;
+    };
+
+    await ensureWorkspace("shop", { services: {} }, snapshot, request);
+
+    expect(etags).toEqual(['"7"', '"8"']);
+  });
+
+  test("does not retry a generic workspace update failure", async () => {
+    const failure = new KuberApiError("failed", 500, {
+      title: "Internal server error",
+      status: 500,
+      code: "INTERNAL_ERROR",
+    });
+    let calls = 0;
+    const request: ApiRequester = async <T>() => {
+      calls += 1;
+      if (calls === 1)
+        return {
+          metadata: { name: "shop", uid: "uid", resourceVersion: "7" },
+        } as T;
+      throw failure;
+    };
+
+    await expect(
+      ensureWorkspace("shop", { services: {} }, snapshot, request),
+    ).rejects.toBe(failure);
+    expect(calls).toBe(2);
+  });
+
  test("plans, applies, runs the hook, waits, then deletes stale identities", async () => {
    const order: string[] = [];
    const desired = [