astral/kuber
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: recover workspace update races

Browse Source
This commit is contained in:
dmgnr
2026-09-06 09:25:33 +00:00 Unverified
parent 053f28a367
commit f14e976bcd
6 changed files with 483 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/command/up-api.test.ts
+157
View File
@@ -158,6 +158,163 @@ describe("up API pipeline", () => {
expect(new Headers(calls[1]?.init?.headers).get("if-match")).toBe('"7"');
});
test("retries one conflict using a fresh ETag", async () => {
const calls: Array<{ path: string; init?: ApiRequestInit }> = [];
let gets = 0;
const request: ApiRequester = async <T>(
path: string,
init?: ApiRequestInit,
) => {
calls.push({ path, init });
if (!init) {
gets += 1;
return {
metadata: {
name: "shop",
uid: "uid",
resourceVersion: gets === 1 ? "7" : "8",
},
spec: { source: { uri: "cas://old", digest: "old" }, config: {} },
} as T;
}
if (calls.length === 2)
throw new KuberApiError("conflict", 409, {
title: "Conflict",
status: 409,
code: "WORKSPACE_CONFLICT",
});
return {
metadata: { name: "shop", uid: "uid", resourceVersion: "9" },
} as T;
};
await ensureWorkspace("shop", { services: {} }, snapshot, request);
expect(calls.map(({ path }) => path)).toEqual([
"/workspaces/shop",
"/workspaces/shop",
"/workspaces/shop",
"/workspaces/shop",
]);
expect(new Headers(calls[3]?.init?.headers).get("if-match")).toBe('"8"');
});
test("accepts a fresh workspace that already matches after a conflict", async () => {
let gets = 0;
let puts = 0;
const desired = {
source: { uri: `cas://${snapshot.digest}`, digest: snapshot.digest },
config: { compose: { services: {} } },
};
const request: ApiRequester = async <T>(
_path: string,
init?: ApiRequestInit,
) => {
if (!init) {
gets += 1;
return {
metadata: { name: "shop", uid: "uid", resourceVersion: "8" },
spec:
gets === 1
? { ...desired, source: { uri: "cas://old", digest: "old" } }
: desired,
} as T;
}
puts += 1;
throw new KuberApiError("conflict", 409, {
title: "Conflict",
status: 409,
code: "WORKSPACE_CONFLICT",
});
};
await ensureWorkspace("shop", { services: {} }, snapshot, request);
expect({ gets, puts }).toEqual({ gets: 2, puts: 1 });
});
test("recovers an ambiguous transport failure after the update committed", async () => {
let gets = 0;
let puts = 0;
const desired = {
source: { uri: `cas://${snapshot.digest}`, digest: snapshot.digest },
config: { compose: { services: {} } },
};
const request: ApiRequester = async <T>(
_path: string,
init?: ApiRequestInit,
) => {
if (!init) {
gets += 1;
return {
metadata: { name: "shop", uid: "uid", resourceVersion: "8" },
spec:
gets === 1
? { ...desired, source: { uri: "cas://old", digest: "old" } }
: desired,
} as T;
}
puts += 1;
throw new TypeError("connection reset");
};
await ensureWorkspace("shop", { services: {} }, snapshot, request);
expect({ gets, puts }).toEqual({ gets: 2, puts: 1 });
});
test("recovers an ambiguous transport failure with one fresh-ETag retry", async () => {
const etags: string[] = [];
let gets = 0;
const request: ApiRequester = async <T>(
_path: string,
init?: ApiRequestInit,
) => {
if (!init) {
gets += 1;
return {
metadata: {
name: "shop",
uid: "uid",
resourceVersion: gets === 1 ? "7" : "8",
},
spec: { source: { uri: "cas://old", digest: "old" }, config: {} },
} as T;
}
etags.push(new Headers(init.headers).get("if-match")!);
if (etags.length === 1) throw new TypeError("connection reset");
return {
metadata: { name: "shop", uid: "uid", resourceVersion: "9" },
} as T;
};
await ensureWorkspace("shop", { services: {} }, snapshot, request);
expect(etags).toEqual(['"7"', '"8"']);
});
test("does not retry a generic workspace update failure", async () => {
const failure = new KuberApiError("failed", 500, {
title: "Internal server error",
status: 500,
code: "INTERNAL_ERROR",
});
let calls = 0;
const request: ApiRequester = async <T>() => {
calls += 1;
if (calls === 1)
return {
metadata: { name: "shop", uid: "uid", resourceVersion: "7" },
} as T;
throw failure;
};
await expect(
ensureWorkspace("shop", { services: {} }, snapshot, request),
).rejects.toBe(failure);
expect(calls).toBe(2);
});
test("plans, applies, runs the hook, waits, then deletes stale identities", async () => {
const order: string[] = [];
const desired = [
tests/server/app.test.ts
+97 -3
View File
@@ -681,6 +681,83 @@ describe("kuber v2 HTTP routes", () => {
expect(await operationStore.list("demo")).toHaveLength(1);
});
test("redacts provider status and reason in correlated generic failure logs", async () => {
const workspaceStore = new MemoryWorkspaceStore({
uid: () => "workspace-uid",
});
await workspaceStore.create({
id: "demo",
source: { uri: "oci://example/demo", digest: "sha256:abc" },
});
const logs: unknown[] = [];
spyOn(workspaceStore, "update").mockRejectedValue(
Object.assign(
new Error(
'provider failed password=top-secret config={"compose":"private"}',
),
{
name: "KubernetesError",
body: {
status: "Failure",
reason: "InternalError",
code: 500,
message: "contains top-secret",
},
},
),
);
const app = createApp({
store: await authenticatedStore("operator"),
workspaceStore,
requestId: () => "request-123",
logger: { error: (entry) => logs.push(entry) },
});
const result = await app(
request(
"/api/v2/workspaces/demo",
{
method: "PUT",
headers: { "if-match": '"1"' },
body: JSON.stringify({
source: { uri: "cas://secret-source", digest: "secret-digest" },
config: { compose: "private-config", password: "top-secret" },
}),
},
"token",
),
);
expect(result.status).toBe(500);
expect(await result.json()).toMatchObject({
code: "INTERNAL_ERROR",
requestId: "request-123",
});
expect(logs).toEqual([
{
event: "request.failed",
requestId: "request-123",
method: "PUT",
pathname: "/api/v2/workspaces/demo",
workspaceId: "demo",
status: 500,
code: "INTERNAL_ERROR",
errorName: "KubernetesError",
message: "The request could not be completed",
kubernetesStatus: {
status: "[REDACTED]",
reason: "[REDACTED]",
code: 500,
},
},
]);
expect(JSON.stringify(logs)).not.toContain("top-secret");
expect(JSON.stringify(logs)).not.toContain("private-config");
expect(JSON.stringify(logs)).not.toContain("secret-source");
expect(JSON.stringify(logs)).not.toContain('"Failure"');
expect(JSON.stringify(logs)).not.toContain('"InternalError"');
});
test("lists persisted operation events with capability and workspace scope checks", async () => {
const store = await authenticatedStore("operator");
const workspaceStore = new MemoryWorkspaceStore({
@@ -801,7 +878,7 @@ describe("kuber v2 HTTP routes", () => {
});
test("does not expose request internals or corrupt success when auditing fails", async () => {
const reported = spyOn(console, "error").mockImplementation(() => {});
const logs: unknown[] = [];
const workspaceStore = new MemoryWorkspaceStore({
uid: () => "workspace-uid",
});
@@ -824,6 +901,8 @@ describe("kuber v2 HTTP routes", () => {
},
list: async () => [],
},
requestId: () => "audit-request-123",
logger: { error: (entry) => logs.push(entry) },
});
const result = await app(
request(
@@ -843,8 +922,23 @@ describe("kuber v2 HTTP routes", () => {
action: "workspace.stop",
});
expect(body.operation.status.state).toBe("succeeded");
expect(reported).toHaveBeenCalledTimes(1);
reported.mockRestore();
expect(logs).toEqual([
{
event: "operation.audit.failed",
requestId: "audit-request-123",
method: "POST",
pathname: "/api/v2/workspaces/demo/lifecycle",
workspaceId: "demo",
operationId: "operation-operation-uid",
status: 500,
code: "AUDIT_APPEND_FAILED",
errorName: "Error",
message: "Successful operation audit event could not be appended",
},
]);
expect(JSON.stringify(logs)).not.toContain("audit unavailable");
expect(JSON.stringify(logs)).not.toContain("private-request");
expect(JSON.stringify(logs)).not.toContain("do-not-store");
expect(
await operationStore.get("operation-operation-uid"),
).not.toHaveProperty("spec.request");