astral/kuber
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: recover workspace update races

Browse Source
This commit is contained in:
dmgnr
2026-09-06 09:25:33 +00:00 Unverified
parent 053f28a367
commit f14e976bcd
6 changed files with 483 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app.ts
+174 -21
View File
@@ -57,13 +57,14 @@ import {
WorkspaceConflictError,
WorkspaceNotFoundError,
WorkspaceValidationError,
WORKSPACE_ID_PATTERN,
workspaceEtag,
type CreateWorkspaceInput,
type UpdateWorkspaceInput,
type Workspace,
type WorkspaceStore,
} from "./workspace-store";
import { redactString } from "./redact";
import { REDACTED, redactString } from "./redact";
const API_PREFIX = "/api/v2";
const RUNTIME_SESSION_MS = 24 * 60 * 60 * 1000;
@@ -107,6 +108,33 @@ export type AppOptions = {
hashPassword?: (password: string) => Promise<string>;
now?: () => number;
requestId?: () => string;
logger?: AppLogger;
};
export type UnknownFailureLog = {
event: "request.failed";
requestId: string;
method: string;
pathname: string;
workspaceId?: string;
status: number;
code: string;
errorName: string;
message: string;
kubernetesStatus?: { status?: string; reason?: string; code?: number };
};
export type RequestErrorLog = Omit<UnknownFailureLog, "event"> & {
event: string;
operationId?: string;
};
export interface AppLogger {
error(entry: RequestErrorLog): void;
}
const defaultAppLogger: AppLogger = {
error: (entry) => console.error(JSON.stringify(entry)),
};
type LoginFailures = { count: number; resetAt: number };
@@ -236,6 +264,7 @@ export function createApp(
Bun.password.hash(password, { algorithm: "argon2id" }));
const now = options.now ?? Date.now;
const makeRequestId = options.requestId ?? randomUUID;
const logger = options.logger ?? defaultAppLogger;
const bodyLimit = options.jsonBodyLimit ?? DEFAULT_JSON_LIMIT;
const allowedOrigins = new Set(options.allowedOrigins ?? []);
const hasOriginConfiguration = options.allowedOrigins !== undefined;
@@ -282,6 +311,70 @@ export function createApp(
return result;
}
function workspaceIdFromPath(pathname: string): string | undefined {
const match = new RegExp(`^${API_PREFIX}/workspaces/([^/]+)`).exec(
pathname,
);
if (!match) return;
try {
const workspaceId = decodeURIComponent(match[1]!);
return WORKSPACE_ID_PATTERN.test(workspaceId) ? workspaceId : undefined;
} catch {
return;
}
}
function kubernetesStatus(
error: unknown,
): UnknownFailureLog["kubernetesStatus"] | undefined {
if (!isRecord(error)) return;
const candidate = isRecord(error.status)
? error.status
: isRecord(error.body)
? error.body
: undefined;
if (!candidate) return;
const status = typeof candidate.status === "string" ? REDACTED : undefined;
const reason = typeof candidate.reason === "string" ? REDACTED : undefined;
const code =
typeof candidate.code === "number" ? candidate.code : undefined;
return status || reason || code !== undefined
? { status, reason, code }
: undefined;
}
function logRequestError(
request: Request,
error: unknown,
{
event,
code,
message,
workspaceId,
operationId,
}: Pick<RequestErrorLog, "event" | "code" | "message"> & {
workspaceId?: string;
operationId?: string;
},
): void {
const pathname = new URL(request.url).pathname;
logger.error({
event,
requestId: requestIds.get(request) ?? makeRequestId(),
method: request.method,
pathname,
...(workspaceId && { workspaceId }),
...(operationId && { operationId }),
status: 500,
code,
errorName: error instanceof Error ? error.name : "UnknownError",
message: redactString(message),
...(kubernetesStatus(error) && {
kubernetesStatus: kubernetesStatus(error),
}),
});
}
async function readJson(
request: Request,
limit = bodyLimit,
@@ -911,10 +1004,13 @@ export function createApp(
operation.metadata.name,
);
} catch (error) {
console.error(
"Failed to append successful operation audit event",
error,
);
logRequestError(request, error, {
event: "operation.audit.failed",
code: "AUDIT_APPEND_FAILED",
message: "Successful operation audit event could not be appended",
workspaceId: workspace.metadata.name,
operationId: operation.metadata.name,
});
}
return response(operationBody(completed, result), 200, {
location: `${API_PREFIX}/operations/${operation.metadata.name}`,
@@ -948,10 +1044,13 @@ export function createApp(
operation.metadata.name,
);
} catch (auditError) {
console.error(
"Failed to append failed operation audit event",
auditError,
);
logRequestError(request, auditError, {
event: "operation.audit.failed",
code: "AUDIT_APPEND_FAILED",
message: "Failed operation audit event could not be appended",
workspaceId: workspace.metadata.name,
operationId: operation.metadata.name,
});
}
throw new HttpError(
failureCode === "WORKSPACE_LEASE_LOST" ? 409 : 500,
@@ -968,16 +1067,25 @@ export function createApp(
try {
await lease?.release();
} catch (error) {
console.error("Failed to release workspace operation lease", error);
logRequestError(request, error, {
event: "operation.lease_release.failed",
code: "LEASE_RELEASE_FAILED",
message: "Workspace operation lease could not be released",
workspaceId: workspace.metadata.name,
operationId: operation.metadata.name,
});
}
}
};
if (request.headers.get("prefer") === "respond-async") {
void executeClaimed().catch((error) => {
console.error(
`Background operation '${operation.metadata.name}' failed`,
error,
);
logRequestError(request, error, {
event: "operation.background.failed",
code: "BACKGROUND_OPERATION_FAILED",
message: "Background operation could not be completed",
workspaceId: workspace.metadata.name,
operationId: operation.metadata.name,
});
});
return response(
{
@@ -2122,7 +2230,6 @@ export function createApp(
);
if (error instanceof Error && /already exists/i.test(error.message))
return new HttpError(409, "Conflict", "ALREADY_EXISTS", error.message);
console.error(error);
return new HttpError(
500,
"Internal server error",
@@ -2195,7 +2302,15 @@ export function createApp(
result = await handleAuthenticated(request, url, identity);
}
} catch (error) {
result = problem(normalizeError(error), requestId);
const normalized = normalizeError(error);
if (normalized.code === "INTERNAL_ERROR")
logRequestError(request, error, {
event: "request.failed",
code: normalized.code,
message: normalized.message,
workspaceId: workspaceIdFromPath(url.pathname),
});
result = problem(normalized, requestId);
}
result.headers.set("x-request-id", requestId);
if (origin && originAllowed) {
@@ -2220,13 +2335,14 @@ export type ExecConnection = {
export type ExecAuthOptions = Pick<
AppOptions,
"store" | "workspaceStore" | "auditStore" | "now"
"store" | "workspaceStore" | "auditStore" | "now" | "logger" | "requestId"
>;
export async function authorizeExecConnection(
options: ExecAuthOptions,
request: Request,
workspaceId: string,
requestId?: string,
): Promise<ExecConnection> {
const now = options.now ?? Date.now;
const identity = await authenticateRequest(options, request, now);
@@ -2261,7 +2377,22 @@ export async function authorizeExecConnection(
workspaceId,
});
} catch (error) {
console.error("Failed to append denied exec audit event", error);
const suppliedRequestId = request.headers.get("x-request-id");
(options.logger ?? defaultAppLogger).error({
event: "exec.audit.failed",
requestId:
requestId ??
(suppliedRequestId && /^[\x21-\x7e]{1,128}$/.test(suppliedRequestId)
? suppliedRequestId
: (options.requestId ?? randomUUID)()),
method: request.method,
pathname: new URL(request.url).pathname,
...(WORKSPACE_ID_PATTERN.test(workspaceId) && { workspaceId }),
status: 500,
code: "AUDIT_APPEND_FAILED",
errorName: error instanceof Error ? error.name : "UnknownError",
message: "Denied exec audit event could not be appended",
});
}
}
throw new HttpError(
@@ -2289,7 +2420,6 @@ export function execProblem(error: unknown, requestId = ""): Response {
let httpError: HttpError;
if (error instanceof HttpError) httpError = error;
else {
console.error(error);
httpError = new HttpError(
500,
"Internal server error",
@@ -2571,11 +2701,33 @@ export async function handleExecUpgrade(
workspaceId: string,
upgrade: (request: Request, data: ExecConnection) => boolean,
): Promise<Response | undefined> {
const suppliedRequestId = request.headers.get("x-request-id");
const requestId =
suppliedRequestId && /^[\x21-\x7e]{1,128}$/.test(suppliedRequestId)
? suppliedRequestId
: (options.requestId ?? randomUUID)();
let connection: ExecConnection;
try {
connection = await authorizeExecConnection(options, request, workspaceId);
connection = await authorizeExecConnection(
options,
request,
workspaceId,
requestId,
);
} catch (error) {
return execProblem(error);
if (!(error instanceof HttpError))
(options.logger ?? defaultAppLogger).error({
event: "request.failed",
requestId,
method: request.method,
pathname: new URL(request.url).pathname,
...(WORKSPACE_ID_PATTERN.test(workspaceId) && { workspaceId }),
status: 500,
code: "INTERNAL_ERROR",
errorName: error instanceof Error ? error.name : "UnknownError",
message: "The request could not be completed",
});
return execProblem(error, requestId);
}
if (!upgrade(request, connection))
return execProblem(
@@ -2585,6 +2737,7 @@ export async function handleExecUpgrade(
"UPGRADE_FAILED",
"WebSocket upgrade failed",
),
requestId,
);
return;
}