feat: prepare 2.6.1-rc5 shared databases and build SSE

server/app.ts

@@ -1,9 +1,10 @@
 import type { KubernetesObject } from "@kubernetes/client-node";
 import { randomUUID } from "node:crypto";
 import type { ComposeSpecification } from "../schema/docker.d";
-import { DatabaseReconciliationError } from "../lib/database";
+import { DATABASE_RECONCILE_PHASES, DatabaseReconciliationError } from "../lib/database";
 import type { Operation as PublicOperation } from "../shared/api";
 import type { BuildRequest, Sha256Digest } from "../shared/build-protocol";
+import { KUBER_VERSION, KUBER_VERSION_HEADER } from "../shared/version";
 import {
   createToken,
   hashToken,
@@ -28,6 +29,7 @@ import {
   BuildValidationError,
   type BuildController,
 } from "./build-controller";
+import { BuildEventStreamHub } from "./build-event-stream";
 import { KubernetesLogError, type LogService } from "./log-service";
 import {
   ExecService,
@@ -142,6 +144,11 @@ export type UnknownFailureLog = {
 export type RequestErrorLog = Omit<UnknownFailureLog, "event"> & {
   event: string;
   operationId?: string;
+  phase?: string;
+  errorClass?: string;
+  providerStatus?: number;
+  providerCode?: string;
+  topFrame?: string;
 };
 
 export interface AppLogger {
@@ -200,6 +207,62 @@ class HttpError extends Error {
   }
 }
 
+const DATABASE_PHASES = new Set<string>(DATABASE_RECONCILE_PHASES);
+const DATABASE_ERROR_CLASSES = new Set([
+  "Error", "TypeError", "RangeError", "SyntaxError", "AbortError",
+  "ApiException", "ResponseError", "FetchError", "TimeoutError",
+]);
+const DATABASE_ERROR_CODES = new Set([
+  "ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "EHOSTUNREACH",
+  "ENOTFOUND", "EAI_AGAIN", "ABORT_ERR", "UND_ERR_CONNECT_TIMEOUT",
+]);
+
+/** Never serialize an unknown error: its message, stack and provider fields may contain secrets. */
+function databaseFailureDiagnostics(error: unknown): Record<string, unknown> {
+  let current = error;
+  let phase = "operation execution";
+  const seen = new Set<unknown>();
+  for (
+    let depth = 0;
+    depth < 5 && current && typeof current === "object" && !seen.has(current);
+    depth++
+  ) {
+    seen.add(current);
+    if (
+      current instanceof DatabaseReconciliationError &&
+      DATABASE_PHASES.has(current.phase)
+    ) {
+      phase = current.phase;
+    }
+    const cause = (current as { cause?: unknown }).cause;
+    if (!cause || typeof cause !== "object" || seen.has(cause)) break;
+    current = cause;
+  }
+  const source = current && typeof current === "object"
+    ? current as Record<string, unknown> : {};
+  const errorClass = current instanceof Error && DATABASE_ERROR_CLASSES.has(current.name)
+    ? current.name
+    : "UnknownError";
+  const status = [source.statusCode, source.status, source.code].find(
+    (value) => typeof value === "number" && Number.isInteger(value) &&
+      value >= 400 && value < 600,
+  );
+  const errorCode = typeof source.code === "string" && DATABASE_ERROR_CODES.has(source.code)
+    ? source.code
+    : undefined;
+  // Only the first frame and a fixed set of our source files; never log raw stacks or paths.
+  const firstFrame = current instanceof Error ? current.stack?.split("\n")[1] : undefined;
+  const frame = firstFrame &&
+    /(?:^|\/)\b((?:lib\/database|server\/management|server\/app)\.ts):(\d+):(\d+)\b/.exec(firstFrame);
+  return {
+    phase,
+    errorClass,
+    ...(status !== undefined && { providerStatus: status }),
+    ...(errorCode && { providerCode: errorCode }),
+    ...(frame && { topFrame: `${frame[1]}:${frame[2]}:${frame[3]}` }),
+  };
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
@@ -282,6 +345,9 @@ export function createApp(
   const now = options.now ?? Date.now;
   const makeRequestId = options.requestId ?? randomUUID;
   const logger = options.logger ?? defaultAppLogger;
+  const buildEventStreams = options.builds
+    ? new BuildEventStreamHub(options.builds)
+    : undefined;
   const bodyLimit = options.jsonBodyLimit ?? DEFAULT_JSON_LIMIT;
   const allowedOrigins = new Set(options.allowedOrigins ?? []);
   const hasOriginConfiguration = options.allowedOrigins !== undefined;
@@ -1150,25 +1216,43 @@ export function createApp(
         });
       } catch (error) {
         if (!operationStarted) throw error;
-        const message = error instanceof Error ? error.message : String(error);
         const leaseLost =
           leaseOwnershipLost ||
           (error instanceof HttpError && error.code === "WORKSPACE_LEASE_LOST");
+        const databaseFailure =
+          action === "databases.reconcile" && !leaseLost
+            ? error instanceof DatabaseReconciliationError
+              ? error
+              : new DatabaseReconciliationError("operation execution", error)
+            : undefined;
+        if (databaseFailure) {
+          logger.error({
+            event: "operation.database_reconcile.failed",
+            requestId: requestIds.get(request) ?? makeRequestId(),
+            method: "POST",
+            pathname: "/api/v2/workspaces/:workspaceId/databases",
+            operationId: operation.metadata.name,
+            status: 500,
+            code: "DATABASE_RECONCILE_FAILED",
+            errorName: "DatabaseReconciliationError",
+            message: "Database reconciliation failed",
+            ...databaseFailureDiagnostics(databaseFailure),
+          });
+        }
+        const message =
+          databaseFailure?.message ??
+          (error instanceof Error ? error.message : String(error));
         const failed = await transitionOperationToFailure(
           operation.metadata.name,
           {
             code: leaseLost
               ? "WORKSPACE_LEASE_LOST"
-              : action === "databases.reconcile" &&
-                  error instanceof DatabaseReconciliationError
+              : databaseFailure
                 ? "DATABASE_RECONCILE_FAILED"
                 : "OPERATION_FAILED",
             message: leaseLost
               ? "Workspace operation lease ownership was lost"
-              : action === "databases.reconcile" &&
-                  !(error instanceof DatabaseReconciliationError)
-                ? "Database reconciliation failed"
-                : message,
+              : message,
           },
         );
         const failure = failed.status.error;
@@ -1470,13 +1554,31 @@ export function createApp(
       );
       if (!action && request.method === "GET")
         return response(await builds.getBuildStatus(id));
-      if (action === "events" && request.method === "GET")
+      if (action === "events" && request.method === "GET") {
+        if (request.headers.get("accept")?.split(",").some((value) =>
+          value.trim().split(";")[0]?.trim().toLowerCase() === "text/event-stream",
+        )) {
+          const cursor = request.headers.get("last-event-id") ?? url.searchParams.get("after");
+          if (
+            cursor !== null &&
+            (!/^(0|[1-9]\d*)$/.test(cursor) ||
+              !Number.isSafeInteger(Number(cursor)))
+          )
+            throw new HttpError(
+              400,
+              "Invalid cursor",
+              "INVALID_QUERY",
+              "Event cursor must be a non-negative safe integer",
+            );
+          return buildEventStreams!.open(id, request);
+        }
         return response(
           await builds.getBuildEvents(
             id,
             nonNegativeInteger(url.searchParams.get("after"), "after") ?? 0,
           ),
         );
+      }
       if (action === "reconcile" && request.method === "POST")
         return response(await builds.reconcileBuild(id));
       if (action === "cancel" && request.method === "POST")
@@ -2553,6 +2655,7 @@ export function createApp(
           result = problem(normalized, requestId);
         }
         result.headers.set("x-request-id", requestId);
+        result.headers.set(KUBER_VERSION_HEADER, KUBER_VERSION);
         if (origin && originAllowed) {
           result.headers.set("access-control-allow-origin", origin);
           result.headers.set("vary", "Origin");
@@ -2689,6 +2792,7 @@ export function execProblem(error: unknown, requestId = ""): Response {
     },
   );
   result.headers.set("content-type", "application/problem+json");
+  result.headers.set(KUBER_VERSION_HEADER, KUBER_VERSION);
   return result;
 }