feat: improve resource reconciliation and observability

server/index.ts

@@ -1,4 +1,5 @@
 import { createApp, cleanupExpiredSessions } from "./app";
+import { logServerRequest, processLogger, safeLog } from "../lib/request-log";
 import { RedactingAuditStore } from "./audit-store";
 import { readFile } from "node:fs/promises";
 import { IMAGE_REGISTRY } from "../const";
@@ -37,6 +38,11 @@ import {
 import { KubernetesLogs } from "./kubernetes-logs";
 import { createLogService } from "./log-service";
 import { resolveRegistryDigest, type RegistryCredentials } from "./registry";
+import {
+  BackgroundBuildReconciler,
+  buildReconcileIntervalMs,
+  buildReconcileTimeoutMs,
+} from "./build-reconciler";
 
 const clients = createKubernetesClients();
 const store = new KubernetesAuthStore(clients.objects);
@@ -102,6 +108,10 @@ const buildStore = new KubernetesBuildStore(
   namespace,
   `${dataRoot}/uploads`,
 );
+const leases = new KubernetesWorkspaceLeaseProvider(
+  createKubernetesLeaseObjects(clients.coordination),
+  namespace,
+);
 const builds = new BuildController({
   cas: new FilesystemCas(`${dataRoot}/cas`),
   store: buildStore,
@@ -130,6 +140,13 @@ const builds = new BuildController({
       origin: registryResolveOrigin,
       insecure: registryResolveOrigin?.startsWith("http://"),
     }),
+  onReconcileFailure: (buildId, error) =>
+    safeLog(processLogger, {
+      event: "build.reconcile.failed",
+      buildId,
+      error,
+    }),
+  reconcileLeases: leases,
 });
 const logs = createLogService(
   new KubernetesLogs(clients.config, clients.apps, clients.core),
@@ -152,10 +169,6 @@ if (bootstrapUsername && bootstrapPassword) {
   }
 }
 
-const leases = new KubernetesWorkspaceLeaseProvider(
-  createKubernetesLeaseObjects(clients.coordination),
-);
-
 try {
   const recovered = await recoverStaleOperations(operationStore);
   if (recovered > 0)
@@ -181,6 +194,20 @@ const sessionCleanupTimer = setInterval(
 );
 sessionCleanupTimer.unref?.();
 
+const buildReconciler = new BackgroundBuildReconciler({
+  runner: builds,
+  timeoutMs: buildReconcileTimeoutMs(
+    process.env.KUBER_BUILD_RECONCILE_TIMEOUT_MS,
+  ),
+  onFailure: (error) =>
+    safeLog(processLogger, { event: "build.reconcile.scan.failed", error }),
+});
+const buildReconcileTimer = setInterval(
+  () => buildReconciler.tick(),
+  buildReconcileIntervalMs(process.env.KUBER_BUILD_RECONCILE_MS),
+);
+buildReconcileTimer.unref?.();
+
 const app = createApp({
   store,
   workspaceStore,
@@ -223,16 +250,20 @@ const server = Bun.serve<ExecConnectionData>({
     const url = new URL(request.url);
     const workspaceId = execUpgradeMatch(url);
     if (workspaceId && request.method === "GET") {
-      return handleExecUpgrade(
-        {
-          store,
-          workspaceStore,
-          auditStore,
-        },
-        request,
-        workspaceId,
-        (upgradeRequest, connection) =>
-          server.upgrade(upgradeRequest, { data: { connection } }),
+      const requestId =
+        request.headers.get("x-request-id") ?? crypto.randomUUID();
+      return logServerRequest(request, requestId, () =>
+        handleExecUpgrade(
+          {
+            store,
+            workspaceStore,
+            auditStore,
+          },
+          request,
+          workspaceId,
+          (upgradeRequest, connection) =>
+            server.upgrade(upgradeRequest, { data: { connection } }),
+        ),
       );
     }
     return app(request);