feat: improve resource reconciliation and observability

server/build-reconciler.ts

@@ -0,0 +1,84 @@
+export const DEFAULT_BUILD_RECONCILE_MS = 30 * 1000;
+export const DEFAULT_BUILD_RECONCILE_TIMEOUT_MS = 20 * 1000;
+/** Leaves time for the 30-second reconciliation lease to be renewed or released. */
+export const MAX_BUILD_RECONCILE_TIMEOUT_MS = 25 * 1000;
+/** Node clamps timer delays above this value to approximately one millisecond. */
+export const MAX_BUILD_RECONCILE_MS = 2_147_483_647;
+
+export function buildReconcileIntervalMs(value: string | undefined): number {
+  const parsed = Number(value ?? DEFAULT_BUILD_RECONCILE_MS);
+  return Number.isFinite(parsed) &&
+    parsed > 0 &&
+    parsed <= MAX_BUILD_RECONCILE_MS
+    ? parsed
+    : DEFAULT_BUILD_RECONCILE_MS;
+}
+
+export function buildReconcileTimeoutMs(value: string | undefined): number {
+  const parsed = Number(value ?? DEFAULT_BUILD_RECONCILE_TIMEOUT_MS);
+  return Number.isFinite(parsed) &&
+    parsed > 0 &&
+    parsed <= MAX_BUILD_RECONCILE_TIMEOUT_MS
+    ? parsed
+    : DEFAULT_BUILD_RECONCILE_TIMEOUT_MS;
+}
+
+export interface BuildReconcileRunner {
+  reconcilePendingBuilds(options?: { signal?: AbortSignal }): Promise<void>;
+}
+
+export interface BackgroundBuildReconcilerOptions {
+  runner: BuildReconcileRunner;
+  timeoutMs: number;
+  onFailure(error: unknown): void;
+  setTimeout?: typeof setTimeout;
+  clearTimeout?: typeof clearTimeout;
+}
+
+/**
+ * Runs one scan at a time. A deadline cancels its generation and reports the
+ * timeout, but the generation remains active until its promise settles. Some
+ * Kubernetes adapters cannot abort an in-flight request, so clearing it at the
+ * deadline could otherwise overlap work for the same build.
+ */
+export class BackgroundBuildReconciler {
+  private current:
+    | {
+        controller: AbortController;
+        timeout: ReturnType<typeof setTimeout>;
+      }
+    | undefined;
+  private readonly setTimeout: typeof setTimeout;
+  private readonly clearTimeout: typeof clearTimeout;
+
+  constructor(private readonly options: BackgroundBuildReconcilerOptions) {
+    this.setTimeout = options.setTimeout ?? setTimeout;
+    this.clearTimeout = options.clearTimeout ?? clearTimeout;
+  }
+
+  tick(): void {
+    if (this.current) return;
+    const controller = new AbortController();
+    const timeout = this.setTimeout(() => {
+      if (this.current?.controller !== controller) return;
+      controller.abort("Build reconciliation scan timed out");
+      this.options.onFailure(
+        new Error(
+          `Build reconciliation scan timed out after ${this.options.timeoutMs}ms`,
+        ),
+      );
+    }, this.options.timeoutMs);
+    this.current = { controller, timeout };
+
+    void this.options.runner
+      .reconcilePendingBuilds({ signal: controller.signal })
+      .catch((error) => {
+        if (!controller.signal.aborted) this.options.onFailure(error);
+      })
+      .finally(() => {
+        if (this.current?.controller !== controller) return;
+        this.clearTimeout(timeout);
+        this.current = undefined;
+      });
+  }
+}