feat: harden self-managed reconciliation

server/operation-store.ts

@@ -48,14 +48,26 @@ export class OperationNotFoundError extends Error {
 
 /** createIdempotent must atomically index workspaceId + idempotencyKey. */
 export interface OperationPersistence {
-  createIdempotent(operation: Operation): Promise<Operation>;
+  createIdempotent(operation: Operation): Promise<CreatedOperation>;
   get(id: string): Promise<Operation | undefined>;
   list(workspaceId?: string): Promise<Operation[]>;
   replace(operation: Operation, expectedResourceVersion: string): Promise<void>;
 }
 
+/** The idempotent operation and whether this caller created its record. */
+export interface CreatedOperation {
+  operation: Operation;
+  created: boolean;
+}
+
 export interface OperationStore {
   create(input: CreateOperationInput): Promise<Operation>;
+  createOrReuse(input: CreateOperationInput): Promise<CreatedOperation>;
+  /**
+   * Atomically claims a pending operation for execution. A false result means
+   * another request has already moved it out of pending.
+   */
+  claimExecution(id: string): Promise<Operation | undefined>;
   get(id: string): Promise<Operation | undefined>;
   list(workspaceId?: string): Promise<Operation[]>;
   transition(
@@ -193,6 +205,20 @@ export function sanitizeOperationResult(
   );
 }
 
+/** Keep operation error codes stable while preventing provider details from escaping. */
+export function sanitizeOperationError(
+  error: OperationError,
+  action?: string,
+): OperationError {
+  const message =
+    action === "databases.reconcile"
+      ? "Database reconciliation failed"
+      : action === "storage.reconcile"
+        ? "Storage reconciliation failed"
+        : redactString(error.message);
+  return { code: error.code, message };
+}
+
 export class PersistentOperationStore implements OperationStore {
   constructor(
     private readonly persistence: OperationPersistence,
@@ -201,6 +227,10 @@ export class PersistentOperationStore implements OperationStore {
   ) {}
 
   async create(input: CreateOperationInput): Promise<Operation> {
+    return (await this.createOrReuse(input)).operation;
+  }
+
+  async createOrReuse(input: CreateOperationInput): Promise<CreatedOperation> {
     if (!input.workspaceId || !input.action.trim()) {
       throw new OperationValidationError(
         "Workspace ID and action are required",
@@ -233,12 +263,12 @@ export class PersistentOperationStore implements OperationStore {
       status: { state: "pending" },
     };
     const stored = await this.persistence.createIdempotent(operation);
-    if (stored.spec.requestHash !== operation.spec.requestHash) {
+    if (stored.operation.spec.requestHash !== operation.spec.requestHash) {
       throw new OperationConflictError(
         "Idempotency key was already used for a different request",
       );
     }
-    return clone(stored);
+    return { operation: clone(stored.operation), created: stored.created };
   }
 
   async get(id: string) {
@@ -246,6 +276,15 @@ export class PersistentOperationStore implements OperationStore {
     return operation && clone(operation);
   }
 
+  async claimExecution(id: string): Promise<Operation | undefined> {
+    try {
+      return await this.transition(id, "running");
+    } catch (error) {
+      if (!(error instanceof OperationConflictError)) throw error;
+      return;
+    }
+  }
+
   async list(workspaceId?: string) {
     return clone(await this.persistence.list(workspaceId));
   }
@@ -288,7 +327,9 @@ export class PersistentOperationStore implements OperationStore {
           sanitizeOperationResult(options.result, current.spec.action),
         ),
       }),
-      ...(options.error && { error: clone(options.error) }),
+      ...(options.error && {
+        error: sanitizeOperationError(options.error, current.spec.action),
+      }),
     };
     await this.persistence.replace(operation, current.metadata.resourceVersion);
     return clone(operation);
@@ -302,10 +343,14 @@ export class MemoryOperationPersistence implements OperationPersistence {
   async createIdempotent(operation: Operation) {
     const key = `${operation.spec.workspaceId}\0${operation.spec.idempotencyKey}`;
     const existingId = this.idempotency.get(key);
-    if (existingId) return clone(this.operations.get(existingId)!);
+    if (existingId)
+      return {
+        operation: clone(this.operations.get(existingId)!),
+        created: false,
+      };
     this.operations.set(operation.metadata.name, clone(operation));
     this.idempotency.set(key, operation.metadata.name);
-    return clone(operation);
+    return { operation: clone(operation), created: true };
   }
 
   async get(id: string) {
@@ -355,11 +400,7 @@ export class MemoryWorkspaceLeaseProvider implements WorkspaceLeaseProvider {
       throw new OperationValidationError("Invalid workspace lease request");
     }
     const current = this.leases.get(workspaceId);
-    if (
-      current &&
-      current.expiresAt > this.now() &&
-      current.holder !== holder
-    ) {
+    if (current && current.expiresAt > this.now()) {
       return undefined;
     }
     const token = randomUUID();