fix: flush CONCAT_FIELDS when payload exceeds MAX_FIELD_LENGTH

src/v1/merger.ts

@@ -32,27 +32,21 @@ export function webhookMerger(app: App) {
       if (existing) {
         clearTimeout(existing.timer);
 
-        existing.data = merge(existing.data, body);
+        const oldData = existing.data;
+        const flushed = new Set<string>();
+        existing.data = merge(oldData, body, undefined, MAX_FIELD_LENGTH, flushed);
         existing.count++;
 
+        if (flushed.size > 0) {
+          existing.count = 1;
+          forward(id, url, oldData);
+          existing.timer = setTimeout(() => forward(id, url, existing.data), bufferTime);
+          return status(202);
+        }
+
         log.debug`collected ${existing}`;
 
-        existing.timer = setTimeout(async () => {
+        existing.timer = setTimeout(() => forward(id, url, existing.data), bufferTime);
-          log.debug`running ${existing}`;
-          try {
-            const res = await fetch(url, {
-              method: "POST",
-              headers: {
-                "content-type": "application/json",
-              },
-              body: JSON.stringify(existing.data),
-            });
-            if (!res.ok)
-              log.warn`Could not forward webhook: ${res.status}\n${await res.json().catch(() => res.text())}`;
-          } finally {
-            pending.delete(id);
-          }
-        }, bufferTime);
 
         return status(202);
       }
@@ -65,22 +59,7 @@ export function webhookMerger(app: App) {
 
       log.debug`collected ${state}`;
 
-      state.timer = setTimeout(async () => {
+      state.timer = setTimeout(() => forward(id, url, state.data), bufferTime);
-        log.debug`running ${state}`;
-        try {
-          const res = await fetch(url, {
-            method: "POST",
-            headers: {
-              "content-type": "application/json",
-            },
-            body: JSON.stringify(state.data),
-          });
-          if (!res.ok)
-            log.warn`Could not forward webhook: ${res.status}\n${await res.json().catch(() => res.text())}`;
-        } finally {
-          pending.delete(id);
-        }
-      }, bufferTime);
 
       pending.set(id, state);
 
@@ -104,22 +83,54 @@ async function getRoutes(): Promise<RouteMap> {
   return res;
 }
 
+const MAX_FIELD_LENGTH = Number(env.MAX_FIELD_LENGTH) || 4000;
+
+async function forward(id: string, url: string, data: unknown) {
+  log.debug`running ${id}`;
+  try {
+    const res = await fetch(url, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(data),
+    });
+    if (!res.ok)
+      log.warn`Could not forward webhook: ${res.status}\n${await res.json().catch(() => res.text())}`;
+  } finally {
+    pending.delete(id);
+  }
+}
+
 const CONCAT_FIELDS = new Set(["content"]);
 
-function merge(a: unknown, b: unknown, key?: string): unknown {
+function merge(
+  a: unknown,
+  b: unknown,
+  key?: string,
+  maxLength = Infinity,
+  flushed?: Set<string>,
+): unknown {
   if (
     key &&
     CONCAT_FIELDS.has(key) &&
     typeof a === "string" &&
     typeof b === "string"
   ) {
-    const CODEBLOCK_MATCHER = /^([^`]+)```(?:([a-z]*)\n)?([^`]+)```\n?/m;
+    const combined = (() => {
-    const [ma, mb] = [a.match(CODEBLOCK_MATCHER), b.match(CODEBLOCK_MATCHER)];
+      const CODEBLOCK_MATCHER = /^([^`]+)```(?:([a-z]*)\n)?([^`]+)```\n?/m;
-    log.debug`${ma} ${mb}`;
+      const [ma, mb] = [a.match(CODEBLOCK_MATCHER), b.match(CODEBLOCK_MATCHER)];
-    if (ma && mb && ma[1] === mb[1]) {
+      log.debug`${ma} ${mb}`;
-      return `${ma[1]}\`\`\`${ma[2] ? `${ma[2]}\n` : ""}${ma[3]}${mb[3]}\`\`\``;
+      if (ma && mb && ma[1] === mb[1]) {
+        return `${ma[1]}\`\`\`${ma[2] ? `${ma[2]}\n` : ""}${ma[3]}${mb[3]}\`\`\``;
+      }
+      return `${a}${a.endsWith("\n") || (a.endsWith("```") && !b.startsWith("`") && !b.startsWith("-# ")) ? "" : "\n"}${b}`;
+    })();
+
+    if (flushed && combined.length > maxLength) {
+      flushed.add(key ?? "");
+      return b;
     }
-    return `${a}${a.endsWith("\n") || (a.endsWith("```") && !b.startsWith("`") && !b.startsWith("-# ")) ? "" : "\n"}${b}`;
+
+    return combined;
   }
 
   if (Array.isArray(a) && Array.isArray(b)) {
@@ -139,7 +150,7 @@ function merge(a: unknown, b: unknown, key?: string): unknown {
     };
 
     for (const [k, v] of Object.entries(b)) {
-      out[k] = k in out ? merge(out[k], v, k) : v;
+      out[k] = k in out ? merge(out[k], v, k, maxLength, flushed) : v;
     }
 
     return out;