fix(webapp): fail loud on mollifier drainer misconfiguration

The bootstrap in mollifierDrainerWorker.server.ts wrapped getMollifierDrainer()
in a try/catch that logged-and-continued on any error, which absorbed the two
designed-to-crash throws in initializeMollifierDrainer():

  - "MollifierDrainer initialised without a buffer" (missing buffer client)
  - "TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS must be at least ... below
    GRACEFUL_SHUTDOWN_TIMEOUT" (shutdown-timeout reconciliation)

Both are deploy-time mistakes: silently disabling the drainer means the
gate keeps writing to the buffer, the drainer never reads, and entries
TTL out in 10min. Bounded in phase 1 (monitoring-only) but customer-
visible data loss in phase 2/3 where the drainer replays into engine.trigger.
Better to fail loud now than retrofit the contract later.

Introduce MollifierConfigurationError for the two deterministic throws.
The bootstrap's catch now rethrows that class (process crashes at module
top-level → orchestrator health check fails → deploy rolls back) while
still logging-and-continuing on transient errors (Redis blip during init
shouldn't take the whole webapp down). instanceof + name fallback covers
the Remix dev hot-reload realm edge case.

Author: d-cs

apps/webapp/app/v3/mollifier/mollifierDrainer.server.ts

@@ -6,6 +6,25 @@ import { singleton } from "~/utils/singleton";
 import { getMollifierBuffer } from "./mollifierBuffer.server";
 import type { BufferedTriggerPayload } from "./bufferedTriggerPayload.server";
 
+// Distinct error class for the deterministic "fail loud at boot" throws
+// below. The bootstrap in `mollifierDrainerWorker.server.ts` catches
+// transient/init errors and logs them so an unrelated Redis blip doesn't
+// crash the webapp, but it RETHROWS this class — a misconfigured
+// shutdown timeout or missing buffer is a deploy-time mistake that
+// should fail health checks and roll back, not silently disable a
+// half-rolled-out feature.
+//
+// The `name` getter is set explicitly so cross-realm `instanceof` checks
+// (e.g. when Remix dev hot-reloads the module and the consumer keeps a
+// reference to the old class) can fall back to `error.name === ...` and
+// still recognise the marker.
+export class MollifierConfigurationError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "MollifierConfigurationError";
+  }
+}
+
 function initializeMollifierDrainer(): MollifierDrainer<BufferedTriggerPayload> {
   const buffer = getMollifierBuffer();
   if (!buffer) {
@@ -15,7 +34,9 @@ function initializeMollifierDrainer(): MollifierDrainer<BufferedTriggerPayload>
     // the buffer can't initialise (e.g. TRIGGER_MOLLIFIER_REDIS_HOST resolves
     // to nothing). Crashing surfaces the misconfig immediately rather
     // than silently leaving entries un-drained.
-    throw new Error("MollifierDrainer initialised without a buffer — env vars inconsistent");
+    throw new MollifierConfigurationError(
+      "MollifierDrainer initialised without a buffer — env vars inconsistent",
+    );
   }
 
   // Validate BEFORE start() so a misconfigured shutdown timeout fails
@@ -37,7 +58,7 @@ function initializeMollifierDrainer(): MollifierDrainer<BufferedTriggerPayload>
     env.TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS >=
     env.GRACEFUL_SHUTDOWN_TIMEOUT - shutdownMarginMs
   ) {
-    throw new Error(
+    throw new MollifierConfigurationError(
       `TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS (${env.TRIGGER_MOLLIFIER_DRAIN_SHUTDOWN_TIMEOUT_MS}) must be at least ${shutdownMarginMs}ms below GRACEFUL_SHUTDOWN_TIMEOUT (${env.GRACEFUL_SHUTDOWN_TIMEOUT}); otherwise the primary's hard exit shadows the drainer's deadline.`,
     );
   }

apps/webapp/app/v3/mollifierDrainerWorker.server.ts

@@ -1,7 +1,10 @@
 import { env } from "~/env.server";
 import { logger } from "~/services/logger.server";
 import { signalsEmitter } from "~/services/signals.server";
-import { getMollifierDrainer } from "./mollifier/mollifierDrainer.server";
+import {
+  getMollifierDrainer,
+  MollifierConfigurationError,
+} from "./mollifier/mollifierDrainer.server";
 
 declare global {
   // eslint-disable-next-line no-var
@@ -79,6 +82,30 @@ export function initMollifierDrainerWorker(): void {
       drainer.start();
     }
   } catch (error) {
+    // Deterministic misconfig (shutdown-timeout vs GRACEFUL_SHUTDOWN_TIMEOUT,
+    // missing buffer client) is a deploy-time mistake the operator must
+    // see immediately — rethrow so the process crashes, health checks
+    // fail, and the orchestrator rolls the deploy back. Phase 1 is
+    // monitoring-only and the silent-fallback was tempting, but Phase 2/3
+    // make the drainer the source of truth for diverted triggers, where a
+    // silently-disabled drainer means data loss. Better to fail loud now
+    // than retrofit later.
+    //
+    // We accept both `instanceof` and `error.name === ...` so Remix dev
+    // hot-reload (where the consumer can hold a stale class reference)
+    // still recognises the marker.
+    if (
+      error instanceof MollifierConfigurationError ||
+      (error instanceof Error && error.name === "MollifierConfigurationError")
+    ) {
+      logger.error("Mollifier drainer misconfiguration — failing loud", {
+        error: error.message,
+      });
+      throw error;
+    }
+    // Anything else (transient Redis blip, unexpected runtime error) is
+    // logged but kept non-fatal — the rest of the webapp shouldn't go
+    // down because the buffer's Redis cluster is briefly unreachable.
     logger.error("Failed to initialise mollifier drainer", { error });
   }
 }