From d53803180b92bccb3f376bc730ff028d90941f7e Mon Sep 17 00:00:00 2001 From: autodev-bot Date: Mon, 8 Jun 2026 20:34:33 +0800 Subject: [PATCH] Fix #1897: fix(memos-local-plugin): add LLM circuit breaker for terminal provider errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #1897 reported ~12,900 paid LLM requests in 24 h on Hermes against a DeepSeek key with insufficient balance. The local `system_model_status` row count (12,900) closely tracked the provider-side `request_count` (11,344) for the same billing window. The naming is misleading: `system_model_status` is not a health probe; it is the audit row written once per LLM call (ok / fallback / error) inside `core/llm/client.ts`. With no circuit breaker, every pipeline subscriber (capture / session-relation / reward / L2 / L3 / skill / retrieval LLM filter / world-model) kept firing on every turn / closed episode / induction, generating one paid request each. Add a per-`LlmClient` circuit breaker: - Trips on terminal errors: HTTP 401/402/403 or messages containing `insufficient balance` / `invalid api key` / `unauthorized` / `account suspended` / `billing`. - Open: short-circuits subsequent calls inside the facade without contacting the provider. Throws `MemosError(LLM_UNAVAILABLE)` with `details.circuitOpen=true` so existing catch blocks still work. - Half-open after cool-down (default 5 min, configurable, min 30 s): next call probes the provider; success closes the breaker, terminal failure re-opens it for another cool-down. - Host fallback rescues a call without tripping the breaker — fallback exists precisely to keep going when the primary is down. - Coalesces `system_model_status="circuit_open"` audit rows to at most one per ~25 s while the breaker stays open, so we don't replace paid spam with audit-row spam. - Exposes `circuitOpen` / `circuitOpenUntil` / `circuitOpenedReason` via `LlmClientStats` for the Overview viewer card. - Enabled by default; legacy behaviour available via `circuitBreaker.enabled = false`. Tests: 9 new vitest cases under `tests/unit/llm/client.test.ts` covering trip on 402, trip on "insufficient balance" message, no trip on generic transient, coalescing, half-open close on success, host-fallback rescues without trip, disabled mode, stats fields, and re-open on terminal probe failure. All 59 LLM and 28 pipeline tests pass; `tsc --noEmit` clean. Out of scope (tracked separately): 429 `Retry-After` handling (issue #1620), per-tool rate limits, daily budget caps. --- apps/memos-local-plugin/core/llm/client.ts | 182 ++++++++++++++ apps/memos-local-plugin/core/llm/index.ts | 2 + apps/memos-local-plugin/core/llm/types.ts | 40 ++- .../tests/unit/llm/client.test.ts | 232 ++++++++++++++++++ 4 files changed, 455 insertions(+), 1 deletion(-) diff --git a/apps/memos-local-plugin/core/llm/client.ts b/apps/memos-local-plugin/core/llm/client.ts index 7749a572f..38ac2a395 100644 --- a/apps/memos-local-plugin/core/llm/client.ts +++ b/apps/memos-local-plugin/core/llm/client.ts @@ -71,6 +71,112 @@ export function createLlmClientWithProvider( let lastFallbackAt: number | null = null; let lastError: { at: number; message: string } | null = null; + // ─── Circuit breaker state (issue #1897) ───────────────────────────────── + // Per-client breaker that trips on terminal provider errors (401/402/403, + // "insufficient balance", "invalid api key", "unauthorized", "account + // suspended", "billing"). Short-circuits subsequent calls inside the + // facade so the broken provider is not contacted again until cool-down + // elapses. Half-open: the next call after `circuitOpenUntil` probes the + // provider; success closes the breaker, terminal failure re-opens it. + const breakerCfg = config.circuitBreaker ?? {}; + const breakerEnabled = breakerCfg.enabled !== false; + const breakerCooldownMs = Math.max(30_000, breakerCfg.cooldownMs ?? 300_000); + const breakerIsTerminal = breakerCfg.isTerminal ?? defaultIsTerminal; + const breakerNow = breakerCfg.now ?? Date.now; + let circuitOpenUntil: number | null = null; + let circuitOpenedReason: string | null = null; + let lastCircuitOpenStatusAt: number | null = null; + + function breakerIsOpen(): boolean { + if (!breakerEnabled) return false; + if (circuitOpenUntil === null) return false; + if (breakerNow() >= circuitOpenUntil) { + // Cool-down elapsed → transition to half-open. We do NOT clear + // `circuitOpenUntil` yet so the very first probe attempt that + // races with the cool-down boundary doesn't fall through to "no + // breaker" twice. The next call's success/failure handler resets + // or re-opens the breaker explicitly. + return false; + } + return true; + } + + function breakerTrip(err: unknown): void { + if (!breakerEnabled) return; + circuitOpenUntil = breakerNow() + breakerCooldownMs; + circuitOpenedReason = summarizeErrMessage(err); + // Reset the coalescer so the first suppressed call after a fresh + // trip always emits a `circuit_open` row. + lastCircuitOpenStatusAt = null; + facadeLog.warn("circuit_breaker.trip", { + provider: provider.name, + model: config.model, + until: circuitOpenUntil, + reason: circuitOpenedReason, + }); + } + + function breakerRecordSuccess(): void { + if (!breakerEnabled) return; + if (circuitOpenUntil !== null) { + facadeLog.info("circuit_breaker.close", { + provider: provider.name, + model: config.model, + }); + } + circuitOpenUntil = null; + circuitOpenedReason = null; + lastCircuitOpenStatusAt = null; + } + + /** + * Emit a coalesced `circuit_open` audit row. At most one row per + * `cooldownMs/12` window per client — bounds audit-row spam while + * still surfacing the suppressed-call event in the Logs viewer. + * The first suppressed call after a fresh trip always emits. + */ + function maybeEmitCircuitOpenStatus(opts: LlmCallOptions | undefined, op: string): void { + if (!config.onStatus) return; + const at = breakerNow(); + const coalesceWindow = Math.max(5_000, Math.floor(breakerCooldownMs / 12)); + if ( + lastCircuitOpenStatusAt !== null && + at - lastCircuitOpenStatusAt < coalesceWindow + ) { + return; + } + lastCircuitOpenStatusAt = at; + try { + config.onStatus({ + status: "circuit_open", + provider: provider.name, + model: config.model, + message: circuitOpenedReason ?? "(unknown reason)", + at, + durationMs: 0, + op, + episodeId: opts?.episodeId, + phase: opts?.phase, + }); + } catch { + /* status sink errors are non-fatal */ + } + } + + function throwBreakerOpen(): never { + const until = circuitOpenUntil ?? breakerNow(); + throw new MemosError( + ERROR_CODES.LLM_UNAVAILABLE, + `circuit_open: ${circuitOpenedReason ?? "terminal provider error"}`, + { + circuitOpen: true, + until, + provider: provider.name, + model: config.model, + }, + ); + } + /** * Mark a successful primary-provider call. We **do not** clear * `lastError` / `lastFallbackAt` here — the viewer picks the most @@ -151,6 +257,15 @@ export function createLlmClientWithProvider( opts: LlmCallOptions | undefined, op: string, ): Promise<{ completion: LlmCompletion }> { + // ── Circuit breaker short-circuit ── + // When the breaker is open we never reach the provider, so no paid + // request is generated. We still emit (coalesced) `circuit_open` + // status rows so the Logs viewer / Overview can surface that + // suppression is happening. + if (breakerIsOpen()) { + maybeEmitCircuitOpenStatus(opts, op); + throwBreakerOpen(); + } requests++; const startedAt = Date.now(); try { @@ -166,6 +281,7 @@ export function createLlmClientWithProvider( }; record(completion, op, messages); const okAt = markOk(); + breakerRecordSuccess(); notifyStatus({ status: "ok", provider: provider.name, @@ -202,7 +318,16 @@ export function createLlmClientWithProvider( // bridge saved this call. Tag the slot yellow (`lastFallbackAt`) // and surface the upstream error to the user via the // system_error log so they can see *why* fallback engaged. + // + // The circuit breaker stays CLOSED here: from the caller's + // perspective the call was rescued, and tripping the breaker + // on host-fallback success would defeat the point of the + // bridge (it exists precisely to keep going when the primary + // is down). The fallback path also already records the + // primary's failure, so the operator still sees the red trail + // in the Logs viewer. const fallbackAt = markFallback(err); + breakerRecordSuccess(); notifyOnError(err); notifyStatus({ status: "fallback", @@ -225,6 +350,10 @@ export function createLlmClientWithProvider( primary: summarizeErr(err), host: summarizeErr(hostErr), }); + // Primary AND host bridge both failed terminally. Trip on the + // primary error (the one the operator typically needs to fix + // — host bridge failures are usually transient stdio issues). + if (breakerIsTerminal(err)) breakerTrip(err); notifyOnError(hostErr); notifyStatus({ status: "error", @@ -249,6 +378,7 @@ export function createLlmClientWithProvider( } failures++; const failAt = markFail(err); + if (breakerIsTerminal(err)) breakerTrip(err); notifyOnError(err); notifyStatus({ status: "error", @@ -415,6 +545,12 @@ export function createLlmClientWithProvider( const call = buildCallInput(opts, opts?.jsonMode === true); const ctx = makeCtx(opts, asProviderLog(providerLog)); + // Short-circuit stream calls when the breaker is open. We do not + // count a suppressed call against `requests` (no network hit). + if (breakerIsOpen()) { + maybeEmitCircuitOpenStatus(opts, opts?.op ?? "stream"); + throwBreakerOpen(); + } requests++; const start = Date.now(); let acc = ""; @@ -448,6 +584,7 @@ export function createLlmClientWithProvider( if (usage?.promptTokens) totalPromptTokens += usage.promptTokens; if (usage?.completionTokens) totalCompletionTokens += usage.completionTokens; const okAt = markOk(); + breakerRecordSuccess(); notifyStatus({ status: "ok", provider: provider.name, @@ -461,6 +598,7 @@ export function createLlmClientWithProvider( } catch (err) { failures++; const failAt = markFail(err); + if (breakerIsTerminal(err)) breakerTrip(err); facadeLog.error("stream.failed", { err: summarizeErr(err) }); notifyOnError(err); notifyStatus({ @@ -497,6 +635,9 @@ export function createLlmClientWithProvider( lastOkAt, lastFallbackAt, lastError, + circuitOpen: breakerIsOpen(), + circuitOpenUntil, + circuitOpenedReason, }; }, resetStats(): void { @@ -509,6 +650,9 @@ export function createLlmClientWithProvider( lastOkAt = null; lastFallbackAt = null; lastError = null; + circuitOpenUntil = null; + circuitOpenedReason = null; + lastCircuitOpenStatusAt = null; }, async close(): Promise { await provider.close?.(); @@ -522,6 +666,10 @@ export function createLlmClientWithProvider( timeoutMs: config.timeoutMs, maxRetries: config.maxRetries, fallbackToHost: config.fallbackToHost, + circuitBreaker: { + enabled: breakerEnabled, + cooldownMs: breakerCooldownMs, + }, }); return client; @@ -562,6 +710,40 @@ function shouldFallback(err: unknown, config: LlmConfig, providerName: LlmProvid ); } +/** + * Default circuit-breaker classifier for terminal provider errors. + * + * A "terminal" error is one that will keep failing until the operator + * intervenes (top up balance, fix API key, fix model name). Retrying + * such an error just burns paid quota and pollutes the audit log, so + * the breaker opens and short-circuits further calls for the cool- + * down window. Issue #1897 reports the symptom — ~12,900 paid LLM + * requests in 24 h against a key with insufficient balance. + * + * Detection sources, in order: + * 1. `MemosError(LLM_UNAVAILABLE)` with `details.status` ∈ 401/402/403 + * — set by `core/llm/fetcher.ts::httpPostJson` for non-ok HTTP + * responses. + * 2. Well-known lowercase phrases in the error message (so providers + * that return 400 for "Insufficient Balance" — looking at you, + * DeepSeek — are still recognized). + */ +function defaultIsTerminal(err: unknown): boolean { + if (!(err instanceof MemosError)) return false; + if (err.code !== ERROR_CODES.LLM_UNAVAILABLE) return false; + const status = Number((err.details as { status?: unknown } | undefined)?.status); + if (status === 401 || status === 402 || status === 403) return true; + const msg = (err.message ?? "").toLowerCase(); + return ( + msg.includes("insufficient balance") || + msg.includes("invalid api key") || + msg.includes("invalid_api_key") || + msg.includes("unauthorized") || + msg.includes("account suspended") || + msg.includes("billing") + ); +} + // ─── Logger adapter ────────────────────────────────────────────────────────── function asProviderLog(log: Logger): LlmProviderLogger { diff --git a/apps/memos-local-plugin/core/llm/index.ts b/apps/memos-local-plugin/core/llm/index.ts index 847c965f0..a295cd4a4 100644 --- a/apps/memos-local-plugin/core/llm/index.ts +++ b/apps/memos-local-plugin/core/llm/index.ts @@ -28,6 +28,7 @@ export { LocalOnlyLlmProvider } from "./providers/local-only.js"; export * from "./prompts/index.js"; export type { LlmCallOptions, + LlmCircuitBreakerConfig, LlmCompleteJsonOptions, LlmCompletion, LlmClient, @@ -40,6 +41,7 @@ export type { LlmProviderLogger, LlmProviderName, LlmRole, + LlmStatusDetail, LlmStreamChunk, LlmUsage, ProviderCallInput, diff --git a/apps/memos-local-plugin/core/llm/types.ts b/apps/memos-local-plugin/core/llm/types.ts index ddfe80c1e..4a835caee 100644 --- a/apps/memos-local-plugin/core/llm/types.ts +++ b/apps/memos-local-plugin/core/llm/types.ts @@ -47,6 +47,33 @@ export interface LlmConfig { * daemon can display status produced by a separate stdio bridge. */ onStatus?: (detail: LlmStatusDetail) => void; + /** + * Optional circuit breaker config. The breaker trips on terminal + * provider errors (HTTP 401/402/403, or well-known phrases like + * "insufficient balance" / "invalid api key" / "unauthorized" / + * "account suspended" / "billing") and short-circuits subsequent + * calls for a cool-down window. Defaults to enabled. See + * `apps/memos-local-plugin/openspec/changes/.../design.md` + * (issue #1897) for the full state machine. + */ + circuitBreaker?: LlmCircuitBreakerConfig; +} + +export interface LlmCircuitBreakerConfig { + /** Default true. Set false to restore legacy (no-breaker) behavior. */ + enabled?: boolean; + /** + * Cool-down window before the breaker enters half-open. Default + * 300_000 ms (5 minutes); minimum clamped to 30_000 ms. + */ + cooldownMs?: number; + /** + * Override the default classifier. Returns true if the error should + * trip the breaker (terminal / non-recoverable). + */ + isTerminal?: (err: unknown) => boolean; + /** Injected clock for tests. Default `Date.now`. */ + now?: () => number; } export interface LlmErrorDetail { @@ -67,7 +94,7 @@ export interface LlmErrorDetail { } export interface LlmStatusDetail { - status: "ok" | "fallback" | "error"; + status: "ok" | "fallback" | "error" | "circuit_open"; provider: LlmProviderName | string; model: string; message?: string; @@ -260,6 +287,17 @@ export interface LlmClientStats extends LastCallStatus { retries: number; totalPromptTokens: number; totalCompletionTokens: number; + /** + * True while the per-client circuit breaker is open (and any + * cooldown timer has not yet elapsed). When true, further calls are + * short-circuited inside the facade and throw immediately without + * touching the provider. See issue #1897. + */ + circuitOpen: boolean; + /** Epoch ms at which the open breaker becomes eligible for half-open probe. */ + circuitOpenUntil: number | null; + /** Free-text reason from the error that opened the breaker. */ + circuitOpenedReason: string | null; } export interface LlmClient { diff --git a/apps/memos-local-plugin/tests/unit/llm/client.test.ts b/apps/memos-local-plugin/tests/unit/llm/client.test.ts index cd5cf6106..f103ccf5d 100644 --- a/apps/memos-local-plugin/tests/unit/llm/client.test.ts +++ b/apps/memos-local-plugin/tests/unit/llm/client.test.ts @@ -14,6 +14,7 @@ import type { LlmProvider, LlmProviderCtx, LlmProviderName, + LlmStatusDetail, LlmStreamChunk, ProviderCallInput, ProviderCompletion, @@ -277,4 +278,235 @@ describe("llm/client", () => { const client = createLlmClientWithProvider(cfg(), fake); await expect(client.complete([] as LlmMessage[])).rejects.toBeInstanceOf(MemosError); }); + + // ─── Circuit breaker (issue #1897) ────────────────────────────────────── + describe("circuit breaker", () => { + function statusSink(): { rows: LlmStatusDetail[]; push: (d: LlmStatusDetail) => void } { + const rows: LlmStatusDetail[] = []; + return { rows, push: (d) => rows.push(d) }; + } + + it("trips on terminal 402 and short-circuits subsequent calls", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "HTTP 402 from openai_compatible", { + provider: "openai_compatible", + status: 402, + }), + ); + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 300_000, now: tick }, + }), + provider, + ); + // First call: real provider hit, fails terminally → breaker trips. + await expect(client.complete("first")).rejects.toBeInstanceOf(MemosError); + expect(provider.calls).toBe(1); + // Second call: should be short-circuited; provider must NOT be invoked. + now += 100; + await expect(client.complete("second")).rejects.toMatchObject({ + code: ERROR_CODES.LLM_UNAVAILABLE, + details: { circuitOpen: true }, + }); + expect(provider.calls).toBe(1); + // Stats expose circuit state. + const stats = client.stats(); + expect(stats.circuitOpen).toBe(true); + expect(stats.circuitOpenUntil).toBe(1_000_000 + 300_000); + expect(stats.circuitOpenedReason).toMatch(/402/); + // Audit rows: at least one `error` and one `circuit_open`. + const statuses = sink.rows.map((r) => r.status); + expect(statuses).toContain("error"); + expect(statuses).toContain("circuit_open"); + }); + + it("trips on 'insufficient balance' message regardless of HTTP status", async () => { + const sink = statusSink(); + const provider = new ThrowingProvider( + new MemosError( + ERROR_CODES.LLM_UNAVAILABLE, + "HTTP 400 from openai_compatible: Insufficient Balance", + { provider: "openai_compatible", status: 400 }, + ), + ); + const client = createLlmClientWithProvider( + cfg({ onStatus: sink.push, circuitBreaker: { enabled: true } }), + provider, + ); + await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); + await expect(client.complete("y")).rejects.toMatchObject({ + details: { circuitOpen: true }, + }); + expect(provider.calls).toBe(1); + }); + + it("does NOT trip on generic LLM_UNAVAILABLE without terminal markers", async () => { + const sink = statusSink(); + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "transient network blip"), + ); + const client = createLlmClientWithProvider( + cfg({ onStatus: sink.push, circuitBreaker: { enabled: true } }), + provider, + ); + // Two consecutive failures with non-terminal classification → both + // calls reach the provider, breaker stays closed. + await expect(client.complete("x")).rejects.toBeInstanceOf(MemosError); + await expect(client.complete("y")).rejects.toBeInstanceOf(MemosError); + expect(provider.calls).toBe(2); + expect(client.stats().circuitOpen).toBe(false); + }); + + it("coalesces circuit_open status rows within cooldown", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "401", { status: 401 }), + ); + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 300_000, now: tick }, + }), + provider, + ); + await expect(client.complete("trip")).rejects.toBeTruthy(); + // 20 suppressed calls within 1 second → at most a small number of + // `circuit_open` rows (we expect 1, but tolerate up to 2 in case the + // coalescer counts the very first short-circuit as a separate row). + for (let i = 0; i < 20; i++) { + now += 50; + await expect(client.complete(`spam-${i}`)).rejects.toBeTruthy(); + } + const openRows = sink.rows.filter((r) => r.status === "circuit_open"); + expect(openRows.length).toBeGreaterThanOrEqual(1); + expect(openRows.length).toBeLessThanOrEqual(2); + // Provider was only touched once (the very first call that tripped). + expect(provider.calls).toBe(1); + }); + + it("half-open probes the provider after cooldown and closes on success", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + let attempt = 0; + const provider: LlmProvider = { + name: "openai_compatible", + async complete() { + attempt++; + if (attempt === 1) { + throw new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "401", { status: 401 }); + } + return { text: "ok", durationMs: 1 }; + }, + }; + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 60_000, now: tick }, + }), + provider, + ); + await expect(client.complete("trip")).rejects.toBeTruthy(); + expect(client.stats().circuitOpen).toBe(true); + // Suppressed call before cooldown elapses. + now += 30_000; + await expect(client.complete("suppressed")).rejects.toMatchObject({ + details: { circuitOpen: true }, + }); + expect(attempt).toBe(1); + // After cooldown, the next call probes the provider. + now += 31_000; // total 61_000 since trip + const r = await client.complete("probe"); + expect(r.text).toBe("ok"); + expect(attempt).toBe(2); + // Breaker closes on success. + expect(client.stats().circuitOpen).toBe(false); + }); + + it("does NOT trip when host fallback rescues the call", async () => { + const sink = statusSink(); + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "402", { status: 402 }), + ); + registerHostLlmBridge({ + id: "test.host", + async complete() { + return { text: "rescued", model: "host-m", durationMs: 1 }; + }, + }); + const client = createLlmClientWithProvider( + cfg({ + fallbackToHost: true, + onStatus: sink.push, + circuitBreaker: { enabled: true }, + }), + provider, + ); + const r = await client.complete("call-1"); + expect(r.servedBy).toBe("host_fallback"); + // Breaker still closed: fallback rescued the call. + expect(client.stats().circuitOpen).toBe(false); + const r2 = await client.complete("call-2"); + expect(r2.servedBy).toBe("host_fallback"); + // Provider hit twice; not short-circuited. + expect(provider.calls).toBe(2); + }); + + it("disabled when circuitBreaker.enabled=false (legacy behavior)", async () => { + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "402", { status: 402 }), + ); + const client = createLlmClientWithProvider( + cfg({ circuitBreaker: { enabled: false } }), + provider, + ); + await expect(client.complete("a")).rejects.toBeTruthy(); + await expect(client.complete("b")).rejects.toBeTruthy(); + await expect(client.complete("c")).rejects.toBeTruthy(); + // All three calls reached the provider. + expect(provider.calls).toBe(3); + expect(client.stats().circuitOpen).toBe(false); + }); + + it("LlmClientStats exposes circuit fields when closed", async () => { + const fake = new FakeProvider("openai_compatible", () => ({ text: "ok", durationMs: 1 })); + const client = createLlmClientWithProvider(cfg(), fake); + await client.complete("x"); + const s = client.stats(); + expect(s.circuitOpen).toBe(false); + expect(s.circuitOpenUntil).toBeNull(); + expect(s.circuitOpenedReason).toBeNull(); + }); + + it("re-opens the breaker if the half-open probe fails terminally again", async () => { + const sink = statusSink(); + let now = 1_000_000; + const tick = () => now; + const provider = new ThrowingProvider( + new MemosError(ERROR_CODES.LLM_UNAVAILABLE, "402", { status: 402 }), + ); + const client = createLlmClientWithProvider( + cfg({ + onStatus: sink.push, + circuitBreaker: { enabled: true, cooldownMs: 60_000, now: tick }, + }), + provider, + ); + await expect(client.complete("trip")).rejects.toBeTruthy(); + expect(client.stats().circuitOpen).toBe(true); + now += 61_000; + // Half-open probe still fails terminally → breaker re-opens. + await expect(client.complete("probe")).rejects.toBeTruthy(); + expect(client.stats().circuitOpen).toBe(true); + expect(client.stats().circuitOpenUntil).toBe(now + 60_000); + // Provider was touched twice total (initial trip + probe). + expect(provider.calls).toBe(2); + }); + }); });