From ac5e5bd02b26d22c4967c055ef3267c9b7a29232 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Thu, 16 Jul 2026 09:25:55 +0300 Subject: [PATCH 1/2] docs(planning): design for autovacuum cost knobs + honest docs Adds optional vacuum_cost_delay/vacuum_cost_limit to outbox_autovacuum_ddl (the throughput lever a churn measurement showed is the binding constraint under heavy churn; off by default). Rewrites the autovacuum docs to scope the benefit honestly: scale_factor/threshold control eligibility (necessary not sufficient); cost_delay/cost_limit control throughput. No probe change. --- .../2026-07-16.01-autovacuum-cost-knobs.md | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 planning/changes/2026-07-16.01-autovacuum-cost-knobs.md diff --git a/planning/changes/2026-07-16.01-autovacuum-cost-knobs.md b/planning/changes/2026-07-16.01-autovacuum-cost-knobs.md new file mode 100644 index 0000000..c21370f --- /dev/null +++ b/planning/changes/2026-07-16.01-autovacuum-cost-knobs.md @@ -0,0 +1,103 @@ +--- +summary: Add optional `vacuum_cost_delay` / `vacuum_cost_limit` params to `outbox_autovacuum_ddl` (the vacuum-throughput lever, off by default), and rewrite the autovacuum docs to scope the benefit honestly — the shipped scale_factor/threshold settings control vacuum *eligibility* (necessary but not sufficient), while cost_delay/cost_limit control vacuum *throughput*, which a churn measurement showed is the binding constraint under heavy load. +--- + +# Design: Autovacuum cost (throughput) knobs + honest docs + +## Summary + +`2026-07-15.02` shipped `outbox_autovacuum_ddl` with `scale_factor=0` + a constant +threshold — settings that make autovacuum fire *sooner* and independent of table +size (eligibility). A follow-up churn measurement against real Postgres showed +those settings are **necessary but not sufficient**: under heavy sustained churn, +vacuum *throughput* is the binding constraint, and throughput is governed by +`autovacuum_vacuum_cost_delay` / `autovacuum_vacuum_cost_limit`, which the shipped +helper did not expose. + +This change adds those two as **optional** per-table params on +`outbox_autovacuum_ddl` (default `None` = not emitted = today's output unchanged), +and rewrites the autovacuum docs to scope the benefit honestly. + +## Motivation + +A three-run churn measurement (documented below, not committed as a benchmark): + +- Under throttled vacuum (Postgres default `cost_delay`), both default and + `scale_factor=0`-tuned tables bloated to ~450 MB / ~1M dead tuples — vacuum could + not keep pace with the churn. +- Under unthrottled vacuum (`cost_delay=0`), both stayed lean (~106 MB / ~7k dead). + +The dominant lever was `cost_delay` (throughput), not the shipped `scale_factor` / +`threshold` (eligibility). The measurement also had a flaw that *understated* +eligibility's value — it used `autovacuum_naptime=1s` to compress time, which made +the daemon so responsive that the eligibility threshold rarely became binding. With +the production default `naptime=60s`, the eligibility settings matter more (fire at +every 60s wake vs fire rarely), but that only manifests over multi-minute sustained +load — the un-gateable scenario `2026-07-15.02` anticipated when it chose to +document rather than benchmark. + +Net: the shipped settings are defensible standard queue-table hygiene (they break +the stale-`reltuples` death-spiral and give size-independent, predictable vacuum), +but they are not a complete bloat solution under heavy churn. Exposing the +throughput knob completes the story and lets the docs be honest about it. + +## Design + +Add two optional keyword params to `outbox_autovacuum_ddl`: + +``` +outbox_autovacuum_ddl( + table_name="outbox", *, schema=None, + vacuum_threshold=1000, insert_threshold=1000, + vacuum_cost_delay: int | None = None, # ms; None -> not emitted (cluster default) + vacuum_cost_limit: int | None = None, # None -> not emitted (cluster default) +) -> str +``` + +- `None` (default) means the reloption is **not** emitted, so the rendered SQL is + byte-identical to today for existing callers. +- When set, the statement gains `autovacuum_vacuum_cost_delay = N` / + `autovacuum_vacuum_cost_limit = N`. `vacuum_cost_delay=0` makes vacuum run + unthrottled (fast, but I/O-heavy) — the lever that bounds bloat under heavy churn. + +No change to the validation probe: `validate_schema(check_autovacuum=True)` still +checks only the **structural** `scale_factor=0` + threshold-present. `cost_delay` / +`cost_limit` are situational tuning, not a structural requirement, so they are not +enforced — a user who tunes throughput their own way must not trip the check. + +## Non-goals + +- **No default for the cost knobs.** `cost_delay=0` spikes I/O on a shared cluster; + hardcoding it would be reckless. Off by default; opt-in with documented guidance. +- **No probe change.** Throughput is not a structural correctness requirement. +- **No churn benchmark in CI.** Unchanged from `2026-07-15.02` — the benefit is + time-and-throughput-dependent and un-gateable. The measurement above is a one-off + demonstration, documented, not committed. + +## Testing + +- **Unit (no DB):** `outbox_autovacuum_ddl` with `vacuum_cost_delay` / `vacuum_cost_limit` + set renders the extra reloptions; with both `None` (default), the output is + byte-identical to before (a regression guard on the default path). + +## Docs + +Rewrite the `docs/operations/alembic.md` autovacuum section and the +`architecture/schema.md` note to scope the benefit honestly: + +- The `scale_factor=0` + threshold settings control vacuum **eligibility** — they + break the stale-`reltuples` death-spiral and give size-independent, predictable + vacuum. Most valuable under the default 60s autovacuum daemon with variable / + bursty backlogs. +- Under **heavy sustained churn**, vacuum **throughput** is the binding constraint; + use `vacuum_cost_delay` (e.g. `0`) / `vacuum_cost_limit` to let vacuum keep pace — + with a caution that unthrottled vacuum is I/O-heavy on a shared cluster. +- Frame the whole thing as **standard queue-table hygiene and insurance**, not a + measured silver bullet — remove any wording that implies a guaranteed bloat win. + +## Risk + +- **The cost knobs can hurt if misused** (`cost_delay=0` I/O spikes). Mitigated by + off-by-default + explicit docs caution; the user opts in deliberately. +- **The honesty rewrite slightly walks back the prior docs' bloat framing.** That is + the point — the prior wording oversold a benefit the measurement could not isolate. From 40bb39cfb42234775b2aee7e76ea38a447ce37d2 Mon Sep 17 00:00:00 2001 From: Artur Shiriev Date: Thu, 16 Jul 2026 09:30:40 +0300 Subject: [PATCH 2/2] feat(autovacuum): add optional vacuum_cost_delay/vacuum_cost_limit knobs outbox_autovacuum_ddl already set scale_factor/threshold for vacuum eligibility; a churn measurement showed vacuum throughput is the binding constraint under heavy sustained load, governed instead by autovacuum_vacuum_cost_delay/cost_limit. Add both as optional keyword-only params, defaulting to None (not emitted, so existing callers' rendered SQL is unchanged), and rewrite the autovacuum docs to scope the eligibility settings honestly as hygiene/insurance rather than a guaranteed bloat fix. --- architecture/schema.md | 8 +++++ docs/operations/alembic.md | 56 +++++++++++++++++++++++++-------- faststream_outbox/autovacuum.py | 17 ++++++++-- tests/test_unit.py | 16 ++++++++++ 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/architecture/schema.md b/architecture/schema.md index 70f8581..e89f309 100644 --- a/architecture/schema.md +++ b/architecture/schema.md @@ -101,3 +101,11 @@ table lacks the settings — separate from the "Outbox schema mismatch: " prefix an operator can tell the two apart. Because it rides `validate_schema()`, the check is coupled to the `[validate]` (Alembic) extra. `fillfactor` is excluded on evidence (HOT is impossible — the claim `UPDATE` mutates both partial indexes' key columns). + +`scale_factor`/threshold control vacuum *eligibility* — this is the structural fix +above, shipped and enforced by the probe. `outbox_autovacuum_ddl()` also accepts +optional `vacuum_cost_delay`/`vacuum_cost_limit`, which control vacuum *throughput* +instead; they default to unset, are not checked by the probe (situational tuning, +not a structural requirement), and are the binding constraint under heavy sustained +churn — eligibility alone cannot keep vacuum ahead of the dead-tuple rate if it runs +throttled. diff --git a/docs/operations/alembic.md b/docs/operations/alembic.md index 84b03b0..cdcb00d 100644 --- a/docs/operations/alembic.md +++ b/docs/operations/alembic.md @@ -171,15 +171,17 @@ consequence that bites is a missing `server_default=now()` on The outbox is a high-churn queue table: every message is one `INSERT`, one lease `UPDATE`, and one terminal `DELETE`, so dead tuples accumulate at roughly **twice -the message rate**. Postgres' default `autovacuum_vacuum_scale_factor = 0.2` fires -vacuum only after a *fraction of the table* is dead — on a queue table that lets -bloat grow, and if the table bloats the fraction is *of the bloated size*, so vacuum -fires ever less often. Set the scale factor to `0` so vacuum triggers on a constant -dead-tuple count instead, tracking churn rather than table size. +the message rate**, and autovacuum has to reclaim them. Two independent levers +matter, and they do different things. -SQLAlchemy's `Table` cannot carry these reloptions, so `alembic revision ---autogenerate` will never emit them — apply them with an explicit statement. -`faststream_outbox` renders it for you: +### Eligibility — when autovacuum fires + +Postgres' default `autovacuum_vacuum_scale_factor = 0.2` fires vacuum only after a +*fraction of the table* is dead. On a queue table whose `reltuples` estimate can go +stale (a table that backed up once keeps a high estimate), that fraction is a high, +size-dependent bar that fires rarely — the classic queue-table death-spiral. +Setting the scale factor to `0` with a constant threshold makes vacuum eligible on a +fixed dead-tuple count instead, independent of table size and of a stale estimate: ```python from alembic import op @@ -193,16 +195,44 @@ def upgrade() -> None: This sets `autovacuum_vacuum_scale_factor = 0` and `autovacuum_vacuum_threshold = 1000` (plus the insert-triggered pair, Postgres 13+). Tune the thresholds for your message rate — `outbox_autovacuum_ddl("outbox", vacuum_threshold=5000, -insert_threshold=5000)` — a higher threshold vacuums less often. `fillfactor` is -intentionally not set: the lease `UPDATE` touches indexed columns, so HOT updates -are impossible and `fillfactor` buys almost nothing here. +insert_threshold=5000)`. + +This is standard queue-table hygiene: it keeps vacuum behavior predictable and +size-independent, and it matters most under the **default 60-second autovacuum +daemon** with variable or bursty backlogs, where being eligible at *every* wake +(rather than rarely) bounds how many dead tuples pile up between vacuums. Treat it as +insurance against a size-dependent bar going stale, not a guaranteed bloat +reduction. + +### Throughput — how fast autovacuum reclaims + +Eligibility is necessary but not sufficient. Under **heavy sustained churn** the +binding constraint is vacuum *throughput*: a throttled autovacuum (Postgres default +`autovacuum_vacuum_cost_delay`) falls behind the dead-tuple rate no matter how +eagerly it is eligible, and the table bloats anyway. The `vacuum_cost_delay` / +`vacuum_cost_limit` params let vacuum keep pace: + +```python +# heavy-churn outbox: let autovacuum run unthrottled so it keeps up +op.execute(outbox_autovacuum_ddl("outbox", vacuum_cost_delay=0)) +``` + +`vacuum_cost_delay=0` removes autovacuum's I/O throttle for this table — the lever +that actually bounds bloat under heavy churn. But it is **I/O-heavy**: on a shared +cluster, an unthrottled vacuum of a large table can spike disk load, so raise it +deliberately and measure. Both cost params default to unset (the cluster default), +so a plain `outbox_autovacuum_ddl("outbox")` changes only eligibility. + +`fillfactor` is intentionally not set: the lease `UPDATE` touches indexed columns, so +HOT updates are impossible and `fillfactor` buys almost nothing here. If your outbox table lives in a non-default `MetaData(schema=...)`, pass the same schema — `outbox_autovacuum_ddl("outbox", schema="app")` — so the `ALTER TABLE` targets that table rather than an unqualified name resolved via `search_path`. -To catch a table that never had the settings applied, pass `check_autovacuum=True` -to `validate_schema()`. Requires the `[validate]` (Alembic) extra: +To catch a table that never had the eligibility settings applied, pass +`check_autovacuum=True` to `validate_schema()` (it checks `scale_factor = 0` + a +threshold, not the cost knobs). Requires the `[validate]` (Alembic) extra: ```python # In a startup hook or /health check -- raises if the outbox table is not tuned diff --git a/faststream_outbox/autovacuum.py b/faststream_outbox/autovacuum.py index 9d46392..d93a2e8 100644 --- a/faststream_outbox/autovacuum.py +++ b/faststream_outbox/autovacuum.py @@ -41,6 +41,8 @@ def outbox_autovacuum_ddl( schema: str | None = None, vacuum_threshold: int = 1000, insert_threshold: int = 1000, + vacuum_cost_delay: int | None = None, + vacuum_cost_limit: int | None = None, ) -> str: """Render the recommended ``ALTER TABLE … SET (autovacuum_*)`` statement. @@ -49,6 +51,13 @@ def outbox_autovacuum_ddl( (resp. inserted) tuples trigger autovacuum; the scale factors are fixed at 0 -- that is the structural fix, not a knob. The insert-triggered reloptions require Postgres 13+. + ``vacuum_cost_delay`` / ``vacuum_cost_limit`` control vacuum *throughput* (how fast + autovacuum reclaims dead tuples once eligible), not eligibility. Both default to + ``None``, which omits the reloption entirely (cluster default) -- so a plain call + renders byte-identical output to before these params existed. ``vacuum_cost_delay=0`` + runs vacuum unthrottled (fast but I/O-heavy) -- the lever that bounds bloat under + heavy sustained churn. + ``schema`` defaults to ``None``, which renders an unqualified table name that resolves via the connection's ``search_path`` -- matching both ``Table.schema=None`` and the ``validate_schema(check_autovacuum=True)`` reloptions lookup's @@ -58,12 +67,16 @@ def outbox_autovacuum_ddl( """ quoted_table = _IDENTIFIER_PREPARER.quote(table_name) quoted_name = quoted_table if schema is None else f"{_IDENTIFIER_PREPARER.quote(schema)}.{quoted_table}" - options = ( + options = [ (_SCALE_FACTOR_KEYS[0], "0"), (_VACUUM_THRESHOLD_KEY, str(vacuum_threshold)), (_SCALE_FACTOR_KEYS[1], "0"), (_INSERT_THRESHOLD_KEY, str(insert_threshold)), - ) + ] + if vacuum_cost_delay is not None: + options.append(("autovacuum_vacuum_cost_delay", str(vacuum_cost_delay))) + if vacuum_cost_limit is not None: + options.append(("autovacuum_vacuum_cost_limit", str(vacuum_cost_limit))) settings = ", ".join(f"{key} = {value}" for key, value in options) return f"ALTER TABLE {quoted_name} SET ({settings})" diff --git a/tests/test_unit.py b/tests/test_unit.py index d38d71b..9c30d1e 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -4145,3 +4145,19 @@ def test_outbox_autovacuum_ddl_schema_qualified() -> None: "autovacuum_vacuum_insert_scale_factor = 0, " "autovacuum_vacuum_insert_threshold = 1000)" ) + + +def test_outbox_autovacuum_ddl_cost_delay_emitted() -> None: + sql = outbox_autovacuum_ddl("outbox", vacuum_cost_delay=0) + assert "autovacuum_vacuum_cost_delay = 0" in sql + + +def test_outbox_autovacuum_ddl_cost_limit_emitted() -> None: + sql = outbox_autovacuum_ddl("outbox", vacuum_cost_limit=2000) + assert "autovacuum_vacuum_cost_limit = 2000" in sql + + +def test_outbox_autovacuum_ddl_default_omits_cost_options() -> None: + sql = outbox_autovacuum_ddl("outbox") + assert "cost_delay" not in sql + assert "cost_limit" not in sql