diff --git a/.github/scripts/compare-jmh.py b/.github/scripts/compare-jmh.py
deleted file mode 100644
index 5010938f1..000000000
--- a/.github/scripts/compare-jmh.py
+++ /dev/null
@@ -1,588 +0,0 @@
-#!/usr/bin/env python3
-"""Compare two sets of JMH JSON results and emit a markdown summary.
-
-Used by `.github/workflows/benchmarks.yml` to diff the latest scheduled
-`main` benchmark run against the run that just finished for a PR.
-
-For each (benchmark, params) pair common to both runs we report two
-metrics:
-
-* `Time` — `primaryMetric.score`. In `SampleTime` mode this is the
- mean sampled latency per op; it's our best available proxy for CPU
- work since no dedicated CPU profiler is configured in
- `BenchmarkRunner`.
-* `Alloc/op` — `secondaryMetrics["gc.alloc.rate.norm"]`, populated by
- JMH's `GCProfiler`. This is bytes allocated per benchmark op and is
- the standard, low-noise JMH memory metric. (Note: JMH 1.37 stores the
- key with no prefix; some visualizer tools display it as
- `·gc.alloc.rate.norm` but the raw JSON does not contain that dot.)
-
-Both metrics are "lower is better", so a positive delta indicates the
-PR is worse than the baseline. A run is considered failed when **any**
-benchmark's worst metric delta exceeds `--threshold-pct` in the worse
-direction. The script writes a `regressions=...`/`improvements=...`
-summary file the workflow uses to set step outputs and decide whether
-to fail the job.
-"""
-
-from __future__ import annotations
-
-import argparse
-import glob
-import json
-import os
-import sys
-from collections import defaultdict
-from dataclasses import dataclass
-from typing import Any, Callable, Dict, List, Optional, Tuple
-
-Key = Tuple[str, str]
-
-
-# ---------------------------------------------------------------------------
-# Metric model
-# ---------------------------------------------------------------------------
-
-# JMH's `GCProfiler` reports allocation rate normalised per op under this
-# secondary metric key. Verified against jmh-core 1.37 sources: the key
-# is the literal `gc.alloc.rate.norm` with no prefix. Some visualizers
-# render it as `·gc.alloc.rate.norm`, which is a display convention, not
-# what JMH writes to JSON.
-ALLOC_NORM_KEY = "gc.alloc.rate.norm"
-
-# Tolerated prefix-bearing variants we'll fall back to if the canonical
-# key ever moves. Lets a future JMH (or a non-Oracle JVM profiler that
-# decorates the label) keep working without us being aware.
-ALLOC_NORM_VARIANTS = (
- ALLOC_NORM_KEY,
- "\u00b7gc.alloc.rate.norm", # pre-emptive middle-dot variant
- "+gc.alloc.rate.norm",
-)
-
-# Set to True the first time we fail to find any allocation data; used
-# to emit a one-time diagnostic listing the secondary keys present so
-# the cause is obvious in workflow logs.
-_alloc_warn_printed = False
-
-
-@dataclass(frozen=True)
-class Metric:
- id: str
- label: str
- # Pull the `{score, scoreError, scoreUnit}`-shaped dict from a JMH record.
- extract: Callable[[Dict[str, Any]], Optional[Dict[str, Any]]]
- # True when a higher score is worse (regression). Both of our metrics
- # are lower-is-better so this is always True today, but the model
- # supports e.g. Throughput mode trivially.
- higher_is_worse: bool = True
-
-
-def _primary(record: Dict[str, Any]) -> Optional[Dict[str, Any]]:
- pm = record.get("primaryMetric")
- return pm if isinstance(pm, dict) else None
-
-
-def _secondary(record: Dict[str, Any], key: str) -> Optional[Dict[str, Any]]:
- sm = record.get("secondaryMetrics") or {}
- val = sm.get(key)
- return val if isinstance(val, dict) else None
-
-
-def _alloc(record: Dict[str, Any]) -> Optional[Dict[str, Any]]:
- """Pull the GC allocation-per-op metric, tolerating prefix variants.
-
- Falls back to a suffix match so even an unrecognised prefix (e.g. a
- third-party JMH profiler that decorates the label) still works."""
- global _alloc_warn_printed
- sm = record.get("secondaryMetrics") or {}
- for k in ALLOC_NORM_VARIANTS:
- v = sm.get(k)
- if isinstance(v, dict):
- return v
- # Last-resort suffix match.
- for k, v in sm.items():
- if isinstance(v, dict) and k.lower().endswith("gc.alloc.rate.norm"):
- return v
- if sm and not _alloc_warn_printed:
- print(
- "warn: no `gc.alloc.rate.norm` (or prefix-variant) found in "
- "secondaryMetrics. Available keys: "
- + ", ".join(sorted(sm.keys())),
- file=sys.stderr,
- )
- _alloc_warn_printed = True
- return None
-
-
-METRICS: List[Metric] = [
- Metric(id="time", label="Time", extract=_primary),
- Metric(id="alloc", label="Alloc/op", extract=_alloc),
-]
-
-
-# ---------------------------------------------------------------------------
-# Loading & helpers
-# ---------------------------------------------------------------------------
-
-
-def load_results(directory: str) -> Dict[Key, Dict[str, Any]]:
- by_key: Dict[Key, Dict[str, Any]] = {}
- paths = sorted(
- glob.glob(os.path.join(directory, "**", "jmh-results-*.json"), recursive=True)
- )
- for path in paths:
- try:
- with open(path, "r", encoding="utf-8") as fh:
- data = json.load(fh)
- except (OSError, json.JSONDecodeError) as exc:
- print(f"warn: could not load {path}: {exc}", file=sys.stderr)
- continue
- if not isinstance(data, list):
- continue
- for record in data:
- bench = record.get("benchmark")
- if not bench:
- continue
- params = record.get("params") or {}
- param_str = ", ".join(f"{k}={params[k]}" for k in sorted(params))
- by_key[(bench, param_str)] = record
- return by_key
-
-
-def _float(d: Optional[Dict[str, Any]], key: str) -> Optional[float]:
- if not d:
- return None
- val = d.get(key)
- try:
- return float(val) if val is not None else None
- except (TypeError, ValueError):
- return None
-
-
-def short_bench(name: str) -> str:
- parts = name.split(".")
- return ".".join(parts[-2:]) if len(parts) >= 2 else name
-
-
-def fmt_score(v: Optional[float], err: Optional[float], unit: str) -> str:
- if v is None:
- return "—"
- body = f"{v:.3g} ± {err:.2g}" if err is not None else f"{v:.3g}"
- return f"{body} {unit}".rstrip()
-
-
-def fmt_delta(d: Optional[float]) -> str:
- if d is None:
- return "—"
- sign = "+" if d >= 0 else ""
- return f"{sign}{d:.2f}%"
-
-
-# ---------------------------------------------------------------------------
-# Comparison
-# ---------------------------------------------------------------------------
-
-
-@dataclass
-class MetricDelta:
- metric: Metric
- baseline: Optional[float]
- current: Optional[float]
- baseline_err: Optional[float]
- current_err: Optional[float]
- unit: str
- delta_pct: Optional[float]
-
- def regression(self, threshold: float) -> bool:
- if self.delta_pct is None:
- return False
- signed = self.delta_pct if self.metric.higher_is_worse else -self.delta_pct
- return signed > threshold
-
- def improvement(self, threshold: float) -> bool:
- if self.delta_pct is None:
- return False
- signed = self.delta_pct if self.metric.higher_is_worse else -self.delta_pct
- return signed < -threshold
-
- def cell(self) -> str:
- b = fmt_score(self.baseline, self.baseline_err, self.unit)
- c = fmt_score(self.current, self.current_err, self.unit)
- return f"{b} → {c} ({fmt_delta(self.delta_pct)})"
-
-
-def metric_delta(metric: Metric, baseline_rec: Dict[str, Any], current_rec: Dict[str, Any]) -> MetricDelta:
- b = metric.extract(baseline_rec)
- c = metric.extract(current_rec)
- bs = _float(b, "score")
- cs = _float(c, "score")
- be = _float(b, "scoreError")
- ce = _float(c, "scoreError")
- unit = (c or {}).get("scoreUnit") or (b or {}).get("scoreUnit") or ""
- if bs is None or cs is None or bs == 0:
- delta_pct: Optional[float] = None
- else:
- delta_pct = (cs - bs) / bs * 100.0
- return MetricDelta(
- metric=metric,
- baseline=bs,
- current=cs,
- baseline_err=be,
- current_err=ce,
- unit=unit,
- delta_pct=delta_pct,
- )
-
-
-@dataclass
-class Row:
- key: Key
- deltas: List[MetricDelta]
-
- def worst_signed_pct(self) -> float:
- worst = 0.0
- for d in self.deltas:
- if d.delta_pct is None:
- continue
- signed = d.delta_pct if d.metric.higher_is_worse else -d.delta_pct
- if signed > worst:
- worst = signed
- return worst
-
- def sort_key(self) -> float:
- best = 0.0
- for d in self.deltas:
- if d.delta_pct is None:
- continue
- if abs(d.delta_pct) > best:
- best = abs(d.delta_pct)
- return best
-
- def status(self, threshold: float) -> str:
- regressed = [d for d in self.deltas if d.regression(threshold)]
- improved = [d for d in self.deltas if d.improvement(threshold)]
- if regressed:
- labels = ", ".join(d.metric.label for d in regressed)
- return f"REGRESSION ({labels})"
- if improved:
- labels = ", ".join(d.metric.label for d in improved)
- return f"improvement ({labels})"
- return ""
-
-
-def build_rows(
- baseline: Dict[Key, Dict[str, Any]],
- current: Dict[Key, Dict[str, Any]],
-) -> List[Row]:
- rows: List[Row] = []
- for key in sorted(set(current) & set(baseline)):
- b = baseline[key]
- c = current[key]
- deltas = [metric_delta(m, b, c) for m in METRICS]
- rows.append(Row(key=key, deltas=deltas))
- return rows
-
-
-# ---------------------------------------------------------------------------
-# Rendering
-# ---------------------------------------------------------------------------
-
-
-def compute_discriminators(
- rows: List[Row], current: Dict[Key, Dict[str, Any]]
-) -> Dict[Key, str]:
- """For each row, return a short string of only the params that differ
- between rows sharing the same fully-qualified benchmark name.
-
- When a benchmark is run with only one combination of params, its
- discriminator is the empty string (no need to disambiguate).
- """
- by_bench: Dict[str, List[Row]] = defaultdict(list)
- for r in rows:
- bench, _ = r.key
- by_bench[bench].append(r)
-
- out: Dict[Key, str] = {}
- for bench, group in by_bench.items():
- if len(group) <= 1:
- out[group[0].key] = ""
- continue
- params_dicts = [(current.get(r.key) or {}).get("params") or {} for r in group]
- all_keys = sorted({k for p in params_dicts for k in p.keys()})
- varying = [
- k for k in all_keys if len({p.get(k) for p in params_dicts}) > 1
- ]
- for r, p in zip(group, params_dicts):
- out[r.key] = ", ".join(f"{k}={p[k]}" for k in varying if k in p)
- return out
-
-
-# Visual markers used in rendered output. "Performance went up" =
-# improvement (less time / less memory per op).
-ARROW_REGRESS = "\u2b07\ufe0f" # ⬇️
-ARROW_IMPROVE = "\u2b06\ufe0f" # ⬆️
-ARROW_NOISE = "\u2796" # ➖
-
-DOT_REGRESS = "\U0001f534" # 🔴
-DOT_IMPROVE = "\U0001f7e2" # 🟢
-
-
-def _row_arrow(r: Row, threshold: float) -> str:
- if any(d.regression(threshold) for d in r.deltas):
- return ARROW_REGRESS
- if any(d.improvement(threshold) for d in r.deltas):
- return ARROW_IMPROVE
- return ARROW_NOISE
-
-
-def _short_delta(d: MetricDelta, threshold: float) -> str:
- """One-line metric delta for the brief bullet list.
-
- Returns "" for noise (caller drops it). Regressions are 🔴-marked and
- bold so they stand out on a packed PR comment; improvements are
- 🟢-marked but unbolded.
- """
- if d.delta_pct is None:
- return ""
- label = d.metric.label
- delta = fmt_delta(d.delta_pct)
- if d.regression(threshold):
- return f"{DOT_REGRESS} **{label} {delta}**"
- if d.improvement(threshold):
- return f"{DOT_IMPROVE} {label} {delta}"
- return ""
-
-
-def _stats_cell(r: Row, threshold: float) -> str:
- """Render the joined Stats cell for one row in the detail table.
-
- Each metric occupies a `
`-separated line. Regressions are
- bolded and 🔴-tagged; improvements are 🟢-tagged. Metrics with no
- baseline/current data are rendered grey ("—") so the cell still
- shows which dimension is missing.
- """
- lines: List[str] = []
- for d in r.deltas:
- label = d.metric.label
- if d.delta_pct is None:
- base = fmt_score(d.baseline, d.baseline_err, d.unit)
- curr = fmt_score(d.current, d.current_err, d.unit)
- lines.append(f"{label} {base} → {curr} (—)")
- continue
- base = fmt_score(d.baseline, d.baseline_err, d.unit)
- curr = fmt_score(d.current, d.current_err, d.unit)
- delta = fmt_delta(d.delta_pct)
- if d.regression(threshold):
- lines.append(f"{DOT_REGRESS} **{label}** {base} → {curr} (**{delta}**)")
- elif d.improvement(threshold):
- lines.append(f"{DOT_IMPROVE} **{label}** {base} → {curr} ({delta})")
- else:
- lines.append(f"{label} {base} → {curr} ({delta})")
- return "
".join(lines)
-
-
-def build_markdown(
- rows: List[Row],
- only_current: List[Key],
- only_baseline: List[Key],
- current: Dict[Key, Dict[str, Any]],
- *,
- threshold: float,
- repo: str,
- server_url: str,
- baseline_run_id: str,
- current_run_id: str,
-) -> Tuple[str, int, int]:
- rows = sorted(rows, key=lambda r: r.sort_key(), reverse=True)
- regressions = sum(1 for r in rows if any(d.regression(threshold) for d in r.deltas))
- improvements = sum(
- 1 for r in rows
- if not any(d.regression(threshold) for d in r.deltas)
- and any(d.improvement(threshold) for d in r.deltas)
- )
- discriminators = compute_discriminators(rows, current)
-
- out: List[str] = [""]
- if regressions:
- out.append(
- f"## {DOT_REGRESS} JMH benchmark comparison — {regressions} regression(s) over {threshold:g}%"
- )
- elif improvements:
- out.append(
- f"## {DOT_IMPROVE} JMH benchmark comparison — no regressions, {improvements} improvement(s) over {threshold:g}%"
- )
- else:
- out.append(f"## {DOT_IMPROVE} JMH benchmark comparison — no changes over {threshold:g}%")
- out.append("")
-
- if repo and baseline_run_id and current_run_id:
- base_url = f"{server_url}/{repo}/actions/runs/{baseline_run_id}"
- curr_url = f"{server_url}/{repo}/actions/runs/{current_run_id}"
- out.append(
- f"Baseline: [`main` run #{baseline_run_id}]({base_url}) — "
- f"PR: [run #{current_run_id}]({curr_url})"
- )
- out.append("")
-
- # ---- brief per-benchmark summary --------------------------------------
- if rows:
- # Stable order: regressions first, then improvements, then noise.
- # Within each bucket, biggest |Δ| first.
- def bucket(r: Row) -> int:
- if any(d.regression(threshold) for d in r.deltas):
- return 0
- if any(d.improvement(threshold) for d in r.deltas):
- return 1
- return 2
-
- rows = sorted(rows, key=lambda r: (bucket(r), -r.sort_key()))
-
- for r in rows:
- bench, _ = r.key
- disc = discriminators.get(r.key, "")
- arrow = _row_arrow(r, threshold)
-
- # Only mention metrics that actually crossed the threshold
- # in the brief view — keeps noisy rows to a single line.
- bits = [s for s in (_short_delta(d, threshold) for d in r.deltas) if s]
-
- line = f"- {arrow} `{short_bench(bench)}`"
- if disc:
- line += f" `[{disc}]`"
- if bits:
- line += " — " + ", ".join(bits)
- out.append(line)
- out.append("")
- else:
- out.append("_No benchmarks matched between baseline and PR._")
- out.append("")
-
- # ---- detailed table (collapsed) ---------------------------------------
- if rows:
- out.append(
- f"Detailed metrics "
- f"(threshold ±{threshold:g}%, Time = primaryMetric.score from SampleTime, "
- f"Alloc/op = {ALLOC_NORM_KEY}; positive Δ% = worse than baseline)"
- "
"
- )
- out.append("")
- out.append("| Benchmark | Stats |")
- out.append("|---|---|")
- for r in rows:
- bench, params = r.key
- bench_cell = f"`{short_bench(bench)}`"
- if params:
- bench_cell += f"
{params}"
- out.append(f"| {bench_cell} | {_stats_cell(r, threshold)} |")
- out.append("")
- out.append(" ")
- out.append("")
-
- if only_current:
- out.append("Benchmarks only in PR run
")
- out.append("")
- for k in only_current:
- bench, params = k
- rec = current[k]
- time_d = _primary(rec) or {}
- alloc_d = _alloc(rec) or {}
- out.append(
- f"- `{short_bench(bench)}` ({params or '—'}): "
- f"time={fmt_score(_float(time_d, 'score'), _float(time_d, 'scoreError'), time_d.get('scoreUnit', ''))}, "
- f"alloc={fmt_score(_float(alloc_d, 'score'), _float(alloc_d, 'scoreError'), alloc_d.get('scoreUnit', ''))}"
- )
- out.append("")
- out.append(" ")
- out.append("")
-
- if only_baseline:
- out.append("Benchmarks only in baseline run
")
- out.append("")
- for k in only_baseline:
- bench, params = k
- out.append(f"- `{short_bench(bench)}` ({params or '—'})")
- out.append("")
- out.append(" ")
- out.append("")
-
- return "\n".join(out), regressions, improvements
-
-
-# ---------------------------------------------------------------------------
-# Entry point
-# ---------------------------------------------------------------------------
-
-
-def main() -> int:
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--baseline", required=True, help="Directory containing baseline JMH JSON files")
- parser.add_argument("--current", required=True, help="Directory containing current JMH JSON files")
- parser.add_argument("--baseline-run-id", default="", help="Baseline workflow run id (for links)")
- parser.add_argument("--current-run-id", default="", help="Current workflow run id (for links)")
- parser.add_argument("--repo", default="", help="owner/name for run links")
- parser.add_argument("--server-url", default="https://github.com")
- parser.add_argument("--output", required=True, help="Output markdown file")
- parser.add_argument(
- "--threshold-pct",
- type=float,
- default=10.0,
- help="Δ%% beyond which a metric is flagged as a regression / improvement (default: 10)",
- )
- parser.add_argument(
- "--summary-output",
- default="",
- help="Optional path to write a key=value summary the workflow can source",
- )
- args = parser.parse_args()
-
- if args.threshold_pct < 0:
- print("error: --threshold-pct must be non-negative", file=sys.stderr)
- return 2
-
- baseline = load_results(args.baseline)
- current = load_results(args.current)
-
- if not current:
- print("error: no current JMH result files found", file=sys.stderr)
- return 2
-
- rows = build_rows(baseline, current)
- only_current = sorted(set(current) - set(baseline))
- only_baseline = sorted(set(baseline) - set(current))
-
- md, regressions, improvements = build_markdown(
- rows,
- only_current,
- only_baseline,
- current,
- threshold=args.threshold_pct,
- repo=args.repo,
- server_url=args.server_url,
- baseline_run_id=args.baseline_run_id,
- current_run_id=args.current_run_id,
- )
-
- with open(args.output, "w", encoding="utf-8") as fh:
- fh.write(md)
-
- if args.summary_output:
- with open(args.summary_output, "w", encoding="utf-8") as fh:
- fh.write(f"regressions={regressions}\n")
- fh.write(f"improvements={improvements}\n")
- fh.write(f"matched={len(rows)}\n")
- fh.write(f"threshold_pct={args.threshold_pct}\n")
-
- print(
- f"wrote {args.output}: {len(rows)} matched, "
- f"{regressions} regression(s) > {args.threshold_pct:g}%, "
- f"{improvements} improvement(s), "
- f"{len(only_current)} only-PR, {len(only_baseline)} only-baseline"
- )
- # We always exit 0; the workflow uses the summary file to decide
- # whether to fail the job so that the comparison comment is still
- # posted on regressions.
- return 0
-
-
-if __name__ == "__main__":
- sys.exit(main())
diff --git a/.github/scripts/test_compare_jmh.py b/.github/scripts/test_compare_jmh.py
deleted file mode 100644
index d426c1ef2..000000000
--- a/.github/scripts/test_compare_jmh.py
+++ /dev/null
@@ -1,217 +0,0 @@
-#!/usr/bin/env python3
-"""End-to-end tests for `compare-jmh.py`.
-
-Runs `compare-jmh.py` against every scenario in `test_data/` via
-subprocess and asserts on both the rendered markdown and the
-`--summary-output` counters. Designed to be run locally
-(`python3 .github/scripts/test_compare_jmh.py`) and from CI without
-extra dependencies — only the standard library is used.
-"""
-
-from __future__ import annotations
-
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-import unittest
-from pathlib import Path
-from typing import Dict, List
-
-HERE = Path(__file__).resolve().parent
-SCRIPT = HERE / "compare-jmh.py"
-DATA = HERE / "test_data"
-
-
-def _parse_summary(path: Path) -> Dict[str, str]:
- out: Dict[str, str] = {}
- for line in path.read_text(encoding="utf-8").splitlines():
- if "=" in line:
- k, v = line.split("=", 1)
- out[k.strip()] = v.strip()
- return out
-
-
-def _run_compare(
- case: str,
- *,
- threshold: float = 10.0,
- extra_args: List[str] | None = None,
-) -> Dict[str, object]:
- """Invoke compare-jmh.py against a fixture; return its outputs.
-
- Returns a dict with: returncode, stdout, stderr, markdown, summary.
- """
- case_dir = DATA / case
- assert case_dir.is_dir(), f"missing test fixture: {case_dir}"
-
- tmp = Path(tempfile.mkdtemp(prefix=f"cmp-jmh-{case}-"))
- try:
- out_md = tmp / "out.md"
- summary = tmp / "summary.env"
- cmd = [
- sys.executable,
- str(SCRIPT),
- "--baseline",
- str(case_dir / "baseline"),
- "--current",
- str(case_dir / "current"),
- "--threshold-pct",
- str(threshold),
- "--output",
- str(out_md),
- "--summary-output",
- str(summary),
- ]
- if extra_args:
- cmd.extend(extra_args)
- proc = subprocess.run(cmd, capture_output=True, text=True)
- result: Dict[str, object] = {
- "returncode": proc.returncode,
- "stdout": proc.stdout,
- "stderr": proc.stderr,
- "markdown": out_md.read_text(encoding="utf-8") if out_md.exists() else "",
- "summary": _parse_summary(summary) if summary.exists() else {},
- }
- return result
- finally:
- shutil.rmtree(tmp, ignore_errors=True)
-
-
-# Pre-computed marker strings — keep in sync with compare-jmh.py.
-ARROW_REGRESS = "\u2b07\ufe0f"
-ARROW_IMPROVE = "\u2b06\ufe0f"
-ARROW_NOISE = "\u2796"
-DOT_REGRESS = "\U0001f534"
-DOT_IMPROVE = "\U0001f7e2"
-
-
-class CompareJmhTest(unittest.TestCase):
- """Each case in test_data/ has a dedicated test asserting on the
- headline shape, the bullet markers, and the summary counters."""
-
- def test_all_improvements(self) -> None:
- r = _run_compare("all_improvements")
- self.assertEqual(r["returncode"], 0)
- md = r["markdown"]
- self.assertIn("no regressions", md)
- self.assertIn(ARROW_IMPROVE, md)
- self.assertIn(DOT_IMPROVE, md)
- self.assertNotIn(ARROW_REGRESS, md)
- self.assertNotIn(DOT_REGRESS, md)
- s = r["summary"]
- self.assertEqual(s.get("regressions"), "0")
- self.assertEqual(s.get("improvements"), "2")
- self.assertEqual(s.get("matched"), "2")
-
- def test_all_regressions(self) -> None:
- r = _run_compare("all_regressions")
- self.assertEqual(r["returncode"], 0)
- md = r["markdown"]
- self.assertIn("regression(s) over", md)
- self.assertIn(ARROW_REGRESS, md)
- self.assertIn(DOT_REGRESS, md)
- # Bold markdown around the metric label.
- self.assertIn("**Time", md)
- s = r["summary"]
- self.assertEqual(s.get("regressions"), "2")
- # No row "purely improved" → improvements should be 0.
- self.assertEqual(s.get("improvements"), "0")
-
- def test_mixed(self) -> None:
- r = _run_compare("mixed")
- self.assertEqual(r["returncode"], 0)
- md = r["markdown"]
- # Both directions present.
- self.assertIn(ARROW_REGRESS, md)
- self.assertIn(ARROW_IMPROVE, md)
- self.assertIn(DOT_REGRESS, md)
- self.assertIn(DOT_IMPROVE, md)
- # Discriminator suffix `[limit=…]` appears for the two
- # `queryV2` variants of the same benchmark.
- self.assertIn("[limit=10000]", md)
- self.assertIn("[limit=100000]", md)
- # The unique-named benchmark must NOT get a discriminator.
- self.assertNotIn("`JDBCQuery.selectJDBCV2` `[", md)
- s = r["summary"]
- self.assertEqual(s.get("matched"), "4")
- self.assertGreater(int(s.get("regressions", "0")), 0)
-
- def test_no_alloc(self) -> None:
- # No `gc.alloc.rate.norm` present anywhere → script must
- # still compare Time and emit a diagnostic on stderr.
- r = _run_compare("no_alloc")
- self.assertEqual(r["returncode"], 0)
- self.assertIn("no `gc.alloc.rate.norm`", r["stderr"])
- md = r["markdown"]
- # Time regression should still be detected and 🔴-tagged…
- self.assertIn(DOT_REGRESS, md)
- # …but no Alloc/op metric is ever 🟢/🔴 because we have no
- # baseline/current data for it.
- self.assertIn("Alloc/op", md)
- # The detail-table cell should fall back to "(—)" for alloc.
- self.assertIn("Alloc/op — → — (—)", md)
- s = r["summary"]
- self.assertEqual(s.get("matched"), "2")
-
- def test_noise_only(self) -> None:
- r = _run_compare("noise_only")
- self.assertEqual(r["returncode"], 0)
- md = r["markdown"]
- self.assertIn("no changes over 10%", md)
- # Every row should be on the noise arrow…
- self.assertIn(ARROW_NOISE, md)
- # …and there should be no red/green dots anywhere.
- self.assertNotIn(DOT_REGRESS, md)
- # 🟢 *is* in the header for the OK case, so don't assert on
- # DOT_IMPROVE alone.
- s = r["summary"]
- self.assertEqual(s.get("regressions"), "0")
- self.assertEqual(s.get("improvements"), "0")
-
- def test_only_in_pr(self) -> None:
- r = _run_compare("only_in_pr")
- self.assertEqual(r["returncode"], 0)
- md = r["markdown"]
- self.assertIn("Benchmarks only in PR run", md)
- self.assertIn("QueryClient.queryV3New", md)
- s = r["summary"]
- # one shared row matched.
- self.assertEqual(s.get("matched"), "1")
-
- def test_only_in_baseline(self) -> None:
- r = _run_compare("only_in_baseline")
- self.assertEqual(r["returncode"], 0)
- md = r["markdown"]
- self.assertIn("Benchmarks only in baseline run", md)
- self.assertIn("QueryClient.queryV0Removed", md)
- s = r["summary"]
- self.assertEqual(s.get("matched"), "1")
-
- def test_empty_intersection(self) -> None:
- r = _run_compare("empty_intersection")
- self.assertEqual(r["returncode"], 0)
- md = r["markdown"]
- self.assertIn("_No benchmarks matched between baseline and PR._", md)
- # Both unique-side sections still appear as .
- self.assertIn("Benchmarks only in PR run", md)
- self.assertIn("Benchmarks only in baseline run", md)
- s = r["summary"]
- self.assertEqual(s.get("matched"), "0")
- self.assertEqual(s.get("regressions"), "0")
- self.assertEqual(s.get("improvements"), "0")
-
- def test_threshold_knob(self) -> None:
- # The same fixture flips from "regression" to "ok" when the
- # threshold is widened past the largest delta.
- strict = _run_compare("all_regressions", threshold=10.0)
- lenient = _run_compare("all_regressions", threshold=200.0)
- self.assertGreater(int(strict["summary"]["regressions"]), 0)
- self.assertEqual(lenient["summary"]["regressions"], "0")
- self.assertIn("no changes", lenient["markdown"])
-
-
-if __name__ == "__main__":
- # `-v` prints each scenario name so failures are obvious in CI logs.
- unittest.main(verbosity=2)
diff --git a/.github/scripts/test_data/README.md b/.github/scripts/test_data/README.md
deleted file mode 100644
index 4a0938f09..000000000
--- a/.github/scripts/test_data/README.md
+++ /dev/null
@@ -1,33 +0,0 @@
-# `compare-jmh.py` test fixtures
-
-Each subdirectory is a self-contained scenario for `compare-jmh.py`. The
-layout is always:
-
-```
-/
- baseline/jmh-results-baseline.json
- current/jmh-results-current.json
-```
-
-`compare-jmh.py` discovers result files by globbing for
-`jmh-results-*.json` under the `--baseline` and `--current` directories,
-so any filename starting with `jmh-results-` works.
-
-JSON records mirror the structure produced by JMH 1.37's
-`ResultFormatType.JSON`: an array of objects with `benchmark`, `params`,
-`primaryMetric.{score,scoreError,scoreUnit}`, and optionally
-`secondaryMetrics["gc.alloc.rate.norm"]`.
-
-| Case | What it covers |
-|---|---|
-| `all_improvements` | Multiple benchmarks where both Time and Alloc/op fall well below the threshold; report should be all ⬆️ / 🟢 with no failure. |
-| `all_regressions` | Multiple benchmarks where Time and/or Alloc/op rise well above the threshold; report should be ⬇️ / 🔴, script flags every row as `REGRESSION`, summary `regressions > 0`. |
-| `mixed` | A blend of regressions, improvements, and within-noise rows including multiple variants of the same benchmark — verifies the bucket ordering and the param-discriminator (`[limit=…]`) logic. |
-| `no_alloc` | Records with no `gc.alloc.rate.norm` at all; verifies the script falls back to Time-only and doesn't render `🔴`/`🟢` markers in the absence of the key (plus prints the diagnostic warning). |
-| `noise_only` | All deltas are inside ±10%; report should be the ✅ "no changes" header and every row should carry the ➖ neutral arrow. |
-| `only_in_pr` | A benchmark appears in `current` but not in `baseline`; verifies the "Benchmarks only in PR run" `` block. |
-| `only_in_baseline` | The mirror case — verifies the "Benchmarks only in baseline run" block. |
-| `empty_intersection` | `baseline` and `current` contain different sets of benchmarks so no rows are matched; verifies the "_No benchmarks matched_" path. |
-
-The companion runner `test_compare_jmh.py` exercises every case and
-checks both the rendered markdown and the `--summary-output` counters.
diff --git a/.github/scripts/test_data/all_improvements/baseline/jmh-results-baseline.json b/.github/scripts/test_data/all_improvements/baseline/jmh-results-baseline.json
deleted file mode 100644
index ff9ab2a4d..000000000
--- a/.github/scripts/test_data/all_improvements/baseline/jmh-results-baseline.json
+++ /dev/null
@@ -1,20 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 80.0, "scoreError": 1.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 4096.0, "scoreError": 24.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/all_improvements/current/jmh-results-current.json b/.github/scripts/test_data/all_improvements/current/jmh-results-current.json
deleted file mode 100644
index 034fa71f0..000000000
--- a/.github/scripts/test_data/all_improvements/current/jmh-results-current.json
+++ /dev/null
@@ -1,20 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 16.0, "scoreError": 0.4, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 1536.0, "scoreError": 10.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 64.0, "scoreError": 1.3, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 3200.0, "scoreError": 20.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/all_regressions/baseline/jmh-results-baseline.json b/.github/scripts/test_data/all_regressions/baseline/jmh-results-baseline.json
deleted file mode 100644
index ff9ab2a4d..000000000
--- a/.github/scripts/test_data/all_regressions/baseline/jmh-results-baseline.json
+++ /dev/null
@@ -1,20 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 80.0, "scoreError": 1.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 4096.0, "scoreError": 24.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/all_regressions/current/jmh-results-current.json b/.github/scripts/test_data/all_regressions/current/jmh-results-current.json
deleted file mode 100644
index f6f1aff82..000000000
--- a/.github/scripts/test_data/all_regressions/current/jmh-results-current.json
+++ /dev/null
@@ -1,20 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 26.0, "scoreError": 0.6, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 3300.0, "scoreError": 18.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 100.0, "scoreError": 1.8, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 5400.0, "scoreError": 30.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/empty_intersection/baseline/jmh-results-baseline.json b/.github/scripts/test_data/empty_intersection/baseline/jmh-results-baseline.json
deleted file mode 100644
index e2483a43c..000000000
--- a/.github/scripts/test_data/empty_intersection/baseline/jmh-results-baseline.json
+++ /dev/null
@@ -1,11 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryOld",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/empty_intersection/current/jmh-results-current.json b/.github/scripts/test_data/empty_intersection/current/jmh-results-current.json
deleted file mode 100644
index 40b2e0d86..000000000
--- a/.github/scripts/test_data/empty_intersection/current/jmh-results-current.json
+++ /dev/null
@@ -1,11 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryNew",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 18.0, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 1900.0, "scoreError": 11.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/mixed/baseline/jmh-results-baseline.json b/.github/scripts/test_data/mixed/baseline/jmh-results-baseline.json
deleted file mode 100644
index c967a2f66..000000000
--- a/.github/scripts/test_data/mixed/baseline/jmh-results-baseline.json
+++ /dev/null
@@ -1,38 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"datasetSourceName": "file://default.csv", "limit": "10000"},
- "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"datasetSourceName": "file://default.csv", "limit": "100000"},
- "primaryMetric": {"score": 180.0, "scoreError": 2.0, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 16384.0, "scoreError": 96.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 80.0, "scoreError": 1.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 4096.0, "scoreError": 24.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.JDBCQuery.selectJDBCV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 30.0, "scoreError": 0.7, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 6144.0, "scoreError": 36.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/mixed/current/jmh-results-current.json b/.github/scripts/test_data/mixed/current/jmh-results-current.json
deleted file mode 100644
index a115b036b..000000000
--- a/.github/scripts/test_data/mixed/current/jmh-results-current.json
+++ /dev/null
@@ -1,38 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"datasetSourceName": "file://default.csv", "limit": "10000"},
- "primaryMetric": {"score": 24.0, "scoreError": 0.6, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2100.0, "scoreError": 13.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"datasetSourceName": "file://default.csv", "limit": "100000"},
- "primaryMetric": {"score": 184.0, "scoreError": 2.0, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 24576.0, "scoreError": 110.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 60.0, "scoreError": 1.2, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 3200.0, "scoreError": 20.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.JDBCQuery.selectJDBCV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 31.0, "scoreError": 0.7, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 6200.0, "scoreError": 36.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/no_alloc/baseline/jmh-results-baseline.json b/.github/scripts/test_data/no_alloc/baseline/jmh-results-baseline.json
deleted file mode 100644
index 1358026eb..000000000
--- a/.github/scripts/test_data/no_alloc/baseline/jmh-results-baseline.json
+++ /dev/null
@@ -1,20 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "p0.50": {"score": 19.5, "scoreError": 0.0, "scoreUnit": "ms/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 80.0, "scoreError": 1.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "p0.50": {"score": 79.0, "scoreError": 0.0, "scoreUnit": "ms/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/no_alloc/current/jmh-results-current.json b/.github/scripts/test_data/no_alloc/current/jmh-results-current.json
deleted file mode 100644
index 76629e32a..000000000
--- a/.github/scripts/test_data/no_alloc/current/jmh-results-current.json
+++ /dev/null
@@ -1,20 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 26.0, "scoreError": 0.6, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "p0.50": {"score": 25.0, "scoreError": 0.0, "scoreUnit": "ms/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 82.0, "scoreError": 1.4, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "p0.50": {"score": 81.0, "scoreError": 0.0, "scoreUnit": "ms/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/noise_only/baseline/jmh-results-baseline.json b/.github/scripts/test_data/noise_only/baseline/jmh-results-baseline.json
deleted file mode 100644
index ff9ab2a4d..000000000
--- a/.github/scripts/test_data/noise_only/baseline/jmh-results-baseline.json
+++ /dev/null
@@ -1,20 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 80.0, "scoreError": 1.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 4096.0, "scoreError": 24.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/noise_only/current/jmh-results-current.json b/.github/scripts/test_data/noise_only/current/jmh-results-current.json
deleted file mode 100644
index 1b54b5bef..000000000
--- a/.github/scripts/test_data/noise_only/current/jmh-results-current.json
+++ /dev/null
@@ -1,20 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 20.6, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2080.0, "scoreError": 12.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 78.5, "scoreError": 1.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 4170.0, "scoreError": 24.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/only_in_baseline/baseline/jmh-results-baseline.json b/.github/scripts/test_data/only_in_baseline/baseline/jmh-results-baseline.json
deleted file mode 100644
index 32c7d3f81..000000000
--- a/.github/scripts/test_data/only_in_baseline/baseline/jmh-results-baseline.json
+++ /dev/null
@@ -1,20 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV0Removed",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 30.0, "scoreError": 0.7, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 3072.0, "scoreError": 18.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/only_in_baseline/current/jmh-results-current.json b/.github/scripts/test_data/only_in_baseline/current/jmh-results-current.json
deleted file mode 100644
index e9611885b..000000000
--- a/.github/scripts/test_data/only_in_baseline/current/jmh-results-current.json
+++ /dev/null
@@ -1,11 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 20.4, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2050.0, "scoreError": 12.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/only_in_pr/baseline/jmh-results-baseline.json b/.github/scripts/test_data/only_in_pr/baseline/jmh-results-baseline.json
deleted file mode 100644
index 5c589d4c2..000000000
--- a/.github/scripts/test_data/only_in_pr/baseline/jmh-results-baseline.json
+++ /dev/null
@@ -1,11 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/scripts/test_data/only_in_pr/current/jmh-results-current.json b/.github/scripts/test_data/only_in_pr/current/jmh-results-current.json
deleted file mode 100644
index 3551830f2..000000000
--- a/.github/scripts/test_data/only_in_pr/current/jmh-results-current.json
+++ /dev/null
@@ -1,20 +0,0 @@
-[
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 20.4, "scoreError": 0.5, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 2050.0, "scoreError": 12.0, "scoreUnit": "B/op"}
- }
- },
- {
- "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV3New",
- "mode": "sample",
- "params": {"limit": "10000"},
- "primaryMetric": {"score": 15.0, "scoreError": 0.4, "scoreUnit": "ms/op"},
- "secondaryMetrics": {
- "gc.alloc.rate.norm": {"score": 1500.0, "scoreError": 10.0, "scoreUnit": "B/op"}
- }
- }
-]
diff --git a/.github/workflows/benchmarks-pr-comment.yml b/.github/workflows/benchmarks-pr-comment.yml
deleted file mode 100644
index fe3a01996..000000000
--- a/.github/workflows/benchmarks-pr-comment.yml
+++ /dev/null
@@ -1,41 +0,0 @@
-name: Benchmarks PR Comment
-
-# Posts a one-time instruction comment on newly opened PRs so contributors /
-# reviewers know how to launch a JMH benchmark run for the PR. The actual
-# benchmark workflow lives in `benchmarks.yml` and is triggered by a
-# `/benchmark` slash command (collaborators only).
-
-on:
- pull_request_target:
- types: [opened]
-
-permissions:
- pull-requests: write
-
-jobs:
- comment:
- if: startsWith(github.repository, 'ClickHouse/clickhouse-java')
- runs-on: ubuntu-latest
- steps:
- - name: Post benchmark instructions
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- gh api -X POST \
- "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \
- -f body="$(cat <<'EOF'
- Repository collaborators can run the JMH benchmark suite against this PR by commenting:
-
- ```
- /benchmark
- ```
-
- Optional regression threshold override (Δ% on Time or Alloc/op; defaults to 10%):
-
- ```
- /benchmark threshold=15
- ```
-
- Only one benchmark run per PR is active at a time — issuing a new `/benchmark` comment cancels the previous run. After the run finishes a separate comment will be posted comparing it against the latest scheduled run on `main`; the PR check fails if any benchmark regresses by more than the threshold.
- EOF
- )"
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
deleted file mode 100644
index 95d973f33..000000000
--- a/.github/workflows/benchmarks.yml
+++ /dev/null
@@ -1,258 +0,0 @@
-name: Benchmarks
-description: Runs minimal JMH benchmark
-
-on:
- schedule:
- - cron: "55 15 * * *"
- workflow_dispatch:
- inputs:
- pr:
- description: "Pull request#"
- required: false
- threshold:
- description: "Regression threshold (Δ% on Time or Alloc/op)"
- required: false
- default: "10"
- issue_comment:
- types: [created]
-
-env:
- CHC_BRANCH: "main"
- CH_VERSION: "25.3"
- JAVA_VERSION: 17
- # Default Δ% above which a regression / improvement is flagged and the
- # PR check is failed. Overridable per workflow_dispatch input or per
- # `/benchmark threshold=N` comment.
- DEFAULT_THRESHOLD_PCT: "10"
-
-# NOTE: there is intentionally no workflow-level `concurrency:` block.
-# `issue_comment` events fire for *every* comment on every PR / issue,
-# including those from bots (e.g. sonarqubecloud). A workflow-level
-# `cancel-in-progress` group keyed on the PR number would cancel an
-# in-flight legitimate `/benchmark` run as soon as any unrelated bot
-# commented on the same PR. The per-PR concurrency rule is enforced on
-# the `jmh` job instead, so unrelated comment events leave the job
-# skipped without claiming the concurrency slot.
-
-jobs:
- jmh:
- name: "Mininal JMH Benchmarks"
- runs-on: "ubuntu-latest"
- timeout-minutes: 30
- permissions:
- contents: read
- pull-requests: write
- issues: write
- actions: read
- # Single fan-in filter, modelled on `.github/workflows/claude.yml`:
- # the job runs for the daily schedule, manual `workflow_dispatch`,
- # or a `/benchmark` slash-command from a non-bot repo collaborator
- # on a pull request. Bot comments and chat comments leave the job
- # skipped — no failed run, no notification, no concurrency
- # collision.
- if: |
- startsWith(github.repository, 'ClickHouse/') &&
- (
- github.event_name == 'schedule' ||
- github.event_name == 'workflow_dispatch' ||
- (
- github.event_name == 'issue_comment' &&
- github.event.issue.pull_request != null &&
- github.event.sender.type != 'Bot' &&
- github.event.comment.user.type != 'Bot' &&
- startsWith(github.event.comment.body, '/benchmark') &&
- contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)
- )
- )
- # One running benchmark per PR (and per-SHA for the daily
- # schedule). Concurrency lives on this job, not on the workflow,
- # so unrelated comment events (which the job-level `if` filters
- # out) never claim the slot or cancel an in-flight run.
- concurrency:
- group: ${{ github.workflow }}-jmh-${{ github.event.issue.number || github.event.inputs.pr || github.sha }}
- cancel-in-progress: true
- steps:
- - name: Acknowledge /benchmark trigger
- if: github.event_name == 'issue_comment'
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- gh api -X POST \
- "repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions" \
- -f content='rocket' || true
-
- - name: Resolve PR number and threshold
- id: pr
- env:
- COMMENT_BODY: ${{ github.event.comment.body }}
- DISPATCH_PR: ${{ github.event.inputs.pr }}
- DISPATCH_THRESHOLD: ${{ github.event.inputs.threshold }}
- DEFAULT_THRESHOLD: ${{ env.DEFAULT_THRESHOLD_PCT }}
- run: |
- case "${{ github.event_name }}" in
- issue_comment)
- # Accept `/benchmark threshold=15` or `/benchmark threshold=7.5`.
- T=$(printf '%s' "$COMMENT_BODY" | grep -oE 'threshold=[0-9]+(\.[0-9]+)?' | head -1 | cut -d= -f2 || true)
- [ -z "$T" ] && T="$DEFAULT_THRESHOLD"
- echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT"
- echo "threshold=$T" >> "$GITHUB_OUTPUT"
- ;;
- workflow_dispatch)
- echo "number=$DISPATCH_PR" >> "$GITHUB_OUTPUT"
- echo "threshold=${DISPATCH_THRESHOLD:-$DEFAULT_THRESHOLD}" >> "$GITHUB_OUTPUT"
- ;;
- *)
- echo "number=" >> "$GITHUB_OUTPUT"
- echo "threshold=$DEFAULT_THRESHOLD" >> "$GITHUB_OUTPUT"
- ;;
- esac
-
- - name: Post "started" comment
- if: github.event_name == 'issue_comment' && steps.pr.outputs.number != ''
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- gh api -X POST \
- "repos/${{ github.repository }}/issues/${{ steps.pr.outputs.number }}/comments" \
- -f body="JMH benchmark run started: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" || true
-
- - name: Check out Git repository
- uses: actions/checkout@v4
- with:
- ref: ${{ env.CHC_BRANCH }}
-
- # The benchmark code runs against the PR's working tree (see the
- # PR checkout below), but the comparison tooling lives in this
- # workflow's contract on `main`. Stash a copy now so the compare
- # step still works even when the PR branch was forked before
- # `.github/scripts/compare-jmh.py` existed.
- - name: Stash comparison tooling from main
- run: |
- mkdir -p "$RUNNER_TEMP/jmh-tools"
- cp -v .github/scripts/compare-jmh.py "$RUNNER_TEMP/jmh-tools/"
-
- - name: Check out PR
- if: steps.pr.outputs.number != ''
- run: |
- git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \
- origin pull/${{ steps.pr.outputs.number }}/merge:merged-pr && git checkout merged-pr
-
- - name: Install JDK and Maven
- uses: actions/setup-java@v4
- with:
- distribution: "temurin"
- java-version: ${{ env.JAVA_VERSION }}
- cache: "maven"
-
- - name: Build
- run: mvn --batch-mode --no-transfer-progress -Dj8 -DskipTests=true clean install
-
- - name: Prepare Dataset
- run: |
- cd ./performance &&
- mvn --batch-mode --no-transfer-progress clean compile exec:exec -Dexec.executable=java \
- -Dexec.args="-classpath %classpath com.clickhouse.benchmark.data.DataSetGenerator -input sample_dataset.sql -name default -rows 100000"
-
- - name: Run Benchmarks
- run: |
- cd ./performance &&
- mvn --batch-mode --no-transfer-progress clean compile exec:exec -Dexec.executable=java -Dexec.args="-classpath %classpath com.clickhouse.benchmark.BenchmarkRunner \
- -l 100000,10000 -m 3 -t 15 -b q,i -d file://default.csv"
-
- - name: Upload test results
- uses: actions/upload-artifact@v4
- if: success()
- with:
- name: result ${{ github.job }}
- path: |
- performance/jmh-results*
-
- # Compare against the latest scheduled run on `main` and post a
- # markdown comment. Only relevant when this run is tied to a PR;
- # scheduled / non-PR runs skip these steps. We never fail the
- # workflow if comparison fails — it's reporting, not gating.
- - name: Fetch baseline results (latest successful main schedule)
- id: baseline
- if: steps.pr.outputs.number != ''
- continue-on-error: true
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- mkdir -p baseline-results
- RUN_ID=$(gh run list \
- --workflow benchmarks.yml \
- --branch main \
- --status success \
- --limit 20 \
- --repo "${{ github.repository }}" \
- --json databaseId,event \
- -q 'map(select(.event=="schedule"))[0].databaseId // empty')
- if [ -z "$RUN_ID" ]; then
- echo "No scheduled baseline run found on main"
- echo "found=false" >> "$GITHUB_OUTPUT"
- exit 0
- fi
- echo "Baseline run: $RUN_ID"
- if gh run download "$RUN_ID" --dir baseline-results --repo "${{ github.repository }}"; then
- echo "found=true" >> "$GITHUB_OUTPUT"
- echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT"
- else
- echo "Failed to download baseline artifacts"
- echo "found=false" >> "$GITHUB_OUTPUT"
- fi
-
- - name: Compare benchmark results
- id: compare
- if: steps.pr.outputs.number != '' && steps.baseline.outputs.found == 'true'
- continue-on-error: true
- run: |
- python3 "$RUNNER_TEMP/jmh-tools/compare-jmh.py" \
- --baseline baseline-results \
- --current performance \
- --baseline-run-id "${{ steps.baseline.outputs.run_id }}" \
- --current-run-id "${{ github.run_id }}" \
- --repo "${{ github.repository }}" \
- --server-url "${{ github.server_url }}" \
- --threshold-pct "${{ steps.pr.outputs.threshold }}" \
- --output comparison.md \
- --summary-output compare-summary.env
- # Surface the script's summary file as step outputs so the
- # follow-up "enforce threshold" step can decide whether to
- # fail the job — without skipping the comment post.
- cat compare-summary.env >> "$GITHUB_OUTPUT"
- echo "ok=true" >> "$GITHUB_OUTPUT"
-
- - name: Post baseline-not-found comment
- if: |
- steps.pr.outputs.number != '' &&
- steps.baseline.outputs.found != 'true'
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- gh pr comment "${{ steps.pr.outputs.number }}" \
- --repo "${{ github.repository }}" \
- --body "JMH benchmark comparison skipped: no successful scheduled run on \`main\` was found to use as a baseline." || true
-
- - name: Post comparison comment
- if: steps.compare.outputs.ok == 'true'
- env:
- GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- gh pr comment "${{ steps.pr.outputs.number }}" \
- --repo "${{ github.repository }}" \
- --body-file comparison.md
-
- # Fail the job — and therefore the PR check — when the comparison
- # script flagged at least one regression beyond the threshold.
- # This runs *after* the comment has been posted so reviewers still
- # see the full table on the PR.
- - name: Enforce regression threshold
- if: steps.compare.outputs.ok == 'true'
- run: |
- REGRESSIONS="${{ steps.compare.outputs.regressions }}"
- THRESHOLD="${{ steps.pr.outputs.threshold }}"
- if [ -n "$REGRESSIONS" ] && [ "$REGRESSIONS" -gt 0 ]; then
- echo "::error::$REGRESSIONS benchmark(s) regressed by more than ${THRESHOLD}% vs baseline."
- exit 1
- fi
- echo "No regressions over ${THRESHOLD}%."
diff --git a/.github/workflows/stresshouse-benchmark-compare.yml b/.github/workflows/stresshouse-benchmark-compare.yml
index a3965e934..1da2a463a 100644
--- a/.github/workflows/stresshouse-benchmark-compare.yml
+++ b/.github/workflows/stresshouse-benchmark-compare.yml
@@ -23,6 +23,7 @@ on:
required: true
type: string
+
permissions:
contents: read
pull-requests: write
@@ -36,9 +37,11 @@ jobs:
(github.event.issue.pull_request &&
startsWith(github.event.comment.body, '/benchmark-compare') &&
contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association))
- uses: ClickHouse/integrations-shared-workflows/.github/workflows/clickhouse-benchmark.yml@main
+ uses: ClickHouse/integrations-shared-workflows/.github/workflows/clickhouse-benchmark.yml@51b1407ab4b67f37333823f2c43aa42f0d9da141
with:
client: clickhouse-java
profile: pr-compare
pr_number: ${{ github.event.issue.number || inputs.pr_number }}
- secrets: inherit
+ secrets:
+ WORKFLOW_AUTH_PUBLIC_APP_ID: ${{ secrets.WORKFLOW_AUTH_PUBLIC_APP_ID }}
+ WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY: ${{ secrets.WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY }}