From 3100e5a509f76e43f4ee1e3fec06153ce15b2146 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 13 Jul 2026 09:53:52 -0700 Subject: [PATCH] Removed own benchmark. Updated stresshouse WF --- .github/scripts/compare-jmh.py | 588 ------------------ .github/scripts/test_compare_jmh.py | 217 ------- .github/scripts/test_data/README.md | 33 - .../baseline/jmh-results-baseline.json | 20 - .../current/jmh-results-current.json | 20 - .../baseline/jmh-results-baseline.json | 20 - .../current/jmh-results-current.json | 20 - .../baseline/jmh-results-baseline.json | 11 - .../current/jmh-results-current.json | 11 - .../mixed/baseline/jmh-results-baseline.json | 38 -- .../mixed/current/jmh-results-current.json | 38 -- .../baseline/jmh-results-baseline.json | 20 - .../no_alloc/current/jmh-results-current.json | 20 - .../baseline/jmh-results-baseline.json | 20 - .../current/jmh-results-current.json | 20 - .../baseline/jmh-results-baseline.json | 20 - .../current/jmh-results-current.json | 11 - .../baseline/jmh-results-baseline.json | 11 - .../current/jmh-results-current.json | 20 - .github/workflows/benchmarks-pr-comment.yml | 41 -- .github/workflows/benchmarks.yml | 258 -------- .../stresshouse-benchmark-compare.yml | 7 +- 22 files changed, 5 insertions(+), 1459 deletions(-) delete mode 100644 .github/scripts/compare-jmh.py delete mode 100644 .github/scripts/test_compare_jmh.py delete mode 100644 .github/scripts/test_data/README.md delete mode 100644 .github/scripts/test_data/all_improvements/baseline/jmh-results-baseline.json delete mode 100644 .github/scripts/test_data/all_improvements/current/jmh-results-current.json delete mode 100644 .github/scripts/test_data/all_regressions/baseline/jmh-results-baseline.json delete mode 100644 .github/scripts/test_data/all_regressions/current/jmh-results-current.json delete mode 100644 .github/scripts/test_data/empty_intersection/baseline/jmh-results-baseline.json delete mode 100644 .github/scripts/test_data/empty_intersection/current/jmh-results-current.json delete mode 100644 .github/scripts/test_data/mixed/baseline/jmh-results-baseline.json delete mode 100644 .github/scripts/test_data/mixed/current/jmh-results-current.json delete mode 100644 .github/scripts/test_data/no_alloc/baseline/jmh-results-baseline.json delete mode 100644 .github/scripts/test_data/no_alloc/current/jmh-results-current.json delete mode 100644 .github/scripts/test_data/noise_only/baseline/jmh-results-baseline.json delete mode 100644 .github/scripts/test_data/noise_only/current/jmh-results-current.json delete mode 100644 .github/scripts/test_data/only_in_baseline/baseline/jmh-results-baseline.json delete mode 100644 .github/scripts/test_data/only_in_baseline/current/jmh-results-current.json delete mode 100644 .github/scripts/test_data/only_in_pr/baseline/jmh-results-baseline.json delete mode 100644 .github/scripts/test_data/only_in_pr/current/jmh-results-current.json delete mode 100644 .github/workflows/benchmarks-pr-comment.yml delete mode 100644 .github/workflows/benchmarks.yml diff --git a/.github/scripts/compare-jmh.py b/.github/scripts/compare-jmh.py deleted file mode 100644 index 5010938f1..000000000 --- a/.github/scripts/compare-jmh.py +++ /dev/null @@ -1,588 +0,0 @@ -#!/usr/bin/env python3 -"""Compare two sets of JMH JSON results and emit a markdown summary. - -Used by `.github/workflows/benchmarks.yml` to diff the latest scheduled -`main` benchmark run against the run that just finished for a PR. - -For each (benchmark, params) pair common to both runs we report two -metrics: - -* `Time` — `primaryMetric.score`. In `SampleTime` mode this is the - mean sampled latency per op; it's our best available proxy for CPU - work since no dedicated CPU profiler is configured in - `BenchmarkRunner`. -* `Alloc/op` — `secondaryMetrics["gc.alloc.rate.norm"]`, populated by - JMH's `GCProfiler`. This is bytes allocated per benchmark op and is - the standard, low-noise JMH memory metric. (Note: JMH 1.37 stores the - key with no prefix; some visualizer tools display it as - `·gc.alloc.rate.norm` but the raw JSON does not contain that dot.) - -Both metrics are "lower is better", so a positive delta indicates the -PR is worse than the baseline. A run is considered failed when **any** -benchmark's worst metric delta exceeds `--threshold-pct` in the worse -direction. The script writes a `regressions=...`/`improvements=...` -summary file the workflow uses to set step outputs and decide whether -to fail the job. -""" - -from __future__ import annotations - -import argparse -import glob -import json -import os -import sys -from collections import defaultdict -from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Optional, Tuple - -Key = Tuple[str, str] - - -# --------------------------------------------------------------------------- -# Metric model -# --------------------------------------------------------------------------- - -# JMH's `GCProfiler` reports allocation rate normalised per op under this -# secondary metric key. Verified against jmh-core 1.37 sources: the key -# is the literal `gc.alloc.rate.norm` with no prefix. Some visualizers -# render it as `·gc.alloc.rate.norm`, which is a display convention, not -# what JMH writes to JSON. -ALLOC_NORM_KEY = "gc.alloc.rate.norm" - -# Tolerated prefix-bearing variants we'll fall back to if the canonical -# key ever moves. Lets a future JMH (or a non-Oracle JVM profiler that -# decorates the label) keep working without us being aware. -ALLOC_NORM_VARIANTS = ( - ALLOC_NORM_KEY, - "\u00b7gc.alloc.rate.norm", # pre-emptive middle-dot variant - "+gc.alloc.rate.norm", -) - -# Set to True the first time we fail to find any allocation data; used -# to emit a one-time diagnostic listing the secondary keys present so -# the cause is obvious in workflow logs. -_alloc_warn_printed = False - - -@dataclass(frozen=True) -class Metric: - id: str - label: str - # Pull the `{score, scoreError, scoreUnit}`-shaped dict from a JMH record. - extract: Callable[[Dict[str, Any]], Optional[Dict[str, Any]]] - # True when a higher score is worse (regression). Both of our metrics - # are lower-is-better so this is always True today, but the model - # supports e.g. Throughput mode trivially. - higher_is_worse: bool = True - - -def _primary(record: Dict[str, Any]) -> Optional[Dict[str, Any]]: - pm = record.get("primaryMetric") - return pm if isinstance(pm, dict) else None - - -def _secondary(record: Dict[str, Any], key: str) -> Optional[Dict[str, Any]]: - sm = record.get("secondaryMetrics") or {} - val = sm.get(key) - return val if isinstance(val, dict) else None - - -def _alloc(record: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """Pull the GC allocation-per-op metric, tolerating prefix variants. - - Falls back to a suffix match so even an unrecognised prefix (e.g. a - third-party JMH profiler that decorates the label) still works.""" - global _alloc_warn_printed - sm = record.get("secondaryMetrics") or {} - for k in ALLOC_NORM_VARIANTS: - v = sm.get(k) - if isinstance(v, dict): - return v - # Last-resort suffix match. - for k, v in sm.items(): - if isinstance(v, dict) and k.lower().endswith("gc.alloc.rate.norm"): - return v - if sm and not _alloc_warn_printed: - print( - "warn: no `gc.alloc.rate.norm` (or prefix-variant) found in " - "secondaryMetrics. Available keys: " - + ", ".join(sorted(sm.keys())), - file=sys.stderr, - ) - _alloc_warn_printed = True - return None - - -METRICS: List[Metric] = [ - Metric(id="time", label="Time", extract=_primary), - Metric(id="alloc", label="Alloc/op", extract=_alloc), -] - - -# --------------------------------------------------------------------------- -# Loading & helpers -# --------------------------------------------------------------------------- - - -def load_results(directory: str) -> Dict[Key, Dict[str, Any]]: - by_key: Dict[Key, Dict[str, Any]] = {} - paths = sorted( - glob.glob(os.path.join(directory, "**", "jmh-results-*.json"), recursive=True) - ) - for path in paths: - try: - with open(path, "r", encoding="utf-8") as fh: - data = json.load(fh) - except (OSError, json.JSONDecodeError) as exc: - print(f"warn: could not load {path}: {exc}", file=sys.stderr) - continue - if not isinstance(data, list): - continue - for record in data: - bench = record.get("benchmark") - if not bench: - continue - params = record.get("params") or {} - param_str = ", ".join(f"{k}={params[k]}" for k in sorted(params)) - by_key[(bench, param_str)] = record - return by_key - - -def _float(d: Optional[Dict[str, Any]], key: str) -> Optional[float]: - if not d: - return None - val = d.get(key) - try: - return float(val) if val is not None else None - except (TypeError, ValueError): - return None - - -def short_bench(name: str) -> str: - parts = name.split(".") - return ".".join(parts[-2:]) if len(parts) >= 2 else name - - -def fmt_score(v: Optional[float], err: Optional[float], unit: str) -> str: - if v is None: - return "—" - body = f"{v:.3g} ± {err:.2g}" if err is not None else f"{v:.3g}" - return f"{body} {unit}".rstrip() - - -def fmt_delta(d: Optional[float]) -> str: - if d is None: - return "—" - sign = "+" if d >= 0 else "" - return f"{sign}{d:.2f}%" - - -# --------------------------------------------------------------------------- -# Comparison -# --------------------------------------------------------------------------- - - -@dataclass -class MetricDelta: - metric: Metric - baseline: Optional[float] - current: Optional[float] - baseline_err: Optional[float] - current_err: Optional[float] - unit: str - delta_pct: Optional[float] - - def regression(self, threshold: float) -> bool: - if self.delta_pct is None: - return False - signed = self.delta_pct if self.metric.higher_is_worse else -self.delta_pct - return signed > threshold - - def improvement(self, threshold: float) -> bool: - if self.delta_pct is None: - return False - signed = self.delta_pct if self.metric.higher_is_worse else -self.delta_pct - return signed < -threshold - - def cell(self) -> str: - b = fmt_score(self.baseline, self.baseline_err, self.unit) - c = fmt_score(self.current, self.current_err, self.unit) - return f"{b} → {c} ({fmt_delta(self.delta_pct)})" - - -def metric_delta(metric: Metric, baseline_rec: Dict[str, Any], current_rec: Dict[str, Any]) -> MetricDelta: - b = metric.extract(baseline_rec) - c = metric.extract(current_rec) - bs = _float(b, "score") - cs = _float(c, "score") - be = _float(b, "scoreError") - ce = _float(c, "scoreError") - unit = (c or {}).get("scoreUnit") or (b or {}).get("scoreUnit") or "" - if bs is None or cs is None or bs == 0: - delta_pct: Optional[float] = None - else: - delta_pct = (cs - bs) / bs * 100.0 - return MetricDelta( - metric=metric, - baseline=bs, - current=cs, - baseline_err=be, - current_err=ce, - unit=unit, - delta_pct=delta_pct, - ) - - -@dataclass -class Row: - key: Key - deltas: List[MetricDelta] - - def worst_signed_pct(self) -> float: - worst = 0.0 - for d in self.deltas: - if d.delta_pct is None: - continue - signed = d.delta_pct if d.metric.higher_is_worse else -d.delta_pct - if signed > worst: - worst = signed - return worst - - def sort_key(self) -> float: - best = 0.0 - for d in self.deltas: - if d.delta_pct is None: - continue - if abs(d.delta_pct) > best: - best = abs(d.delta_pct) - return best - - def status(self, threshold: float) -> str: - regressed = [d for d in self.deltas if d.regression(threshold)] - improved = [d for d in self.deltas if d.improvement(threshold)] - if regressed: - labels = ", ".join(d.metric.label for d in regressed) - return f"REGRESSION ({labels})" - if improved: - labels = ", ".join(d.metric.label for d in improved) - return f"improvement ({labels})" - return "" - - -def build_rows( - baseline: Dict[Key, Dict[str, Any]], - current: Dict[Key, Dict[str, Any]], -) -> List[Row]: - rows: List[Row] = [] - for key in sorted(set(current) & set(baseline)): - b = baseline[key] - c = current[key] - deltas = [metric_delta(m, b, c) for m in METRICS] - rows.append(Row(key=key, deltas=deltas)) - return rows - - -# --------------------------------------------------------------------------- -# Rendering -# --------------------------------------------------------------------------- - - -def compute_discriminators( - rows: List[Row], current: Dict[Key, Dict[str, Any]] -) -> Dict[Key, str]: - """For each row, return a short string of only the params that differ - between rows sharing the same fully-qualified benchmark name. - - When a benchmark is run with only one combination of params, its - discriminator is the empty string (no need to disambiguate). - """ - by_bench: Dict[str, List[Row]] = defaultdict(list) - for r in rows: - bench, _ = r.key - by_bench[bench].append(r) - - out: Dict[Key, str] = {} - for bench, group in by_bench.items(): - if len(group) <= 1: - out[group[0].key] = "" - continue - params_dicts = [(current.get(r.key) or {}).get("params") or {} for r in group] - all_keys = sorted({k for p in params_dicts for k in p.keys()}) - varying = [ - k for k in all_keys if len({p.get(k) for p in params_dicts}) > 1 - ] - for r, p in zip(group, params_dicts): - out[r.key] = ", ".join(f"{k}={p[k]}" for k in varying if k in p) - return out - - -# Visual markers used in rendered output. "Performance went up" = -# improvement (less time / less memory per op). -ARROW_REGRESS = "\u2b07\ufe0f" # ⬇️ -ARROW_IMPROVE = "\u2b06\ufe0f" # ⬆️ -ARROW_NOISE = "\u2796" # ➖ - -DOT_REGRESS = "\U0001f534" # 🔴 -DOT_IMPROVE = "\U0001f7e2" # 🟢 - - -def _row_arrow(r: Row, threshold: float) -> str: - if any(d.regression(threshold) for d in r.deltas): - return ARROW_REGRESS - if any(d.improvement(threshold) for d in r.deltas): - return ARROW_IMPROVE - return ARROW_NOISE - - -def _short_delta(d: MetricDelta, threshold: float) -> str: - """One-line metric delta for the brief bullet list. - - Returns "" for noise (caller drops it). Regressions are 🔴-marked and - bold so they stand out on a packed PR comment; improvements are - 🟢-marked but unbolded. - """ - if d.delta_pct is None: - return "" - label = d.metric.label - delta = fmt_delta(d.delta_pct) - if d.regression(threshold): - return f"{DOT_REGRESS} **{label} {delta}**" - if d.improvement(threshold): - return f"{DOT_IMPROVE} {label} {delta}" - return "" - - -def _stats_cell(r: Row, threshold: float) -> str: - """Render the joined Stats cell for one row in the detail table. - - Each metric occupies a `
`-separated line. Regressions are - bolded and 🔴-tagged; improvements are 🟢-tagged. Metrics with no - baseline/current data are rendered grey ("—") so the cell still - shows which dimension is missing. - """ - lines: List[str] = [] - for d in r.deltas: - label = d.metric.label - if d.delta_pct is None: - base = fmt_score(d.baseline, d.baseline_err, d.unit) - curr = fmt_score(d.current, d.current_err, d.unit) - lines.append(f"{label} {base} → {curr} (—)") - continue - base = fmt_score(d.baseline, d.baseline_err, d.unit) - curr = fmt_score(d.current, d.current_err, d.unit) - delta = fmt_delta(d.delta_pct) - if d.regression(threshold): - lines.append(f"{DOT_REGRESS} **{label}** {base} → {curr} (**{delta}**)") - elif d.improvement(threshold): - lines.append(f"{DOT_IMPROVE} **{label}** {base} → {curr} ({delta})") - else: - lines.append(f"{label} {base} → {curr} ({delta})") - return "
".join(lines) - - -def build_markdown( - rows: List[Row], - only_current: List[Key], - only_baseline: List[Key], - current: Dict[Key, Dict[str, Any]], - *, - threshold: float, - repo: str, - server_url: str, - baseline_run_id: str, - current_run_id: str, -) -> Tuple[str, int, int]: - rows = sorted(rows, key=lambda r: r.sort_key(), reverse=True) - regressions = sum(1 for r in rows if any(d.regression(threshold) for d in r.deltas)) - improvements = sum( - 1 for r in rows - if not any(d.regression(threshold) for d in r.deltas) - and any(d.improvement(threshold) for d in r.deltas) - ) - discriminators = compute_discriminators(rows, current) - - out: List[str] = [""] - if regressions: - out.append( - f"## {DOT_REGRESS} JMH benchmark comparison — {regressions} regression(s) over {threshold:g}%" - ) - elif improvements: - out.append( - f"## {DOT_IMPROVE} JMH benchmark comparison — no regressions, {improvements} improvement(s) over {threshold:g}%" - ) - else: - out.append(f"## {DOT_IMPROVE} JMH benchmark comparison — no changes over {threshold:g}%") - out.append("") - - if repo and baseline_run_id and current_run_id: - base_url = f"{server_url}/{repo}/actions/runs/{baseline_run_id}" - curr_url = f"{server_url}/{repo}/actions/runs/{current_run_id}" - out.append( - f"Baseline: [`main` run #{baseline_run_id}]({base_url}) — " - f"PR: [run #{current_run_id}]({curr_url})" - ) - out.append("") - - # ---- brief per-benchmark summary -------------------------------------- - if rows: - # Stable order: regressions first, then improvements, then noise. - # Within each bucket, biggest |Δ| first. - def bucket(r: Row) -> int: - if any(d.regression(threshold) for d in r.deltas): - return 0 - if any(d.improvement(threshold) for d in r.deltas): - return 1 - return 2 - - rows = sorted(rows, key=lambda r: (bucket(r), -r.sort_key())) - - for r in rows: - bench, _ = r.key - disc = discriminators.get(r.key, "") - arrow = _row_arrow(r, threshold) - - # Only mention metrics that actually crossed the threshold - # in the brief view — keeps noisy rows to a single line. - bits = [s for s in (_short_delta(d, threshold) for d in r.deltas) if s] - - line = f"- {arrow} `{short_bench(bench)}`" - if disc: - line += f" `[{disc}]`" - if bits: - line += " — " + ", ".join(bits) - out.append(line) - out.append("") - else: - out.append("_No benchmarks matched between baseline and PR._") - out.append("") - - # ---- detailed table (collapsed) --------------------------------------- - if rows: - out.append( - f"
Detailed metrics " - f"(threshold ±{threshold:g}%, Time = primaryMetric.score from SampleTime, " - f"Alloc/op = {ALLOC_NORM_KEY}; positive Δ% = worse than baseline)" - "" - ) - out.append("") - out.append("| Benchmark | Stats |") - out.append("|---|---|") - for r in rows: - bench, params = r.key - bench_cell = f"`{short_bench(bench)}`" - if params: - bench_cell += f"
{params}" - out.append(f"| {bench_cell} | {_stats_cell(r, threshold)} |") - out.append("") - out.append("
") - out.append("") - - if only_current: - out.append("
Benchmarks only in PR run") - out.append("") - for k in only_current: - bench, params = k - rec = current[k] - time_d = _primary(rec) or {} - alloc_d = _alloc(rec) or {} - out.append( - f"- `{short_bench(bench)}` ({params or '—'}): " - f"time={fmt_score(_float(time_d, 'score'), _float(time_d, 'scoreError'), time_d.get('scoreUnit', ''))}, " - f"alloc={fmt_score(_float(alloc_d, 'score'), _float(alloc_d, 'scoreError'), alloc_d.get('scoreUnit', ''))}" - ) - out.append("") - out.append("
") - out.append("") - - if only_baseline: - out.append("
Benchmarks only in baseline run") - out.append("") - for k in only_baseline: - bench, params = k - out.append(f"- `{short_bench(bench)}` ({params or '—'})") - out.append("") - out.append("
") - out.append("") - - return "\n".join(out), regressions, improvements - - -# --------------------------------------------------------------------------- -# Entry point -# --------------------------------------------------------------------------- - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--baseline", required=True, help="Directory containing baseline JMH JSON files") - parser.add_argument("--current", required=True, help="Directory containing current JMH JSON files") - parser.add_argument("--baseline-run-id", default="", help="Baseline workflow run id (for links)") - parser.add_argument("--current-run-id", default="", help="Current workflow run id (for links)") - parser.add_argument("--repo", default="", help="owner/name for run links") - parser.add_argument("--server-url", default="https://github.com") - parser.add_argument("--output", required=True, help="Output markdown file") - parser.add_argument( - "--threshold-pct", - type=float, - default=10.0, - help="Δ%% beyond which a metric is flagged as a regression / improvement (default: 10)", - ) - parser.add_argument( - "--summary-output", - default="", - help="Optional path to write a key=value summary the workflow can source", - ) - args = parser.parse_args() - - if args.threshold_pct < 0: - print("error: --threshold-pct must be non-negative", file=sys.stderr) - return 2 - - baseline = load_results(args.baseline) - current = load_results(args.current) - - if not current: - print("error: no current JMH result files found", file=sys.stderr) - return 2 - - rows = build_rows(baseline, current) - only_current = sorted(set(current) - set(baseline)) - only_baseline = sorted(set(baseline) - set(current)) - - md, regressions, improvements = build_markdown( - rows, - only_current, - only_baseline, - current, - threshold=args.threshold_pct, - repo=args.repo, - server_url=args.server_url, - baseline_run_id=args.baseline_run_id, - current_run_id=args.current_run_id, - ) - - with open(args.output, "w", encoding="utf-8") as fh: - fh.write(md) - - if args.summary_output: - with open(args.summary_output, "w", encoding="utf-8") as fh: - fh.write(f"regressions={regressions}\n") - fh.write(f"improvements={improvements}\n") - fh.write(f"matched={len(rows)}\n") - fh.write(f"threshold_pct={args.threshold_pct}\n") - - print( - f"wrote {args.output}: {len(rows)} matched, " - f"{regressions} regression(s) > {args.threshold_pct:g}%, " - f"{improvements} improvement(s), " - f"{len(only_current)} only-PR, {len(only_baseline)} only-baseline" - ) - # We always exit 0; the workflow uses the summary file to decide - # whether to fail the job so that the comparison comment is still - # posted on regressions. - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/scripts/test_compare_jmh.py b/.github/scripts/test_compare_jmh.py deleted file mode 100644 index d426c1ef2..000000000 --- a/.github/scripts/test_compare_jmh.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -"""End-to-end tests for `compare-jmh.py`. - -Runs `compare-jmh.py` against every scenario in `test_data/` via -subprocess and asserts on both the rendered markdown and the -`--summary-output` counters. Designed to be run locally -(`python3 .github/scripts/test_compare_jmh.py`) and from CI without -extra dependencies — only the standard library is used. -""" - -from __future__ import annotations - -import os -import shutil -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path -from typing import Dict, List - -HERE = Path(__file__).resolve().parent -SCRIPT = HERE / "compare-jmh.py" -DATA = HERE / "test_data" - - -def _parse_summary(path: Path) -> Dict[str, str]: - out: Dict[str, str] = {} - for line in path.read_text(encoding="utf-8").splitlines(): - if "=" in line: - k, v = line.split("=", 1) - out[k.strip()] = v.strip() - return out - - -def _run_compare( - case: str, - *, - threshold: float = 10.0, - extra_args: List[str] | None = None, -) -> Dict[str, object]: - """Invoke compare-jmh.py against a fixture; return its outputs. - - Returns a dict with: returncode, stdout, stderr, markdown, summary. - """ - case_dir = DATA / case - assert case_dir.is_dir(), f"missing test fixture: {case_dir}" - - tmp = Path(tempfile.mkdtemp(prefix=f"cmp-jmh-{case}-")) - try: - out_md = tmp / "out.md" - summary = tmp / "summary.env" - cmd = [ - sys.executable, - str(SCRIPT), - "--baseline", - str(case_dir / "baseline"), - "--current", - str(case_dir / "current"), - "--threshold-pct", - str(threshold), - "--output", - str(out_md), - "--summary-output", - str(summary), - ] - if extra_args: - cmd.extend(extra_args) - proc = subprocess.run(cmd, capture_output=True, text=True) - result: Dict[str, object] = { - "returncode": proc.returncode, - "stdout": proc.stdout, - "stderr": proc.stderr, - "markdown": out_md.read_text(encoding="utf-8") if out_md.exists() else "", - "summary": _parse_summary(summary) if summary.exists() else {}, - } - return result - finally: - shutil.rmtree(tmp, ignore_errors=True) - - -# Pre-computed marker strings — keep in sync with compare-jmh.py. -ARROW_REGRESS = "\u2b07\ufe0f" -ARROW_IMPROVE = "\u2b06\ufe0f" -ARROW_NOISE = "\u2796" -DOT_REGRESS = "\U0001f534" -DOT_IMPROVE = "\U0001f7e2" - - -class CompareJmhTest(unittest.TestCase): - """Each case in test_data/ has a dedicated test asserting on the - headline shape, the bullet markers, and the summary counters.""" - - def test_all_improvements(self) -> None: - r = _run_compare("all_improvements") - self.assertEqual(r["returncode"], 0) - md = r["markdown"] - self.assertIn("no regressions", md) - self.assertIn(ARROW_IMPROVE, md) - self.assertIn(DOT_IMPROVE, md) - self.assertNotIn(ARROW_REGRESS, md) - self.assertNotIn(DOT_REGRESS, md) - s = r["summary"] - self.assertEqual(s.get("regressions"), "0") - self.assertEqual(s.get("improvements"), "2") - self.assertEqual(s.get("matched"), "2") - - def test_all_regressions(self) -> None: - r = _run_compare("all_regressions") - self.assertEqual(r["returncode"], 0) - md = r["markdown"] - self.assertIn("regression(s) over", md) - self.assertIn(ARROW_REGRESS, md) - self.assertIn(DOT_REGRESS, md) - # Bold markdown around the metric label. - self.assertIn("**Time", md) - s = r["summary"] - self.assertEqual(s.get("regressions"), "2") - # No row "purely improved" → improvements should be 0. - self.assertEqual(s.get("improvements"), "0") - - def test_mixed(self) -> None: - r = _run_compare("mixed") - self.assertEqual(r["returncode"], 0) - md = r["markdown"] - # Both directions present. - self.assertIn(ARROW_REGRESS, md) - self.assertIn(ARROW_IMPROVE, md) - self.assertIn(DOT_REGRESS, md) - self.assertIn(DOT_IMPROVE, md) - # Discriminator suffix `[limit=…]` appears for the two - # `queryV2` variants of the same benchmark. - self.assertIn("[limit=10000]", md) - self.assertIn("[limit=100000]", md) - # The unique-named benchmark must NOT get a discriminator. - self.assertNotIn("`JDBCQuery.selectJDBCV2` `[", md) - s = r["summary"] - self.assertEqual(s.get("matched"), "4") - self.assertGreater(int(s.get("regressions", "0")), 0) - - def test_no_alloc(self) -> None: - # No `gc.alloc.rate.norm` present anywhere → script must - # still compare Time and emit a diagnostic on stderr. - r = _run_compare("no_alloc") - self.assertEqual(r["returncode"], 0) - self.assertIn("no `gc.alloc.rate.norm`", r["stderr"]) - md = r["markdown"] - # Time regression should still be detected and 🔴-tagged… - self.assertIn(DOT_REGRESS, md) - # …but no Alloc/op metric is ever 🟢/🔴 because we have no - # baseline/current data for it. - self.assertIn("Alloc/op", md) - # The detail-table cell should fall back to "(—)" for alloc. - self.assertIn("Alloc/op — → — (—)", md) - s = r["summary"] - self.assertEqual(s.get("matched"), "2") - - def test_noise_only(self) -> None: - r = _run_compare("noise_only") - self.assertEqual(r["returncode"], 0) - md = r["markdown"] - self.assertIn("no changes over 10%", md) - # Every row should be on the noise arrow… - self.assertIn(ARROW_NOISE, md) - # …and there should be no red/green dots anywhere. - self.assertNotIn(DOT_REGRESS, md) - # 🟢 *is* in the header for the OK case, so don't assert on - # DOT_IMPROVE alone. - s = r["summary"] - self.assertEqual(s.get("regressions"), "0") - self.assertEqual(s.get("improvements"), "0") - - def test_only_in_pr(self) -> None: - r = _run_compare("only_in_pr") - self.assertEqual(r["returncode"], 0) - md = r["markdown"] - self.assertIn("Benchmarks only in PR run", md) - self.assertIn("QueryClient.queryV3New", md) - s = r["summary"] - # one shared row matched. - self.assertEqual(s.get("matched"), "1") - - def test_only_in_baseline(self) -> None: - r = _run_compare("only_in_baseline") - self.assertEqual(r["returncode"], 0) - md = r["markdown"] - self.assertIn("Benchmarks only in baseline run", md) - self.assertIn("QueryClient.queryV0Removed", md) - s = r["summary"] - self.assertEqual(s.get("matched"), "1") - - def test_empty_intersection(self) -> None: - r = _run_compare("empty_intersection") - self.assertEqual(r["returncode"], 0) - md = r["markdown"] - self.assertIn("_No benchmarks matched between baseline and PR._", md) - # Both unique-side sections still appear as
. - self.assertIn("Benchmarks only in PR run", md) - self.assertIn("Benchmarks only in baseline run", md) - s = r["summary"] - self.assertEqual(s.get("matched"), "0") - self.assertEqual(s.get("regressions"), "0") - self.assertEqual(s.get("improvements"), "0") - - def test_threshold_knob(self) -> None: - # The same fixture flips from "regression" to "ok" when the - # threshold is widened past the largest delta. - strict = _run_compare("all_regressions", threshold=10.0) - lenient = _run_compare("all_regressions", threshold=200.0) - self.assertGreater(int(strict["summary"]["regressions"]), 0) - self.assertEqual(lenient["summary"]["regressions"], "0") - self.assertIn("no changes", lenient["markdown"]) - - -if __name__ == "__main__": - # `-v` prints each scenario name so failures are obvious in CI logs. - unittest.main(verbosity=2) diff --git a/.github/scripts/test_data/README.md b/.github/scripts/test_data/README.md deleted file mode 100644 index 4a0938f09..000000000 --- a/.github/scripts/test_data/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# `compare-jmh.py` test fixtures - -Each subdirectory is a self-contained scenario for `compare-jmh.py`. The -layout is always: - -``` -/ - baseline/jmh-results-baseline.json - current/jmh-results-current.json -``` - -`compare-jmh.py` discovers result files by globbing for -`jmh-results-*.json` under the `--baseline` and `--current` directories, -so any filename starting with `jmh-results-` works. - -JSON records mirror the structure produced by JMH 1.37's -`ResultFormatType.JSON`: an array of objects with `benchmark`, `params`, -`primaryMetric.{score,scoreError,scoreUnit}`, and optionally -`secondaryMetrics["gc.alloc.rate.norm"]`. - -| Case | What it covers | -|---|---| -| `all_improvements` | Multiple benchmarks where both Time and Alloc/op fall well below the threshold; report should be all ⬆️ / 🟢 with no failure. | -| `all_regressions` | Multiple benchmarks where Time and/or Alloc/op rise well above the threshold; report should be ⬇️ / 🔴, script flags every row as `REGRESSION`, summary `regressions > 0`. | -| `mixed` | A blend of regressions, improvements, and within-noise rows including multiple variants of the same benchmark — verifies the bucket ordering and the param-discriminator (`[limit=…]`) logic. | -| `no_alloc` | Records with no `gc.alloc.rate.norm` at all; verifies the script falls back to Time-only and doesn't render `🔴`/`🟢` markers in the absence of the key (plus prints the diagnostic warning). | -| `noise_only` | All deltas are inside ±10%; report should be the ✅ "no changes" header and every row should carry the ➖ neutral arrow. | -| `only_in_pr` | A benchmark appears in `current` but not in `baseline`; verifies the "Benchmarks only in PR run" `
` block. | -| `only_in_baseline` | The mirror case — verifies the "Benchmarks only in baseline run" block. | -| `empty_intersection` | `baseline` and `current` contain different sets of benchmarks so no rows are matched; verifies the "_No benchmarks matched_" path. | - -The companion runner `test_compare_jmh.py` exercises every case and -checks both the rendered markdown and the `--summary-output` counters. diff --git a/.github/scripts/test_data/all_improvements/baseline/jmh-results-baseline.json b/.github/scripts/test_data/all_improvements/baseline/jmh-results-baseline.json deleted file mode 100644 index ff9ab2a4d..000000000 --- a/.github/scripts/test_data/all_improvements/baseline/jmh-results-baseline.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 80.0, "scoreError": 1.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 4096.0, "scoreError": 24.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/all_improvements/current/jmh-results-current.json b/.github/scripts/test_data/all_improvements/current/jmh-results-current.json deleted file mode 100644 index 034fa71f0..000000000 --- a/.github/scripts/test_data/all_improvements/current/jmh-results-current.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 16.0, "scoreError": 0.4, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 1536.0, "scoreError": 10.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 64.0, "scoreError": 1.3, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 3200.0, "scoreError": 20.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/all_regressions/baseline/jmh-results-baseline.json b/.github/scripts/test_data/all_regressions/baseline/jmh-results-baseline.json deleted file mode 100644 index ff9ab2a4d..000000000 --- a/.github/scripts/test_data/all_regressions/baseline/jmh-results-baseline.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 80.0, "scoreError": 1.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 4096.0, "scoreError": 24.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/all_regressions/current/jmh-results-current.json b/.github/scripts/test_data/all_regressions/current/jmh-results-current.json deleted file mode 100644 index f6f1aff82..000000000 --- a/.github/scripts/test_data/all_regressions/current/jmh-results-current.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 26.0, "scoreError": 0.6, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 3300.0, "scoreError": 18.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 100.0, "scoreError": 1.8, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 5400.0, "scoreError": 30.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/empty_intersection/baseline/jmh-results-baseline.json b/.github/scripts/test_data/empty_intersection/baseline/jmh-results-baseline.json deleted file mode 100644 index e2483a43c..000000000 --- a/.github/scripts/test_data/empty_intersection/baseline/jmh-results-baseline.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryOld", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/empty_intersection/current/jmh-results-current.json b/.github/scripts/test_data/empty_intersection/current/jmh-results-current.json deleted file mode 100644 index 40b2e0d86..000000000 --- a/.github/scripts/test_data/empty_intersection/current/jmh-results-current.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryNew", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 18.0, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 1900.0, "scoreError": 11.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/mixed/baseline/jmh-results-baseline.json b/.github/scripts/test_data/mixed/baseline/jmh-results-baseline.json deleted file mode 100644 index c967a2f66..000000000 --- a/.github/scripts/test_data/mixed/baseline/jmh-results-baseline.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"datasetSourceName": "file://default.csv", "limit": "10000"}, - "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"datasetSourceName": "file://default.csv", "limit": "100000"}, - "primaryMetric": {"score": 180.0, "scoreError": 2.0, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 16384.0, "scoreError": 96.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 80.0, "scoreError": 1.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 4096.0, "scoreError": 24.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.JDBCQuery.selectJDBCV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 30.0, "scoreError": 0.7, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 6144.0, "scoreError": 36.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/mixed/current/jmh-results-current.json b/.github/scripts/test_data/mixed/current/jmh-results-current.json deleted file mode 100644 index a115b036b..000000000 --- a/.github/scripts/test_data/mixed/current/jmh-results-current.json +++ /dev/null @@ -1,38 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"datasetSourceName": "file://default.csv", "limit": "10000"}, - "primaryMetric": {"score": 24.0, "scoreError": 0.6, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2100.0, "scoreError": 13.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"datasetSourceName": "file://default.csv", "limit": "100000"}, - "primaryMetric": {"score": 184.0, "scoreError": 2.0, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 24576.0, "scoreError": 110.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 60.0, "scoreError": 1.2, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 3200.0, "scoreError": 20.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.JDBCQuery.selectJDBCV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 31.0, "scoreError": 0.7, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 6200.0, "scoreError": 36.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/no_alloc/baseline/jmh-results-baseline.json b/.github/scripts/test_data/no_alloc/baseline/jmh-results-baseline.json deleted file mode 100644 index 1358026eb..000000000 --- a/.github/scripts/test_data/no_alloc/baseline/jmh-results-baseline.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "p0.50": {"score": 19.5, "scoreError": 0.0, "scoreUnit": "ms/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 80.0, "scoreError": 1.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "p0.50": {"score": 79.0, "scoreError": 0.0, "scoreUnit": "ms/op"} - } - } -] diff --git a/.github/scripts/test_data/no_alloc/current/jmh-results-current.json b/.github/scripts/test_data/no_alloc/current/jmh-results-current.json deleted file mode 100644 index 76629e32a..000000000 --- a/.github/scripts/test_data/no_alloc/current/jmh-results-current.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 26.0, "scoreError": 0.6, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "p0.50": {"score": 25.0, "scoreError": 0.0, "scoreUnit": "ms/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 82.0, "scoreError": 1.4, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "p0.50": {"score": 81.0, "scoreError": 0.0, "scoreUnit": "ms/op"} - } - } -] diff --git a/.github/scripts/test_data/noise_only/baseline/jmh-results-baseline.json b/.github/scripts/test_data/noise_only/baseline/jmh-results-baseline.json deleted file mode 100644 index ff9ab2a4d..000000000 --- a/.github/scripts/test_data/noise_only/baseline/jmh-results-baseline.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 80.0, "scoreError": 1.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 4096.0, "scoreError": 24.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/noise_only/current/jmh-results-current.json b/.github/scripts/test_data/noise_only/current/jmh-results-current.json deleted file mode 100644 index 1b54b5bef..000000000 --- a/.github/scripts/test_data/noise_only/current/jmh-results-current.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 20.6, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2080.0, "scoreError": 12.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.InsertClient.insertV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 78.5, "scoreError": 1.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 4170.0, "scoreError": 24.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/only_in_baseline/baseline/jmh-results-baseline.json b/.github/scripts/test_data/only_in_baseline/baseline/jmh-results-baseline.json deleted file mode 100644 index 32c7d3f81..000000000 --- a/.github/scripts/test_data/only_in_baseline/baseline/jmh-results-baseline.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV0Removed", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 30.0, "scoreError": 0.7, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 3072.0, "scoreError": 18.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/only_in_baseline/current/jmh-results-current.json b/.github/scripts/test_data/only_in_baseline/current/jmh-results-current.json deleted file mode 100644 index e9611885b..000000000 --- a/.github/scripts/test_data/only_in_baseline/current/jmh-results-current.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 20.4, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2050.0, "scoreError": 12.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/only_in_pr/baseline/jmh-results-baseline.json b/.github/scripts/test_data/only_in_pr/baseline/jmh-results-baseline.json deleted file mode 100644 index 5c589d4c2..000000000 --- a/.github/scripts/test_data/only_in_pr/baseline/jmh-results-baseline.json +++ /dev/null @@ -1,11 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 20.0, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2048.0, "scoreError": 12.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/scripts/test_data/only_in_pr/current/jmh-results-current.json b/.github/scripts/test_data/only_in_pr/current/jmh-results-current.json deleted file mode 100644 index 3551830f2..000000000 --- a/.github/scripts/test_data/only_in_pr/current/jmh-results-current.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV2", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 20.4, "scoreError": 0.5, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 2050.0, "scoreError": 12.0, "scoreUnit": "B/op"} - } - }, - { - "benchmark": "com.clickhouse.benchmark.clients.QueryClient.queryV3New", - "mode": "sample", - "params": {"limit": "10000"}, - "primaryMetric": {"score": 15.0, "scoreError": 0.4, "scoreUnit": "ms/op"}, - "secondaryMetrics": { - "gc.alloc.rate.norm": {"score": 1500.0, "scoreError": 10.0, "scoreUnit": "B/op"} - } - } -] diff --git a/.github/workflows/benchmarks-pr-comment.yml b/.github/workflows/benchmarks-pr-comment.yml deleted file mode 100644 index fe3a01996..000000000 --- a/.github/workflows/benchmarks-pr-comment.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Benchmarks PR Comment - -# Posts a one-time instruction comment on newly opened PRs so contributors / -# reviewers know how to launch a JMH benchmark run for the PR. The actual -# benchmark workflow lives in `benchmarks.yml` and is triggered by a -# `/benchmark` slash command (collaborators only). - -on: - pull_request_target: - types: [opened] - -permissions: - pull-requests: write - -jobs: - comment: - if: startsWith(github.repository, 'ClickHouse/clickhouse-java') - runs-on: ubuntu-latest - steps: - - name: Post benchmark instructions - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api -X POST \ - "repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \ - -f body="$(cat <<'EOF' - Repository collaborators can run the JMH benchmark suite against this PR by commenting: - - ``` - /benchmark - ``` - - Optional regression threshold override (Δ% on Time or Alloc/op; defaults to 10%): - - ``` - /benchmark threshold=15 - ``` - - Only one benchmark run per PR is active at a time — issuing a new `/benchmark` comment cancels the previous run. After the run finishes a separate comment will be posted comparing it against the latest scheduled run on `main`; the PR check fails if any benchmark regresses by more than the threshold. - EOF - )" diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml deleted file mode 100644 index 95d973f33..000000000 --- a/.github/workflows/benchmarks.yml +++ /dev/null @@ -1,258 +0,0 @@ -name: Benchmarks -description: Runs minimal JMH benchmark - -on: - schedule: - - cron: "55 15 * * *" - workflow_dispatch: - inputs: - pr: - description: "Pull request#" - required: false - threshold: - description: "Regression threshold (Δ% on Time or Alloc/op)" - required: false - default: "10" - issue_comment: - types: [created] - -env: - CHC_BRANCH: "main" - CH_VERSION: "25.3" - JAVA_VERSION: 17 - # Default Δ% above which a regression / improvement is flagged and the - # PR check is failed. Overridable per workflow_dispatch input or per - # `/benchmark threshold=N` comment. - DEFAULT_THRESHOLD_PCT: "10" - -# NOTE: there is intentionally no workflow-level `concurrency:` block. -# `issue_comment` events fire for *every* comment on every PR / issue, -# including those from bots (e.g. sonarqubecloud). A workflow-level -# `cancel-in-progress` group keyed on the PR number would cancel an -# in-flight legitimate `/benchmark` run as soon as any unrelated bot -# commented on the same PR. The per-PR concurrency rule is enforced on -# the `jmh` job instead, so unrelated comment events leave the job -# skipped without claiming the concurrency slot. - -jobs: - jmh: - name: "Mininal JMH Benchmarks" - runs-on: "ubuntu-latest" - timeout-minutes: 30 - permissions: - contents: read - pull-requests: write - issues: write - actions: read - # Single fan-in filter, modelled on `.github/workflows/claude.yml`: - # the job runs for the daily schedule, manual `workflow_dispatch`, - # or a `/benchmark` slash-command from a non-bot repo collaborator - # on a pull request. Bot comments and chat comments leave the job - # skipped — no failed run, no notification, no concurrency - # collision. - if: | - startsWith(github.repository, 'ClickHouse/') && - ( - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' || - ( - github.event_name == 'issue_comment' && - github.event.issue.pull_request != null && - github.event.sender.type != 'Bot' && - github.event.comment.user.type != 'Bot' && - startsWith(github.event.comment.body, '/benchmark') && - contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) - ) - ) - # One running benchmark per PR (and per-SHA for the daily - # schedule). Concurrency lives on this job, not on the workflow, - # so unrelated comment events (which the job-level `if` filters - # out) never claim the slot or cancel an in-flight run. - concurrency: - group: ${{ github.workflow }}-jmh-${{ github.event.issue.number || github.event.inputs.pr || github.sha }} - cancel-in-progress: true - steps: - - name: Acknowledge /benchmark trigger - if: github.event_name == 'issue_comment' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api -X POST \ - "repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions" \ - -f content='rocket' || true - - - name: Resolve PR number and threshold - id: pr - env: - COMMENT_BODY: ${{ github.event.comment.body }} - DISPATCH_PR: ${{ github.event.inputs.pr }} - DISPATCH_THRESHOLD: ${{ github.event.inputs.threshold }} - DEFAULT_THRESHOLD: ${{ env.DEFAULT_THRESHOLD_PCT }} - run: | - case "${{ github.event_name }}" in - issue_comment) - # Accept `/benchmark threshold=15` or `/benchmark threshold=7.5`. - T=$(printf '%s' "$COMMENT_BODY" | grep -oE 'threshold=[0-9]+(\.[0-9]+)?' | head -1 | cut -d= -f2 || true) - [ -z "$T" ] && T="$DEFAULT_THRESHOLD" - echo "number=${{ github.event.issue.number }}" >> "$GITHUB_OUTPUT" - echo "threshold=$T" >> "$GITHUB_OUTPUT" - ;; - workflow_dispatch) - echo "number=$DISPATCH_PR" >> "$GITHUB_OUTPUT" - echo "threshold=${DISPATCH_THRESHOLD:-$DEFAULT_THRESHOLD}" >> "$GITHUB_OUTPUT" - ;; - *) - echo "number=" >> "$GITHUB_OUTPUT" - echo "threshold=$DEFAULT_THRESHOLD" >> "$GITHUB_OUTPUT" - ;; - esac - - - name: Post "started" comment - if: github.event_name == 'issue_comment' && steps.pr.outputs.number != '' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api -X POST \ - "repos/${{ github.repository }}/issues/${{ steps.pr.outputs.number }}/comments" \ - -f body="JMH benchmark run started: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" || true - - - name: Check out Git repository - uses: actions/checkout@v4 - with: - ref: ${{ env.CHC_BRANCH }} - - # The benchmark code runs against the PR's working tree (see the - # PR checkout below), but the comparison tooling lives in this - # workflow's contract on `main`. Stash a copy now so the compare - # step still works even when the PR branch was forked before - # `.github/scripts/compare-jmh.py` existed. - - name: Stash comparison tooling from main - run: | - mkdir -p "$RUNNER_TEMP/jmh-tools" - cp -v .github/scripts/compare-jmh.py "$RUNNER_TEMP/jmh-tools/" - - - name: Check out PR - if: steps.pr.outputs.number != '' - run: | - git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ - origin pull/${{ steps.pr.outputs.number }}/merge:merged-pr && git checkout merged-pr - - - name: Install JDK and Maven - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: ${{ env.JAVA_VERSION }} - cache: "maven" - - - name: Build - run: mvn --batch-mode --no-transfer-progress -Dj8 -DskipTests=true clean install - - - name: Prepare Dataset - run: | - cd ./performance && - mvn --batch-mode --no-transfer-progress clean compile exec:exec -Dexec.executable=java \ - -Dexec.args="-classpath %classpath com.clickhouse.benchmark.data.DataSetGenerator -input sample_dataset.sql -name default -rows 100000" - - - name: Run Benchmarks - run: | - cd ./performance && - mvn --batch-mode --no-transfer-progress clean compile exec:exec -Dexec.executable=java -Dexec.args="-classpath %classpath com.clickhouse.benchmark.BenchmarkRunner \ - -l 100000,10000 -m 3 -t 15 -b q,i -d file://default.csv" - - - name: Upload test results - uses: actions/upload-artifact@v4 - if: success() - with: - name: result ${{ github.job }} - path: | - performance/jmh-results* - - # Compare against the latest scheduled run on `main` and post a - # markdown comment. Only relevant when this run is tied to a PR; - # scheduled / non-PR runs skip these steps. We never fail the - # workflow if comparison fails — it's reporting, not gating. - - name: Fetch baseline results (latest successful main schedule) - id: baseline - if: steps.pr.outputs.number != '' - continue-on-error: true - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - mkdir -p baseline-results - RUN_ID=$(gh run list \ - --workflow benchmarks.yml \ - --branch main \ - --status success \ - --limit 20 \ - --repo "${{ github.repository }}" \ - --json databaseId,event \ - -q 'map(select(.event=="schedule"))[0].databaseId // empty') - if [ -z "$RUN_ID" ]; then - echo "No scheduled baseline run found on main" - echo "found=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - echo "Baseline run: $RUN_ID" - if gh run download "$RUN_ID" --dir baseline-results --repo "${{ github.repository }}"; then - echo "found=true" >> "$GITHUB_OUTPUT" - echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" - else - echo "Failed to download baseline artifacts" - echo "found=false" >> "$GITHUB_OUTPUT" - fi - - - name: Compare benchmark results - id: compare - if: steps.pr.outputs.number != '' && steps.baseline.outputs.found == 'true' - continue-on-error: true - run: | - python3 "$RUNNER_TEMP/jmh-tools/compare-jmh.py" \ - --baseline baseline-results \ - --current performance \ - --baseline-run-id "${{ steps.baseline.outputs.run_id }}" \ - --current-run-id "${{ github.run_id }}" \ - --repo "${{ github.repository }}" \ - --server-url "${{ github.server_url }}" \ - --threshold-pct "${{ steps.pr.outputs.threshold }}" \ - --output comparison.md \ - --summary-output compare-summary.env - # Surface the script's summary file as step outputs so the - # follow-up "enforce threshold" step can decide whether to - # fail the job — without skipping the comment post. - cat compare-summary.env >> "$GITHUB_OUTPUT" - echo "ok=true" >> "$GITHUB_OUTPUT" - - - name: Post baseline-not-found comment - if: | - steps.pr.outputs.number != '' && - steps.baseline.outputs.found != 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh pr comment "${{ steps.pr.outputs.number }}" \ - --repo "${{ github.repository }}" \ - --body "JMH benchmark comparison skipped: no successful scheduled run on \`main\` was found to use as a baseline." || true - - - name: Post comparison comment - if: steps.compare.outputs.ok == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh pr comment "${{ steps.pr.outputs.number }}" \ - --repo "${{ github.repository }}" \ - --body-file comparison.md - - # Fail the job — and therefore the PR check — when the comparison - # script flagged at least one regression beyond the threshold. - # This runs *after* the comment has been posted so reviewers still - # see the full table on the PR. - - name: Enforce regression threshold - if: steps.compare.outputs.ok == 'true' - run: | - REGRESSIONS="${{ steps.compare.outputs.regressions }}" - THRESHOLD="${{ steps.pr.outputs.threshold }}" - if [ -n "$REGRESSIONS" ] && [ "$REGRESSIONS" -gt 0 ]; then - echo "::error::$REGRESSIONS benchmark(s) regressed by more than ${THRESHOLD}% vs baseline." - exit 1 - fi - echo "No regressions over ${THRESHOLD}%." diff --git a/.github/workflows/stresshouse-benchmark-compare.yml b/.github/workflows/stresshouse-benchmark-compare.yml index a3965e934..1da2a463a 100644 --- a/.github/workflows/stresshouse-benchmark-compare.yml +++ b/.github/workflows/stresshouse-benchmark-compare.yml @@ -23,6 +23,7 @@ on: required: true type: string + permissions: contents: read pull-requests: write @@ -36,9 +37,11 @@ jobs: (github.event.issue.pull_request && startsWith(github.event.comment.body, '/benchmark-compare') && contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association)) - uses: ClickHouse/integrations-shared-workflows/.github/workflows/clickhouse-benchmark.yml@main + uses: ClickHouse/integrations-shared-workflows/.github/workflows/clickhouse-benchmark.yml@51b1407ab4b67f37333823f2c43aa42f0d9da141 with: client: clickhouse-java profile: pr-compare pr_number: ${{ github.event.issue.number || inputs.pr_number }} - secrets: inherit + secrets: + WORKFLOW_AUTH_PUBLIC_APP_ID: ${{ secrets.WORKFLOW_AUTH_PUBLIC_APP_ID }} + WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY: ${{ secrets.WORKFLOW_AUTH_PUBLIC_PRIVATE_KEY }}