Add some performance benchmarks which will test various scenarios and compare them to performance of master

.circleci/config.yml

@@ -73,10 +73,88 @@ jobs:
       - after_failure:
           when : "on_fail"
 
+  perf_bench:
+    description: "Run insert/select performance benchmarks on master and on this branch, print the diff"
+    docker:
+      - image: nuodb/nuodb:latest
+        user: root
+    resource_class: xlarge
+    environment:
+      TZ : America/New_York
+      NUO_SET_TLS : disable
+      NUOCMD_CLIENT_KEY : ""
+      NUOCMD_VERIFY_SERVER : ""
+      NUOCMD_PLUGINS : ""
+    steps:
+      - checkout
+      - run:
+          name: Install build tools
+          command: |
+            PYVER=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+            dnf install -y git make gcc "python${PYVER}-devel"
+      - run:
+          name: Install pip
+          command: |
+            curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
+            python3 /tmp/get-pip.py --user
+      - run:
+          name: Make artifact directories
+          command: mkdir -p artifacts results
+      - run:
+          name: Start NuoDB Admin
+          command: |
+            sudo -u nuodb /opt/nuodb/etc/nuoadmin tls $NUO_SET_TLS
+            sudo -u nuodb /opt/nuodb/etc/nuoadmin tls status
+            sudo -u nuodb /opt/nuodb/etc/nuoadmin start
+            sudo -u nuodb /opt/nuodb/bin/nuocmd --show-json get effective-license
+      # Run master then this branch on the same runner and let compare.py
+      # print a delta table.  Hardware noise mostly cancels because both
+      # runs share the container; xlarge gives us dedicated cores.
+      - run:
+          name: Baseline benchmarks on master
+          command: |
+            GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+                git fetch --no-tags --depth=1 origin master
+            git worktree add /tmp/base FETCH_HEAD
+            # Copy the perf suite + conftest hooks from this branch so the
+            # master worktree's driver is exercised by the same benchmarks.
+            cp -a tests/perf /tmp/base/tests/
+            cp tests/conftest.py /tmp/base/tests/conftest.py
+            cd /tmp/base
+            make PYTHON=python3 install
+            $HOME/.local/bin/pip install pytest-benchmark
+            python3 -m pytest tests/perf --run-perf --benchmark-only \
+                --benchmark-json=/tmp/master.json \
+                --benchmark-columns=min,mean,median,stddev,rounds
+      - run:
+          name: Reset DB
+          command: |
+            sudo -u nuodb /opt/nuodb/bin/nuocmd shutdown database \
+                --db-name pynuodb_test 2>/dev/null || true
+      - run:
+          name: Branch benchmarks + diff vs master
+          command: |
+            make PYTHON=python3 install
+            $HOME/.local/bin/pip install pytest-benchmark
+            python3 -m pytest tests/perf --run-perf --benchmark-only \
+                --benchmark-json=artifacts/branch.json \
+                --benchmark-columns=min,mean,median,stddev,rounds
+            python3 tests/perf/compare.py \
+                /tmp/master.json artifacts/branch.json \
+                | tee artifacts/perf_diff.txt
+      - store_artifacts:
+          path: artifacts
+      - after_failure:
+          when : "on_fail"
+
 workflows:
   build-project:
     jobs:
       - build_n_run:
           name: "Build and run regression tests"
           context:
             - common-config
+      - perf_bench:
+          name: "Run performance benchmarks, comparing results to master"
+          context:
+            - common-config

test_requirements.txt

@@ -1,6 +1,7 @@
 mock>=1.0
 nose>=1.3
 pytest>=2.7
+pytest-benchmark>=4.0
 coverage>=3.7
 pytest-cov>=1.8.1
 coveralls>=0.5

tests/conftest.py

@@ -30,6 +30,26 @@
 
 from . import nuocmd, cvtjson
 
+
+def pytest_addoption(parser):
+    parser.addoption("--run-perf", action="store_true", default=False,
+                     help="run performance benchmarks under tests/perf")
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "perf: performance benchmark; skipped unless --run-perf is passed")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--run-perf"):
+        return
+    skip = pytest.mark.skip(reason="need --run-perf to run performance tests")
+    for item in items:
+        if "perf" in item.keywords:
+            item.add_marker(skip)
+
 _log = logging.getLogger("pynuodbtest")
 
 DB_OPTIONS = []  # type: List[str]

tests/perf/compare.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""Compare two pytest-benchmark JSON files (master vs branch).
+
+Prints a table of master min, branch min, and the absolute + percentage
+delta per test.  Exits non-zero if any test regressed by more than
+--fail-threshold (default 10%), so CI turns a real regression into a
+failed build.  Improvements never fail the build.
+"""
+from __future__ import print_function
+
+import argparse
+import json
+import sys
+
+
+def _load(path):
+    with open(path) as f:
+        data = json.load(f)
+    return {b['name']: b['stats']['min'] for b in data['benchmarks']}
+
+
+def main(args):
+    master = _load(args.master)
+    branch = _load(args.branch)
+
+    print("%-40s %16s %16s %14s %10s" % ( "Test", "master min (ms)", "branch min (ms)", "delta (ms)", "delta %"))
+    print("-" * 100)
+
+    regressed = []
+    for name in sorted(set(master) & set(branch)):
+        m = master[name] * 1000.0
+        b = branch[name] * 1000.0
+        d = b - m
+        p = (d / m) * 100.0 if m else float('nan')
+        print("%-40s %16.3f %16.3f %+14.3f %+9.2f%%" % (name, m, b, d, p))
+        if p > args.fail_threshold:
+            regressed.append((name, p))
+
+    only_master = sorted(set(master) - set(branch))
+    only_branch = sorted(set(branch) - set(master))
+    if only_master:
+        print("\nOnly in master: %s" % ", ".join(only_master))
+    if only_branch:
+        print("Only in branch: %s" % ", ".join(only_branch))
+
+    print()
+    if regressed:
+        print("FAIL: %d test(s) regressed by more than %.2f%%:" % (len(regressed), args.fail_threshold))
+        for name, p in regressed:
+            print("  %s: %+.2f%%" % (name, p))
+        sys.exit(1)
+    print("OK: no test regressed by more than %.2f%%" % args.fail_threshold)
+
+
+def _parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument('master', help='pytest-benchmark JSON for master')
+    p.add_argument('branch', help='pytest-benchmark JSON for this branch')
+    p.add_argument('--fail-threshold', type=float, default=15.0,
+                   help='percent slowdown that fails the build (default 15)')
+    return p.parse_args()
+
+
+if __name__ == '__main__':
+    main(_parse_args())