Generate workflow summary report for MG7 cross-section checks

.github/scripts/build_xsec_summary.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+"""Build a GitHub Actions job summary for the systematic mg7 cross-section
+checks (.github/workflows/check_xsec_processes_mg7.yml).
+
+Reads the per-process JSON records written by
+tests/acceptance_tests/test_check_xsec_processes_mg7.py (one file per
+process, via $MG7_XSEC_RESULTS_DIR) plus the reference file that defines
+which processes should have been tested, and prints a markdown report:
+a table of every process (with pass/fail, cross-sections, deviation and
+pull), followed by the scraped error message for every process that did
+not succeed.
+
+Usage:
+    build_xsec_summary.py <reference.json> <results_dir> [--tolerance PCT] \
+        >> "$GITHUB_STEP_SUMMARY"
+"""
+import argparse
+import glob
+import json
+import math
+import os
+
+pjoin = os.path.join
+
+
+def fmt_value(x, sig=4):
+    if x is None:
+        return '—'
+    return '{:.{sig}g}'.format(x, sig=sig)
+
+
+def fmt_signed(x, decimals=2):
+    if x is None or math.isnan(x) or math.isinf(x):
+        return '—'
+    return '{:+.{d}f}'.format(x, d=decimals)
+
+
+def slug(section, id_):
+    return 'fail-%s-%s' % (section, id_)
+
+
+def load_results(results_dir):
+    """Return {(section, id): record} from every *.json under results_dir."""
+    results = {}
+    for path in sorted(glob.glob(pjoin(results_dir, '**', '*.json'), recursive=True)):
+        try:
+            with open(path) as f:
+                record = json.load(f)
+        except Exception:
+            continue
+        results[(record.get('section'), record.get('id'))] = record
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('reference', help='path to check_xsec_processes_reference.json')
+    parser.add_argument('results_dir', help='directory with the per-process result JSON files')
+    parser.add_argument('--tolerance', type=float, default=None,
+                         help='tolerance (fraction, e.g. 0.01) used for this run, for display only')
+    parser.add_argument('--events', default=None,
+                         help='events/process used for this run, for display only')
+    args = parser.parse_args()
+
+    with open(args.reference) as f:
+        ref = json.load(f)
+
+    results = load_results(args.results_dir)
+
+    rows = []
+    failures = []
+    n_pass = 0
+    n_fail = 0
+    for section, entries in ref['sections'].items():
+        for entry in entries:
+            key = (section, entry['id'])
+            record = results.get(key)
+            ref_cross = entry['cross']
+            ref_error = entry.get('error')
+
+            if record is None:
+                status = 'fail'
+                got = err = None
+                message = ('No result recorded for this process -- the job '
+                            'likely crashed or was cancelled before it '
+                            'completed. Check the workflow run log for '
+                            '`test_%s_%s_mg7`.' % (section, entry['id']))
+            else:
+                status = record.get('status', 'fail')
+                got = record.get('cross')
+                err = record.get('error')
+                message = record.get('message')
+
+            if status == 'pass':
+                n_pass += 1
+            else:
+                n_fail += 1
+
+            reldev_pct = None
+            pull = None
+            if got is not None and ref_cross:
+                reldev_pct = 100.0 * (got - ref_cross) / ref_cross
+            if got is not None and err is not None and ref_error is not None:
+                denom = math.sqrt(err ** 2 + ref_error ** 2)
+                if denom > 0:
+                    pull = (got - ref_cross) / denom
+
+            anchor = slug(section, entry['id'])
+            status_cell = '✅' if status == 'pass' else ('[❌](#%s)' % anchor)
+
+            rows.append({
+                'status_cell': status_cell,
+                'section': section,
+                'id': entry['id'],
+                'process': entry['process'],
+                'cross': fmt_value(got),
+                'error': fmt_value(err),
+                'ref_cross': fmt_value(ref_cross),
+                'ref_error': fmt_value(ref_error),
+                'reldev': fmt_signed(reldev_pct),
+                'pull': fmt_signed(pull),
+            })
+
+            if status != 'pass':
+                failures.append({
+                    'anchor': anchor,
+                    'section': section,
+                    'id': entry['id'],
+                    'process': entry['process'],
+                    'message': message or '(no error message captured)',
+                })
+
+    out = []
+    out.append('## Systematic cross-section checks (mg7)')
+    out.append('')
+    detail = []
+    if args.tolerance is not None:
+        detail.append('tolerance: %.2f%%' % (100 * args.tolerance))
+    if args.events is not None:
+        detail.append('events/process: %s' % args.events)
+    if detail:
+        out.append('_' + ' &middot; '.join(detail) + '_')
+        out.append('')
+    out.append('**%d/%d processes passed**' % (n_pass, n_pass + n_fail))
+    out.append('')
+    out.append('| | Section | Process | σ [pb] | ± σ | Ref σ [pb] | ± Ref | Δ [%] | Pull |')
+    out.append('|---|---|---|---|---|---|---|---|---|')
+    for r in rows:
+        out.append('| %s | %s | `%s`<br>%s | %s | %s | %s | %s | %s | %s |' % (
+            r['status_cell'], r['section'], r['id'], r['process'],
+            r['cross'], r['error'], r['ref_cross'], r['ref_error'],
+            r['reldev'], r['pull']))
+
+    if failures:
+        out.append('')
+        out.append('## Failure details')
+        for fr in failures:
+            out.append('')
+            out.append('### <a id="%s"></a>%s / `%s`' % (fr['anchor'], fr['section'], fr['id']))
+            out.append('')
+            out.append('`%s`' % fr['process'])
+            out.append('')
+            out.append('```')
+            out.append(fr['message'])
+            out.append('```')
+
+    print('\n'.join(out))
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/check_xsec_processes_mg7.yml

@@ -78,6 +78,41 @@ jobs:
       # Provides the NNPDF23 grid + LHAPDF_DATA_PATH the mg7 runtime needs.
       - uses: ./.github/actions/install_lhapdf_pdfset
       - name: run systematic cross-section test (${{ matrix.section }})
+        env:
+          MG7_XSEC_RESULTS_DIR: ${{ github.workspace }}/xsec_results/${{ matrix.section }}
         run: |
           cd $GITHUB_WORKSPACE
+          mkdir -p "$MG7_XSEC_RESULTS_DIR"
           ./tests/test_manager.py "test_${{ matrix.section }}_.*_mg7" -pA -t0 -l INFO
+
+      # Runs even if the test step above failed, so a crashed/timed-out
+      # section still contributes whatever per-process results it managed to
+      # record before dying.
+      - name: upload cross-section results (${{ matrix.section }})
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: xsec-results-${{ matrix.section }}
+          path: xsec_results/${{ matrix.section }}/*.json
+          if-no-files-found: warn
+
+  xsec_summary:
+    needs: check_xsec_processes
+    if: always()
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: download cross-section results
+        uses: actions/download-artifact@v5
+        with:
+          pattern: xsec-results-*
+          path: xsec_results
+          merge-multiple: true
+      - name: build job summary
+        run: |
+          python3 .github/scripts/build_xsec_summary.py \
+            tests/acceptance_tests/check_xsec_processes_reference.json \
+            xsec_results \
+            --tolerance "${{ inputs.tolerance || '0.01' }}" \
+            --events "${{ inputs.events || '100000' }}" \
+            >> "$GITHUB_STEP_SUMMARY"

madgraph/iolibs/template_files/mg7/gridpack.py

@@ -31,6 +31,16 @@
 import tomllib
 import argparse
 
+
+def resolve_verbosity(verbosity: str) -> str:
+    """Resolve the run_card "auto" verbosity to "pretty"/"log" depending on
+    whether stdout is attached to a terminal; other values pass through
+    unchanged."""
+    if verbosity == "auto":
+        return "pretty" if sys.stdout.isatty() else "log"
+    return verbosity
+
+
 def main() -> None:
     # load run card and metadata. Use the RunCardMG7 representation when the
     # madgraph package is importable; gridpacks are meant to be portable, so
@@ -70,7 +80,7 @@ def main() -> None:
         "--verbosity",
         type=str,
         default=run_args["verbosity"],
-        choices=["none", "pretty", "log"]
+        choices=["none", "pretty", "log", "auto"]
     )
     parser.add_argument(
         "--output_format",
@@ -133,7 +143,7 @@ def main() -> None:
     config.freeze_max_weight_after = args.freeze_max_weight_after
     config.cpu_batch_size = args.cpu_batch_size
     config.gpu_batch_size = args.gpu_batch_size
-    config.verbosity = args.verbosity
+    config.verbosity = resolve_verbosity(args.verbosity)
     config.combine_thread_count = run_args["combine_thread_pool_size"]
     config.cut_efficiency_threshold = gen_args["cut_efficiency_threshold"]
     config.max_cut_repetitions = gen_args["max_cut_repetitions"]

madgraph/iolibs/template_files/mg7/madevent.py

@@ -97,6 +97,15 @@ def format_time(t: int, centi: bool = False):
         return f"{int(hours):02}:{int(minutes):02}:{seconds:02.0f}"
 
 
+def resolve_verbosity(verbosity: str) -> str:
+    """Resolve the run_card "auto" verbosity to "pretty"/"log" depending on
+    whether stdout is attached to a terminal; other values pass through
+    unchanged."""
+    if verbosity == "auto":
+        return "pretty" if sys.stdout.isatty() else "log"
+    return verbosity
+
+
 @dataclass
 class Channel:
     phasespace_mapping: ms.PhaseSpaceMapping
@@ -372,7 +381,7 @@ def init_generator_config(self) -> None:
         cfg.optimization_threshold = vegas_args["optimization_threshold"]
         cfg.cpu_batch_size = gen_args["cpu_batch_size"]
         cfg.gpu_batch_size = gen_args["gpu_batch_size"]
-        cfg.verbosity = run_args["verbosity"]
+        cfg.verbosity = resolve_verbosity(run_args["verbosity"])
         cfg.combine_thread_count = run_args["combine_thread_pool_size"]
         cfg.cut_efficiency_threshold = gen_args["cut_efficiency_threshold"]
         cfg.max_cut_repetitions = gen_args["max_cut_repetitions"]
@@ -492,7 +501,7 @@ def train_madnis(self) -> None:
         run_args = self.run_card["run"]
 
         config = ms.MadnisConfig()
-        config.verbosity = run_args["verbosity"]
+        config.verbosity = resolve_verbosity(run_args["verbosity"])
         config.learning_rate = madnis_args["lr"]
         config.batches = madnis_args["train_batches"]
         config.log_interval = madnis_args["log_interval"]

madgraph/iolibs/template_files/mg7/run_card.toml

@@ -12,7 +12,7 @@ cpu_thread_pool_size = %(run.cpu_thread_pool_size)s
 gpu_thread_pool_size = %(run.gpu_thread_pool_size)s
 combine_thread_pool_size = %(run.combine_thread_pool_size)s
 output_format = %(run.output_format)s # options: compact_npy, lhe_npy, lhe
-verbosity = %(run.verbosity)s # options: silent, pretty, log
+verbosity = %(run.verbosity)s # options: silent, pretty, log, auto (pretty if attached to a terminal, log otherwise)
 dummy_matrix_element = %(run.dummy_matrix_element)s
 
 [gridpack]

madgraph/various/banner.py

@@ -6446,8 +6446,8 @@ def default_setup(self):
         self.add_toml_param('run', 'combine_thread_pool_size', -1, gridpack=True)
         self.add_toml_param('run', 'output_format', "lhe", gridpack=True,
             allowed=['compact_npy', 'lhe_npy', 'lhe'])
-        self.add_toml_param('run', 'verbosity', "pretty", gridpack=True,
-            allowed=['silent', 'pretty', 'log'])
+        self.add_toml_param('run', 'verbosity', "auto", gridpack=True,
+            allowed=['silent', 'pretty', 'log', 'auto'])
         self.add_toml_param('run', 'dummy_matrix_element', False)
 
         # ---------------------------- [gridpack] ----------------------