feat: merge-train/spartan

.github/workflows/deploy-irm.yml

@@ -6,7 +6,7 @@ on:
   workflow_call:
     inputs:
       network:
-        description: "Network that IRM will be monitoring e.g. testnet, staging-ignition..."
+        description: "Network that IRM will be monitoring e.g. testnet..."
         required: true
         type: string
         default: "testnet"
@@ -41,7 +41,7 @@ on:
   workflow_dispatch:
     inputs:
       network:
-        description: "Network that IRM will be monitoring e.g. testnet, staging-ignition..."
+        description: "Network that IRM will be monitoring e.g. testnet..."
         required: true
         type: string
         default: "testnet"

.github/workflows/deploy-network.yml

@@ -46,7 +46,7 @@ on:
   workflow_dispatch:
     inputs:
       network:
-        description: "Network to deploy (e.g., staging-public, staging-ignition, testnet, next-net)"
+        description: "Network to deploy (e.g., staging-public, testnet, next-net)"
         required: true
         type: choice
         options:

.github/workflows/deploy-staging-public.yml

@@ -1,10 +1,6 @@
 name: Deploy to staging-public
 
 on:
-  workflow_run:
-    workflows: ["Nightly Release Tag (v4-next)"]
-    types:
-      - completed
   workflow_dispatch:
     inputs:
       tag:

.github/workflows/ensure-funded-environment.yml

@@ -6,7 +6,7 @@ on:
   workflow_call:
     inputs:
       environment:
-        description: 'Environment to fund (e.g., staging-public, next-net, staging-ignition, testnet)'
+        description: 'Environment to fund (e.g., staging-public, next-net, testnet)'
         required: true
         type: string
       low_watermark:
@@ -32,7 +32,6 @@ on:
         options:
         - staging-public
         - next-net
-        - staging-ignition
         - testnet
         - devnet
         - tps-scenario

.github/workflows/metrics-deploy.yml

@@ -100,7 +100,6 @@ jobs:
       GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME: ${{ inputs.grafana_dashboard_password_secret_name }}
       SLACK_WEBHOOK_URL_SECRET_NAME: slack-webhook-url
       SLACK_WEBHOOK_STAGING_PUBLIC_SECRET_NAME: slack-webhook-staging-public-url
-      SLACK_WEBHOOK_STAGING_IGNITION_SECRET_NAME: slack-webhook-staging-ignition-url
       SLACK_WEBHOOK_NEXT_SCENARIO_SECRET_NAME: slack-webhook-next-scenario-url
       SLACK_WEBHOOK_NEXT_NET_SECRET_NAME: slack-webhook-next-net-url
       SLACK_WEBHOOK_TESTNET_SECRET_NAME: slack-webhook-testnet-url
@@ -164,7 +163,6 @@ jobs:
             -var="GRAFANA_PASSWORD_SECRET_NAME=${{ env.GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_SECRET_NAME=${{ env.SLACK_WEBHOOK_URL_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_STAGING_PUBLIC_SECRET_NAME=${{ env.SLACK_WEBHOOK_STAGING_PUBLIC_SECRET_NAME }}" \
-            -var="SLACK_WEBHOOK_STAGING_IGNITION_SECRET_NAME=${{ env.SLACK_WEBHOOK_STAGING_IGNITION_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_NEXT_SCENARIO_SECRET_NAME=${{ env.SLACK_WEBHOOK_NEXT_SCENARIO_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_NEXT_NET_SECRET_NAME=${{ env.SLACK_WEBHOOK_NEXT_NET_SECRET_NAME }}" \
             -lock=${{ inputs.respect_tf_lock }}
@@ -179,7 +177,6 @@ jobs:
             -var="GRAFANA_PASSWORD_SECRET_NAME=${{ env.GRAFANA_DASHBOARD_PASSWORD_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_SECRET_NAME=${{ env.SLACK_WEBHOOK_URL_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_STAGING_PUBLIC_SECRET_NAME=${{ env.SLACK_WEBHOOK_STAGING_PUBLIC_SECRET_NAME }}" \
-            -var="SLACK_WEBHOOK_STAGING_IGNITION_SECRET_NAME=${{ env.SLACK_WEBHOOK_STAGING_IGNITION_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_NEXT_SCENARIO_SECRET_NAME=${{ env.SLACK_WEBHOOK_NEXT_SCENARIO_SECRET_NAME }}" \
             -var="SLACK_WEBHOOK_NEXT_NET_SECRET_NAME=${{ env.SLACK_WEBHOOK_NEXT_NET_SECRET_NAME }}" \
             -out=tfplan \

.test_patterns.yml

@@ -116,9 +116,7 @@ tests:
     error_regex: "✕ Repay"
     owners:
       - *lasse
-  # http://ci.aztec-labs.com/e85ff6c5da65a1e7
-  - regex: "src/e2e_cross_chain_messaging/l1_to_l2.test.ts"
-    error_regex: "Failed to advance block"
+  - regex: "src/e2e_cross_chain_messaging/l1_to_l2"
     owners:
       - *palla
   - regex: "src/e2e_cross_chain_messaging/token_bridge_private"
@@ -141,6 +139,10 @@ tests:
     error_regex: "Anvil failed to stop in time|Cannot read properties of undefined"
     owners:
       - *palla
+  - regex: "src/e2e_bot.test.ts"
+    error_regex: "Tx dropped"
+    owners:
+      - *palla
 
   # yarn-project tests
   # Attempt to catch all kv-store browser test failures (consider them quarantined for now)
@@ -164,6 +166,10 @@ tests:
     error_regex: "Error: Timeout of 2000ms exceeded"
     owners:
       - *martin
+  - regex: "kv-store/.*store\\.test"
+    error_regex: "guards against too many cursors"
+    owners:
+      - *palla
   - regex: "ethereum/src/test/tx_delayer.test.ts"
     error_regex: "delays a transaction until a given L1 timestamp"
     owners:
@@ -172,6 +178,18 @@ tests:
     error_regex: "ContractFunctionExecutionError: The contract function"
     owners:
       - *mitch
+  # Under proposer pipelining each validator votes in its own slot and the votes
+  # don't aggregate into the same round, so the slashing quorum (3) is never
+  # reached within the 414s budget; the test consistently times out at the docker
+  # outer 600s (exit 124). The publisher refactor lands all vote-offenses tx's
+  # on L1 successfully — voteCount on the slasher proposer simply stays at 1
+  # per round. This is a slashing-payload aggregation issue independent of
+  # publisher work; skip until the slashing team addresses it separately.
+  - regex: "e2e_p2p/valid_epoch_pruned_slash.test.ts"
+    skip: true
+    owners:
+      - *mitch
+      - *palla
   - regex: "archiver/src/archiver/archiver.test.ts"
     error_regex: "Received number of calls: 1"
     owners:
@@ -185,14 +203,6 @@ tests:
       - *phil
       - *palla
 
-  # http://ci.aztec-labs.com/64a972aafaa40dd0
-  # ProvingBroker › Retries › does not retry if job is stale — kv-store closes
-  # before the broker's final reportProvingJobError write lands.
-  - regex: "prover-client/src/proving_broker/proving_broker.test.ts"
-    error_regex: "does not retry if job is stale|Store is closed"
-    owners:
-      - *alex
-
   # Nightly GKE tests
   - regex: "spartan/bootstrap.sh"
     owners:
@@ -347,11 +357,42 @@ tests:
     owners:
       - *esau
 
+  # http://ci.aztec-labs.com/6db6103599cb22e6
+  # Under pipelining, the lazy validator's attestation can arrive after the
+  # p2p checkpoint-attestation-validator's acceptance window, getting rejected
+  # before the slasher's gossip callback fires. The slasher then never records
+  # ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL and the test times out.
+  - regex: "src/e2e_slashing/attested_invalid_proposal.test.ts"
+    error_regex: "Timeout awaiting honest validator slash offenses for invalid proposal attestation"
+    owners:
+      - *palla
+
+  # HA full suite is unstable under HA pipelining: multiple distinct failure
+  # modes (governance vote coord, work distribution, afterAll teardown hook
+  # timeouts, peer races on checkpoint proposals — see #23344, #23541,
+  # ci.aztec-labs.com/136431da99834194). Flagging individual error_regex
+  # entries and even the whole-suite flake match still let it dequeue PRs
+  # because ci3 retries once and the suite fails both attempts. Skip
+  # outright on merge-train/spartan until HA pipelining stabilises.
   - regex: "yarn-project/end-to-end/scripts/run_test.sh ha src/composed/ha/e2e_ha_full.test.ts"
-    error_regex: "✕ should coordinate governance voting across HA nodes"
+    skip: true
     owners:
       - *spyros
 
+  # Multi-validator web3signer suite is unstable under proposer pipelining:
+  # two distinct failure modes have been seen in the same "should build blocks
+  # & attest with multiple validator keys" case — Promise.all index-0
+  # waitForTx aborting with "Tx dropped by P2P node"
+  # (ci.aztec-labs.com/9a5fa90aa18f62e7), and the proposer missing the slot's
+  # 5-attestation deadline ("AttestationTimeoutError" / "Block .* not found .*
+  # reorg") on PR #23344 run 26370196367 (ci.aztec-labs.com/b91d3218b5e88ae4).
+  # Error-regex flake matches still let ci3 retry-and-fail both attempts. Skip
+  # outright on merge-train/spartan until proposer pipelining stabilises.
+  - regex: "yarn-project/end-to-end/scripts/run_test.sh web3signer src/composed/web3signer/e2e_multi_validator_node_key_store.test.ts"
+    skip: true
+    owners:
+      - *palla
+
   # http://ci.aztec-labs.com/98d59d04f85223f8
   # Build-cache flake: module not found during Jest startup
   - regex: "src/e2e_sequencer/gov_proposal.parallel.test.ts"
@@ -374,3 +415,11 @@ tests:
     error_regex: "✕ verifies transactions at 10 TPS"
     owners:
       - *phil
+
+  # http://ci.aztec-labs.com/930ddc9ede87059f
+  # Pipelining race: slasher doesn't record the duplicate proposal offense before
+  # the test's wait timeout — same family as #23501.
+  - regex: "src/e2e_p2p/duplicate_proposal_slash.test.ts"
+    error_regex: "TimeoutError: Timeout awaiting duplicate proposal offense"
+    owners:
+      - *palla

aztec-up/bootstrap.sh

@@ -20,6 +20,9 @@ function build {
   echo
 
   # Create Verdaccio config.
+  # publish.allow_offline lets verdaccio accept publishes when the npmjs
+  # uplink is briefly unreachable, instead of returning 503. We never want
+  # the upstream existence check to gate these local fake-publishes.
   cat > /tmp/verdaccio-config.yaml <<EOF
 storage: $PWD/verdaccio-storage
 max_body_size: 1000mb
@@ -28,7 +31,23 @@ uplinks:
   npmjs:
     url: https://registry.npmjs.org/
 
+publish:
+  allow_offline: true
+
 packages:
+  # @aztec/viem is a third-party fork published to npm, not built locally.
+  # Match it before @aztec/* so it falls through to the npmjs uplink.
+  "@aztec/viem":
+    access: \$all
+    publish: \$all
+    unpublish: \$all
+    proxy: npmjs
+
+  "@aztec/*":
+    access: \$all
+    publish: \$all
+    unpublish: \$all
+
   "@*/*":
     access: \$all
     publish: \$all

aztec-up/test/amm_flow.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+export SEQ_ENABLE_PROPOSER_PIPELINING=true
+
 # Start local network and wait for port to open.
 aztec start --local-network &
 local_network_pid=$!

aztec-up/test/basic_install.sh

@@ -10,6 +10,7 @@ echo
 
 export LOG_LEVEL=silent
 export PXE_PROVER=none
+export SEQ_ENABLE_PROPOSER_PIPELINING=true
 
 # Start local network and wait for port to open.
 aztec start --local-network &

aztec-up/test/bridge_and_claim.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+export SEQ_ENABLE_PROPOSER_PIPELINING=true
+
 # Start local network and wait for port to open.
 aztec start --local-network &
 local_network_pid=$!

docs/developer_versioned_docs/version-v4.3.0/docs/aztec-nr/debugging.md

@@ -19,7 +19,7 @@ Enable different levels of logging on the local network or node by setting `LOG_
 
 ```bash
 # Set log level (options: fatal, error, warn, info, verbose, debug, trace)
-LOG_LEVEL=debug aztec start --local-network
+LOG_LEVEL="debug; info: json-rpc, simulator" aztec start --local-network
 
 # Different levels for different services
 LOG_LEVEL="verbose;info:sequencer" aztec start --local-network

docs/docs-operate/operators/reference/ethereum-rpc-reference.md

@@ -270,7 +270,7 @@ Aztec automatically retries failed requests on alternative endpoints when multip
 Enable detailed RPC logging:
 
 ```bash
-LOG_LEVEL=debug # or verbose
+LOG_LEVEL="debug; info: json-rpc, simulator" # or verbose
 ```
 
 Look for log entries related to:

docs/docs-operate/operators/sequencer-management/slashing-configuration.md

@@ -161,7 +161,7 @@ SLASH_DATA_WITHHOLDING_PENALTY=0  # Set to >0 to enable
 
 # Invalid attestations and blocks
 SLASH_PROPOSE_INVALID_ATTESTATIONS_PENALTY=2000000000000000000000  # 2000 tokens
-SLASH_ATTEST_DESCENDANT_OF_INVALID_PENALTY=2000000000000000000000  # 2000 tokens
+SLASH_PROPOSE_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS_PENALTY=2000000000000000000000  # 2000 tokens
 SLASH_INVALID_BLOCK_PENALTY=2000000000000000000000  # 2000 tokens
 
 # Offense expiration

docs/examples/bootstrap.sh

@@ -210,7 +210,10 @@ function execute-examples {
 }
 
 function test_cmds {
-  echo "$hash:ONLY_TERM_PARENT=1 docs/examples/bootstrap.sh execute"
+  # Bumped from the default 600s by ~50% (now 15m) to absorb cumulative-runtime growth
+  # under merge-queue load — example_swap's `wait-for-proven` poll was tipping SIGTERM
+  # near the old limit. See PR #23253 dequeue log http://ci.aztec-labs.com/b08ac48286302949.
+  echo "$hash:ONLY_TERM_PARENT=1:TIMEOUT=15m docs/examples/bootstrap.sh execute"
 }
 
 function test {

docs/examples/ts/docker-compose.yml

@@ -28,6 +28,7 @@ services:
       WS_BLOCK_CHECK_INTERVAL_MS: 500
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
       P2P_MIN_TX_POOL_AGE_MS: 0
+      SEQ_ENABLE_PROPOSER_PIPELINING: 'true'
       HARDWARE_CONCURRENCY: ${HARDWARE_CONCURRENCY:-}
 
   docs-examples:

docs/network_versioned_docs/version-v4.3.0/operators/reference/ethereum-rpc-reference.md

@@ -270,7 +270,7 @@ Aztec automatically retries failed requests on alternative endpoints when multip
 Enable detailed RPC logging:
 
 ```bash
-LOG_LEVEL=debug # or verbose
+LOG_LEVEL="debug; info: json-rpc, simulator" # or verbose
 ```
 
 Look for log entries related to:

docs/static/typescript-api/mainnet/foundation.md

@@ -1422,7 +1422,7 @@ type NetworkConfigMap = z.infer<typeof NetworkConfigMapSchema>
 
 ### NetworkNames
 ```typescript
-type NetworkNames = "local" | "staging-ignition" | "staging-public" | "testnet" | "mainnet" | "next-net" | "devnet"
+type NetworkNames = "local" | "staging-public" | "testnet" | "mainnet" | "next-net" | "devnet"
 ```
 
 ### PartialBy

l1-contracts/bootstrap.sh

@@ -18,8 +18,11 @@ function download_solc {
   echo_stderr "Downloading solc $solc_version via svm..."
   # svm-rs always uses ~/.svm if it exists. Make sure it does for a consistent path across OS/architecture.
   mkdir -p "$HOME/.svm"
-  # We build a minimal file to trigger svm download of solc.
-  forge build --use "$solc_version" src/core/libraries/ConstantsGen.sol 2>/dev/null
+  # We build a minimal file to trigger svm download of solc. svm fetches the
+  # binary from binaries.soliditylang.org, which intermittently fails to resolve
+  # under heavy parallel CI load; retry to ride out transient DNS drops. (The
+  # merge queue disables the cache above, so this download path runs every time.)
+  retry "forge build --use \"$solc_version\" src/core/libraries/ConstantsGen.sol 2>/dev/null"
 
   # Copy from svm cache to local path
   local svm_path="$HOME/.svm/$solc_version/solc-$solc_version"

l1-contracts/src/core/slashing/SlashingProposer.sol

@@ -50,12 +50,11 @@ import {SafeCast} from "@oz/utils/math/SafeCast.sol";
  *
  *      About SLASH_OFFSET_IN_ROUNDS:
  *      - This offset gives us time to detect an offense and then vote on it in a later
- *        round. For instance, an `VALID_EPOCH_PRUNED` offense for epoch N is only triggered after
- *        `PROOF_SUBMISSION_WINDOW` epochs. Consider the following:
- *        - Epoch 1 is valid
- *        - At the end of epoch 3, the proof for 1 has not landed, so epoch 1 is pruned
- *        - Network decides to slash the committee of epoch 1
- *        - This means that only starting from epoch 4 we should be voting for slashing the committee of epoch 1
+ *        round. For instance, a `DATA_WITHHOLDING` offense for slot S is only triggered after
+ *        `DATA_WITHHOLDING_TOLERANCE_SLOTS` slots. Consider:
+ *        - Slot S publishes a checkpoint
+ *        - At slot S + tolerance, observers find missing data and want to slash the attesters
+ *        - Voting on that slash needs to happen in a round that starts after detection
  *      - In terms of voting, this parameter means that in round R we are voting for the committee of epochs starting
  *        from (R - SLASH_OFFSET_IN_ROUNDS) * ROUND_SIZE_IN_EPOCHS.
  *      - For example, with SLASH_OFFSET_IN_ROUNDS=2, ROUND_SIZE=10, and EPOCH_DURATION=2

noir-projects/bootstrap.sh

@@ -10,9 +10,14 @@ function build {
   function prep {
     set -eu
     (cd noir-protocol-circuits && yarn && node ./scripts/generate_variants.js)
-    for dir in noir-contracts noir-protocol-circuits mock-protocol-circuits aztec-nr; do
-      (cd $dir && ../../noir/noir-repo/target/release/nargo fmt --check)
-    done
+    # nargo downloads its git dependencies (e.g. noir-lang/poseidon) on first use.
+    # Under heavy parallel CI load the VPC DNS resolver drops lookups
+    # ("Could not resolve host: github.com"), leaving a half-cloned dependency dir
+    # that then fails with "Cannot read file .../Nargo.toml". Retry the download,
+    # wiping the partial dependency cache after a failure so the next attempt
+    # re-clones cleanly. A warm cache is left intact on success.
+    local fmt_check='( set -e; for dir in noir-contracts noir-protocol-circuits mock-protocol-circuits aztec-nr; do (cd "$dir" && ../../noir/noir-repo/target/release/nargo fmt --check); done )'
+    RETRY_SLEEP=10 retry "$fmt_check || { rm -rf \"\$HOME/nargo\"; exit 1; }"
   }
   export -f prep
 

playground/docker-compose.yml

@@ -27,6 +27,7 @@ services:
       WS_BLOCK_CHECK_INTERVAL_MS: 50
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
       P2P_MIN_TX_POOL_AGE_MS: 0
+      SEQ_ENABLE_PROPOSER_PIPELINING: 'true'
     healthcheck:
       test: ['CMD', 'curl', '-fSs', 'http://127.0.0.1:8080/status']
       interval: 3s