diff --git a/yarn-project/end-to-end/src/e2e_block_building.test.ts b/yarn-project/end-to-end/src/e2e_block_building.test.ts index d0be0f54f429..b5b2645777e0 100644 --- a/yarn-project/end-to-end/src/e2e_block_building.test.ts +++ b/yarn-project/end-to-end/src/e2e_block_building.test.ts @@ -562,11 +562,7 @@ describe('e2e_block_building', () => { // The culprit is a nullifier not being cleared up from world state during block building if a tx fails processing, // which translates in an incorrect end state for world state. We can easily detect this by checking whether the nullifier // tree next available leaf index is a multiple of 64. - // TODO(kill-non-pipelined): under pipelining, an AVM failure mid-block triggers a - // `DELETE_FORK failed: Fork not found` loop in world-state and the sequencer's publisher - // is left in `Transaction sending is interrupted`. This needs a source-level fix in the - // pipelined checkpoint job's fork-cleanup path; the test invariant is still relevant. - it.skip('clears up all nullifiers if tx processing fails', async () => { + it('clears up all nullifiers if tx processing fails', async () => { const context = await setup(1, { ...PIPELINING_SETUP_OPTS, minTxsPerBlock: 1, numberOfInitialFundedAccounts: 1 }); ({ teardown, @@ -598,23 +594,28 @@ describe('e2e_block_building', () => { const txHashResults = await Promise.all(batches.map(batch => batch.send({ from: ownerAddress, wait: NO_WAIT }))); const txHashes = txHashResults.map(({ txHash }) => txHash); logger.warn(`Sent two txs to test contract`, { txs: txHashes.map(hash => hash.toString()) }); - await Promise.race(txHashes.map(txHash => waitForTx(aztecNode, txHash, { timeout: 60 }))); + // Use Promise.any (not Promise.race): exactly one of the two txs will be dropped (the one that hits + // the fake AVM error in tx processing), so the dropped-tx rejection would settle Promise.race first. + // We want the first *successful* mine. + const minedTxHash = await Promise.any( + txHashes.map(async txHash => { + await waitForTx(aztecNode, txHash, { timeout: 60 }); + return txHash; + }), + ); - logger.warn(`At least one tx has been mined`); - const lastBlock = (await context.aztecNode.getBlockData('latest'))?.header; - expect(lastBlock).toBeDefined(); + logger.warn(`At least one tx has been mined`, { minedTxHash: minedTxHash.toString() }); + const minedReceipt = await aztecNode.getTxReceipt(minedTxHash); + const block = await context.aztecNode.getBlock(minedReceipt.blockNumber!); + expect(block).toBeDefined(); - logger.warn(`Latest block is ${lastBlock!.getBlockNumber()}`, { state: lastBlock?.state.partial }); - const nextNullifierIndex = lastBlock!.state.partial.nullifierTree.nextAvailableLeafIndex; + logger.warn(`Mined block is ${block!.header.getBlockNumber()}`, { state: block!.header.state.partial }); + const nextNullifierIndex = block!.header.state.partial.nullifierTree.nextAvailableLeafIndex; expect(nextNullifierIndex % 64).toEqual(0); }); }); - // TODO(kill-non-pipelined): reorg path under pipelined sequencer hangs to wallclock after - // `advanceToNextEpoch` + `markAsProven`. The world-state hits a `DELETE_FORK failed: Fork not - // found` loop and PXE catch-up never completes. Needs source-level fix in the pipelined - // checkpoint job's fork-cleanup path on prune. - describe.skip('reorgs', () => { + describe('reorgs', () => { let contract: StatefulTestContract; let cheatCodes: CheatCodes; let ownerAddress: AztecAddress; @@ -630,7 +631,7 @@ describe('e2e_block_building', () => { cheatCodes, watcher, accounts: [ownerAddress], - } = await setup(1, { ...PIPELINING_SETUP_OPTS })); + } = await setup(1, { ...PIPELINING_SETUP_OPTS, minTxsPerBlock: 1 })); ({ contract } = await StatefulTestContract.deploy(wallet, ownerAddress, 1).send({ from: ownerAddress })); initialBlockNumber = await aztecNode.getBlockNumber(); @@ -638,10 +639,12 @@ describe('e2e_block_building', () => { await cheatCodes.rollup.advanceToNextEpoch(); + // Mark all blocks up to the current pending tip as proven so the contract-deployment block + // is anchored against a proven checkpoint. The e2e fixture's AnvilTestWatcher does NOT + // auto-prove under interval mining (only under automining), so we must drive proven manually. + await cheatCodes.rollup.markAsProven(); const bn = await aztecNode.getBlockNumber(); - while ((await aztecNode.getBlockNumber('proven')) < bn) { - await sleep(1000); - } + await retryUntil(async () => (await aztecNode.getBlockNumber('proven')) >= bn, 'wait-proven', 60, 1); watcher.setIsMarkingAsProven(false); }); diff --git a/yarn-project/end-to-end/src/e2e_multi_validator/e2e_multi_validator_node.test.ts b/yarn-project/end-to-end/src/e2e_multi_validator/e2e_multi_validator_node.test.ts index e41b54eba63f..5b662846434c 100644 --- a/yarn-project/end-to-end/src/e2e_multi_validator/e2e_multi_validator_node.test.ts +++ b/yarn-project/end-to-end/src/e2e_multi_validator/e2e_multi_validator_node.test.ts @@ -1,12 +1,12 @@ +import type { InitialAccountData } from '@aztec/accounts/testing'; import type { Archiver } from '@aztec/archiver'; import type { AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node'; +import { NO_FROM } from '@aztec/aztec.js/account'; import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses'; -import { waitForProven } from '@aztec/aztec.js/contracts'; import { ContractDeployer } from '@aztec/aztec.js/deployment'; import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import type { AztecNode } from '@aztec/aztec.js/node'; -import type { Wallet } from '@aztec/aztec.js/wallet'; import type { CheatCodes } from '@aztec/aztec/testing'; import { createExtendedL1Client } from '@aztec/ethereum/client'; import { getL1ContractsConfigEnvVars } from '@aztec/ethereum/config'; @@ -19,22 +19,30 @@ import { retryUntil } from '@aztec/foundation/retry'; import { RollupAbi } from '@aztec/l1-artifacts/RollupAbi'; import { StatefulTestContractArtifact } from '@aztec/noir-test-contracts.js/StatefulTest'; import { CheckpointAttestation, ConsensusPayload } from '@aztec/stdlib/p2p'; +import { TxStatus } from '@aztec/stdlib/tx'; +import { jest } from '@jest/globals'; import { getContract } from 'viem'; import { privateKeyToAccount } from 'viem/accounts'; +import { PIPELINING_SETUP_OPTS } from '../fixtures/fixtures.js'; import { getPrivateKeyFromIndex, setup } from '../fixtures/utils.js'; +import type { TestWallet } from '../test-wallet/test_wallet.js'; const VALIDATOR_COUNT = 5; const COMMITTEE_SIZE = VALIDATOR_COUNT - 2; const PUBLISHER_COUNT = 2; describe('e2e_multi_validator_node', () => { + // Each test starts its own multi-validator network and waits for checkpointed L2 transactions. + jest.setTimeout(15 * 60 * 1000); + let initialValidatorPrivateKeys: `0x${string}`[]; let validatorAddresses: `0x${string}`[]; let teardown: () => Promise; - let wallet: Wallet; + let wallet: TestWallet; let ownerAddress: AztecAddress; + let initialFundedAccounts: InitialAccountData[]; let aztecNode: AztecNode; let config: AztecNodeConfig; let logger: Logger; @@ -67,26 +75,22 @@ describe('e2e_multi_validator_node', () => { }); const { aztecSlotDuration: _aztecSlotDuration } = getL1ContractsConfigEnvVars(); - ({ - teardown, - logger, - wallet, - accounts: [ownerAddress], - aztecNode, - config, - deployL1ContractsValues, - cheatCodes, - } = await setup(1, { - initialValidators, - aztecTargetCommitteeSize: COMMITTEE_SIZE, - sequencerPublisherPrivateKeys: publisherPrivateKeys.map(k => new SecretValue(k)), - minTxsPerBlock: 1, - archiverPollingIntervalMS: 200, - sequencerPollingIntervalMS: 200, - worldStateBlockCheckIntervalMS: 200, - blockCheckIntervalMS: 200, - startProverNode: true, - })); + ({ teardown, logger, wallet, initialFundedAccounts, aztecNode, config, deployL1ContractsValues, cheatCodes } = + await setup( + 1, + { + ...PIPELINING_SETUP_OPTS, + initialValidators, + aztecTargetCommitteeSize: COMMITTEE_SIZE, + sequencerPublisherPrivateKeys: publisherPrivateKeys.map(k => new SecretValue(k)), + archiverPollingIntervalMS: 200, + sequencerPollingIntervalMS: 200, + worldStateBlockCheckIntervalMS: 200, + blockCheckIntervalMS: 200, + skipAccountDeployment: true, + }, + { syncChainTip: 'checkpointed' }, + )); rollup = new RollupContract( deployL1ContractsValues.l1Client, @@ -109,16 +113,34 @@ describe('e2e_multi_validator_node', () => { await teardown(); }); + const deployOwnerAccount = async () => { + const accountData = initialFundedAccounts[0]; + const accountManager = await wallet.createSchnorrAccount( + accountData.secret, + accountData.salt, + accountData.signingKey, + ); + const deployMethod = await accountManager.getDeployMethod(); + await deployMethod.send({ + from: NO_FROM, + wait: { + waitForStatus: TxStatus.CHECKPOINTED, + timeout: (config.aztecProofSubmissionEpochs + 1) * config.aztecEpochDuration * config.aztecSlotDuration, + }, + }); + await wallet.sync(); + ownerAddress = accountManager.address; + }; + it('should build blocks & attest with multiple validator keys', async () => { + await deployOwnerAccount(); + const deployer = new ContractDeployer(artifact, wallet); logger.info(`Deploying contract from ${ownerAddress}`); const { receipt: tx } = await deployer .deploy([ownerAddress, 1], { salt: new Fr(BigInt(1)) }) .send({ from: ownerAddress }); - await waitForProven(aztecNode, tx, { - provenTimeout: (config.aztecProofSubmissionEpochs + 1) * config.aztecEpochDuration * config.aztecSlotDuration, - }); expect(tx.blockNumber).toBeDefined(); const dataStore = (aztecNode as AztecNodeService).getBlockSource() as Archiver; @@ -166,6 +188,7 @@ describe('e2e_multi_validator_node', () => { BigInt(await cheatCodes.rollup.getEpoch()) + BigInt(config.lagInEpochsForValidatorSet + 1), ), ); + await deployOwnerAccount(); // check that the committee is undefined const committee = await rollup.getCurrentEpochCommittee(); @@ -177,9 +200,6 @@ describe('e2e_multi_validator_node', () => { const { receipt: tx } = await deployer .deploy([ownerAddress, 1], { salt: new Fr(BigInt(1)) }) .send({ from: ownerAddress }); - await waitForProven(aztecNode, tx, { - provenTimeout: (config.aztecProofSubmissionEpochs + 1) * config.aztecEpochDuration * config.aztecSlotDuration, - }); expect(tx.blockNumber).toBeDefined(); const dataStore = (aztecNode as AztecNodeService).getBlockSource() as Archiver; @@ -196,8 +216,13 @@ describe('e2e_multi_validator_node', () => { expect(attestations.length).toBeGreaterThanOrEqual((COMMITTEE_SIZE * 2) / 3 + 1); - const signers = attestations.map(att => att.getSender()!.toString()); + const signers = attestations.map(att => att.getSender()!.toString().toLowerCase()); + const validatorAddressesLower = validatorAddresses.map(a => a.toLowerCase()); + expect(signers.every(s => validatorAddressesLower.includes(s))).toBe(true); - expect(signers).toEqual(expect.arrayContaining(validatorAddresses.slice(0, COMMITTEE_SIZE))); + const committeeAtCheckpoint = await rollup.getCommitteeAt(publishedCheckpoint.checkpoint.header.timestamp); + expect(committeeAtCheckpoint?.length).toBe(COMMITTEE_SIZE); + const committeeAtCheckpointLower = committeeAtCheckpoint!.map(a => a.toString().toLowerCase()); + expect(signers.every(s => committeeAtCheckpointLower.includes(s))).toBe(true); }); }); diff --git a/yarn-project/end-to-end/src/e2e_publisher_funding_multi.test.ts b/yarn-project/end-to-end/src/e2e_publisher_funding_multi.test.ts index e48477c8fefa..940174e33e24 100644 --- a/yarn-project/end-to-end/src/e2e_publisher_funding_multi.test.ts +++ b/yarn-project/end-to-end/src/e2e_publisher_funding_multi.test.ts @@ -17,6 +17,7 @@ import { join } from 'path'; import { type Hex, parseEther } from 'viem'; import { privateKeyToAccount } from 'viem/accounts'; +import { PIPELINING_SETUP_OPTS } from './fixtures/fixtures.js'; import { getPrivateKeyFromIndex, setup } from './fixtures/utils.js'; // Key indices from the test MNEMONIC (all pre-funded by Anvil): @@ -89,6 +90,7 @@ describe('e2e_publisher_funding_multi', () => { sequencer: sequencerClient, ethCheatCodes, } = await setup(0, { + ...PIPELINING_SETUP_OPTS, initialValidators, keyStoreDirectory, publisherFundingThreshold: FUNDING_THRESHOLD, @@ -156,9 +158,11 @@ describe('e2e_publisher_funding_multi', () => { // Funder should have sent 2 * FUNDING_AMOUNT plus gas costs (single multicall) expect(funderSpent).toBeGreaterThanOrEqual(2n * FUNDING_AMOUNT); - // Second round: after funding, publishers are above threshold. The active publisher will - // spend gas publishing blocks and eventually drop below threshold again, triggering re-funding - // for that one publisher. + // Second round: deterministically drop one publisher below threshold after the first refill. + // Waiting for organic gas depletion is brittle under pipelined publisher rotation: the exact + // publisher cadence and L1 gas burn vary enough that the balance may not cross the threshold + // before the test timeout, even though the periodic funding loop is healthy. + await ethCheatCodes.setBalance(publisher1Address, LOW_BALANCE); const funderBalanceBefore2 = await ethCheatCodes.getBalance(funderAddress); logger.info(`Waiting for second funding round`); diff --git a/yarn-project/end-to-end/src/e2e_slashing/broadcasted_invalid_checkpoint_proposal_slash.test.ts b/yarn-project/end-to-end/src/e2e_slashing/broadcasted_invalid_checkpoint_proposal_slash.test.ts index bd9186286012..6a9d4fab22d1 100644 --- a/yarn-project/end-to-end/src/e2e_slashing/broadcasted_invalid_checkpoint_proposal_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_slashing/broadcasted_invalid_checkpoint_proposal_slash.test.ts @@ -221,7 +221,8 @@ describe('e2e_slashing_broadcasted_invalid_checkpoint_proposal_slash', () => { aztecSlotDuration: AZTEC_SLOT_DURATION, aztecTargetCommitteeSize: COMMITTEE_SIZE, aztecProofSubmissionEpochs: 1024, - enableProposerPipelining: false, + enableProposerPipelining: true, + inboxLag: 2, mockGossipSubNetwork: true, slashingQuorum: SLASHING_QUORUM, slashingRoundSizeInEpochs: SLASHING_ROUND_SIZE / AZTEC_EPOCH_DURATION, @@ -264,7 +265,7 @@ describe('e2e_slashing_broadcasted_invalid_checkpoint_proposal_slash', () => { { ...t.ctx.aztecNodeConfig, dontStartSequencer: true, - enableProposerPipelining: false, + enableProposerPipelining: true, slashBroadcastedInvalidCheckpointProposalPenalty: slashingUnit, slashSelfAllowed: true, },