diff --git a/epoch.go b/epoch.go index 23dc513d..bd71b759 100644 --- a/epoch.go +++ b/epoch.go @@ -73,7 +73,7 @@ type Epoch struct { EpochConfig // Runtime oneTimeVerifier *oneTimeVerifier - sched *scheduler + sched *Scheduler lock sync.Mutex lastBlock *VerifiedFinalizedBlock // latest block & fcert commited canReceiveMessages atomic.Bool diff --git a/epoch_failover_test.go b/epoch_failover_test.go index db53f761..11375117 100644 --- a/epoch_failover_test.go +++ b/epoch_failover_test.go @@ -25,33 +25,15 @@ import ( // notarize and finalize block for round 1 // we expect the future empty notarization for round 2 to increment the round func TestEpochLeaderFailoverWithEmptyNotarization(t *testing.T) { - l := testutil.MakeLogger(t, 1) - - bb := &testBlockBuilder{ - out: make(chan *testBlock, 2), - blockShouldBeBuilt: make(chan struct{}, 1), - in: make(chan *testBlock, 2), + bb := &testutil.TestBlockBuilder{ + Out: make(chan *testutil.TestBlock, 2), + BlockShouldBeBuilt: make(chan struct{}, 1), + In: make(chan *testutil.TestBlock, 2), } - storage := newInMemStorage() nodes := []NodeID{{1}, {2}, {3}, {4}} - wal := newTestWAL(t) - - start := time.Now() - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - StartTime: start, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -83,12 +65,12 @@ func TestEpochLeaderFailoverWithEmptyNotarization(t *testing.T) { require.True(t, ok) // Artificially force the block builder to output the blocks we want. - for len(bb.out) > 0 { - <-bb.out + for len(bb.Out) > 0 { + <-bb.Out } for _, block := range []VerifiedBlock{block1, block2} { - bb.out <- block.(*testBlock) - bb.in <- block.(*testBlock) + bb.Out <- block.(*testutil.TestBlock) + bb.In <- block.(*testutil.TestBlock) } emptyNotarization := newEmptyNotarization(nodes[:3], 2, 1) @@ -99,11 +81,11 @@ func TestEpochLeaderFailoverWithEmptyNotarization(t *testing.T) { notarizeAndFinalizeRound(t, e, bb) - wal.assertNotarization(2) + wal.AssertNotarization(2) nextBlockSeqToCommit := uint64(2) nextRoundToCommit := uint64(3) - runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testBlockBuilder, storage *InMemStorage, wal *testWAL) { + runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder, storage *testutil.InMemStorage, wal *testutil.TestWAL) { // Ensure our node proposes block with sequence 3 for round 4 block, _ := notarizeAndFinalizeRound(t, e, bb) require.Equal(t, nextBlockSeqToCommit, block.BlockHeader().Seq) @@ -114,7 +96,7 @@ func TestEpochLeaderFailoverWithEmptyNotarization(t *testing.T) { // newEmptyNotarization creates a new empty notarization func newEmptyNotarization(nodes []NodeID, round uint64, seq uint64) *EmptyNotarization { - var qc testQC + var qc testutil.TestQC for i, node := range nodes { qc = append(qc, Signature{Signer: node, Value: []byte{byte(i)}}) @@ -130,30 +112,12 @@ func newEmptyNotarization(nodes []NodeID, round uint64, seq uint64) *EmptyNotari } func TestEpochLeaderFailoverReceivesEmptyVotesEarly(t *testing.T) { - l := testutil.MakeLogger(t, 1) - - bb := &testBlockBuilder{out: make(chan *testBlock, 1), blockShouldBeBuilt: make(chan struct{}, 1)} - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1), BlockShouldBeBuilt: make(chan struct{}, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - wal := newTestWAL(t) - - start := time.Now() - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - StartTime: start, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -191,22 +155,20 @@ func TestEpochLeaderFailoverReceivesEmptyVotesEarly(t *testing.T) { EmptyVoteMessage: emptyVoteFrom2, }, nodes[2]) - bb.blockShouldBeBuilt <- struct{}{} + bb.BlockShouldBeBuilt <- struct{}{} - waitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) + testutil.WaitForBlockProposerTimeout(t, e, &e.StartTime, e.Metadata().Round) - runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testBlockBuilder, storage *InMemStorage, wal *testWAL) { - wal.lock.Lock() + runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder, storage *testutil.InMemStorage, wal *testutil.TestWAL) { walContent, err := wal.ReadAll() require.NoError(t, err) - wal.lock.Unlock() rawEmptyVote, rawEmptyNotarization, rawProposal := walContent[len(walContent)-3], walContent[len(walContent)-2], walContent[len(walContent)-1] emptyVote, err := ParseEmptyVoteRecord(rawEmptyVote) require.NoError(t, err) require.Equal(t, createEmptyVote(emptyBlockMd, nodes[0]).Vote, emptyVote) - emptyNotarization, err := EmptyNotarizationFromRecord(rawEmptyNotarization, &testQCDeserializer{t: t}) + emptyNotarization, err := EmptyNotarizationFromRecord(rawEmptyNotarization, e.QCDeserializer) require.NoError(t, err) require.Equal(t, emptyVoteFrom1.Vote, emptyNotarization.Vote) require.Equal(t, uint64(3), emptyNotarization.Vote.Round) @@ -219,13 +181,13 @@ func TestEpochLeaderFailoverReceivesEmptyVotesEarly(t *testing.T) { require.Equal(t, uint64(3), header.Seq) // Ensure our node proposes block with sequence 3 for round 4 - block := <-bb.out + block := <-bb.Out for i := 1; i <= quorum; i++ { injectTestFinalization(t, e, block, nodes[i]) } - block2 := storage.waitForBlockCommit(3) + block2 := storage.WaitForBlockCommit(3) require.Equal(t, block, block2) require.Equal(t, uint64(4), storage.Height()) require.Equal(t, uint64(4), block2.BlockHeader().Round) @@ -235,29 +197,12 @@ func TestEpochLeaderFailoverReceivesEmptyVotesEarly(t *testing.T) { } func TestEpochLeaderFailover(t *testing.T) { - l := testutil.MakeLogger(t, 1) - - bb := &testBlockBuilder{out: make(chan *testBlock, 1), blockShouldBeBuilt: make(chan struct{}, 1)} - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1), BlockShouldBeBuilt: make(chan struct{}, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - wal := newTestWAL(t) - start := time.Now() - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - StartTime: start, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -274,11 +219,11 @@ func TestEpochLeaderFailover(t *testing.T) { notarizeAndFinalizeRound(t, e, bb) } - bb.blockShouldBeBuilt <- struct{}{} + bb.BlockShouldBeBuilt <- struct{}{} - waitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) + testutil.WaitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) - runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testBlockBuilder, storage *InMemStorage, wal *testWAL) { + runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder, storage *testutil.InMemStorage, wal *testutil.TestWAL) { lastBlock, _, ok := storage.Retrieve(storage.Height() - 1) require.True(t, ok) @@ -300,17 +245,15 @@ func TestEpochLeaderFailover(t *testing.T) { EmptyVoteMessage: emptyVoteFrom2, }, nodes[2]) - wal.lock.Lock() walContent, err := wal.ReadAll() require.NoError(t, err) - wal.lock.Unlock() rawEmptyVote, rawEmptyNotarization := walContent[len(walContent)-2], walContent[len(walContent)-1] emptyVote, err := ParseEmptyVoteRecord(rawEmptyVote) require.NoError(t, err) require.Equal(t, createEmptyVote(emptyBlockMd, nodes[0]).Vote, emptyVote) - emptyNotarization, err := EmptyNotarizationFromRecord(rawEmptyNotarization, &testQCDeserializer{t: t}) + emptyNotarization, err := EmptyNotarizationFromRecord(rawEmptyNotarization, e.QCDeserializer) require.NoError(t, err) require.Equal(t, emptyVoteFrom1.Vote, emptyNotarization.Vote) require.Equal(t, uint64(3), emptyNotarization.Vote.Round) @@ -330,33 +273,16 @@ func TestEpochLeaderFailover(t *testing.T) { } func TestEpochNoFinalizationAfterEmptyVote(t *testing.T) { - l := testutil.MakeLogger(t, 1) - - bb := &testBlockBuilder{out: make(chan *testBlock, 1), blockShouldBeBuilt: make(chan struct{}, 1)} - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1), BlockShouldBeBuilt: make(chan struct{}, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - wal := newTestWAL(t) - recordedMessages := make(chan *Message, 7) - comm := &recordingComm{Communication: noopComm(nodes), BroadcastMessages: recordedMessages} + comm := &testutil.RecordingComm{Communication: testutil.NewNoopComm(nodes), BroadcastMessages: recordedMessages} - start := time.Now() - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - StartTime: start, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: comm, - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) + start := conf.StartTime e, err := NewEpoch(conf) require.NoError(t, err) @@ -370,8 +296,8 @@ func TestEpochNoFinalizationAfterEmptyVote(t *testing.T) { <-recordedMessages } - bb.blockShouldBeBuilt <- struct{}{} - waitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) + bb.BlockShouldBeBuilt <- struct{}{} + testutil.WaitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) b, _, ok := storage.Retrieve(0) require.True(t, ok) @@ -383,9 +309,9 @@ func TestEpochNoFinalizationAfterEmptyVote(t *testing.T) { }) require.True(t, ok) - block := <-bb.out + block := <-bb.Out - vote, err := newTestVote(block, leader) + vote, err := newTestVote(block, leader, e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -399,14 +325,14 @@ func TestEpochNoFinalizationAfterEmptyVote(t *testing.T) { injectTestVote(t, e, block, nodes[i]) } - wal.assertNotarization(1) + wal.AssertNotarization(1) for i := 1; i < quorum; i++ { injectTestFinalization(t, e, block, nodes[i]) } // A block should not have been committed because we do not include our own finalization. - storage.ensureNoBlockCommit(t, 1) + storage.EnsureNoBlockCommit(t, 1) // There should only two messages sent, which are an empty vote and a notarization. // This proves that a finalization or a regular vote were never sent by us. @@ -420,29 +346,11 @@ func TestEpochNoFinalizationAfterEmptyVote(t *testing.T) { } func TestEpochLeaderFailoverAfterProposal(t *testing.T) { - bb := &testBlockBuilder{out: make(chan *testBlock, 1), blockShouldBeBuilt: make(chan struct{}, 1)} - storage := newInMemStorage() - + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1), BlockShouldBeBuilt: make(chan struct{}, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - wal := newTestWAL(t) - - logger := testutil.MakeLogger(t, 1) - start := time.Now() - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - StartTime: start, - Logger: logger, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -458,7 +366,7 @@ func TestEpochLeaderFailoverAfterProposal(t *testing.T) { notarizeAndFinalizeRound(t, e, bb) } - wal.assertWALSize(6) // (block, notarization) x 3 rounds + wal.AssertWALSize(6) // (block, notarization) x 3 rounds // leader is the proposer of the new block for the given round leader := LeaderForRound(nodes, 3) @@ -467,9 +375,9 @@ func TestEpochLeaderFailoverAfterProposal(t *testing.T) { require.True(t, ok) require.Equal(t, md.Round, md.Seq) - block := <-bb.out + block := <-bb.Out - vote, err := newTestVote(block, leader) + vote, err := newTestVote(block, leader, e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -480,13 +388,13 @@ func TestEpochLeaderFailoverAfterProposal(t *testing.T) { require.NoError(t, err) // Wait until we have verified the block and written it to the WAL - wal.assertWALSize(7) + wal.AssertWALSize(7) // Send a timeout from the application - bb.blockShouldBeBuilt <- struct{}{} - waitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) + bb.BlockShouldBeBuilt <- struct{}{} + testutil.WaitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) - runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testBlockBuilder, storage *InMemStorage, wal *testWAL) { + runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder, storage *testutil.InMemStorage, wal *testutil.TestWAL) { lastBlock, _, ok := storage.Retrieve(storage.Height() - 1) require.True(t, ok) @@ -528,7 +436,7 @@ func TestEpochLeaderFailoverAfterProposal(t *testing.T) { require.NoError(t, err) require.Equal(t, createEmptyVote(md, nodes[0]).Vote, emptyVote) - emptyNotarization, err := EmptyNotarizationFromRecord(rawEmptyNotarization, &testQCDeserializer{t: t}) + emptyNotarization, err := EmptyNotarizationFromRecord(rawEmptyNotarization, e.QCDeserializer) require.NoError(t, err) require.Equal(t, emptyVoteFrom1.Vote, emptyNotarization.Vote) require.Equal(t, uint64(3), emptyNotarization.Vote.Round) @@ -538,29 +446,11 @@ func TestEpochLeaderFailoverAfterProposal(t *testing.T) { } func TestEpochLeaderFailoverTwice(t *testing.T) { - l := testutil.MakeLogger(t, 1) - - bb := &testBlockBuilder{out: make(chan *testBlock, 1), blockShouldBeBuilt: make(chan struct{}, 1)} - storage := newInMemStorage() - + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1), BlockShouldBeBuilt: make(chan struct{}, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - wal := newTestWAL(t) - start := time.Now() - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - StartTime: start, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -573,11 +463,11 @@ func TestEpochLeaderFailoverTwice(t *testing.T) { t.Log("Node 2 crashes, leader failover to node 3") - bb.blockShouldBeBuilt <- struct{}{} + bb.BlockShouldBeBuilt <- struct{}{} - waitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) + testutil.WaitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) - runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testBlockBuilder, storage *InMemStorage, wal *testWAL) { + runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder, storage *testutil.InMemStorage, wal *testutil.TestWAL) { lastBlock, _, ok := storage.Retrieve(storage.Height() - 1) require.True(t, ok) @@ -599,15 +489,15 @@ func TestEpochLeaderFailoverTwice(t *testing.T) { EmptyVoteMessage: emptyVoteFrom3, }, nodes[3]) - wal.assertNotarization(2) + wal.AssertNotarization(2) t.Log("Node 3 crashes and node 2 comes back up (just in time)") - bb.blockShouldBeBuilt <- struct{}{} + bb.BlockShouldBeBuilt <- struct{}{} - waitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) + testutil.WaitForBlockProposerTimeout(t, e, &start, e.Metadata().Round) - runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testBlockBuilder, storage *InMemStorage, wal *testWAL) { + runCrashAndRestartExecution(t, e, bb, wal, storage, func(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder, storage *testutil.InMemStorage, wal *testutil.TestWAL) { md := ProtocolMetadata{ Round: 3, Seq: 1, @@ -624,7 +514,7 @@ func TestEpochLeaderFailoverTwice(t *testing.T) { EmptyVoteMessage: emptyVoteFrom3, }, nodes[3]) - wal.assertNotarization(3) + wal.AssertNotarization(3) // Ensure our node proposes block with sequence 2 for round 4 nextRoundToCommit := uint64(4) @@ -644,7 +534,7 @@ func TestEpochLeaderFailoverTwice(t *testing.T) { require.NoError(t, err) require.Equal(t, createEmptyVote(md, nodes[0]).Vote, emptyVote) - emptyNotarization, err := EmptyNotarizationFromRecord(rawEmptyNotarization, &testQCDeserializer{t: t}) + emptyNotarization, err := EmptyNotarizationFromRecord(rawEmptyNotarization, e.QCDeserializer) require.NoError(t, err) require.Equal(t, emptyVoteFrom1.Vote, emptyNotarization.Vote) require.Equal(t, uint64(3), emptyNotarization.Vote.Round) @@ -666,25 +556,6 @@ func createEmptyVote(md ProtocolMetadata, signer NodeID) *EmptyVote { return emptyVoteFrom2 } -func waitForBlockProposerTimeout(t *testing.T, e *Epoch, startTime *time.Time, startRound uint64) { - timeout := time.NewTimer(time.Minute) - defer timeout.Stop() - - for { - if e.WAL.(*testWAL).containsEmptyVote(startRound) || e.WAL.(*testWAL).containsEmptyNotarization(startRound) { - return - } - *startTime = startTime.Add(e.EpochConfig.MaxProposalWait / 5) - e.AdvanceTime(*startTime) - select { - case <-time.After(time.Millisecond * 10): - continue - case <-timeout.C: - require.Fail(t, "timed out waiting for event") - } - } -} - func TestEpochLeaderFailoverNotNeeded(t *testing.T) { var timedOut atomic.Bool @@ -696,29 +567,15 @@ func TestEpochLeaderFailoverNotNeeded(t *testing.T) { return nil }) - bb := &testBlockBuilder{out: make(chan *testBlock, 1), blockShouldBeBuilt: make(chan struct{}, 1)} - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1), BlockShouldBeBuilt: make(chan struct{}, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - wal := newTestWAL(t) - start := time.Now() - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - StartTime: start, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, wal, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) + conf.Logger = l e, err := NewEpoch(conf) require.NoError(t, err) @@ -733,16 +590,16 @@ func TestEpochLeaderFailoverNotNeeded(t *testing.T) { for round := uint64(0); round < rounds; round++ { notarizeAndFinalizeRound(t, e, bb) } - bb.blockShouldBeBuilt <- struct{}{} + bb.BlockShouldBeBuilt <- struct{}{} e.AdvanceTime(start.Add(conf.MaxProposalWait / 2)) md := e.Metadata() _, ok := bb.BuildBlock(context.Background(), md) require.True(t, ok) - block := <-bb.out + block := <-bb.Out - vote, err := newTestVote(block, nodes[3]) + vote, err := newTestVote(block, nodes[3], e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -757,7 +614,7 @@ func TestEpochLeaderFailoverNotNeeded(t *testing.T) { injectTestVote(t, e, block, nodes[i]) } - wal.assertNotarization(3) + wal.AssertNotarization(3) e.AdvanceTime(start.Add(conf.MaxProposalWait / 2)) e.AdvanceTime(start.Add(conf.MaxProposalWait / 2)) @@ -765,7 +622,7 @@ func TestEpochLeaderFailoverNotNeeded(t *testing.T) { require.False(t, timedOut.Load()) } -func runCrashAndRestartExecution(t *testing.T, e *Epoch, bb *testBlockBuilder, wal *testWAL, storage *InMemStorage, f epochExecution) { +func runCrashAndRestartExecution(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder, wal *testutil.TestWAL, storage *testutil.InMemStorage, f epochExecution) { // Split the test into two scenarios: // 1) The node proceeds as usual. // 2) The node crashes and restarts. @@ -775,10 +632,10 @@ func runCrashAndRestartExecution(t *testing.T, e *Epoch, bb *testBlockBuilder, w nodes := e.Comm.ListNodes() // Clone the block builder - bbAfterCrash := &testBlockBuilder{ - out: cloneBlockChan(bb.out), - in: cloneBlockChan(bb.in), - blockShouldBeBuilt: make(chan struct{}, cap(bb.blockShouldBeBuilt)), + bbAfterCrash := &testutil.TestBlockBuilder{ + Out: cloneBlockChan(bb.Out), + In: cloneBlockChan(bb.In), + BlockShouldBeBuilt: make(chan struct{}, cap(bb.BlockShouldBeBuilt)), } // Case 1: @@ -788,21 +645,9 @@ func runCrashAndRestartExecution(t *testing.T, e *Epoch, bb *testBlockBuilder, w // Case 2: t.Run(fmt.Sprintf("%s-with-crash", t.Name()), func(t *testing.T) { - conf := EpochConfig{ - QCDeserializer: &testQCDeserializer{t: t}, - BlockDeserializer: &blockDeserializer{}, - MaxProposalWait: DefaultMaxProposalWaitTime, - StartTime: time.Now(), - Logger: testutil.MakeLogger(t, 1), - ID: nodes[0], - Signer: &testSigner{}, - WAL: cloneWAL, - Verifier: &testVerifier{}, - Storage: cloneStorage, - Comm: noopComm(nodes), - BlockBuilder: bbAfterCrash, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bbAfterCrash) + conf.Storage = cloneStorage + conf.WAL = cloneWAL e, err := NewEpoch(conf) require.NoError(t, err) @@ -812,9 +657,9 @@ func runCrashAndRestartExecution(t *testing.T, e *Epoch, bb *testBlockBuilder, w }) } -func cloneBlockChan(in chan *testBlock) chan *testBlock { - tmp := make(chan *testBlock, cap(in)) - out := make(chan *testBlock, cap(in)) +func cloneBlockChan(in chan *testutil.TestBlock) chan *testutil.TestBlock { + tmp := make(chan *testutil.TestBlock, cap(in)) + out := make(chan *testutil.TestBlock, cap(in)) for len(in) > 0 { block := <-in @@ -829,14 +674,4 @@ func cloneBlockChan(in chan *testBlock) chan *testBlock { return out } -type recordingComm struct { - Communication - BroadcastMessages chan *Message -} - -func (rc *recordingComm) Broadcast(msg *Message) { - rc.BroadcastMessages <- msg - rc.Communication.Broadcast(msg) -} - -type epochExecution func(t *testing.T, e *Epoch, bb *testBlockBuilder, storage *InMemStorage, wal *testWAL) +type epochExecution func(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder, storage *testutil.InMemStorage, wal *testutil.TestWAL) diff --git a/epoch_multinode_test.go b/epoch_multinode_test.go index 3f67747a..b51dc52f 100644 --- a/epoch_multinode_test.go +++ b/epoch_multinode_test.go @@ -4,537 +4,27 @@ package simplex_test import ( - "bytes" - "context" - "encoding/binary" . "simplex" - "simplex/record" "simplex/testutil" - "simplex/wal" - "sync" "testing" - "time" - - "github.com/stretchr/testify/require" ) func TestSimplexMultiNodeSimple(t *testing.T) { - bb := newTestControlledBlockBuilder(t) + bb := testutil.NewTestControlledBlockBuilder(t) nodes := []NodeID{{1}, {2}, {3}, {4}} - net := newInMemNetwork(t, nodes) - newSimplexNode(t, nodes[0], net, bb, nil) - newSimplexNode(t, nodes[1], net, bb, nil) - newSimplexNode(t, nodes[2], net, bb, nil) - newSimplexNode(t, nodes[3], net, bb, nil) + net := testutil.NewInMemNetwork(t, nodes) + testutil.NewSimplexNode(t, nodes[0], net, bb, nil) + testutil.NewSimplexNode(t, nodes[1], net, bb, nil) + testutil.NewSimplexNode(t, nodes[2], net, bb, nil) + testutil.NewSimplexNode(t, nodes[3], net, bb, nil) - net.startInstances() + net.StartInstances() for seq := 0; seq < 10; seq++ { - bb.triggerNewBlock() - for _, n := range net.instances { - n.storage.waitForBlockCommit(uint64(seq)) - } - } -} - -func (t *testNode) start() { - go t.handleMessages() - require.NoError(t.t, t.e.Start()) -} - -type testNodeConfig struct { - // optional - initialStorage []VerifiedFinalizedBlock - comm Communication - replicationEnabled bool -} - -// newSimplexNode creates a new testNode and adds it to [net]. -func newSimplexNode(t *testing.T, nodeID NodeID, net *inMemNetwork, bb BlockBuilder, config *testNodeConfig) *testNode { - comm := newTestComm(nodeID, net, allowAllMessages) - - epochConfig := defaultTestNodeEpochConfig(t, nodeID, comm, bb) - - if config != nil { - updateEpochConfig(&epochConfig, config) - } - - e, err := NewEpoch(epochConfig) - require.NoError(t, err) - ti := &testNode{ - wal: epochConfig.WAL.(*testWAL), - e: e, - t: t, - storage: epochConfig.Storage.(*InMemStorage), - ingress: make(chan struct { - msg *Message - from NodeID - }, 100)} - - net.addNode(ti) - return ti -} - -func updateEpochConfig(epochConfig *EpochConfig, testConfig *testNodeConfig) { - // set the initial storage - for _, data := range testConfig.initialStorage { - epochConfig.Storage.Index(data.VerifiedBlock, data.FCert) - } - - // TODO: remove optional replication flag - epochConfig.ReplicationEnabled = testConfig.replicationEnabled - - // custom communication - if testConfig.comm != nil { - epochConfig.Comm = testConfig.comm - } -} - -func defaultTestNodeEpochConfig(t *testing.T, nodeID NodeID, comm Communication, bb BlockBuilder) EpochConfig { - l := testutil.MakeLogger(t, int(nodeID[0])) - storage := newInMemStorage() - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Comm: comm, - Logger: l, - ID: nodeID, - Signer: &testSigner{}, - WAL: newTestWAL(t), - Verifier: &testVerifier{}, - Storage: storage, - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - BlockDeserializer: &blockDeserializer{}, - QCDeserializer: &testQCDeserializer{t: t}, - StartTime: time.Now(), - } - return conf -} - -type testNode struct { - wal *testWAL - storage *InMemStorage - e *Epoch - ingress chan struct { - msg *Message - from NodeID - } - t *testing.T -} - -func (t *testNode) HandleMessage(msg *Message, from NodeID) error { - err := t.e.HandleMessage(msg, from) - require.NoError(t.t, err) - return err -} - -func (t *testNode) handleMessages() { - for msg := range t.ingress { - err := t.HandleMessage(msg.msg, msg.from) - require.NoError(t.t, err) - if err != nil { - return - } - } -} - -type testWAL struct { - WriteAheadLog - t *testing.T - lock sync.Mutex - signal sync.Cond -} - -func newTestWAL(t *testing.T) *testWAL { - var tw testWAL - tw.WriteAheadLog = wal.NewMemWAL(t) - tw.signal = sync.Cond{L: &tw.lock} - tw.t = t - return &tw -} - -func (tw *testWAL) Clone() *testWAL { - tw.lock.Lock() - defer tw.lock.Unlock() - - rawWAL, err := tw.ReadAll() - require.NoError(tw.t, err) - - wal := newTestWAL(tw.t) - - for _, entry := range rawWAL { - wal.Append(entry) - } - - return wal -} - -func (tw *testWAL) Append(b []byte) error { - tw.lock.Lock() - defer tw.lock.Unlock() - - err := tw.WriteAheadLog.Append(b) - tw.signal.Signal() - return err -} - -func (tw *testWAL) assertWALSize(n int) { - tw.lock.Lock() - defer tw.lock.Unlock() - - for { - rawRecords, err := tw.WriteAheadLog.ReadAll() - require.NoError(tw.t, err) - - if len(rawRecords) == n { - return - } - - tw.signal.Wait() - } -} - -func (tw *testWAL) assertNotarization(round uint64) { - tw.lock.Lock() - defer tw.lock.Unlock() - - for { - rawRecords, err := tw.WriteAheadLog.ReadAll() - require.NoError(tw.t, err) - - for _, rawRecord := range rawRecords { - if binary.BigEndian.Uint16(rawRecord[:2]) == record.NotarizationRecordType { - _, vote, err := ParseNotarizationRecord(rawRecord) - require.NoError(tw.t, err) - - if vote.Round == round { - return - } - } - if binary.BigEndian.Uint16(rawRecord[:2]) == record.EmptyNotarizationRecordType { - _, vote, err := ParseEmptyNotarizationRecord(rawRecord) - require.NoError(tw.t, err) - - if vote.Round == round { - return - } - } - } - - tw.signal.Wait() - } - -} - -func (tw *testWAL) containsEmptyVote(round uint64) bool { - tw.lock.Lock() - defer tw.lock.Unlock() - - rawRecords, err := tw.WriteAheadLog.ReadAll() - require.NoError(tw.t, err) - - for _, rawRecord := range rawRecords { - if binary.BigEndian.Uint16(rawRecord[:2]) == record.EmptyVoteRecordType { - vote, err := ParseEmptyVoteRecord(rawRecord) - require.NoError(tw.t, err) - - if vote.Round == round { - return true - } - } - } - - return false -} - -func (tw *testWAL) containsEmptyNotarization(round uint64) bool { - tw.lock.Lock() - defer tw.lock.Unlock() - - rawRecords, err := tw.WriteAheadLog.ReadAll() - require.NoError(tw.t, err) - - for _, rawRecord := range rawRecords { - if binary.BigEndian.Uint16(rawRecord[:2]) == record.EmptyNotarizationRecordType { - _, vote, err := ParseEmptyNotarizationRecord(rawRecord) - require.NoError(tw.t, err) - - if vote.Round == round { - return true - } - } - } - - return false -} - -// messageFilter defines a function that filters -// certain messages from being sent or broadcasted. -type messageFilter func(*Message, NodeID) bool - -// allowAllMessages allows every message to be sent -func allowAllMessages(*Message, NodeID) bool { - return true -} - -// denyFinalizationMessages blocks any messages that would cause nodes in -// a network to index a block in storage. -func denyFinalizationMessages(msg *Message, destination NodeID) bool { - if msg.Finalization != nil { - return false - } - if msg.FinalizationCertificate != nil { - return false - } - - return true -} - -func onlyAllowEmptyRoundMessages(msg *Message, destination NodeID) bool { - if msg.EmptyNotarization != nil { - return true - } - if msg.EmptyVoteMessage != nil { - return true - } - return false -} - -type testComm struct { - from NodeID - net *inMemNetwork - messageFilter messageFilter - lock sync.RWMutex -} - -func newTestComm(from NodeID, net *inMemNetwork, messageFilter messageFilter) *testComm { - return &testComm{ - from: from, - net: net, - messageFilter: messageFilter, - } -} - -func (c *testComm) ListNodes() []NodeID { - return c.net.nodes -} - -func (c *testComm) SendMessage(msg *Message, destination NodeID) { - if !c.isMessagePermitted(msg, destination) { - return - } - - // cannot send if either [from] or [destination] is not connected - if c.net.IsDisconnected(destination) || c.net.IsDisconnected(c.from) { - return - } - - c.maybeTranslateOutoingToIncomingMessageTypes(msg) - - for _, instance := range c.net.instances { - if bytes.Equal(instance.e.ID, destination) { - instance.ingress <- struct { - msg *Message - from NodeID - }{msg: msg, from: c.from} - return + bb.TriggerNewBlock() + for _, n := range net.Instances() { + n.Storage.WaitForBlockCommit(uint64(seq)) } } } - -func (c *testComm) setFilter(filter messageFilter) { - c.lock.Lock() - defer c.lock.Unlock() - - c.messageFilter = filter -} - -func (c *testComm) maybeTranslateOutoingToIncomingMessageTypes(msg *Message) { - if msg.VerifiedReplicationResponse != nil { - data := make([]QuorumRound, 0, len(msg.VerifiedReplicationResponse.Data)) - - for _, verifiedQuorumRound := range msg.VerifiedReplicationResponse.Data { - // Outgoing block is of type verified block but incoming block is of type Block, - // so we do a type cast because the test block implements both. - quorumRound := QuorumRound{} - if verifiedQuorumRound.EmptyNotarization != nil { - quorumRound.EmptyNotarization = verifiedQuorumRound.EmptyNotarization - } else { - quorumRound.Block = verifiedQuorumRound.VerifiedBlock.(Block) - if verifiedQuorumRound.Notarization != nil { - quorumRound.Notarization = verifiedQuorumRound.Notarization - } - if verifiedQuorumRound.FCert != nil { - quorumRound.FCert = verifiedQuorumRound.FCert - } - } - - data = append(data, quorumRound) - } - - var latestRound *QuorumRound - if msg.VerifiedReplicationResponse.LatestRound != nil { - if msg.VerifiedReplicationResponse.LatestRound.EmptyNotarization != nil { - latestRound = &QuorumRound{ - EmptyNotarization: msg.VerifiedReplicationResponse.LatestRound.EmptyNotarization, - } - } else { - latestRound = &QuorumRound{ - Block: msg.VerifiedReplicationResponse.LatestRound.VerifiedBlock.(Block), - Notarization: msg.VerifiedReplicationResponse.LatestRound.Notarization, - FCert: msg.VerifiedReplicationResponse.LatestRound.FCert, - EmptyNotarization: msg.VerifiedReplicationResponse.LatestRound.EmptyNotarization, - } - } - } - - require.Nil( - c.net.t, - msg.ReplicationResponse, - "message cannot include ReplicationResponse & VerifiedReplicationResponse", - ) - - msg.ReplicationResponse = &ReplicationResponse{ - Data: data, - LatestRound: latestRound, - } - } - - if msg.VerifiedBlockMessage != nil { - require.Nil(c.net.t, msg.BlockMessage, "message cannot include BlockMessage & VerifiedBlockMessage") - msg.BlockMessage = &BlockMessage{ - Block: msg.VerifiedBlockMessage.VerifiedBlock.(Block), - Vote: msg.VerifiedBlockMessage.Vote, - } - } -} - -func (c *testComm) isMessagePermitted(msg *Message, destination NodeID) bool { - c.lock.RLock() - defer c.lock.RUnlock() - - return c.messageFilter(msg, destination) -} - -func (c *testComm) Broadcast(msg *Message) { - if c.net.IsDisconnected(c.from) { - return - } - - c.maybeTranslateOutoingToIncomingMessageTypes(msg) - - for _, instance := range c.net.instances { - if !c.isMessagePermitted(msg, instance.e.ID) { - return - } - // Skip sending the message to yourself or disconnected nodes - if bytes.Equal(c.from, instance.e.ID) || c.net.IsDisconnected(instance.e.ID) { - continue - } - - instance.ingress <- struct { - msg *Message - from NodeID - }{msg: msg, from: c.from} - } -} - -type inMemNetwork struct { - t *testing.T - nodes []NodeID - instances []*testNode - lock sync.RWMutex - disconnected map[string]struct{} -} - -// newInMemNetwork creates an in-memory network. Node IDs must be provided before -// adding instances, as nodes require prior knowledge of all participants. -func newInMemNetwork(t *testing.T, nodes []NodeID) *inMemNetwork { - net := &inMemNetwork{ - t: t, - nodes: nodes, - instances: make([]*testNode, 0), - disconnected: make(map[string]struct{}), - } - return net -} - -func (n *inMemNetwork) addNode(node *testNode) { - allowed := false - for _, id := range n.nodes { - if bytes.Equal(id, node.e.ID) { - allowed = true - break - } - } - require.True(node.t, allowed, "node must be declared before adding") - n.instances = append(n.instances, node) -} - -func (n *inMemNetwork) setAllNodesMessageFilter(filter messageFilter) { - for _, instance := range n.instances { - instance.e.Comm.(*testComm).setFilter(filter) - } -} - -func (n *inMemNetwork) IsDisconnected(node NodeID) bool { - n.lock.RLock() - defer n.lock.RUnlock() - - _, ok := n.disconnected[string(node)] - return ok -} - -func (n *inMemNetwork) Connect(node NodeID) { - n.lock.Lock() - defer n.lock.Unlock() - - delete(n.disconnected, string(node)) -} - -func (n *inMemNetwork) Disconnect(node NodeID) { - n.lock.Lock() - defer n.lock.Unlock() - - n.disconnected[string(node)] = struct{}{} -} - -// startInstances starts all instances in the network. -// The first one is typically the leader, so we make sure to start it last. -func (n *inMemNetwork) startInstances() { - require.Equal(n.t, len(n.nodes), len(n.instances)) - - for i := len(n.nodes) - 1; i >= 0; i-- { - n.instances[i].start() - } -} - -// testControlledBlockBuilder is a test block builder that blocks -// block building until a trigger is received -type testControlledBlockBuilder struct { - t *testing.T - control chan struct{} - testBlockBuilder -} - -func newTestControlledBlockBuilder(t *testing.T) *testControlledBlockBuilder { - return &testControlledBlockBuilder{ - t: t, - control: make(chan struct{}, 1), - testBlockBuilder: testBlockBuilder{out: make(chan *testBlock, 1), blockShouldBeBuilt: make(chan struct{}, 1)}, - } -} - -func (t *testControlledBlockBuilder) triggerNewBlock() { - select { - case t.control <- struct{}{}: - default: - - } -} - -func (t *testControlledBlockBuilder) BuildBlock(ctx context.Context, metadata ProtocolMetadata) (VerifiedBlock, bool) { - <-t.control - return t.testBlockBuilder.BuildBlock(ctx, metadata) -} diff --git a/epoch_test.go b/epoch_test.go index b91762d8..50da515f 100644 --- a/epoch_test.go +++ b/epoch_test.go @@ -4,18 +4,14 @@ package simplex_test import ( - "bytes" "context" "crypto/rand" - "crypto/sha256" - "encoding/asn1" "encoding/binary" "fmt" "math" rand2 "math/rand" . "simplex" "simplex/testutil" - "simplex/wal" "strings" "sync" "testing" @@ -27,32 +23,17 @@ import ( func TestEpochHandleNotarizationFutureRound(t *testing.T) { l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{} + bb := &testutil.TestBlockBuilder{} nodes := []NodeID{{1}, {2}, {3}, {4}} // Create the two blocks ahead of time blocks := createBlocks(t, nodes, bb, 2) - firstBlock := blocks[0].VerifiedBlock.(*testBlock) - secondBlock := blocks[1].VerifiedBlock.(*testBlock) - bb.out = make(chan *testBlock, 1) - bb.in = make(chan *testBlock, 1) - - storage := newInMemStorage() - - wal := newTestWAL(t) + firstBlock := blocks[0].VerifiedBlock.(*testutil.TestBlock) + secondBlock := blocks[1].VerifiedBlock.(*testutil.TestBlock) + bb.Out = make(chan *testutil.TestBlock, 1) + bb.In = make(chan *testutil.TestBlock, 1) quorum := Quorum(len(nodes)) - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -64,11 +45,11 @@ func TestEpochHandleNotarizationFutureRound(t *testing.T) { //for len(bb.out) > 0 { // <-bb.out //} - bb.in <- firstBlock - bb.out <- firstBlock + bb.In <- firstBlock + bb.Out <- firstBlock // Create a notarization for round 1 which is a future round because we haven't gone through round 0 yet. - notarization, err := newNotarization(l, &testSignatureAggregator{}, secondBlock, nodes) + notarization, err := newNotarization(l, conf.SignatureAggregator, secondBlock, nodes) require.NoError(t, err) // Give the node the notarization message before receiving the first block @@ -80,7 +61,7 @@ func TestEpochHandleNotarizationFutureRound(t *testing.T) { notarizeAndFinalizeRound(t, e, bb) // Emulate round 1 by sending the block - vote, err := newTestVote(secondBlock, nodes[1]) + vote, err := newTestVote(secondBlock, nodes[1], e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -91,38 +72,21 @@ func TestEpochHandleNotarizationFutureRound(t *testing.T) { require.NoError(t, err) // The node should store the notarization of the second block once it gets the block. - wal.assertNotarization(1) + wal.AssertNotarization(1) for i := 1; i < quorum; i++ { injectTestFinalization(t, e, secondBlock, nodes[i]) } - blockCommitted := storage.waitForBlockCommit(1) + blockCommitted := storage.WaitForBlockCommit(1) require.Equal(t, secondBlock, blockCommitted) } func TestEpochConsecutiveProposalsDoNotGetVerified(t *testing.T) { - l := testutil.MakeLogger(t, 1) - - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() - - wal := newTestWAL(t) + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[1], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[1], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -137,12 +101,12 @@ func TestEpochConsecutiveProposalsDoNotGetVerified(t *testing.T) { require.Equal(t, md.Round, md.Seq) onlyVerifyOnce := make(chan struct{}) - block := <-bb.out - block.onVerify = func() { + block := <-bb.Out + block.OnVerify = func() { close(onlyVerifyOnce) } - vote, err := newTestVote(block, leader) + vote, err := newTestVote(block, leader, e.Signer) require.NoError(t, err) var wg sync.WaitGroup @@ -171,49 +135,32 @@ func TestEpochConsecutiveProposalsDoNotGetVerified(t *testing.T) { } func TestEpochNotarizeTwiceThenFinalize(t *testing.T) { - l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() - - wal := newTestWAL(t) - + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} recordedMessages := make(chan *Message, 100) - comm := &recordingComm{Communication: noopComm(nodes), BroadcastMessages: recordedMessages} - - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: comm, - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + comm := &testutil.RecordingComm{Communication: testutil.NewNoopComm(nodes), BroadcastMessages: recordedMessages} + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) e, err := NewEpoch(conf) require.NoError(t, err) require.NoError(t, e.Start()) // Round 0 - block0 := <-bb.out + block0 := <-bb.Out injectTestVote(t, e, block0, nodes[1]) injectTestVote(t, e, block0, nodes[2]) - wal.assertNotarization(0) + wal.AssertNotarization(0) // Round 1 md := e.Metadata() _, ok := bb.BuildBlock(context.Background(), md) require.True(t, ok) - block1 := <-bb.out + block1 := <-bb.Out - vote, err := newTestVote(block1, nodes[1]) + vote, err := newTestVote(block1, nodes[1], e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -225,15 +172,15 @@ func TestEpochNotarizeTwiceThenFinalize(t *testing.T) { injectTestVote(t, e, block1, nodes[2]) - wal.assertNotarization(1) + wal.AssertNotarization(1) // Round 2 md = e.Metadata() _, ok = bb.BuildBlock(context.Background(), md) require.True(t, ok) - block2 := <-bb.out + block2 := <-bb.Out - vote, err = newTestVote(block2, nodes[2]) + vote, err = newTestVote(block2, nodes[2], e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -245,14 +192,14 @@ func TestEpochNotarizeTwiceThenFinalize(t *testing.T) { injectTestVote(t, e, block2, nodes[1]) - wal.assertNotarization(2) + wal.AssertNotarization(2) // drain the recorded messages for len(recordedMessages) > 0 { <-recordedMessages } - blocks := []*testBlock{block0, block1} + blocks := []*testutil.TestBlock{block0, block1} var wg sync.WaitGroup wg.Add(1) @@ -281,36 +228,20 @@ func TestEpochNotarizeTwiceThenFinalize(t *testing.T) { injectTestFinalization(t, e, block2, nodes[1]) injectTestFinalization(t, e, block2, nodes[2]) - storage.waitForBlockCommit(0) - storage.waitForBlockCommit(1) - storage.waitForBlockCommit(2) + storage.WaitForBlockCommit(0) + storage.WaitForBlockCommit(1) + storage.WaitForBlockCommit(2) close(finish) wg.Wait() } func TestEpochFinalizeThenNotarize(t *testing.T) { - l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() - - wal := newTestWAL(t) + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } - + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -319,7 +250,7 @@ func TestEpochFinalizeThenNotarize(t *testing.T) { t.Run("commit without notarization, only with finalization", func(t *testing.T) { for round := 0; round < 100; round++ { advanceRoundFromFinalization(t, e, bb) - storage.waitForBlockCommit(uint64(round)) + storage.WaitForBlockCommit(uint64(round)) } }) @@ -333,9 +264,9 @@ func TestEpochFinalizeThenNotarize(t *testing.T) { require.True(t, ok) } - block := <-bb.out + block := <-bb.Out - vote, err := newTestVote(block, nodes[0]) + vote, err := newTestVote(block, nodes[0], e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -349,30 +280,16 @@ func TestEpochFinalizeThenNotarize(t *testing.T) { injectTestVote(t, e, block, nodes[i]) } - wal.assertNotarization(100) + wal.AssertNotarization(100) }) } func TestEpochSimpleFlow(t *testing.T) { - l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: newTestWAL(t), - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } - + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -385,23 +302,10 @@ func TestEpochSimpleFlow(t *testing.T) { } func TestEpochStartedTwice(t *testing.T) { - l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal.NewMemWAL(t), - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -410,22 +314,22 @@ func TestEpochStartedTwice(t *testing.T) { require.ErrorIs(t, e.Start(), ErrAlreadyStarted) } -func advanceRoundFromNotarization(t *testing.T, e *Epoch, bb *testBlockBuilder) (VerifiedBlock, *Notarization) { +func advanceRoundFromNotarization(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder) (VerifiedBlock, *Notarization) { return advanceRound(t, e, bb, true, false) } -func advanceRoundFromFinalization(t *testing.T, e *Epoch, bb *testBlockBuilder) VerifiedBlock { +func advanceRoundFromFinalization(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder) VerifiedBlock { block, _ := advanceRound(t, e, bb, false, true) return block } -func notarizeAndFinalizeRound(t *testing.T, e *Epoch, bb *testBlockBuilder) (VerifiedBlock, *Notarization) { +func notarizeAndFinalizeRound(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder) (VerifiedBlock, *Notarization) { return advanceRound(t, e, bb, true, true) } // advanceRound progresses [e] to a new round. If [notarize] is set, the round will progress due to a notarization. // If [finalize] is set, the round will advance and the block will be indexed to storage. -func advanceRound(t *testing.T, e *Epoch, bb *testBlockBuilder, notarize bool, finalize bool) (VerifiedBlock, *Notarization) { +func advanceRound(t *testing.T, e *Epoch, bb *testutil.TestBlockBuilder, notarize bool, finalize bool) (VerifiedBlock, *Notarization) { require.True(t, notarize || finalize, "must either notarize or finalize a round to advance") nodes := e.Comm.ListNodes() quorum := Quorum(len(nodes)) @@ -439,11 +343,11 @@ func advanceRound(t *testing.T, e *Epoch, bb *testBlockBuilder, notarize bool, f require.True(t, ok) } - block := <-bb.out + block := <-bb.Out if !isEpochNode { // send node a message from the leader - vote, err := newTestVote(block, leader) + vote, err := newTestVote(block, leader, e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -460,7 +364,7 @@ func advanceRound(t *testing.T, e *Epoch, bb *testBlockBuilder, notarize bool, f n, err := newNotarization(e.Logger, e.SignatureAggregator, block, nodes[0:quorum]) injectTestNotarization(t, e, n, nodes[1]) - e.WAL.(*testWAL).assertNotarization(block.metadata.Round) + e.WAL.(*testutil.TestWAL).AssertNotarization(block.BlockHeader().Round) require.NoError(t, err) notarization = &n } @@ -469,7 +373,7 @@ func advanceRound(t *testing.T, e *Epoch, bb *testBlockBuilder, notarize bool, f for i := 1; i <= quorum; i++ { injectTestFinalization(t, e, block, nodes[i]) } - blockFromStorage := e.Storage.(*InMemStorage).waitForBlockCommit(block.metadata.Seq) + blockFromStorage := e.Storage.(*testutil.InMemStorage).WaitForBlockCommit(block.BlockHeader().Seq) require.Equal(t, block, blockFromStorage) } @@ -494,26 +398,12 @@ func TestEpochInterleavingMessages(t *testing.T) { } func testEpochInterleavingMessages(t *testing.T, seed int64) { - l := testutil.MakeLogger(t, 1) rounds := 10 - bb := &testBlockBuilder{in: make(chan *testBlock, rounds)} - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{In: make(chan *testutil.TestBlock, rounds)} nodes := []NodeID{{1}, {2}, {3}, {4}} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal.NewMemWAL(t), - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } - + conf, _, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -531,17 +421,17 @@ func testEpochInterleavingMessages(t *testing.T, seed int64) { for i := 0; i < rounds; i++ { t.Log("Waiting for commit of round", i) - storage.waitForBlockCommit(uint64(i)) + storage.WaitForBlockCommit(uint64(i)) } } -func createCallbacks(t *testing.T, rounds int, protocolMetadata ProtocolMetadata, nodes []NodeID, e *Epoch, bb *testBlockBuilder) []func() { +func createCallbacks(t *testing.T, rounds int, protocolMetadata ProtocolMetadata, nodes []NodeID, e *Epoch, bb *testutil.TestBlockBuilder) []func() { blocks := make([]VerifiedBlock, 0, rounds) callbacks := make([]func(), 0, rounds*4+len(blocks)) for i := 0; i < rounds; i++ { - block := newTestBlock(protocolMetadata) + block := testutil.NewTestBlock(protocolMetadata) blocks = append(blocks, block) protocolMetadata.Seq++ @@ -551,7 +441,7 @@ func createCallbacks(t *testing.T, rounds int, protocolMetadata ProtocolMetadata leader := LeaderForRound(nodes, uint64(i)) if !leader.Equals(e.ID) { - vote, err := newTestVote(block, leader) + vote, err := newTestVote(block, leader, e.Signer) require.NoError(t, err) callbacks = append(callbacks, func() { @@ -564,12 +454,12 @@ func createCallbacks(t *testing.T, rounds int, protocolMetadata ProtocolMetadata }, leader) }) } else { - bb.in <- block + bb.In <- block } for j := 1; j <= 2; j++ { node := nodes[j] - vote, err := newTestVote(block, node) + vote, err := newTestVote(block, node, e.Signer) require.NoError(t, err) msg := Message{ VoteMessage: vote, @@ -585,7 +475,7 @@ func createCallbacks(t *testing.T, rounds int, protocolMetadata ProtocolMetadata for j := 1; j <= 2; j++ { node := nodes[j] - finalization := newTestFinalization(t, block, node) + finalization := newTestFinalization(t, block, node, e.Signer) msg := Message{ Finalization: finalization, } @@ -616,25 +506,10 @@ func TestEpochBlockSentTwice(t *testing.T) { return nil }) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() - - wal := newTestWAL(t) - + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[1], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } - + conf, wal, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[1], testutil.NewNoopComm(nodes), bb) + conf.Logger = l e, err := NewEpoch(conf) require.NoError(t, err) @@ -648,7 +523,7 @@ func TestEpochBlockSentTwice(t *testing.T) { block := b.(Block) - vote, err := newTestVote(block, nodes[2]) + vote, err := newTestVote(block, nodes[2], e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -658,7 +533,7 @@ func TestEpochBlockSentTwice(t *testing.T) { }, nodes[2]) require.NoError(t, err) - wal.assertWALSize(0) + wal.AssertWALSize(0) require.True(t, tooFarMsg) err = e.HandleMessage(&Message{ @@ -669,7 +544,7 @@ func TestEpochBlockSentTwice(t *testing.T) { }, nodes[2]) require.NoError(t, err) - wal.assertWALSize(0) + wal.AssertWALSize(0) require.True(t, alreadyReceivedMsg) } @@ -725,36 +600,22 @@ func TestEpochQCSignedByNonExistentNodes(t *testing.T) { return nil }) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() - - wal := newTestWAL(t) + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) require.NoError(t, e.Start()) - block := <-bb.out + block := <-bb.Out - wal.assertWALSize(1) + wal.AssertWALSize(1) t.Run("notarization with unknown signer isn't taken into account", func(t *testing.T) { - notarization, err := newNotarization(l, &testSignatureAggregator{}, block, []NodeID{{2}, {3}, {5}}) + notarization, err := newNotarization(l, e.SignatureAggregator, block, []NodeID{{2}, {3}, {5}}) require.NoError(t, err) err = e.HandleMessage(&Message{ @@ -767,11 +628,11 @@ func TestEpochQCSignedByNonExistentNodes(t *testing.T) { require.NoError(t, err) fmt.Println(">>>", len(rawWAL)) - wal.assertWALSize(1) + wal.AssertWALSize(1) }) t.Run("notarization with double signer isn't taken into account", func(t *testing.T) { - notarization, err := newNotarization(l, &testSignatureAggregator{}, block, []NodeID{{2}, {3}, {2}}) + notarization, err := newNotarization(l, e.SignatureAggregator, block, []NodeID{{2}, {3}, {2}}) require.NoError(t, err) err = e.HandleMessage(&Message{ @@ -779,11 +640,11 @@ func TestEpochQCSignedByNonExistentNodes(t *testing.T) { }, nodes[1]) require.NoError(t, err) - wal.assertWALSize(1) + wal.AssertWALSize(1) }) t.Run("empty notarization with unknown signer isn't taken into account", func(t *testing.T) { - var qc testQC + var qc testutil.TestQC for i, n := range []NodeID{{2}, {3}, {5}} { qc = append(qc, Signature{Signer: n, Value: []byte{byte(i)}}) } @@ -799,11 +660,11 @@ func TestEpochQCSignedByNonExistentNodes(t *testing.T) { }, nodes[1]) require.NoError(t, err) - wal.assertWALSize(1) + wal.AssertWALSize(1) }) t.Run("empty notarization with double signer isn't taken into account", func(t *testing.T) { - var qc testQC + var qc testutil.TestQC for i, n := range []NodeID{{2}, {3}, {2}} { qc = append(qc, Signature{Signer: n, Value: []byte{byte(i)}}) } @@ -819,29 +680,29 @@ func TestEpochQCSignedByNonExistentNodes(t *testing.T) { }, nodes[1]) require.NoError(t, err) - wal.assertWALSize(1) + wal.AssertWALSize(1) }) t.Run("finalization certificate with unknown signer isn't taken into account", func(t *testing.T) { - fCert, _ := newFinalizationRecord(t, l, &testSignatureAggregator{}, block, []NodeID{{2}, {3}, {5}}) + fCert, _ := newFinalizationRecord(t, l, e.SignatureAggregator, block, []NodeID{{2}, {3}, {5}}) err = e.HandleMessage(&Message{ FinalizationCertificate: &fCert, }, nodes[1]) require.NoError(t, err) - storage.ensureNoBlockCommit(t, 0) + storage.EnsureNoBlockCommit(t, 0) }) t.Run("finalization certificate with double signer isn't taken into account", func(t *testing.T) { - fCert, _ := newFinalizationRecord(t, l, &testSignatureAggregator{}, block, []NodeID{{2}, {3}, {3}}) + fCert, _ := newFinalizationRecord(t, l, e.SignatureAggregator, block, []NodeID{{2}, {3}, {3}}) err = e.HandleMessage(&Message{ FinalizationCertificate: &fCert, }, nodes[1]) require.NoError(t, err) - storage.ensureNoBlockCommit(t, 0) + storage.EnsureNoBlockCommit(t, 0) }) } @@ -856,23 +717,10 @@ func TestEpochBlockSentFromNonLeader(t *testing.T) { return nil }) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() - wal := newTestWAL(t) + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[1], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } - + conf, wal, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[1], testutil.NewNoopComm(nodes), bb) + conf.Logger = l e, err := NewEpoch(conf) require.NoError(t, err) @@ -885,7 +733,7 @@ func TestEpochBlockSentFromNonLeader(t *testing.T) { block := b.(Block) notLeader := nodes[3] - vote, err := newTestVote(block, notLeader) + vote, err := newTestVote(block, notLeader, e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -912,24 +760,11 @@ func TestEpochBlockTooHighRound(t *testing.T) { return nil }) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() - - wal := newTestWAL(t) + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[1], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - } + conf, wal, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[1], testutil.NewNoopComm(nodes), bb) + conf.Logger = l e, err := NewEpoch(conf) require.NoError(t, err) @@ -949,7 +784,7 @@ func TestEpochBlockTooHighRound(t *testing.T) { block := b.(Block) - vote, err := newTestVote(block, nodes[0]) + vote, err := newTestVote(block, nodes[0], e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -960,7 +795,7 @@ func TestEpochBlockTooHighRound(t *testing.T) { require.NoError(t, err) require.True(t, rejectedBlock) - wal.assertWALSize(0) + wal.AssertWALSize(0) }) t.Run("block is accepted", func(t *testing.T) { @@ -974,7 +809,7 @@ func TestEpochBlockTooHighRound(t *testing.T) { block := b.(Block) - vote, err := newTestVote(block, nodes[0]) + vote, err := newTestVote(block, nodes[0], e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -985,7 +820,7 @@ func TestEpochBlockTooHighRound(t *testing.T) { require.NoError(t, err) require.False(t, rejectedBlock) - wal.assertWALSize(1) + wal.AssertWALSize(1) }) } @@ -994,11 +829,11 @@ type AnyBlock interface { BlockHeader() BlockHeader } -func newTestVote(block AnyBlock, id NodeID) (*Vote, error) { +func newTestVote(block AnyBlock, id NodeID, signer Signer) (*Vote, error) { vote := ToBeSignedVote{ BlockHeader: block.BlockHeader(), } - sig, err := vote.Sign(&testSigner{}) + sig, err := vote.Sign(signer) if err != nil { return nil, err } @@ -1013,7 +848,7 @@ func newTestVote(block AnyBlock, id NodeID) (*Vote, error) { } func injectTestVote(t *testing.T, e *Epoch, block VerifiedBlock, id NodeID) { - vote, err := newTestVote(block, id) + vote, err := newTestVote(block, id, e.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ VoteMessage: vote, @@ -1021,9 +856,9 @@ func injectTestVote(t *testing.T, e *Epoch, block VerifiedBlock, id NodeID) { require.NoError(t, err) } -func newTestFinalization(t *testing.T, block VerifiedBlock, id NodeID) *Finalization { +func newTestFinalization(t *testing.T, block VerifiedBlock, id NodeID, signer Signer) *Finalization { f := ToBeSignedFinalization{BlockHeader: block.BlockHeader()} - sig, err := f.Sign(&testSigner{}) + sig, err := f.Sign(signer) require.NoError(t, err) return &Finalization{ Signature: Signature{ @@ -1038,7 +873,7 @@ func newTestFinalization(t *testing.T, block VerifiedBlock, id NodeID) *Finaliza func injectTestFinalization(t *testing.T, e *Epoch, block VerifiedBlock, id NodeID) { err := e.HandleMessage(&Message{ - Finalization: newTestFinalization(t, block, id), + Finalization: newTestFinalization(t, block, id, e.Signer), }, id) require.NoError(t, err) } @@ -1049,376 +884,3 @@ func injectTestNotarization(t *testing.T, e *Epoch, notarization Notarization, i }, id) require.NoError(t, err) } - -type testQCDeserializer struct { - t *testing.T -} - -func (t *testQCDeserializer) DeserializeQuorumCertificate(bytes []byte) (QuorumCertificate, error) { - var qc []Signature - rest, err := asn1.Unmarshal(bytes, &qc) - require.NoError(t.t, err) - require.Empty(t.t, rest) - return testQC(qc), err -} - -type testSignatureAggregator struct { - err error -} - -func (t *testSignatureAggregator) Aggregate(signatures []Signature) (QuorumCertificate, error) { - return testQC(signatures), t.err -} - -type testQC []Signature - -func (t testQC) Signers() []NodeID { - res := make([]NodeID, 0, len(t)) - for _, sig := range t { - res = append(res, sig.Signer) - } - return res -} - -func (t testQC) Verify(msg []byte) error { - return nil -} - -func (t testQC) Bytes() []byte { - bytes, err := asn1.Marshal(t) - if err != nil { - panic(err) - } - return bytes -} - -type testSigner struct { -} - -func (t *testSigner) Sign([]byte) ([]byte, error) { - return []byte{1, 2, 3}, nil -} - -type testVerifier struct { -} - -func (t *testVerifier) VerifyBlock(VerifiedBlock) error { - return nil -} - -func (t *testVerifier) Verify(_ []byte, _ []byte, _ NodeID) error { - return nil -} - -type noopComm []NodeID - -func (n noopComm) ListNodes() []NodeID { - return n -} - -func (n noopComm) SendMessage(*Message, NodeID) { - -} - -func (n noopComm) Broadcast(msg *Message) { - -} - -// ListnerComm is a comm that listens for incoming messages -// and sends them to the [in] channel -type listnerComm struct { - noopComm - in chan *Message -} - -func NewListenerComm(nodeIDs []NodeID) *listnerComm { - return &listnerComm{ - noopComm: noopComm(nodeIDs), - in: make(chan *Message, 1), - } -} - -func (b *listnerComm) SendMessage(msg *Message, id NodeID) { - b.in <- msg -} - -type testBlockBuilder struct { - out chan *testBlock - in chan *testBlock - blockShouldBeBuilt chan struct{} -} - -// BuildBlock builds a new testblock and sends it to the BlockBuilder channel -func (t *testBlockBuilder) BuildBlock(_ context.Context, metadata ProtocolMetadata) (VerifiedBlock, bool) { - if len(t.in) > 0 { - block := <-t.in - return block, true - } - - tb := newTestBlock(metadata) - - select { - case t.out <- tb: - default: - } - - return tb, true -} - -func (t *testBlockBuilder) IncomingBlock(ctx context.Context) { - select { - case <-t.blockShouldBeBuilt: - case <-ctx.Done(): - } -} - -type testBlock struct { - data []byte - metadata ProtocolMetadata - digest [32]byte - onVerify func() - verificationDelay chan struct{} -} - -func (tb *testBlock) Verify(context.Context) (VerifiedBlock, error) { - defer func() { - if tb.onVerify != nil { - tb.onVerify() - } - }() - if tb.verificationDelay == nil { - return tb, nil - } - - <-tb.verificationDelay - - return tb, nil -} - -func newTestBlock(metadata ProtocolMetadata) *testBlock { - tb := testBlock{ - metadata: metadata, - data: make([]byte, 32), - } - - _, err := rand.Read(tb.data) - if err != nil { - panic(err) - } - - tb.computeDigest() - - return &tb -} - -func (tb *testBlock) computeDigest() { - var bb bytes.Buffer - bb.Write(tb.Bytes()) - tb.digest = sha256.Sum256(bb.Bytes()) -} - -func (t *testBlock) BlockHeader() BlockHeader { - return BlockHeader{ - ProtocolMetadata: t.metadata, - Digest: t.digest, - } -} - -func (t *testBlock) Bytes() []byte { - bh := BlockHeader{ - ProtocolMetadata: t.metadata, - } - - mdBuff := bh.Bytes() - - buff := make([]byte, len(t.data)+len(mdBuff)+4) - binary.BigEndian.PutUint32(buff, uint32(len(t.data))) - copy(buff[4:], t.data) - copy(buff[4+len(t.data):], mdBuff) - return buff -} - -type InMemStorage struct { - data map[uint64]struct { - VerifiedBlock - FinalizationCertificate - } - - lock sync.Mutex - signal sync.Cond -} - -func newInMemStorage() *InMemStorage { - s := &InMemStorage{ - data: make(map[uint64]struct { - VerifiedBlock - FinalizationCertificate - }), - } - - s.signal = *sync.NewCond(&s.lock) - - return s -} - -func (mem *InMemStorage) Clone() *InMemStorage { - clone := newInMemStorage() - mem.lock.Lock() - height := mem.Height() - mem.lock.Unlock() - for seq := uint64(0); seq < height; seq++ { - mem.lock.Lock() - block, fCert, ok := mem.Retrieve(seq) - if !ok { - panic(fmt.Sprintf("failed retrieving block %d", seq)) - } - mem.lock.Unlock() - clone.Index(block, fCert) - } - return clone -} - -func (mem *InMemStorage) waitForBlockCommit(seq uint64) VerifiedBlock { - mem.lock.Lock() - defer mem.lock.Unlock() - - for { - if data, exists := mem.data[seq]; exists { - return data.VerifiedBlock - } - - mem.signal.Wait() - } -} - -func (mem *InMemStorage) ensureNoBlockCommit(t *testing.T, seq uint64) { - require.Never(t, func() bool { - mem.lock.Lock() - defer mem.lock.Unlock() - - _, exists := mem.data[seq] - return exists - }, time.Second, time.Millisecond*100, "block %d has been committed but shouldn't have been", seq) -} - -func (mem *InMemStorage) Height() uint64 { - return uint64(len(mem.data)) -} - -func (mem *InMemStorage) Retrieve(seq uint64) (VerifiedBlock, FinalizationCertificate, bool) { - item, ok := mem.data[seq] - if !ok { - return nil, FinalizationCertificate{}, false - } - return item.VerifiedBlock, item.FinalizationCertificate, true -} - -func (mem *InMemStorage) Index(block VerifiedBlock, certificate FinalizationCertificate) { - mem.lock.Lock() - defer mem.lock.Unlock() - - seq := block.BlockHeader().Seq - - _, ok := mem.data[seq] - if ok { - panic(fmt.Sprintf("block with seq %d already indexed!", seq)) - } - mem.data[seq] = struct { - VerifiedBlock - FinalizationCertificate - }{block, - certificate, - } - - mem.signal.Signal() -} - -type blockDeserializer struct { -} - -func (b *blockDeserializer) DeserializeBlock(buff []byte) (VerifiedBlock, error) { - blockLen := binary.BigEndian.Uint32(buff[:4]) - bh := BlockHeader{} - if err := bh.FromBytes(buff[4+blockLen:]); err != nil { - return nil, err - } - - tb := testBlock{ - data: buff[4 : 4+blockLen], - metadata: bh.ProtocolMetadata, - } - - tb.computeDigest() - - return &tb, nil -} - -func TestBlockDeserializer(t *testing.T) { - var blockDeserializer blockDeserializer - - tb := newTestBlock(ProtocolMetadata{Seq: 1, Round: 2, Epoch: 3}) - tb2, err := blockDeserializer.DeserializeBlock(tb.Bytes()) - require.NoError(t, err) - require.Equal(t, tb, tb2) -} - -func TestQuorum(t *testing.T) { - for _, testCase := range []struct { - n int - f int - q int - }{ - { - n: 1, f: 0, - q: 1, - }, - { - n: 2, f: 0, - q: 2, - }, - { - n: 3, f: 0, - q: 2, - }, - { - n: 4, f: 1, - q: 3, - }, - { - n: 5, f: 1, - q: 4, - }, - { - n: 6, f: 1, - q: 4, - }, - { - n: 7, f: 2, - q: 5, - }, - { - n: 8, f: 2, - q: 6, - }, - { - n: 9, f: 2, - q: 6, - }, - { - n: 10, f: 3, - q: 7, - }, - { - n: 11, f: 3, - q: 8, - }, - { - n: 12, f: 3, - q: 8, - }, - } { - t.Run(fmt.Sprintf("%d", testCase.n), func(t *testing.T) { - require.Equal(t, testCase.q, Quorum(testCase.n)) - }) - } -} diff --git a/msg_test.go b/msg_test.go index 3e315fe4..60c2c57a 100644 --- a/msg_test.go +++ b/msg_test.go @@ -2,6 +2,7 @@ package simplex_test import ( "simplex" + "simplex/testutil" "testing" "github.com/stretchr/testify/require" @@ -31,21 +32,21 @@ func TestQuorumRoundMalformed(t *testing.T) { }, { name: "block and notarization", qr: simplex.QuorumRound{ - Block: &testBlock{}, + Block: &testutil.TestBlock{}, Notarization: &simplex.Notarization{}, }, expectedErr: false, }, { name: "block and fcert", qr: simplex.QuorumRound{ - Block: &testBlock{}, + Block: &testutil.TestBlock{}, FCert: &simplex.FinalizationCertificate{}, }, expectedErr: false, }, { name: "block and empty notarization", qr: simplex.QuorumRound{ - Block: &testBlock{}, + Block: &testutil.TestBlock{}, EmptyNotarization: &simplex.EmptyNotarization{}, }, expectedErr: true, @@ -53,7 +54,7 @@ func TestQuorumRoundMalformed(t *testing.T) { { name: "block and notarization and fcert", qr: simplex.QuorumRound{ - Block: &testBlock{}, + Block: &testutil.TestBlock{}, Notarization: &simplex.Notarization{}, FCert: &simplex.FinalizationCertificate{}, }, @@ -76,7 +77,7 @@ func TestQuorumRoundMalformed(t *testing.T) { { name: "just block", qr: simplex.QuorumRound{ - Block: &testBlock{}, + Block: &testutil.TestBlock{}, }, expectedErr: true, }, diff --git a/notarization_test.go b/notarization_test.go index d9a402e9..4fe556dc 100644 --- a/notarization_test.go +++ b/notarization_test.go @@ -17,7 +17,8 @@ var errorSigAggregation = errors.New("signature error") func TestNewNotarization(t *testing.T) { l := testutil.MakeLogger(t, 1) - testBlock := &testBlock{} + testBlock := &testutil.TestBlock{} + signer := &testutil.TestSigner{} tests := []struct { name string votesForCurrentRound map[string]*simplex.Vote @@ -31,21 +32,21 @@ func TestNewNotarization(t *testing.T) { votes := make(map[string]*simplex.Vote) nodeIds := [][]byte{{1}, {2}, {3}, {4}, {5}} for _, nodeId := range nodeIds { - vote, err := newTestVote(testBlock, nodeId) + vote, err := newTestVote(testBlock, nodeId, &testutil.TestSigner{}) require.NoError(t, err) votes[string(nodeId)] = vote } return votes }(), block: testBlock, - signatureAggregator: &testSignatureAggregator{}, + signatureAggregator: &testutil.TestSignatureAggregator{}, expectError: nil, }, { name: "no votes", votesForCurrentRound: map[string]*simplex.Vote{}, block: testBlock, - signatureAggregator: &testSignatureAggregator{}, + signatureAggregator: &testutil.TestSignatureAggregator{}, expectError: simplex.ErrorNoVotes, }, { @@ -54,14 +55,14 @@ func TestNewNotarization(t *testing.T) { votes := make(map[string]*simplex.Vote) nodeIds := [][]byte{{1}, {2}, {3}, {4}, {5}} for _, nodeId := range nodeIds { - vote, err := newTestVote(testBlock, nodeId) + vote, err := newTestVote(testBlock, nodeId, signer) require.NoError(t, err) votes[string(nodeId)] = vote } return votes }(), block: testBlock, - signatureAggregator: &testSignatureAggregator{err: errorSigAggregation}, + signatureAggregator: &testutil.TestSignatureAggregator{Err: errorSigAggregation}, expectError: errorSigAggregation, }, } @@ -86,6 +87,7 @@ func TestNewNotarization(t *testing.T) { func TestNewFinalizationCertificate(t *testing.T) { l := testutil.MakeLogger(t, 1) + signer := &testutil.TestSigner{} tests := []struct { name string finalizations []*simplex.Finalization @@ -97,47 +99,47 @@ func TestNewFinalizationCertificate(t *testing.T) { { name: "valid finalizations in order", finalizations: []*simplex.Finalization{ - newTestFinalization(t, &testBlock{}, []byte{1}), - newTestFinalization(t, &testBlock{}, []byte{2}), - newTestFinalization(t, &testBlock{}, []byte{3}), + newTestFinalization(t, &testutil.TestBlock{}, []byte{1}, signer), + newTestFinalization(t, &testutil.TestBlock{}, []byte{2}, signer), + newTestFinalization(t, &testutil.TestBlock{}, []byte{3}, signer), }, - signatureAggregator: &testSignatureAggregator{}, - expectedFinalization: &newTestFinalization(t, &testBlock{}, []byte{1}).Finalization, + signatureAggregator: &testutil.TestSignatureAggregator{}, + expectedFinalization: &newTestFinalization(t, &testutil.TestBlock{}, []byte{1}, signer).Finalization, expectError: nil, }, { name: "unsorted finalizations", finalizations: []*simplex.Finalization{ - newTestFinalization(t, &testBlock{}, []byte{3}), - newTestFinalization(t, &testBlock{}, []byte{1}), - newTestFinalization(t, &testBlock{}, []byte{2}), + newTestFinalization(t, &testutil.TestBlock{}, []byte{3}, signer), + newTestFinalization(t, &testutil.TestBlock{}, []byte{1}, signer), + newTestFinalization(t, &testutil.TestBlock{}, []byte{2}, signer), }, - signatureAggregator: &testSignatureAggregator{}, - expectedFinalization: &newTestFinalization(t, &testBlock{}, []byte{1}).Finalization, + signatureAggregator: &testutil.TestSignatureAggregator{}, + expectedFinalization: &newTestFinalization(t, &testutil.TestBlock{}, []byte{1}, signer).Finalization, expectError: nil, }, { name: "finalizations with different digests", finalizations: []*simplex.Finalization{ - newTestFinalization(t, &testBlock{digest: [32]byte{1}}, []byte{1}), - newTestFinalization(t, &testBlock{digest: [32]byte{2}}, []byte{2}), - newTestFinalization(t, &testBlock{digest: [32]byte{3}}, []byte{3}), + newTestFinalization(t, &testutil.TestBlock{Digest: [32]byte{1}}, []byte{1}, signer), + newTestFinalization(t, &testutil.TestBlock{Digest: [32]byte{2}}, []byte{2}, signer), + newTestFinalization(t, &testutil.TestBlock{Digest: [32]byte{3}}, []byte{3}, signer), }, - signatureAggregator: &testSignatureAggregator{}, + signatureAggregator: &testutil.TestSignatureAggregator{}, expectError: simplex.ErrorInvalidFinalizationDigest, }, { name: "signature aggregator errors", finalizations: []*simplex.Finalization{ - newTestFinalization(t, &testBlock{}, []byte{1}), + newTestFinalization(t, &testutil.TestBlock{}, []byte{1}, signer), }, - signatureAggregator: &testSignatureAggregator{err: errorSigAggregation}, + signatureAggregator: &testutil.TestSignatureAggregator{Err: errorSigAggregation}, expectError: errorSigAggregation, }, { name: "no votes", finalizations: []*simplex.Finalization{}, - signatureAggregator: &testSignatureAggregator{}, + signatureAggregator: &testutil.TestSignatureAggregator{}, expectError: simplex.ErrorNoVotes, }, } diff --git a/record_test.go b/record_test.go index 6ea23c75..866cc80f 100644 --- a/record_test.go +++ b/record_test.go @@ -6,6 +6,7 @@ package simplex_test import ( "simplex" "simplex/record" + "simplex/testutil" "testing" "github.com/stretchr/testify/require" @@ -14,7 +15,7 @@ import ( func newNotarization(logger simplex.Logger, signatureAggregator simplex.SignatureAggregator, block simplex.VerifiedBlock, ids []simplex.NodeID) (simplex.Notarization, error) { votesForCurrentRound := make(map[string]*simplex.Vote) for _, id := range ids { - vote, err := newTestVote(block, id) + vote, err := newTestVote(block, id, &testutil.TestSigner{}) if err != nil { return simplex.Notarization{}, err } @@ -40,7 +41,7 @@ func newNotarizationRecord(logger simplex.Logger, signatureAggregator simplex.Si func newFinalizationRecord(t *testing.T, logger simplex.Logger, signatureAggregator simplex.SignatureAggregator, block simplex.VerifiedBlock, ids []simplex.NodeID) (simplex.FinalizationCertificate, []byte) { finalizations := make([]*simplex.Finalization, len(ids)) for i, id := range ids { - finalizations[i] = newTestFinalization(t, block, id) + finalizations[i] = newTestFinalization(t, block, id, &testutil.TestSigner{}) } fCert, err := simplex.NewFinalizationCertificate(logger, signatureAggregator, finalizations) diff --git a/recovery_test.go b/recovery_test.go index 7c8d141d..d2120a30 100644 --- a/recovery_test.go +++ b/recovery_test.go @@ -9,7 +9,6 @@ import ( . "simplex" "simplex/record" "simplex/testutil" - "simplex/wal" "testing" "time" @@ -19,28 +18,12 @@ import ( // TestRecoverFromWALProposed tests that the epoch can recover from // a wal with a single block record written to it(that we have proposed). func TestRecoverFromWALProposed(t *testing.T) { - l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - wal := newTestWAL(t) - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} ctx := context.Background() nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - BlockDeserializer: &blockDeserializer{}, - QCDeserializer: &testQCDeserializer{t: t}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -72,14 +55,14 @@ func TestRecoverFromWALProposed(t *testing.T) { require.NotEqual(t, 0, rounds) } - block := <-bb.out + block := <-bb.Out if rounds == 0 { require.Equal(t, firstBlock, block) } if !isEpochNode { // send node a message from the leader - vote, err := newTestVote(block, leader) + vote, err := newTestVote(block, leader, conf.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -99,7 +82,7 @@ func TestRecoverFromWALProposed(t *testing.T) { injectTestFinalization(t, e, block, nodes[i]) } - block2 := storage.waitForBlockCommit(i) + block2 := storage.WaitForBlockCommit(i) require.Equal(t, block, block2) } @@ -111,27 +94,11 @@ func TestRecoverFromWALProposed(t *testing.T) { // with a block record written to it, and a notarization record. func TestRecoverFromNotarization(t *testing.T) { l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - wal := wal.NewMemWAL(t) - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} ctx := context.Background() nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - sigAggregrator := &testSignatureAggregator{} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: sigAggregrator, - BlockDeserializer: &blockDeserializer{}, - QCDeserializer: &testQCDeserializer{t: t}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -145,7 +112,7 @@ func TestRecoverFromNotarization(t *testing.T) { require.NoError(t, wal.Append(blockRecord)) // lets add some notarizations - notarizationRecord, err := newNotarizationRecord(l, sigAggregrator, block, nodes[0:quorum]) + notarizationRecord, err := newNotarizationRecord(l, conf.SignatureAggregator, block, nodes[0:quorum]) require.NoError(t, err) // when we start this we should kickoff the finalization process by broadcasting a finalization message and then waiting for incoming finalization messages @@ -167,7 +134,7 @@ func TestRecoverFromNotarization(t *testing.T) { injectTestFinalization(t, e, block, nodes[i]) } - committedData := storage.data[0].VerifiedBlock.Bytes() + committedData := storage.Data[0].VerifiedBlock.Bytes() require.Equal(t, block.Bytes(), committedData) require.Equal(t, uint64(1), e.Storage.Height()) } @@ -176,29 +143,13 @@ func TestRecoverFromNotarization(t *testing.T) { // with a block already stored in the storage func TestRecoverFromWalWithStorage(t *testing.T) { l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - wal := wal.NewMemWAL(t) - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} ctx := context.Background() nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - sigAggregrator := &testSignatureAggregator{} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: sigAggregrator, - BlockDeserializer: &blockDeserializer{}, - QCDeserializer: &testQCDeserializer{t: t}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) - storage.Index(newTestBlock(ProtocolMetadata{Seq: 0, Round: 0, Epoch: 0}), FinalizationCertificate{}) + storage.Index(testutil.NewTestBlock(ProtocolMetadata{Seq: 0, Round: 0, Epoch: 0}), FinalizationCertificate{}) e, err := NewEpoch(conf) require.NoError(t, err) require.Equal(t, uint64(1), e.Metadata().Round) @@ -212,7 +163,7 @@ func TestRecoverFromWalWithStorage(t *testing.T) { require.NoError(t, wal.Append(record)) // lets add some notarizations - notarizationRecord, err := newNotarizationRecord(l, sigAggregrator, block, nodes[0:quorum]) + notarizationRecord, err := newNotarizationRecord(l, conf.SignatureAggregator, block, nodes[0:quorum]) require.NoError(t, err) require.NoError(t, wal.Append(notarizationRecord)) @@ -237,7 +188,7 @@ func TestRecoverFromWalWithStorage(t *testing.T) { injectTestFinalization(t, e, block, nodes[i]) } - committedData := storage.data[1].VerifiedBlock.Bytes() + committedData := storage.Data[1].VerifiedBlock.Bytes() require.Equal(t, block.Bytes(), committedData) require.Equal(t, uint64(2), e.Storage.Height()) } @@ -245,28 +196,10 @@ func TestRecoverFromWalWithStorage(t *testing.T) { // TestWalCreated tests that the epoch correctly writes to the WAL func TestWalCreatedProperly(t *testing.T) { l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() - + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - signatureAggregator := &testSignatureAggregator{} - qd := &testQCDeserializer{t: t} - wal := newTestWAL(t) - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: signatureAggregator, - QCDeserializer: qd, - BlockDeserializer: &blockDeserializer{}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -279,13 +212,13 @@ func TestWalCreatedProperly(t *testing.T) { require.NoError(t, e.Start()) // ensure a block record is written to the WAL - wal.assertWALSize(1) + wal.AssertWALSize(1) records, err = e.WAL.ReadAll() require.NoError(t, err) require.Len(t, records, 1) blockFromWal, err := BlockFromRecord(conf.BlockDeserializer, records[0]) require.NoError(t, err) - block := <-bb.out + block := <-bb.Out require.Equal(t, blockFromWal, block) // start at one since our node has already voted @@ -296,7 +229,7 @@ func TestWalCreatedProperly(t *testing.T) { records, err = e.WAL.ReadAll() require.NoError(t, err) require.Len(t, records, 2) - expectedNotarizationRecord, err := newNotarizationRecord(l, signatureAggregator, block, nodes[0:quorum]) + expectedNotarizationRecord, err := newNotarizationRecord(l, conf.SignatureAggregator, block, nodes[0:quorum]) require.NoError(t, err) require.Equal(t, expectedNotarizationRecord, records[1]) @@ -309,32 +242,16 @@ func TestWalCreatedProperly(t *testing.T) { require.NoError(t, err) require.Len(t, records, 2) - committedData := storage.data[0].VerifiedBlock.Bytes() + committedData := storage.Data[0].VerifiedBlock.Bytes() require.Equal(t, block.Bytes(), committedData) } // TestWalWritesBlockRecord tests that the epoch correctly writes to the WAL // a block proposed by a node other than the epoch node func TestWalWritesBlockRecord(t *testing.T) { - l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() - blockDeserializer := &blockDeserializer{} + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} - wal := newTestWAL(t) - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[1], // nodes[1] is not the leader for the first round - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: &testSignatureAggregator{}, - BlockDeserializer: blockDeserializer, - } + conf, wal, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[1], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) @@ -354,9 +271,9 @@ func TestWalWritesBlockRecord(t *testing.T) { _, ok := bb.BuildBlock(context.Background(), md) require.True(t, ok) - block := <-bb.out + block := <-bb.Out // send epoch node this block - vote, err := newTestVote(block, nodes[0]) + vote, err := newTestVote(block, nodes[0], conf.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -367,43 +284,27 @@ func TestWalWritesBlockRecord(t *testing.T) { require.NoError(t, err) // ensure a block record is written to the WAL - wal.assertWALSize(1) + wal.AssertWALSize(1) records, err = e.WAL.ReadAll() require.NoError(t, err) require.Len(t, records, 1) - blockFromWal, err := BlockFromRecord(blockDeserializer, records[0]) + blockFromWal, err := BlockFromRecord(conf.BlockDeserializer, records[0]) require.NoError(t, err) require.Equal(t, block, blockFromWal) } func TestWalWritesFinalizationCert(t *testing.T) { l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() - sigAggregrator := &testSignatureAggregator{} + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - wal := newTestWAL(t) - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: sigAggregrator, - BlockDeserializer: &blockDeserializer{}, - QCDeserializer: &testQCDeserializer{t: t}, - } + conf, wal, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) e, err := NewEpoch(conf) require.NoError(t, err) require.NoError(t, e.Start()) - firstBlock := <-bb.out + firstBlock := <-bb.Out // notarize the first block for i := 1; i < quorum; i++ { injectTestVote(t, e, firstBlock, nodes[i]) @@ -414,7 +315,7 @@ func TestWalWritesFinalizationCert(t *testing.T) { blockFromWal, err := BlockFromRecord(conf.BlockDeserializer, records[0]) require.NoError(t, err) require.Equal(t, firstBlock, blockFromWal) - expectedNotarizationRecord, err := newNotarizationRecord(l, sigAggregrator, firstBlock, nodes[0:quorum]) + expectedNotarizationRecord, err := newNotarizationRecord(l, conf.SignatureAggregator, firstBlock, nodes[0:quorum]) require.NoError(t, err) require.Equal(t, expectedNotarizationRecord, records[1]) @@ -425,13 +326,13 @@ func TestWalWritesFinalizationCert(t *testing.T) { md.Prev = firstBlock.BlockHeader().Digest _, ok := bb.BuildBlock(context.Background(), md) require.True(t, ok) - secondBlock := <-bb.out + secondBlock := <-bb.Out // increase the round but don't index storage require.Equal(t, uint64(1), e.Metadata().Round) require.Equal(t, uint64(0), e.Storage.Height()) - vote, err := newTestVote(secondBlock, nodes[1]) + vote, err := newTestVote(secondBlock, nodes[1], conf.Signer) require.NoError(t, err) err = e.HandleMessage(&Message{ BlockMessage: &BlockMessage{ @@ -445,7 +346,7 @@ func TestWalWritesFinalizationCert(t *testing.T) { injectTestVote(t, e, secondBlock, nodes[i]) } - wal.assertWALSize(4) + wal.AssertWALSize(4) records, err = e.WAL.ReadAll() require.NoError(t, err) @@ -453,7 +354,7 @@ func TestWalWritesFinalizationCert(t *testing.T) { blockFromWal, err = BlockFromRecord(conf.BlockDeserializer, records[2]) require.NoError(t, err) require.Equal(t, secondBlock, blockFromWal) - expectedNotarizationRecord, err = newNotarizationRecord(l, sigAggregrator, secondBlock, nodes[0:quorum]) + expectedNotarizationRecord, err = newNotarizationRecord(l, conf.SignatureAggregator, secondBlock, nodes[0:quorum]) require.NoError(t, err) require.Equal(t, expectedNotarizationRecord, records[3]) @@ -468,7 +369,7 @@ func TestWalWritesFinalizationCert(t *testing.T) { recordType := binary.BigEndian.Uint16(records[4]) require.Equal(t, record.FinalizationRecordType, recordType) _, err = FinalizationCertificateFromRecord(records[4], e.QCDeserializer) - _, expectedFinalizationRecord := newFinalizationRecord(t, l, sigAggregrator, secondBlock, nodes[0:quorum]) + _, expectedFinalizationRecord := newFinalizationRecord(t, l, conf.SignatureAggregator, secondBlock, nodes[0:quorum]) require.NoError(t, err) require.Equal(t, expectedFinalizationRecord, records[4]) @@ -480,27 +381,11 @@ func TestWalWritesFinalizationCert(t *testing.T) { // Appends to the wal -> block, notarization, second block, notarization block 2, finalization for block 2. func TestRecoverFromMultipleNotarizations(t *testing.T) { l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - wal := wal.NewMemWAL(t) - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} ctx := context.Background() nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - sigAggregrator := &testSignatureAggregator{} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: sigAggregrator, - BlockDeserializer: &blockDeserializer{}, - QCDeserializer: &testQCDeserializer{t: t}, - } + conf, wal, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) // Create first block and write to WAL e, err := NewEpoch(conf) @@ -512,7 +397,7 @@ func TestRecoverFromMultipleNotarizations(t *testing.T) { record := BlockRecord(firstBlock.BlockHeader(), firstBlock.Bytes()) wal.Append(record) - firstNotarizationRecord, err := newNotarizationRecord(l, sigAggregrator, firstBlock, nodes[0:quorum]) + firstNotarizationRecord, err := newNotarizationRecord(l, conf.SignatureAggregator, firstBlock, nodes[0:quorum]) require.NoError(t, err) wal.Append(firstNotarizationRecord) @@ -524,12 +409,12 @@ func TestRecoverFromMultipleNotarizations(t *testing.T) { wal.Append(record) // Add notarization for second block - secondNotarizationRecord, err := newNotarizationRecord(l, sigAggregrator, secondBlock, nodes[0:quorum]) + secondNotarizationRecord, err := newNotarizationRecord(l, conf.SignatureAggregator, secondBlock, nodes[0:quorum]) require.NoError(t, err) wal.Append(secondNotarizationRecord) // Create finalization record for second block - fCert2, finalizationRecord := newFinalizationRecord(t, l, sigAggregrator, secondBlock, nodes[0:quorum]) + fCert2, finalizationRecord := newFinalizationRecord(t, l, conf.SignatureAggregator, secondBlock, nodes[0:quorum]) wal.Append(finalizationRecord) err = e.Start() @@ -539,44 +424,28 @@ func TestRecoverFromMultipleNotarizations(t *testing.T) { require.Equal(t, uint64(0), e.Storage.Height()) // now if we send fCert for block 1, we should index both 1 & 2 - fCert1, _ := newFinalizationRecord(t, l, sigAggregrator, firstBlock, nodes[0:quorum]) + fCert1, _ := newFinalizationRecord(t, l, conf.SignatureAggregator, firstBlock, nodes[0:quorum]) err = e.HandleMessage(&Message{ FinalizationCertificate: &fCert1, }, nodes[1]) require.NoError(t, err) require.Equal(t, uint64(2), e.Storage.Height()) - require.Equal(t, firstBlock.Bytes(), storage.data[0].VerifiedBlock.Bytes()) - require.Equal(t, secondBlock.Bytes(), storage.data[1].VerifiedBlock.Bytes()) - require.Equal(t, fCert1, storage.data[0].FinalizationCertificate) - require.Equal(t, fCert2, storage.data[1].FinalizationCertificate) + require.Equal(t, firstBlock.Bytes(), storage.Data[0].VerifiedBlock.Bytes()) + require.Equal(t, secondBlock.Bytes(), storage.Data[1].VerifiedBlock.Bytes()) + require.Equal(t, fCert1, storage.Data[0].FinalizationCertificate) + require.Equal(t, fCert2, storage.Data[1].FinalizationCertificate) } // TestRecoversFromMultipleNotarizations tests that the epoch can recover from a wal // with its last notarization record being from a less recent round. func TestRecoveryWithoutNotarization(t *testing.T) { l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - wal := wal.NewMemWAL(t) - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} ctx := context.Background() nodes := []NodeID{{1}, {2}, {3}, {4}} quorum := Quorum(len(nodes)) - sigAggregrator := &testSignatureAggregator{} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal, - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: sigAggregrator, - BlockDeserializer: &blockDeserializer{}, - QCDeserializer: &testQCDeserializer{t: t}, - } + conf, wal, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) protocolMetadata := ProtocolMetadata{Seq: 0, Round: 0, Epoch: 0} firstBlock, ok := bb.BuildBlock(ctx, protocolMetadata) @@ -584,7 +453,7 @@ func TestRecoveryWithoutNotarization(t *testing.T) { record := BlockRecord(firstBlock.BlockHeader(), firstBlock.Bytes()) wal.Append(record) - firstNotarizationRecord, err := newNotarizationRecord(l, sigAggregrator, firstBlock, nodes[0:quorum]) + firstNotarizationRecord, err := newNotarizationRecord(l, conf.SignatureAggregator, firstBlock, nodes[0:quorum]) require.NoError(t, err) wal.Append(firstNotarizationRecord) @@ -602,9 +471,9 @@ func TestRecoveryWithoutNotarization(t *testing.T) { record = BlockRecord(thirdBlock.BlockHeader(), thirdBlock.Bytes()) wal.Append(record) - fCert1, _ := newFinalizationRecord(t, l, sigAggregrator, firstBlock, nodes[0:quorum]) - fCert2, _ := newFinalizationRecord(t, l, sigAggregrator, secondBlock, nodes[0:quorum]) - fCer3, _ := newFinalizationRecord(t, l, sigAggregrator, thirdBlock, nodes[0:quorum]) + fCert1, _ := newFinalizationRecord(t, l, conf.SignatureAggregator, firstBlock, nodes[0:quorum]) + fCert2, _ := newFinalizationRecord(t, l, conf.SignatureAggregator, secondBlock, nodes[0:quorum]) + fCer3, _ := newFinalizationRecord(t, l, conf.SignatureAggregator, thirdBlock, nodes[0:quorum]) conf.Storage.Index(firstBlock, fCert1) conf.Storage.Index(secondBlock, fCert2) @@ -622,25 +491,12 @@ func TestRecoveryWithoutNotarization(t *testing.T) { } func TestEpochCorrectlyInitializesMetadataFromStorage(t *testing.T) { - l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} + nodes := []NodeID{{1}, {2}, {3}, {4}} - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal.NewMemWAL(t), - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - BlockDeserializer: &blockDeserializer{}, - QCDeserializer: &testQCDeserializer{t: t}, - } + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) - block := newTestBlock(ProtocolMetadata{Seq: 0, Round: 0, Epoch: 0}) + block := testutil.NewTestBlock(ProtocolMetadata{Seq: 0, Round: 0, Epoch: 0}) conf.Storage.Index(block, FinalizationCertificate{}) e, err := NewEpoch(conf) require.NoError(t, err) @@ -654,35 +510,23 @@ func TestEpochCorrectlyInitializesMetadataFromStorage(t *testing.T) { } func TestRecoveryAsLeader(t *testing.T) { - l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []NodeID{{1}, {2}, {3}, {4}} finalizedBlocks := createBlocks(t, nodes, bb, 4) - storage := newInMemStorage() + storage := testutil.NewInMemStorage() for _, finalizedBlock := range finalizedBlocks { storage.Index(finalizedBlock.VerifiedBlock, finalizedBlock.FCert) } - conf := EpochConfig{ - MaxProposalWait: DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[0], - Signer: &testSigner{}, - WAL: wal.NewMemWAL(t), - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - BlockDeserializer: &blockDeserializer{}, - QCDeserializer: &testQCDeserializer{t: t}, - } + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], testutil.NewNoopComm(nodes), bb) + conf.Storage = storage e, err := NewEpoch(conf) require.NoError(t, err) require.Equal(t, uint64(4), e.Storage.Height()) require.NoError(t, e.Start()) - <-bb.out + <-bb.Out // wait for the block to finish verifying time.Sleep(50 * time.Millisecond) diff --git a/replication_request_test.go b/replication_request_test.go index 50eca455..671eae05 100644 --- a/replication_request_test.go +++ b/replication_request_test.go @@ -3,6 +3,7 @@ package simplex_test import ( "bytes" "simplex" + "simplex/testutil" "testing" "github.com/stretchr/testify/require" @@ -10,10 +11,10 @@ import ( // TestReplicationRequestIndexedBlocks tests replication requests for indexed blocks. func TestReplicationeRequestIndexedBlocks(t *testing.T) { - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []simplex.NodeID{{1}, {2}, {3}, {4}} - comm := NewListenerComm(nodes) - conf := defaultTestNodeEpochConfig(t, nodes[0], comm, bb) + comm := testutil.NewListenerComm(nodes) + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) conf.ReplicationEnabled = true numBlocks := uint64(10) @@ -35,7 +36,7 @@ func TestReplicationeRequestIndexedBlocks(t *testing.T) { err = e.HandleMessage(req, nodes[1]) require.NoError(t, err) - msg := <-comm.in + msg := <-comm.In resp := msg.VerifiedReplicationResponse require.Nil(t, resp.LatestRound) @@ -55,7 +56,7 @@ func TestReplicationeRequestIndexedBlocks(t *testing.T) { err = e.HandleMessage(req, nodes[1]) require.NoError(t, err) - msg = <-comm.in + msg = <-comm.In resp = msg.VerifiedReplicationResponse require.Zero(t, len(resp.Data)) } @@ -63,10 +64,10 @@ func TestReplicationeRequestIndexedBlocks(t *testing.T) { // TestReplicationRequestNotarizations tests replication requests for notarized blocks. func TestReplicationRequestNotarizations(t *testing.T) { // generate 5 blocks & notarizations - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []simplex.NodeID{{1}, {2}, {3}, {4}} - comm := NewListenerComm(nodes) - conf := defaultTestNodeEpochConfig(t, nodes[0], comm, bb) + comm := testutil.NewListenerComm(nodes) + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) conf.ReplicationEnabled = true e, err := simplex.NewEpoch(conf) @@ -100,7 +101,7 @@ func TestReplicationRequestNotarizations(t *testing.T) { err = e.HandleMessage(req, nodes[1]) require.NoError(t, err) - msg := <-comm.in + msg := <-comm.In resp := msg.VerifiedReplicationResponse require.NoError(t, err) require.NotNil(t, resp) @@ -117,10 +118,10 @@ func TestReplicationRequestNotarizations(t *testing.T) { // TestReplicationRequestMixed ensures the replication response also includes empty notarizations func TestReplicationRequestMixed(t *testing.T) { // generate 5 blocks & notarizations - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []simplex.NodeID{{1}, {2}, {3}, {4}} - comm := NewListenerComm(nodes) - conf := defaultTestNodeEpochConfig(t, nodes[0], comm, bb) + comm := testutil.NewListenerComm(nodes) + conf, _, _ := testutil.DefaultTestNodeEpochConfig(t, nodes[0], comm, bb) conf.ReplicationEnabled = true e, err := simplex.NewEpoch(conf) @@ -138,7 +139,7 @@ func TestReplicationRequestMixed(t *testing.T) { e.HandleMessage(&simplex.Message{ EmptyNotarization: emptyNotarization, }, nodes[1]) - e.WAL.(*testWAL).assertNotarization(uint64(i)) + e.WAL.(*testutil.TestWAL).AssertNotarization(uint64(i)) rounds[i] = simplex.VerifiedQuorumRound{ EmptyNotarization: emptyNotarization, } @@ -168,7 +169,7 @@ func TestReplicationRequestMixed(t *testing.T) { err = e.HandleMessage(req, nodes[1]) require.NoError(t, err) - msg := <-comm.in + msg := <-comm.In resp := msg.VerifiedReplicationResponse require.Equal(t, *resp.LatestRound, rounds[numBlocks-1]) @@ -182,12 +183,12 @@ func TestReplicationRequestMixed(t *testing.T) { } func TestNilReplicationResponse(t *testing.T) { - bb := newTestControlledBlockBuilder(t) + bb := testutil.NewTestControlledBlockBuilder(t) nodes := []simplex.NodeID{{1}, {2}, {3}, {4}} - net := newInMemNetwork(t, nodes) + net := testutil.NewInMemNetwork(t, nodes) - normalNode0 := newSimplexNode(t, nodes[0], net, bb, nil) - normalNode0.start() + normalNode0 := testutil.NewSimplexNode(t, nodes[0], net, bb, nil) + normalNode0.Start() err := normalNode0.HandleMessage(&simplex.Message{ ReplicationResponse: &simplex.ReplicationResponse{ @@ -201,17 +202,17 @@ func TestNilReplicationResponse(t *testing.T) { // This replication response is malformeds since it must also include a notarization or // finalization certificate. func TestMalformedReplicationResponse(t *testing.T) { - bb := newTestControlledBlockBuilder(t) + bb := testutil.NewTestControlledBlockBuilder(t) nodes := []simplex.NodeID{{1}, {2}, {3}, {4}} - net := newInMemNetwork(t, nodes) + net := testutil.NewInMemNetwork(t, nodes) - normalNode0 := newSimplexNode(t, nodes[0], net, bb, nil) - normalNode0.start() + normalNode0 := testutil.NewSimplexNode(t, nodes[0], net, bb, nil) + normalNode0.Start() err := normalNode0.HandleMessage(&simplex.Message{ ReplicationResponse: &simplex.ReplicationResponse{ Data: []simplex.QuorumRound{{ - Block: &testBlock{}, + Block: &testutil.TestBlock{}, }}, }, }, nodes[1]) diff --git a/replication_test.go b/replication_test.go index 93d8c2fc..d661e593 100644 --- a/replication_test.go +++ b/replication_test.go @@ -9,7 +9,6 @@ import ( "fmt" "simplex" "simplex/testutil" - "simplex/wal" "testing" "time" @@ -38,36 +37,36 @@ func TestBasicReplication(t *testing.T) { } func testReplication(t *testing.T, startSeq uint64, nodes []simplex.NodeID) { - bb := newTestControlledBlockBuilder(t) - net := newInMemNetwork(t, nodes) + bb := testutil.NewTestControlledBlockBuilder(t) + net := testutil.NewInMemNetwork(t, nodes) // initiate a network with 4 nodes. one node is behind by 8 blocks - storageData := createBlocks(t, nodes, &bb.testBlockBuilder, startSeq) - testEpochConfig := &testNodeConfig{ - initialStorage: storageData, - replicationEnabled: true, - } - normalNode1 := newSimplexNode(t, nodes[0], net, bb, testEpochConfig) - normalNode2 := newSimplexNode(t, nodes[1], net, bb, testEpochConfig) - normalNode3 := newSimplexNode(t, nodes[2], net, bb, testEpochConfig) - laggingNode := newSimplexNode(t, nodes[3], net, bb, &testNodeConfig{ - replicationEnabled: true, + storageData := createBlocks(t, nodes, &bb.TestBlockBuilder, startSeq) + testEpochConfig := &testutil.TestNodeConfig{ + InitialStorage: storageData, + ReplicationEnabled: true, + } + normalNode1 := testutil.NewSimplexNode(t, nodes[0], net, bb, testEpochConfig) + normalNode2 := testutil.NewSimplexNode(t, nodes[1], net, bb, testEpochConfig) + normalNode3 := testutil.NewSimplexNode(t, nodes[2], net, bb, testEpochConfig) + laggingNode := testutil.NewSimplexNode(t, nodes[3], net, bb, &testutil.TestNodeConfig{ + ReplicationEnabled: true, }) - require.Equal(t, startSeq, normalNode1.storage.Height()) - require.Equal(t, startSeq, normalNode2.storage.Height()) - require.Equal(t, startSeq, normalNode3.storage.Height()) - require.Equal(t, uint64(0), laggingNode.storage.Height()) + require.Equal(t, startSeq, normalNode1.Storage.Height()) + require.Equal(t, startSeq, normalNode2.Storage.Height()) + require.Equal(t, startSeq, normalNode3.Storage.Height()) + require.Equal(t, uint64(0), laggingNode.Storage.Height()) - net.startInstances() - bb.triggerNewBlock() + net.StartInstances() + bb.TriggerNewBlock() // all blocks except the lagging node start at round 8, seq 8. // lagging node starts at round 0, seq 0. // this asserts that the lagging node catches up to the latest round for i := 0; i <= int(startSeq); i++ { - for _, n := range net.instances { - n.storage.waitForBlockCommit(uint64(startSeq)) + for _, n := range net.Instances() { + n.Storage.WaitForBlockCommit(uint64(startSeq)) } } } @@ -78,29 +77,29 @@ func testReplication(t *testing.T, startSeq uint64, nodes []simplex.NodeID) { func TestReplicationAdversarialNode(t *testing.T) { nodes := []simplex.NodeID{{1}, {2}, {3}, []byte("lagging")} quorum := simplex.Quorum(len(nodes)) - bb := newTestControlledBlockBuilder(t) - net := newInMemNetwork(t, nodes) + bb := testutil.NewTestControlledBlockBuilder(t) + net := testutil.NewInMemNetwork(t, nodes) - testEpochConfig := &testNodeConfig{ - replicationEnabled: true, + testEpochConfig := &testutil.TestNodeConfig{ + ReplicationEnabled: true, } // doubleBlockProposalNode will propose two blocks for the same round - doubleBlockProposalNode := newSimplexNode(t, nodes[0], net, bb, testEpochConfig) - normalNode2 := newSimplexNode(t, nodes[1], net, bb, testEpochConfig) - normalNode3 := newSimplexNode(t, nodes[2], net, bb, testEpochConfig) - laggingNode := newSimplexNode(t, nodes[3], net, bb, &testNodeConfig{ - replicationEnabled: true, + doubleBlockProposalNode := testutil.NewSimplexNode(t, nodes[0], net, bb, testEpochConfig) + normalNode2 := testutil.NewSimplexNode(t, nodes[1], net, bb, testEpochConfig) + normalNode3 := testutil.NewSimplexNode(t, nodes[2], net, bb, testEpochConfig) + laggingNode := testutil.NewSimplexNode(t, nodes[3], net, bb, &testutil.TestNodeConfig{ + ReplicationEnabled: true, }) - require.Equal(t, uint64(0), doubleBlockProposalNode.storage.Height()) - require.Equal(t, uint64(0), normalNode2.storage.Height()) - require.Equal(t, uint64(0), normalNode3.storage.Height()) - require.Equal(t, uint64(0), laggingNode.storage.Height()) + require.Equal(t, uint64(0), doubleBlockProposalNode.Storage.Height()) + require.Equal(t, uint64(0), normalNode2.Storage.Height()) + require.Equal(t, uint64(0), normalNode3.Storage.Height()) + require.Equal(t, uint64(0), laggingNode.Storage.Height()) - net.startInstances() - doubleBlock := newTestBlock(doubleBlockProposalNode.e.Metadata()) - doubleBlockVote, err := newTestVote(doubleBlock, doubleBlockProposalNode.e.ID) + net.StartInstances() + doubleBlock := testutil.NewTestBlock(doubleBlockProposalNode.E.Metadata()) + doubleBlockVote, err := newTestVote(doubleBlock, doubleBlockProposalNode.E.ID, doubleBlockProposalNode.E.Signer) require.NoError(t, err) msg := &simplex.Message{ BlockMessage: &simplex.BlockMessage{ @@ -109,33 +108,33 @@ func TestReplicationAdversarialNode(t *testing.T) { }, } - laggingNode.e.HandleMessage(msg, doubleBlockProposalNode.e.ID) - net.Disconnect(laggingNode.e.ID) + laggingNode.E.HandleMessage(msg, doubleBlockProposalNode.E.ID) + net.Disconnect(laggingNode.E.ID) blocks := []simplex.VerifiedBlock{} for i := range 2 { - bb.triggerNewBlock() - block := <-bb.out + bb.TriggerNewBlock() + block := <-bb.Out blocks = append(blocks, block) - for _, n := range net.instances[:3] { - commited := n.storage.waitForBlockCommit(uint64(i)) - require.Equal(t, block, commited.(*testBlock)) + for _, n := range net.Instances()[:3] { + commited := n.Storage.WaitForBlockCommit(uint64(i)) + require.Equal(t, block, commited.(*testutil.TestBlock)) } } // lagging node should not have commited the block - require.Equal(t, uint64(0), laggingNode.storage.Height()) - require.Equal(t, uint64(0), laggingNode.e.Metadata().Round) - net.Connect(laggingNode.e.ID) + require.Equal(t, uint64(0), laggingNode.Storage.Height()) + require.Equal(t, uint64(0), laggingNode.E.Metadata().Round) + net.Connect(laggingNode.E.ID) + fCert, _ := newFinalizationRecord(t, laggingNode.E.Logger, laggingNode.E.SignatureAggregator, blocks[1], nodes[:quorum]) - fCert, _ := newFinalizationRecord(t, laggingNode.e.Logger, laggingNode.e.SignatureAggregator, blocks[1], nodes[:quorum]) fCertMsg := &simplex.Message{ FinalizationCertificate: &fCert, } - laggingNode.e.HandleMessage(fCertMsg, doubleBlockProposalNode.e.ID) + laggingNode.E.HandleMessage(fCertMsg, doubleBlockProposalNode.E.ID) for i := range 2 { - lagBlock := laggingNode.storage.waitForBlockCommit(uint64(i)) + lagBlock := laggingNode.Storage.WaitForBlockCommit(uint64(i)) require.Equal(t, blocks[i], lagBlock) } } @@ -144,98 +143,97 @@ func TestReplicationAdversarialNode(t *testing.T) { // notarizations after lagging behind. func TestReplicationNotarizations(t *testing.T) { nodes := []simplex.NodeID{{1}, {2}, {3}, {4}} - bb := newTestControlledBlockBuilder(t) - net := newInMemNetwork(t, nodes) - - newNodeConfig := func(from simplex.NodeID) *testNodeConfig { - comm := newTestComm(from, net, allowAllMessages) - return &testNodeConfig{ - comm: comm, - replicationEnabled: true, + bb := testutil.NewTestControlledBlockBuilder(t) + net := testutil.NewInMemNetwork(t, nodes) + + newNodeConfig := func(from simplex.NodeID) *testutil.TestNodeConfig { + comm := testutil.NewTestComm(from, net, testutil.AllowAllMessages) + return &testutil.TestNodeConfig{ + Comm: comm, + ReplicationEnabled: true, } } - newSimplexNode(t, nodes[0], net, bb, newNodeConfig(nodes[0])) - newSimplexNode(t, nodes[1], net, bb, newNodeConfig(nodes[1])) - newSimplexNode(t, nodes[2], net, bb, newNodeConfig(nodes[2])) + testutil.NewSimplexNode(t, nodes[0], net, bb, newNodeConfig(nodes[0])) + testutil.NewSimplexNode(t, nodes[1], net, bb, newNodeConfig(nodes[1])) + testutil.NewSimplexNode(t, nodes[2], net, bb, newNodeConfig(nodes[2])) // we do not expect the lagging node to build any blocks - laggingBb := newTestControlledBlockBuilder(t) - laggingNode := newSimplexNode(t, nodes[3], net, laggingBb, newNodeConfig(nodes[3])) + laggingBb := testutil.NewTestControlledBlockBuilder(t) + laggingNode := testutil.NewSimplexNode(t, nodes[3], net, laggingBb, newNodeConfig(nodes[3])) - for _, n := range net.instances { - require.Equal(t, uint64(0), n.storage.Height()) + for _, n := range net.Instances() { + require.Equal(t, uint64(0), n.Storage.Height()) } epochTimes := make([]time.Time, 0, len(nodes)) - for _, n := range net.instances { - epochTimes = append(epochTimes, n.e.StartTime) + for _, n := range net.Instances() { + epochTimes = append(epochTimes, n.E.StartTime) } - net.startInstances() + net.StartInstances() - net.Disconnect(laggingNode.e.ID) + net.Disconnect(laggingNode.E.ID) numNotarizations := 9 missedSeqs := uint64(0) blocks := []simplex.VerifiedBlock{} // finalization for the first block - bb.triggerNewBlock() - block := <-bb.out + bb.TriggerNewBlock() + block := <-bb.Out blocks = append(blocks, block) - for _, n := range net.instances { - if n.e.ID.Equals(laggingNode.e.ID) { + for _, n := range net.Instances() { + if n.E.ID.Equals(laggingNode.E.ID) { continue } - n.storage.waitForBlockCommit(0) + n.Storage.WaitForBlockCommit(0) } - net.setAllNodesMessageFilter(denyFinalizationMessages) + net.SetAllNodesMessageFilter(denyFinalizationMessages) // normal nodes continue to make progress for i := uint64(1); i < uint64(numNotarizations); i++ { - emptyRound := bytes.Equal(simplex.LeaderForRound(nodes, i), laggingNode.e.ID) + emptyRound := bytes.Equal(simplex.LeaderForRound(nodes, i), laggingNode.E.ID) if emptyRound { - advanceWithoutLeader(t, net, bb, epochTimes, i, laggingNode.e.ID) + testutil.AdvanceWithoutLeader(t, net, bb, epochTimes, i, laggingNode.E.ID) missedSeqs++ } else { - bb.triggerNewBlock() - block := <-bb.out + bb.TriggerNewBlock() + block := <-bb.Out blocks = append(blocks, block) - for _, n := range net.instances { - if n.e.ID.Equals(laggingNode.e.ID) { + for _, n := range net.Instances() { + if n.E.ID.Equals(laggingNode.E.ID) { continue } - n.wal.assertNotarization(i) + n.Wal.AssertNotarization(i) } } } - for _, n := range net.instances { - if n.e.ID.Equals(laggingNode.e.ID) { - require.Equal(t, uint64(0), n.storage.Height()) - require.Equal(t, uint64(0), n.e.Metadata().Round) + for _, n := range net.Instances() { + if n.E.ID.Equals(laggingNode.E.ID) { + require.Equal(t, uint64(0), n.Storage.Height()) + require.Equal(t, uint64(0), n.E.Metadata().Round) continue } - // assert metadata - require.Equal(t, uint64(numNotarizations), n.e.Metadata().Round) - require.Equal(t, uint64(1), n.e.Storage.Height()) + require.Equal(t, uint64(numNotarizations), n.E.Metadata().Round) + require.Equal(t, uint64(1), n.E.Storage.Height()) } - net.setAllNodesMessageFilter(allowAllMessages) - net.Connect(laggingNode.e.ID) - bb.triggerNewBlock() + net.SetAllNodesMessageFilter(testutil.AllowAllMessages) + net.Connect(laggingNode.E.ID) + bb.TriggerNewBlock() // lagging node should replicate the first finalized block and subsequent notarizations - laggingNode.storage.waitForBlockCommit(0) + laggingNode.Storage.WaitForBlockCommit(0) for i := 1; i < numNotarizations+1; i++ { - for _, n := range net.instances { + for _, n := range net.Instances() { // lagging node wont have a notarization record if it was the leader leader := simplex.LeaderForRound(nodes, uint64(i)) - if n.e.ID.Equals(leader) && n.e.ID.Equals(nodes[3]) { + if n.E.ID.Equals(leader) && n.E.ID.Equals(nodes[3]) { continue } - n.wal.assertNotarization(uint64(i)) + n.Wal.AssertNotarization(uint64(i)) } } @@ -260,130 +258,153 @@ func TestReplicationEmptyNotarizations(t *testing.T) { } } +// denyFinalizationMessages blocks any messages that would cause nodes in +// a network to index a block in storage. +func denyFinalizationMessages(msg *simplex.Message, destination simplex.NodeID) bool { + if msg.Finalization != nil { + return false + } + if msg.FinalizationCertificate != nil { + return false + } + + return true +} + +func onlyAllowEmptyRoundMessages(msg *simplex.Message, _ simplex.NodeID) bool { + if msg.EmptyNotarization != nil { + return true + } + if msg.EmptyVoteMessage != nil { + return true + } + return false +} + func testReplicationEmptyNotarizations(t *testing.T, nodes []simplex.NodeID, endRound uint64) { - bb := newTestControlledBlockBuilder(t) - laggingBb := newTestControlledBlockBuilder(t) - net := newInMemNetwork(t, nodes) - newNodeConfig := func(from simplex.NodeID) *testNodeConfig { - comm := newTestComm(from, net, allowAllMessages) - return &testNodeConfig{ - comm: comm, - replicationEnabled: true, + bb := testutil.NewTestControlledBlockBuilder(t) + laggingBb := testutil.NewTestControlledBlockBuilder(t) + net := testutil.NewInMemNetwork(t, nodes) + newNodeConfig := func(from simplex.NodeID) *testutil.TestNodeConfig { + comm := testutil.NewTestComm(from, net, testutil.AllowAllMessages) + return &testutil.TestNodeConfig{ + Comm: comm, + ReplicationEnabled: true, } } startTimes := make([]time.Time, 0, len(nodes)) - newSimplexNode(t, nodes[0], net, bb, newNodeConfig(nodes[0])) - newSimplexNode(t, nodes[1], net, bb, newNodeConfig(nodes[1])) - newSimplexNode(t, nodes[2], net, bb, newNodeConfig(nodes[2])) - newSimplexNode(t, nodes[3], net, bb, newNodeConfig(nodes[3])) - newSimplexNode(t, nodes[4], net, bb, newNodeConfig(nodes[4])) - laggingNode := newSimplexNode(t, nodes[5], net, laggingBb, newNodeConfig(nodes[5])) + testutil.NewSimplexNode(t, nodes[0], net, bb, newNodeConfig(nodes[0])) + testutil.NewSimplexNode(t, nodes[1], net, bb, newNodeConfig(nodes[1])) + testutil.NewSimplexNode(t, nodes[2], net, bb, newNodeConfig(nodes[2])) + testutil.NewSimplexNode(t, nodes[3], net, bb, newNodeConfig(nodes[3])) + testutil.NewSimplexNode(t, nodes[4], net, bb, newNodeConfig(nodes[4])) + laggingNode := testutil.NewSimplexNode(t, nodes[5], net, laggingBb, newNodeConfig(nodes[5])) - for _, n := range net.instances { - require.Equal(t, uint64(0), n.storage.Height()) - startTimes = append(startTimes, n.e.StartTime) + for _, n := range net.Instances() { + require.Equal(t, uint64(0), n.Storage.Height()) + startTimes = append(startTimes, n.E.StartTime) } - net.startInstances() + net.StartInstances() - net.Disconnect(laggingNode.e.ID) + net.Disconnect(laggingNode.E.ID) - bb.triggerNewBlock() - for _, n := range net.instances { - if n.e.ID.Equals(laggingNode.e.ID) { + bb.TriggerNewBlock() + for _, n := range net.Instances() { + if n.E.ID.Equals(laggingNode.E.ID) { continue } - n.storage.waitForBlockCommit(0) + n.Storage.WaitForBlockCommit(0) } - net.setAllNodesMessageFilter(onlyAllowEmptyRoundMessages) + net.SetAllNodesMessageFilter(onlyAllowEmptyRoundMessages) // normal nodes continue to make progress for i := uint64(1); i < endRound; i++ { leader := simplex.LeaderForRound(nodes, i) - if !leader.Equals(laggingNode.e.ID) { - bb.triggerNewBlock() + if !leader.Equals(laggingNode.E.ID) { + bb.TriggerNewBlock() } - advanceWithoutLeader(t, net, bb, startTimes, i, laggingNode.e.ID) + testutil.AdvanceWithoutLeader(t, net, bb, startTimes, i, laggingNode.E.ID) } - for _, n := range net.instances { - if n.e.ID.Equals(laggingNode.e.ID) { - require.Equal(t, uint64(0), n.storage.Height()) - require.Equal(t, uint64(0), n.e.Metadata().Round) + for _, n := range net.Instances() { + if n.E.ID.Equals(laggingNode.E.ID) { + require.Equal(t, uint64(0), n.Storage.Height()) + require.Equal(t, uint64(0), n.E.Metadata().Round) continue } // assert metadata - require.Equal(t, uint64(endRound), n.e.Metadata().Round) - require.Equal(t, uint64(1), n.e.Metadata().Seq) - require.Equal(t, uint64(1), n.e.Storage.Height()) + require.Equal(t, uint64(endRound), n.E.Metadata().Round) + require.Equal(t, uint64(1), n.E.Metadata().Seq) + require.Equal(t, uint64(1), n.E.Storage.Height()) } - net.setAllNodesMessageFilter(allowAllMessages) - net.Connect(laggingNode.e.ID) - bb.triggerNewBlock() - for _, n := range net.instances { - n.storage.waitForBlockCommit(1) + net.SetAllNodesMessageFilter(testutil.AllowAllMessages) + net.Connect(laggingNode.E.ID) + bb.TriggerNewBlock() + for _, n := range net.Instances() { + n.Storage.WaitForBlockCommit(1) } - require.Equal(t, uint64(2), laggingNode.storage.Height()) - require.Equal(t, uint64(endRound+1), laggingNode.e.Metadata().Round) - require.Equal(t, uint64(2), laggingNode.e.Metadata().Seq) + require.Equal(t, uint64(2), laggingNode.Storage.Height()) + require.Equal(t, uint64(endRound+1), laggingNode.E.Metadata().Round) + require.Equal(t, uint64(2), laggingNode.E.Metadata().Seq) } // TestReplicationStartsBeforeCurrentRound tests the replication process of a node that // starts replicating in the middle of the current round. func TestReplicationStartsBeforeCurrentRound(t *testing.T) { - bb := newTestControlledBlockBuilder(t) + bb := testutil.NewTestControlledBlockBuilder(t) nodes := []simplex.NodeID{{1}, {2}, {3}, []byte("lagging")} quorum := simplex.Quorum(len(nodes)) - net := newInMemNetwork(t, nodes) + net := testutil.NewInMemNetwork(t, nodes) startSeq := uint64(simplex.DefaultMaxRoundWindow + 3) - storageData := createBlocks(t, nodes, &bb.testBlockBuilder, startSeq) - testEpochConfig := &testNodeConfig{ - initialStorage: storageData, - replicationEnabled: true, - } - normalNode1 := newSimplexNode(t, nodes[0], net, bb, testEpochConfig) - normalNode2 := newSimplexNode(t, nodes[1], net, bb, testEpochConfig) - normalNode3 := newSimplexNode(t, nodes[2], net, bb, testEpochConfig) - laggingNode := newSimplexNode(t, nodes[3], net, bb, &testNodeConfig{ - replicationEnabled: true, + storageData := createBlocks(t, nodes, &bb.TestBlockBuilder, startSeq) + testEpochConfig := &testutil.TestNodeConfig{ + InitialStorage: storageData, + ReplicationEnabled: true, + } + normalNode1 := testutil.NewSimplexNode(t, nodes[0], net, bb, testEpochConfig) + normalNode2 := testutil.NewSimplexNode(t, nodes[1], net, bb, testEpochConfig) + normalNode3 := testutil.NewSimplexNode(t, nodes[2], net, bb, testEpochConfig) + laggingNode := testutil.NewSimplexNode(t, nodes[3], net, bb, &testutil.TestNodeConfig{ + ReplicationEnabled: true, }) firstBlock := storageData[0].VerifiedBlock record := simplex.BlockRecord(firstBlock.BlockHeader(), firstBlock.Bytes()) - laggingNode.wal.Append(record) + laggingNode.Wal.Append(record) - firstNotarizationRecord, err := newNotarizationRecord(laggingNode.e.Logger, laggingNode.e.SignatureAggregator, firstBlock, nodes[0:quorum]) + firstNotarizationRecord, err := newNotarizationRecord(laggingNode.E.Logger, laggingNode.E.SignatureAggregator, firstBlock, nodes[0:quorum]) require.NoError(t, err) - laggingNode.wal.Append(firstNotarizationRecord) + laggingNode.Wal.Append(firstNotarizationRecord) secondBlock := storageData[1].VerifiedBlock record = simplex.BlockRecord(secondBlock.BlockHeader(), secondBlock.Bytes()) - laggingNode.wal.Append(record) + laggingNode.Wal.Append(record) - secondNotarizationRecord, err := newNotarizationRecord(laggingNode.e.Logger, laggingNode.e.SignatureAggregator, secondBlock, nodes[0:quorum]) + secondNotarizationRecord, err := newNotarizationRecord(laggingNode.E.Logger, laggingNode.E.SignatureAggregator, secondBlock, nodes[0:quorum]) require.NoError(t, err) - laggingNode.wal.Append(secondNotarizationRecord) + laggingNode.Wal.Append(secondNotarizationRecord) - require.Equal(t, startSeq, normalNode1.storage.Height()) - require.Equal(t, startSeq, normalNode2.storage.Height()) - require.Equal(t, startSeq, normalNode3.storage.Height()) - require.Equal(t, uint64(0), laggingNode.storage.Height()) + require.Equal(t, startSeq, normalNode1.Storage.Height()) + require.Equal(t, startSeq, normalNode2.Storage.Height()) + require.Equal(t, startSeq, normalNode3.Storage.Height()) + require.Equal(t, uint64(0), laggingNode.Storage.Height()) - net.startInstances() + net.StartInstances() - laggingNodeMd := laggingNode.e.Metadata() + laggingNodeMd := laggingNode.E.Metadata() require.Equal(t, uint64(2), laggingNodeMd.Round) - bb.triggerNewBlock() + bb.TriggerNewBlock() for i := 0; i <= int(startSeq); i++ { - for _, n := range net.instances { - n.storage.waitForBlockCommit(uint64(startSeq)) + for _, n := range net.Instances() { + n.Storage.WaitForBlockCommit(uint64(startSeq)) } } } @@ -391,24 +412,12 @@ func TestReplicationStartsBeforeCurrentRound(t *testing.T) { func TestReplicationFutureFinalizationCertificate(t *testing.T) { // send a block, then simultaneously send a finalization certificate for the block l := testutil.MakeLogger(t, 1) - bb := &testBlockBuilder{out: make(chan *testBlock, 1)} - storage := newInMemStorage() + bb := &testutil.TestBlockBuilder{Out: make(chan *testutil.TestBlock, 1)} nodes := []simplex.NodeID{{1}, {2}, {3}, {4}} quorum := simplex.Quorum(len(nodes)) - signatureAggregator := &testSignatureAggregator{} - conf := simplex.EpochConfig{ - MaxProposalWait: simplex.DefaultMaxProposalWaitTime, - Logger: l, - ID: nodes[1], - Signer: &testSigner{}, - WAL: wal.NewMemWAL(t), - Verifier: &testVerifier{}, - Storage: storage, - Comm: noopComm(nodes), - BlockBuilder: bb, - SignatureAggregator: signatureAggregator, - } + signatureAggregator := &testutil.TestSignatureAggregator{} + conf, _, storage := testutil.DefaultTestNodeEpochConfig(t, nodes[1], testutil.NoopComm(nodes), bb) e, err := simplex.NewEpoch(conf) require.NoError(t, err) @@ -420,10 +429,10 @@ func TestReplicationFutureFinalizationCertificate(t *testing.T) { require.True(t, ok) require.Equal(t, md.Round, md.Seq) - block := <-bb.out - block.verificationDelay = make(chan struct{}) // add a delay to the block verification + block := <-bb.Out + block.VerificationDelay = make(chan struct{}) // add a delay to the block verification - vote, err := newTestVote(block, nodes[0]) + vote, err := newTestVote(block, nodes[0], conf.Signer) require.NoError(t, err) err = e.HandleMessage(&simplex.Message{ @@ -441,9 +450,9 @@ func TestReplicationFutureFinalizationCertificate(t *testing.T) { }, nodes[0]) require.NoError(t, err) - block.verificationDelay <- struct{}{} // unblock the block verification + block.VerificationDelay <- struct{}{} // unblock the block verification - storedBlock := storage.waitForBlockCommit(0) + storedBlock := storage.WaitForBlockCommit(0) require.Equal(t, uint64(1), storage.Height()) require.Equal(t, block, storedBlock) } @@ -476,43 +485,43 @@ func TestReplicationAfterNodeDisconnects(t *testing.T) { } func testReplicationAfterNodeDisconnects(t *testing.T, nodes []simplex.NodeID, startDisconnect, endDisconnect uint64) { - bb := newTestControlledBlockBuilder(t) - laggingBb := newTestControlledBlockBuilder(t) - net := newInMemNetwork(t, nodes) - testConfig := &testNodeConfig{ - replicationEnabled: true, - } - normalNode1 := newSimplexNode(t, nodes[0], net, bb, testConfig) - normalNode2 := newSimplexNode(t, nodes[1], net, bb, testConfig) - normalNode3 := newSimplexNode(t, nodes[2], net, bb, testConfig) - laggingNode := newSimplexNode(t, nodes[3], net, laggingBb, testConfig) - - require.Equal(t, uint64(0), normalNode1.storage.Height()) - require.Equal(t, uint64(0), normalNode2.storage.Height()) - require.Equal(t, uint64(0), normalNode3.storage.Height()) - require.Equal(t, uint64(0), laggingNode.storage.Height()) + bb := testutil.NewTestControlledBlockBuilder(t) + laggingBb := testutil.NewTestControlledBlockBuilder(t) + net := testutil.NewInMemNetwork(t, nodes) + testConfig := &testutil.TestNodeConfig{ + ReplicationEnabled: true, + } + normalNode1 := testutil.NewSimplexNode(t, nodes[0], net, bb, testConfig) + normalNode2 := testutil.NewSimplexNode(t, nodes[1], net, bb, testConfig) + normalNode3 := testutil.NewSimplexNode(t, nodes[2], net, bb, testConfig) + laggingNode := testutil.NewSimplexNode(t, nodes[3], net, laggingBb, testConfig) + + require.Equal(t, uint64(0), normalNode1.Storage.Height()) + require.Equal(t, uint64(0), normalNode2.Storage.Height()) + require.Equal(t, uint64(0), normalNode3.Storage.Height()) + require.Equal(t, uint64(0), laggingNode.Storage.Height()) epochTimes := make([]time.Time, 0, 4) - for _, n := range net.instances { - epochTimes = append(epochTimes, n.e.StartTime) + for _, n := range net.Instances() { + epochTimes = append(epochTimes, n.E.StartTime) } - net.startInstances() + net.StartInstances() for i := uint64(0); i < startDisconnect; i++ { if bytes.Equal(simplex.LeaderForRound(nodes, i), nodes[3]) { - laggingBb.triggerNewBlock() + laggingBb.TriggerNewBlock() } else { - bb.triggerNewBlock() + bb.TriggerNewBlock() } - for _, n := range net.instances { - n.storage.waitForBlockCommit(i) + for _, n := range net.Instances() { + n.Storage.WaitForBlockCommit(i) } } // all nodes have commited `startDisconnect` blocks - for _, n := range net.instances { - require.Equal(t, startDisconnect, n.storage.Height()) + for _, n := range net.Instances() { + require.Equal(t, startDisconnect, n.Storage.Height()) } // lagging node disconnects @@ -520,7 +529,7 @@ func testReplicationAfterNodeDisconnects(t *testing.T, nodes []simplex.NodeID, s isLaggingNodeLeader := bytes.Equal(simplex.LeaderForRound(nodes, startDisconnect), nodes[3]) if isLaggingNodeLeader { - laggingBb.triggerNewBlock() + laggingBb.TriggerNewBlock() } missedSeqs := uint64(0) @@ -528,42 +537,42 @@ func testReplicationAfterNodeDisconnects(t *testing.T, nodes []simplex.NodeID, s for i := startDisconnect; i < endDisconnect; i++ { emptyRound := bytes.Equal(simplex.LeaderForRound(nodes, i), nodes[3]) if emptyRound { - advanceWithoutLeader(t, net, bb, epochTimes, i, laggingNode.e.ID) + testutil.AdvanceWithoutLeader(t, net, bb, epochTimes, i, laggingNode.E.ID) missedSeqs++ } else { - bb.triggerNewBlock() - for _, n := range net.instances[:3] { - n.storage.waitForBlockCommit(i - missedSeqs) + bb.TriggerNewBlock() + for _, n := range net.Instances()[:3] { + n.Storage.WaitForBlockCommit(i - missedSeqs) } } } // all nodes excpet for lagging node have progressed and commited [endDisconnect - missedSeqs] blocks - for _, n := range net.instances[:3] { - require.Equal(t, endDisconnect-missedSeqs, n.storage.Height()) + for _, n := range net.Instances()[:3] { + require.Equal(t, endDisconnect-missedSeqs, n.Storage.Height()) } - require.Equal(t, startDisconnect, laggingNode.storage.Height()) - require.Equal(t, startDisconnect, laggingNode.e.Metadata().Round) + require.Equal(t, startDisconnect, laggingNode.Storage.Height()) + require.Equal(t, startDisconnect, laggingNode.E.Metadata().Round) // lagging node reconnects net.Connect(nodes[3]) - bb.triggerNewBlock() - for _, n := range net.instances { - n.storage.waitForBlockCommit(endDisconnect - missedSeqs) + bb.TriggerNewBlock() + for _, n := range net.Instances() { + n.Storage.WaitForBlockCommit(endDisconnect - missedSeqs) } - for _, n := range net.instances { - require.Equal(t, endDisconnect-missedSeqs, n.storage.Height()-1) - require.Equal(t, endDisconnect+1, n.e.Metadata().Round) + for _, n := range net.Instances() { + require.Equal(t, endDisconnect-missedSeqs, n.Storage.Height()-1) + require.Equal(t, endDisconnect+1, n.E.Metadata().Round) } // the lagging node should build a block when triggered if its the leader if bytes.Equal(simplex.LeaderForRound(nodes, endDisconnect+1), nodes[3]) { - laggingBb.triggerNewBlock() + laggingBb.TriggerNewBlock() } else { - bb.triggerNewBlock() + bb.TriggerNewBlock() } - for _, n := range net.instances { - n.storage.waitForBlockCommit(endDisconnect - missedSeqs + 1) + for _, n := range net.Instances() { + n.Storage.WaitForBlockCommit(endDisconnect - missedSeqs + 1) } } @@ -598,101 +607,46 @@ func TestReplicationNotarizationWithoutFinalizations(t *testing.T) { // TestReplicationNotarizationWithoutFinalizations tests that a lagging node will replicate // blocks that have notarizations but no finalizations. func testReplicationNotarizationWithoutFinalizations(t *testing.T, numBlocks uint64, nodes []simplex.NodeID) { - bb := newTestControlledBlockBuilder(t) - net := newInMemNetwork(t, nodes) - - nodeConfig := func(from simplex.NodeID) *testNodeConfig { - comm := newTestComm(from, net, onlyAllowBlockProposalsAndNotarizations) - return &testNodeConfig{ - comm: comm, - replicationEnabled: true, + bb := testutil.NewTestControlledBlockBuilder(t) + net := testutil.NewInMemNetwork(t, nodes) + + nodeConfig := func(from simplex.NodeID) *testutil.TestNodeConfig { + comm := testutil.NewTestComm(from, net, onlyAllowBlockProposalsAndNotarizations) + return &testutil.TestNodeConfig{ + Comm: comm, + ReplicationEnabled: true, } } - newSimplexNode(t, nodes[0], net, bb, nodeConfig(nodes[0])) - newSimplexNode(t, nodes[1], net, bb, nodeConfig(nodes[1])) - newSimplexNode(t, nodes[2], net, bb, nodeConfig(nodes[2])) + testutil.NewSimplexNode(t, nodes[0], net, bb, nodeConfig(nodes[0])) + testutil.NewSimplexNode(t, nodes[1], net, bb, nodeConfig(nodes[1])) + testutil.NewSimplexNode(t, nodes[2], net, bb, nodeConfig(nodes[2])) - laggingNode := newSimplexNode(t, nodes[3], net, bb, nodeConfig(nodes[3])) + laggingNode := testutil.NewSimplexNode(t, nodes[3], net, bb, nodeConfig(nodes[3])) - for _, n := range net.instances { - require.Equal(t, uint64(0), n.storage.Height()) + for _, n := range net.Instances() { + require.Equal(t, uint64(0), n.Storage.Height()) } - net.startInstances() + net.StartInstances() // normal nodes continue to make progress for i := uint64(0); i < uint64(numBlocks); i++ { - bb.triggerNewBlock() - for _, n := range net.instances[:3] { - n.storage.waitForBlockCommit(uint64(i)) - } - - } - - laggingNode.wal.assertNotarization(numBlocks - 1) - require.Equal(t, uint64(0), laggingNode.storage.Height()) - require.Equal(t, uint64(numBlocks), laggingNode.e.Metadata().Round) - - net.setAllNodesMessageFilter(allowAllMessages) - bb.triggerNewBlock() - for _, n := range net.instances { - n.storage.waitForBlockCommit(uint64(numBlocks)) - } -} - -func waitToEnterRound(t *testing.T, n *testNode, round uint64) { - timeout := time.NewTimer(time.Minute) - defer timeout.Stop() - - for { - if n.e.Metadata().Round >= round { - return + bb.TriggerNewBlock() + for _, n := range net.Instances()[:3] { + n.Storage.WaitForBlockCommit(uint64(i)) } - select { - case <-time.After(time.Millisecond * 10): - continue - case <-timeout.C: - require.Fail(t, "timed out waiting for event") - } } -} -func advanceWithoutLeader(t *testing.T, net *inMemNetwork, bb *testControlledBlockBuilder, epochTimes []time.Time, round uint64, laggingNodeId simplex.NodeID) { - // we need to ensure all blocks are waiting for the channel before proceeding - // otherwise, we may send to a channel that is not ready to receive - for _, n := range net.instances { - if laggingNodeId.Equals(n.e.ID) { - continue - } + laggingNode.Wal.AssertNotarization(numBlocks - 1) + require.Equal(t, uint64(0), laggingNode.Storage.Height()) + require.Equal(t, uint64(numBlocks), laggingNode.E.Metadata().Round) - waitToEnterRound(t, n, round) - } - - for _, n := range net.instances { - leader := n.e.ID.Equals(simplex.LeaderForRound(net.nodes, n.e.Metadata().Round)) - if leader || laggingNodeId.Equals(n.e.ID) { - continue - } - bb.blockShouldBeBuilt <- struct{}{} - } - - for i, n := range net.instances { - // the leader will not write an empty vote to the wal - // because it cannot both propose a block & send an empty vote in the same round - leader := n.e.ID.Equals(simplex.LeaderForRound(net.nodes, n.e.Metadata().Round)) - if leader || laggingNodeId.Equals(n.e.ID) { - continue - } - waitForBlockProposerTimeout(t, n.e, &epochTimes[i], round) - } - - for _, n := range net.instances { - if laggingNodeId.Equals(n.e.ID) { - continue - } - n.wal.assertNotarization(round) + net.SetAllNodesMessageFilter(testutil.AllowAllMessages) + bb.TriggerNewBlock() + for _, n := range net.Instances() { + n.Storage.WaitForBlockCommit(uint64(numBlocks)) } } @@ -711,7 +665,7 @@ func createBlocks(t *testing.T, nodes []simplex.NodeID, bb simplex.BlockBuilder, block, ok := bb.BuildBlock(ctx, protocolMetadata) require.True(t, ok) prev = block.BlockHeader().Digest - fCert, _ := newFinalizationRecord(t, logger, &testSignatureAggregator{}, block, nodes) + fCert, _ := newFinalizationRecord(t, logger, &testutil.TestSignatureAggregator{}, block, nodes) data = append(data, simplex.VerifiedFinalizedBlock{ VerifiedBlock: block, FCert: fCert, diff --git a/sched.go b/sched.go index 5f81b7ed..edcb2ded 100644 --- a/sched.go +++ b/sched.go @@ -9,18 +9,18 @@ import ( "go.uber.org/zap" ) -type scheduler struct { +type Scheduler struct { logger Logger lock sync.Mutex signal sync.Cond pending dependencies - ready []task + ready []Task close bool } -func NewScheduler(logger Logger) *scheduler { - var as scheduler - as.pending = newDependencies() +func NewScheduler(logger Logger) *Scheduler { + var as Scheduler + as.pending = NewDependencies() as.signal = sync.Cond{L: &as.lock} as.logger = logger @@ -29,14 +29,14 @@ func NewScheduler(logger Logger) *scheduler { return &as } -func (as *scheduler) Size() int { +func (as *Scheduler) Size() int { as.lock.Lock() defer as.lock.Unlock() return as.pending.Size() + len(as.ready) } -func (as *scheduler) Close() { +func (as *Scheduler) Close() { as.lock.Lock() defer as.lock.Unlock() @@ -84,7 +84,7 @@ and A is not scheduled yet because scheduling of tasks is done under a lock. The */ -func (as *scheduler) run() { +func (as *Scheduler) run() { as.lock.Lock() defer as.lock.Unlock() @@ -98,13 +98,13 @@ func (as *scheduler) run() { } taskToRun := as.ready[0] - as.ready[0] = task{} // Cleanup any object references reachable from the closure of the task + as.ready[0] = Task{} // Cleanup any object references reachable from the closure of the task as.ready = as.ready[1:] // (4) numReadyTasks := len(as.ready) as.lock.Unlock() // (5) as.logger.Debug("Running task", zap.Int("remaining ready tasks", numReadyTasks)) - id := taskToRun.f() // (6) + id := taskToRun.F() // (6) as.logger.Debug("Task finished execution", zap.Stringer("taskID", id)) as.lock.Lock() @@ -114,7 +114,7 @@ func (as *scheduler) run() { } } -func (as *scheduler) Schedule(f func() Digest, prev Digest, ready bool) { +func (as *Scheduler) Schedule(f func() Digest, prev Digest, ready bool) { as.lock.Lock() defer as.lock.Unlock() @@ -122,9 +122,9 @@ func (as *scheduler) Schedule(f func() Digest, prev Digest, ready bool) { return } - task := task{ - f: f, - parent: prev, + task := Task{ + F: f, + Parent: prev, } if !ready { @@ -140,18 +140,18 @@ func (as *scheduler) Schedule(f func() Digest, prev Digest, ready bool) { as.signal.Broadcast() // (11) } -type task struct { - f func() Digest - parent Digest +type Task struct { + F func() Digest + Parent Digest } type dependencies struct { - dependsOn map[Digest][]task // values depend on key. + dependsOn map[Digest][]Task // values depend on key. } -func newDependencies() dependencies { +func NewDependencies() dependencies { return dependencies{ - dependsOn: make(map[Digest][]task), + dependsOn: make(map[Digest][]Task), } } @@ -159,12 +159,12 @@ func (d *dependencies) Size() int { return len(d.dependsOn) } -func (d *dependencies) Insert(t task) { - dependency := t.parent +func (d *dependencies) Insert(t Task) { + dependency := t.Parent d.dependsOn[dependency] = append(d.dependsOn[dependency], t) } -func (t *dependencies) Remove(id Digest) []task { +func (t *dependencies) Remove(id Digest) []Task { dependents := t.dependsOn[id] delete(t.dependsOn, id) return dependents diff --git a/sched_test.go b/sched_test.go index cefc90fc..dc036e52 100644 --- a/sched_test.go +++ b/sched_test.go @@ -1,11 +1,12 @@ // Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. // See the file LICENSE for licensing terms. -package simplex +package simplex_test import ( "crypto/rand" rand2 "math/rand" + . "simplex" "simplex/testutil" "sync" "testing" @@ -15,12 +16,12 @@ import ( ) func TestDependencyTree(t *testing.T) { - dt := newDependencies() + dt := NewDependencies() for i := 0; i < 5; i++ { - dt.Insert(task{f: func() Digest { + dt.Insert(Task{F: func() Digest { return Digest{uint8(i + 1)} - }, parent: Digest{uint8(i)}}) + }, Parent: Digest{uint8(i)}}) } require.Equal(t, 5, dt.Size()) @@ -28,7 +29,7 @@ func TestDependencyTree(t *testing.T) { for i := 0; i < 5; i++ { j := dt.Remove(Digest{uint8(i)}) require.Len(t, j, 1) - require.Equal(t, Digest{uint8(i + 1)}, j[0].f()) + require.Equal(t, Digest{uint8(i + 1)}, j[0].F()) } } @@ -106,7 +107,7 @@ func TestAsyncScheduler(t *testing.T) { }) } -func scheduleTask(lock *sync.Mutex, finished map[Digest]struct{}, dependency Digest, id Digest, wg *sync.WaitGroup, as *scheduler, i int) func() { +func scheduleTask(lock *sync.Mutex, finished map[Digest]struct{}, dependency Digest, id Digest, wg *sync.WaitGroup, as *Scheduler, i int) func() { var dep Digest copy(dep[:], dependency[:]) diff --git a/testutil/block_builder.go b/testutil/block_builder.go new file mode 100644 index 00000000..e7a9d8de --- /dev/null +++ b/testutil/block_builder.go @@ -0,0 +1,136 @@ +package testutil + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "simplex" + "testing" +) + +type TestBlockBuilder struct { + Out chan *TestBlock + In chan *TestBlock + BlockShouldBeBuilt chan struct{} +} + +// BuildBlock builds a new testblock and sends it to the BlockBuilder channel +func (t *TestBlockBuilder) BuildBlock(_ context.Context, metadata simplex.ProtocolMetadata) (simplex.VerifiedBlock, bool) { + if len(t.In) > 0 { + block := <-t.In + return block, true + } + + tb := NewTestBlock(metadata) + + select { + case t.Out <- tb: + default: + } + + return tb, true +} + +func (t *TestBlockBuilder) IncomingBlock(ctx context.Context) { + select { + case <-t.BlockShouldBeBuilt: + case <-ctx.Done(): + } +} + +type TestBlock struct { + data []byte + metadata simplex.ProtocolMetadata + OnVerify func() + Digest [32]byte + VerificationDelay chan struct{} +} + +func (tb *TestBlock) Verify(context.Context) (simplex.VerifiedBlock, error) { + defer func() { + if tb.OnVerify != nil { + tb.OnVerify() + } + }() + if tb.VerificationDelay == nil { + return tb, nil + } + + <-tb.VerificationDelay + + return tb, nil +} + +func NewTestBlock(metadata simplex.ProtocolMetadata) *TestBlock { + tb := TestBlock{ + metadata: metadata, + data: make([]byte, 32), + } + + _, err := rand.Read(tb.data) + if err != nil { + panic(err) + } + + tb.computeDigest() + + return &tb +} + +func (tb *TestBlock) computeDigest() { + var bb bytes.Buffer + bb.Write(tb.Bytes()) + tb.Digest = sha256.Sum256(bb.Bytes()) +} + +func (t *TestBlock) BlockHeader() simplex.BlockHeader { + return simplex.BlockHeader{ + ProtocolMetadata: t.metadata, + Digest: t.Digest, + } +} + +func (t *TestBlock) Bytes() []byte { + bh := simplex.BlockHeader{ + ProtocolMetadata: t.metadata, + } + + mdBuff := bh.Bytes() + + buff := make([]byte, len(t.data)+len(mdBuff)+4) + binary.BigEndian.PutUint32(buff, uint32(len(t.data))) + copy(buff[4:], t.data) + copy(buff[4+len(t.data):], mdBuff) + return buff +} + +// testControlledBlockBuilder is a test block builder that blocks +// block building until a trigger is received +type testControlledBlockBuilder struct { + t *testing.T + control chan struct{} + TestBlockBuilder +} + +func NewTestControlledBlockBuilder(t *testing.T) *testControlledBlockBuilder { + return &testControlledBlockBuilder{ + t: t, + control: make(chan struct{}, 1), + TestBlockBuilder: TestBlockBuilder{Out: make(chan *TestBlock, 1), BlockShouldBeBuilt: make(chan struct{}, 1)}, + } +} + +func (t *testControlledBlockBuilder) TriggerNewBlock() { + select { + case t.control <- struct{}{}: + default: + + } +} + +func (t *testControlledBlockBuilder) BuildBlock(ctx context.Context, metadata simplex.ProtocolMetadata) (simplex.VerifiedBlock, bool) { + <-t.control + return t.TestBlockBuilder.BuildBlock(ctx, metadata) +} diff --git a/testutil/comm.go b/testutil/comm.go new file mode 100644 index 00000000..ef081cda --- /dev/null +++ b/testutil/comm.go @@ -0,0 +1,203 @@ +package testutil + +import ( + "bytes" + "simplex" + "sync" + + "github.com/stretchr/testify/require" +) + +type NoopComm []simplex.NodeID + +func NewNoopComm(nodeIDs []simplex.NodeID) NoopComm { + return NoopComm(nodeIDs) +} + +func (n NoopComm) ListNodes() []simplex.NodeID { + return n +} + +func (n NoopComm) SendMessage(*simplex.Message, simplex.NodeID) { + +} + +func (n NoopComm) Broadcast(msg *simplex.Message) { + +} + +// ListnerComm is a comm that listens for incoming messages +// and sends them to the [in] channel +type listnerComm struct { + NoopComm + In chan *simplex.Message +} + +func NewListenerComm(nodeIDs []simplex.NodeID) *listnerComm { + return &listnerComm{ + NoopComm: NoopComm(nodeIDs), + In: make(chan *simplex.Message, 1), + } +} + +func (b *listnerComm) SendMessage(msg *simplex.Message, id simplex.NodeID) { + b.In <- msg +} + +// messageFilter defines a function that filters +// certain messages from being sent or broadcasted. +type MessageFilter func(*simplex.Message, simplex.NodeID) bool + +// allowAllMessages allows every message to be sent +func AllowAllMessages(*simplex.Message, simplex.NodeID) bool { + return true +} + +type testComm struct { + from simplex.NodeID + net *inMemNetwork + messageFilter MessageFilter + lock sync.RWMutex +} + +func NewTestComm(from simplex.NodeID, net *inMemNetwork, messageFilter MessageFilter) *testComm { + return &testComm{ + from: from, + net: net, + messageFilter: messageFilter, + } +} + +func (c *testComm) ListNodes() []simplex.NodeID { + return c.net.nodes +} + +func (c *testComm) SendMessage(msg *simplex.Message, destination simplex.NodeID) { + if !c.isMessagePermitted(msg, destination) { + return + } + + // cannot send if either [from] or [destination] is not connected + if c.net.IsDisconnected(destination) || c.net.IsDisconnected(c.from) { + return + } + + c.maybeTranslateOutoingToIncomingMessageTypes(msg) + + for _, instance := range c.net.instances { + if bytes.Equal(instance.E.ID, destination) { + instance.ingress <- struct { + msg *simplex.Message + from simplex.NodeID + }{msg: msg, from: c.from} + return + } + } +} + +func (c *testComm) setFilter(filter MessageFilter) { + c.lock.Lock() + defer c.lock.Unlock() + + c.messageFilter = filter +} + +func (c *testComm) maybeTranslateOutoingToIncomingMessageTypes(msg *simplex.Message) { + if msg.VerifiedReplicationResponse != nil { + data := make([]simplex.QuorumRound, 0, len(msg.VerifiedReplicationResponse.Data)) + + for _, verifiedQuorumRound := range msg.VerifiedReplicationResponse.Data { + // Outgoing block is of type verified block but incoming block is of type Block, + // so we do a type cast because the test block implements both. + quorumRound := simplex.QuorumRound{} + if verifiedQuorumRound.EmptyNotarization != nil { + quorumRound.EmptyNotarization = verifiedQuorumRound.EmptyNotarization + } else { + quorumRound.Block = verifiedQuorumRound.VerifiedBlock.(simplex.Block) + if verifiedQuorumRound.Notarization != nil { + quorumRound.Notarization = verifiedQuorumRound.Notarization + } + if verifiedQuorumRound.FCert != nil { + quorumRound.FCert = verifiedQuorumRound.FCert + } + } + + data = append(data, quorumRound) + } + + var latestRound *simplex.QuorumRound + if msg.VerifiedReplicationResponse.LatestRound != nil { + if msg.VerifiedReplicationResponse.LatestRound.EmptyNotarization != nil { + latestRound = &simplex.QuorumRound{ + EmptyNotarization: msg.VerifiedReplicationResponse.LatestRound.EmptyNotarization, + } + } else { + latestRound = &simplex.QuorumRound{ + Block: msg.VerifiedReplicationResponse.LatestRound.VerifiedBlock.(simplex.Block), + Notarization: msg.VerifiedReplicationResponse.LatestRound.Notarization, + FCert: msg.VerifiedReplicationResponse.LatestRound.FCert, + EmptyNotarization: msg.VerifiedReplicationResponse.LatestRound.EmptyNotarization, + } + } + } + + require.Nil( + c.net.t, + msg.ReplicationResponse, + "message cannot include ReplicationResponse & VerifiedReplicationResponse", + ) + + msg.ReplicationResponse = &simplex.ReplicationResponse{ + Data: data, + LatestRound: latestRound, + } + } + + if msg.VerifiedBlockMessage != nil { + require.Nil(c.net.t, msg.BlockMessage, "message cannot include BlockMessage & VerifiedBlockMessage") + msg.BlockMessage = &simplex.BlockMessage{ + Block: msg.VerifiedBlockMessage.VerifiedBlock.(simplex.Block), + Vote: msg.VerifiedBlockMessage.Vote, + } + } +} + +func (c *testComm) isMessagePermitted(msg *simplex.Message, destination simplex.NodeID) bool { + c.lock.RLock() + defer c.lock.RUnlock() + + return c.messageFilter(msg, destination) +} + +func (c *testComm) Broadcast(msg *simplex.Message) { + if c.net.IsDisconnected(c.from) { + return + } + + c.maybeTranslateOutoingToIncomingMessageTypes(msg) + + for _, instance := range c.net.instances { + if !c.isMessagePermitted(msg, instance.E.ID) { + return + } + // Skip sending the message to yourself or disconnected nodes + if bytes.Equal(c.from, instance.E.ID) || c.net.IsDisconnected(instance.E.ID) { + continue + } + + instance.ingress <- struct { + msg *simplex.Message + from simplex.NodeID + }{msg: msg, from: c.from} + } +} + +type RecordingComm struct { + simplex.Communication + BroadcastMessages chan *simplex.Message +} + +func (rc *RecordingComm) Broadcast(msg *simplex.Message) { + rc.BroadcastMessages <- msg + rc.Communication.Broadcast(msg) +} diff --git a/testutil/network.go b/testutil/network.go new file mode 100644 index 00000000..9530519d --- /dev/null +++ b/testutil/network.go @@ -0,0 +1,124 @@ +package testutil + +import ( + "bytes" + "simplex" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type inMemNetwork struct { + t *testing.T + nodes []simplex.NodeID + instances []*testNode + lock sync.RWMutex + disconnected map[string]struct{} +} + +// NewInMemNetwork creates an in-memory network. Node IDs must be provided before +// adding instances, as nodes require prior knowledge of all participants. +func NewInMemNetwork(t *testing.T, nodes []simplex.NodeID) *inMemNetwork { + net := &inMemNetwork{ + t: t, + nodes: nodes, + instances: make([]*testNode, 0), + disconnected: make(map[string]struct{}), + } + return net +} + +func (n *inMemNetwork) addNode(node *testNode) { + allowed := false + for _, id := range n.nodes { + if bytes.Equal(id, node.E.ID) { + allowed = true + break + } + } + require.True(node.t, allowed, "node must be declared before adding") + n.instances = append(n.instances, node) +} + +func (n *inMemNetwork) SetAllNodesMessageFilter(filter MessageFilter) { + for _, instance := range n.instances { + instance.E.Comm.(*testComm).setFilter(filter) + } +} + +func (n *inMemNetwork) IsDisconnected(node simplex.NodeID) bool { + n.lock.RLock() + defer n.lock.RUnlock() + + _, ok := n.disconnected[string(node)] + return ok +} + +func (n *inMemNetwork) Connect(node simplex.NodeID) { + n.lock.Lock() + defer n.lock.Unlock() + + delete(n.disconnected, string(node)) +} + +func (n *inMemNetwork) Disconnect(node simplex.NodeID) { + n.lock.Lock() + defer n.lock.Unlock() + + n.disconnected[string(node)] = struct{}{} +} + +func (n *inMemNetwork) Instances() []*testNode { + return n.instances +} + +// startInstances starts all instances in the network. +// The first one is typically the leader, so we make sure to start it last. +func (n *inMemNetwork) StartInstances() { + require.Equal(n.t, len(n.nodes), len(n.instances)) + + for i := len(n.nodes) - 1; i >= 0; i-- { + n.instances[i].Start() + } +} + +// AdvanceWithoutLeader advances the network to the next round without the leader proposing a block. +// This is useful for testing scenarios where the leader is lagging behind. +func AdvanceWithoutLeader(t *testing.T, net *inMemNetwork, bb *testControlledBlockBuilder, epochTimes []time.Time, round uint64, laggingNodeId simplex.NodeID) { + // we need to ensure all blocks are waiting for the channel before proceeding + // otherwise, we may send to a channel that is not ready to receive + for _, n := range net.instances { + if laggingNodeId.Equals(n.E.ID) { + continue + } + + n.WaitToEnterRound(round) + } + + for _, n := range net.instances { + leader := n.E.ID.Equals(simplex.LeaderForRound(net.nodes, n.E.Metadata().Round)) + if leader || laggingNodeId.Equals(n.E.ID) { + continue + } + bb.BlockShouldBeBuilt <- struct{}{} + } + + for i, n := range net.instances { + // the leader will not write an empty vote to the wal + // because it cannot both propose a block & send an empty vote in the same round + leader := n.E.ID.Equals(simplex.LeaderForRound(net.nodes, n.E.Metadata().Round)) + if leader || laggingNodeId.Equals(n.E.ID) { + continue + } + n.waitForBlockProposerTimeout(&epochTimes[i], round) + } + + for _, n := range net.instances { + if laggingNodeId.Equals(n.E.ID) { + continue + } + n.Wal.AssertNotarization(round) + } +} diff --git a/testutil/node.go b/testutil/node.go new file mode 100644 index 00000000..9ebaafeb --- /dev/null +++ b/testutil/node.go @@ -0,0 +1,243 @@ +package testutil + +import ( + "encoding/asn1" + "encoding/binary" + "simplex" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type testNode struct { + Storage *InMemStorage + E *simplex.Epoch + Wal *TestWAL + ingress chan struct { + msg *simplex.Message + from simplex.NodeID + } + t *testing.T +} + +func (t *testNode) Start() { + go t.handleMessages() + require.NoError(t.t, t.E.Start()) +} + +type TestNodeConfig struct { + // optional + InitialStorage []simplex.VerifiedFinalizedBlock + Comm simplex.Communication + ReplicationEnabled bool +} + +// NewSimplexNode creates a new testNode and adds it to [net]. +func NewSimplexNode(t *testing.T, nodeID simplex.NodeID, net *inMemNetwork, bb simplex.BlockBuilder, config *TestNodeConfig) *testNode { + comm := NewTestComm(nodeID, net, AllowAllMessages) + + epochConfig, _, _ := DefaultTestNodeEpochConfig(t, nodeID, comm, bb) + + if config != nil { + updateEpochConfig(&epochConfig, config) + } + + e, err := simplex.NewEpoch(epochConfig) + require.NoError(t, err) + ti := &testNode{ + Wal: epochConfig.WAL.(*TestWAL), + E: e, + t: t, + Storage: epochConfig.Storage.(*InMemStorage), + ingress: make(chan struct { + msg *simplex.Message + from simplex.NodeID + }, 100)} + + net.addNode(ti) + return ti +} + +func updateEpochConfig(epochConfig *simplex.EpochConfig, testConfig *TestNodeConfig) { + // set the initial storage + for _, data := range testConfig.InitialStorage { + epochConfig.Storage.Index(data.VerifiedBlock, data.FCert) + } + + // TODO: remove optional replication flag + epochConfig.ReplicationEnabled = testConfig.ReplicationEnabled + + // custom communication + if testConfig.Comm != nil { + epochConfig.Comm = testConfig.Comm + } +} + +func DefaultTestNodeEpochConfig(t *testing.T, nodeID simplex.NodeID, comm simplex.Communication, bb simplex.BlockBuilder) (simplex.EpochConfig, *TestWAL, *InMemStorage) { + l := MakeLogger(t, int(nodeID[0])) + storage := NewInMemStorage() + wal := newTestWAL(t) + conf := simplex.EpochConfig{ + MaxProposalWait: simplex.DefaultMaxProposalWaitTime, + Logger: l, + ID: nodeID, + Signer: &TestSigner{}, + WAL: wal, + Verifier: &testVerifier{}, + Storage: storage, + Comm: comm, + BlockBuilder: bb, + SignatureAggregator: &TestSignatureAggregator{}, + BlockDeserializer: &blockDeserializer{}, + QCDeserializer: &testQCDeserializer{t: t}, + StartTime: time.Now(), + } + return conf, wal, storage +} + +func (t *testNode) HandleMessage(msg *simplex.Message, from simplex.NodeID) error { + err := t.E.HandleMessage(msg, from) + require.NoError(t.t, err) + return err +} + +func (t *testNode) handleMessages() { + for msg := range t.ingress { + err := t.HandleMessage(msg.msg, msg.from) + require.NoError(t.t, err) + if err != nil { + return + } + } +} + +func (n *testNode) WaitToEnterRound(round uint64) { + timeout := time.NewTimer(time.Minute) + defer timeout.Stop() + + for { + if n.E.Metadata().Round >= round { + return + } + + select { + case <-time.After(time.Millisecond * 10): + continue + case <-timeout.C: + require.Fail(n.t, "timed out waiting for event") + } + } +} + +type TestSigner struct { +} + +func (t *TestSigner) Sign([]byte) ([]byte, error) { + return []byte{1, 2, 3}, nil +} + +type testVerifier struct { +} + +func (t *testVerifier) VerifyBlock(simplex.VerifiedBlock) error { + return nil +} + +func (t *testVerifier) Verify(_ []byte, _ []byte, _ simplex.NodeID) error { + return nil +} + +type testQCDeserializer struct { + t *testing.T +} + +func (t *testQCDeserializer) DeserializeQuorumCertificate(bytes []byte) (simplex.QuorumCertificate, error) { + var qc []simplex.Signature + rest, err := asn1.Unmarshal(bytes, &qc) + require.NoError(t.t, err) + require.Empty(t.t, rest) + return TestQC(qc), err +} + +type TestSignatureAggregator struct { + Err error +} + +func (t *TestSignatureAggregator) Aggregate(signatures []simplex.Signature) (simplex.QuorumCertificate, error) { + return TestQC(signatures), t.Err +} + +type TestQC []simplex.Signature + +func (t TestQC) Signers() []simplex.NodeID { + res := make([]simplex.NodeID, 0, len(t)) + for _, sig := range t { + res = append(res, sig.Signer) + } + return res +} + +func (t TestQC) Verify(msg []byte) error { + return nil +} + +func (t TestQC) Bytes() []byte { + bytes, err := asn1.Marshal(t) + if err != nil { + panic(err) + } + return bytes +} + +type blockDeserializer struct { +} + +func (b *blockDeserializer) DeserializeBlock(buff []byte) (simplex.VerifiedBlock, error) { + blockLen := binary.BigEndian.Uint32(buff[:4]) + bh := simplex.BlockHeader{} + if err := bh.FromBytes(buff[4+blockLen:]); err != nil { + return nil, err + } + + tb := TestBlock{ + data: buff[4 : 4+blockLen], + metadata: bh.ProtocolMetadata, + } + + tb.computeDigest() + + return &tb, nil +} + +func TestBlockDeserializer(t *testing.T) { + var blockDeserializer blockDeserializer + + tb := NewTestBlock(simplex.ProtocolMetadata{Seq: 1, Round: 2, Epoch: 3}) + tb2, err := blockDeserializer.DeserializeBlock(tb.Bytes()) + require.NoError(t, err) + require.Equal(t, tb, tb2) +} + +func (n *testNode) waitForBlockProposerTimeout(startTime *time.Time, startRound uint64) { + WaitForBlockProposerTimeout(n.t, n.E, startTime, startRound) +} + +func WaitForBlockProposerTimeout(t *testing.T, e *simplex.Epoch, startTime *time.Time, startRound uint64) { + timeout := time.NewTimer(time.Minute) + defer timeout.Stop() + + for { + if e.WAL.(*TestWAL).containsEmptyVote(startRound) || e.WAL.(*TestWAL).containsEmptyNotarization(startRound) { + return + } + *startTime = startTime.Add(e.EpochConfig.MaxProposalWait / 5) + e.AdvanceTime(*startTime) + select { + case <-time.After(time.Millisecond * 10): + continue + case <-timeout.C: + require.Fail(t, "timed out waiting for event") + } + } +} diff --git a/testutil/storage.go b/testutil/storage.go new file mode 100644 index 00000000..837bbacc --- /dev/null +++ b/testutil/storage.go @@ -0,0 +1,118 @@ +package testutil + +import ( + "fmt" + "simplex" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type InMemStorage struct { + Data map[uint64]struct { + simplex.VerifiedBlock + simplex.FinalizationCertificate + } + + lock sync.Mutex + signal sync.Cond +} + +func NewInMemStorage() *InMemStorage { + s := &InMemStorage{ + Data: make(map[uint64]struct { + simplex.VerifiedBlock + simplex.FinalizationCertificate + }), + } + + s.signal = *sync.NewCond(&s.lock) + + return s +} + +func (mem *InMemStorage) Clone() *InMemStorage { + clone := NewInMemStorage() + mem.lock.Lock() + height := mem.Height() + mem.lock.Unlock() + for seq := uint64(0); seq < height; seq++ { + mem.lock.Lock() + block, fCert, ok := mem.Retrieve(seq) + if !ok { + panic(fmt.Sprintf("failed retrieving block %d", seq)) + } + mem.lock.Unlock() + clone.Index(block, fCert) + } + return clone +} + +func (mem *InMemStorage) WaitForBlockCommit(seq uint64) simplex.VerifiedBlock { + mem.lock.Lock() + defer mem.lock.Unlock() + + for { + if data, exists := mem.Data[seq]; exists { + return data.VerifiedBlock + } + + mem.signal.Wait() + } +} + +func (mem *InMemStorage) EnsureNoBlockCommit(t *testing.T, seq uint64) { + require.Never(t, func() bool { + mem.lock.Lock() + defer mem.lock.Unlock() + + _, exists := mem.Data[seq] + return exists + }, time.Second, time.Millisecond*100, "block %d has been committed but shouldn't have been", seq) +} + +func (mem *InMemStorage) Height() uint64 { + return uint64(len(mem.Data)) +} + +func (mem *InMemStorage) Retrieve(seq uint64) (simplex.VerifiedBlock, simplex.FinalizationCertificate, bool) { + item, ok := mem.Data[seq] + if !ok { + return nil, simplex.FinalizationCertificate{}, false + } + return item.VerifiedBlock, item.FinalizationCertificate, true +} + +func (mem *InMemStorage) Index(block simplex.VerifiedBlock, certificate simplex.FinalizationCertificate) { + mem.lock.Lock() + defer mem.lock.Unlock() + + seq := block.BlockHeader().Seq + + _, ok := mem.Data[seq] + if ok { + panic(fmt.Sprintf("block with seq %d already indexed!", seq)) + } + mem.Data[seq] = struct { + simplex.VerifiedBlock + simplex.FinalizationCertificate + }{block, + certificate, + } + + mem.signal.Signal() +} + +func (mem *InMemStorage) SetIndex(seq uint64, block simplex.VerifiedBlock, certificate simplex.FinalizationCertificate) { + mem.lock.Lock() + defer mem.lock.Unlock() + + mem.Data[seq] = struct { + simplex.VerifiedBlock + simplex.FinalizationCertificate + }{block, + certificate, + } +} diff --git a/testutil/wal.go b/testutil/wal.go new file mode 100644 index 00000000..1b009a22 --- /dev/null +++ b/testutil/wal.go @@ -0,0 +1,149 @@ +package testutil + +import ( + "encoding/binary" + "simplex" + "simplex/record" + "simplex/wal" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +type TestWAL struct { + simplex.WriteAheadLog + t *testing.T + lock sync.Mutex + signal sync.Cond +} + +func newTestWAL(t *testing.T) *TestWAL { + var tw TestWAL + tw.WriteAheadLog = wal.NewMemWAL(t) + tw.signal = sync.Cond{L: &tw.lock} + tw.t = t + return &tw +} + +func (tw *TestWAL) Clone() *TestWAL { + tw.lock.Lock() + defer tw.lock.Unlock() + + rawWAL, err := tw.WriteAheadLog.ReadAll() + require.NoError(tw.t, err) + + wal := newTestWAL(tw.t) + + for _, entry := range rawWAL { + wal.Append(entry) + } + + return wal +} + +func (tw *TestWAL) ReadAll() ([][]byte, error) { + tw.lock.Lock() + defer tw.lock.Unlock() + + return tw.WriteAheadLog.ReadAll() +} + +func (tw *TestWAL) Append(b []byte) error { + tw.lock.Lock() + defer tw.lock.Unlock() + + err := tw.WriteAheadLog.Append(b) + tw.signal.Signal() + return err +} + +func (tw *TestWAL) AssertWALSize(n int) { + tw.lock.Lock() + defer tw.lock.Unlock() + + for { + rawRecords, err := tw.WriteAheadLog.ReadAll() + require.NoError(tw.t, err) + + if len(rawRecords) == n { + return + } + + tw.signal.Wait() + } +} + +func (tw *TestWAL) AssertNotarization(round uint64) { + tw.lock.Lock() + defer tw.lock.Unlock() + + for { + rawRecords, err := tw.WriteAheadLog.ReadAll() + require.NoError(tw.t, err) + + for _, rawRecord := range rawRecords { + if binary.BigEndian.Uint16(rawRecord[:2]) == record.NotarizationRecordType { + _, vote, err := simplex.ParseNotarizationRecord(rawRecord) + require.NoError(tw.t, err) + + if vote.Round == round { + return + } + } + if binary.BigEndian.Uint16(rawRecord[:2]) == record.EmptyNotarizationRecordType { + _, vote, err := simplex.ParseEmptyNotarizationRecord(rawRecord) + require.NoError(tw.t, err) + + if vote.Round == round { + return + } + } + } + + tw.signal.Wait() + } + +} + +func (tw *TestWAL) containsEmptyVote(round uint64) bool { + tw.lock.Lock() + defer tw.lock.Unlock() + + rawRecords, err := tw.WriteAheadLog.ReadAll() + require.NoError(tw.t, err) + + for _, rawRecord := range rawRecords { + if binary.BigEndian.Uint16(rawRecord[:2]) == record.EmptyVoteRecordType { + vote, err := simplex.ParseEmptyVoteRecord(rawRecord) + require.NoError(tw.t, err) + + if vote.Round == round { + return true + } + } + } + + return false +} + +func (tw *TestWAL) containsEmptyNotarization(round uint64) bool { + tw.lock.Lock() + defer tw.lock.Unlock() + + rawRecords, err := tw.WriteAheadLog.ReadAll() + require.NoError(tw.t, err) + + for _, rawRecord := range rawRecords { + if binary.BigEndian.Uint16(rawRecord[:2]) == record.EmptyNotarizationRecordType { + _, vote, err := simplex.ParseEmptyNotarizationRecord(rawRecord) + require.NoError(tw.t, err) + + if vote.Round == round { + return true + } + } + } + + return false +} diff --git a/util_test.go b/util_test.go index c55a5c4b..22abf18c 100644 --- a/util_test.go +++ b/util_test.go @@ -5,7 +5,7 @@ package simplex_test import ( "errors" - "simplex" + "fmt" . "simplex" "simplex/testutil" "testing" @@ -14,23 +14,21 @@ import ( ) func TestRetrieveFromStorage(t *testing.T) { - brokenStorage := newInMemStorage() - brokenStorage.data[41] = struct { - VerifiedBlock - FinalizationCertificate - }{VerifiedBlock: newTestBlock(ProtocolMetadata{Seq: 41})} + brokenStorage := testutil.NewInMemStorage() + brokenStorage.SetIndex(41, testutil.NewTestBlock(ProtocolMetadata{Seq: 41}), FinalizationCertificate{}) - block := newTestBlock(ProtocolMetadata{Seq: 0}) + block := testutil.NewTestBlock(ProtocolMetadata{Seq: 0}) fCert := FinalizationCertificate{ Finalization: ToBeSignedFinalization{ BlockHeader: block.BlockHeader(), }, } - normalStorage := newInMemStorage() - normalStorage.data[0] = struct { - VerifiedBlock - FinalizationCertificate - }{VerifiedBlock: block, FinalizationCertificate: fCert} + normalStorage := testutil.NewInMemStorage() + normalStorage.SetIndex( + 0, + block, + fCert, + ) for _, testCase := range []struct { description string @@ -40,7 +38,7 @@ func TestRetrieveFromStorage(t *testing.T) { }{ { description: "no blocks in storage", - storage: newInMemStorage(), + storage: testutil.NewInMemStorage(), }, { description: "broken storage", @@ -73,7 +71,7 @@ func TestFinalizationCertificateValidation(t *testing.T) { eligibleSigners[string(n)] = struct{}{} } quorumSize := Quorum(len(nodes)) - signatureAggregator := &testSignatureAggregator{} + signatureAggregator := &testutil.TestSignatureAggregator{} // Test tests := []struct { name string @@ -84,7 +82,7 @@ func TestFinalizationCertificateValidation(t *testing.T) { { name: "valid finalization certificate", fCert: func() FinalizationCertificate { - block := newTestBlock(ProtocolMetadata{}) + block := testutil.NewTestBlock(ProtocolMetadata{}) fCert, _ := newFinalizationRecord(t, l, signatureAggregator, block, nodes[:quorumSize]) return fCert }(), @@ -93,7 +91,7 @@ func TestFinalizationCertificateValidation(t *testing.T) { }, { name: "not enough signers", fCert: func() FinalizationCertificate { - block := newTestBlock(ProtocolMetadata{}) + block := testutil.NewTestBlock(ProtocolMetadata{}) fCert, _ := newFinalizationRecord(t, l, signatureAggregator, block, nodes[:quorumSize-1]) return fCert }(), @@ -103,7 +101,7 @@ func TestFinalizationCertificateValidation(t *testing.T) { { name: "signer signed twice", fCert: func() FinalizationCertificate { - block := newTestBlock(ProtocolMetadata{}) + block := testutil.NewTestBlock(ProtocolMetadata{}) doubleNodes := []NodeID{{1}, {2}, {3}, {4}, {4}} fCert, _ := newFinalizationRecord(t, l, signatureAggregator, block, doubleNodes) return fCert @@ -120,7 +118,7 @@ func TestFinalizationCertificateValidation(t *testing.T) { { name: "nodes are not eligible signers", fCert: func() FinalizationCertificate { - block := newTestBlock(ProtocolMetadata{}) + block := testutil.NewTestBlock(ProtocolMetadata{}) signers := []NodeID{{1}, {2}, {3}, {4}, {6}} fCert, _ := newFinalizationRecord(t, l, signatureAggregator, block, signers) return fCert @@ -131,7 +129,7 @@ func TestFinalizationCertificateValidation(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - valid := simplex.IsFinalizationCertificateValid(eligibleSigners, &tt.fCert, tt.quorumSize, l) + valid := IsFinalizationCertificateValid(eligibleSigners, &tt.fCert, tt.quorumSize, l) require.Equal(t, tt.valid, valid) }) } @@ -141,10 +139,10 @@ func TestGetHighestQuorumRound(t *testing.T) { // Test nodes := []NodeID{{1}, {2}, {3}, {4}, {5}} l := testutil.MakeLogger(t, 0) - signatureAggregator := &testSignatureAggregator{} + signatureAggregator := &testutil.TestSignatureAggregator{} // seq 1 - block1 := newTestBlock(ProtocolMetadata{ + block1 := testutil.NewTestBlock(ProtocolMetadata{ Seq: 1, Round: 1, }) @@ -153,7 +151,7 @@ func TestGetHighestQuorumRound(t *testing.T) { fCert1, _ := newFinalizationRecord(t, l, signatureAggregator, block1, nodes) // seq 10 - block10 := newTestBlock(ProtocolMetadata{Seq: 10, Round: 10}) + block10 := testutil.NewTestBlock(ProtocolMetadata{Seq: 10, Round: 10}) notarization10, err := newNotarization(l, signatureAggregator, block10, nodes) require.NoError(t, err) fCert10, _ := newFinalizationRecord(t, l, signatureAggregator, block10, nodes) @@ -250,8 +248,69 @@ func TestGetHighestQuorumRound(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - qr := simplex.GetLatestVerifiedQuorumRound(tt.round, tt.eNote, tt.lastBlock) + qr := GetLatestVerifiedQuorumRound(tt.round, tt.eNote, tt.lastBlock) require.Equal(t, tt.expectedQr, qr) }) } } + +func TestQuorum(t *testing.T) { + for _, testCase := range []struct { + n int + f int + q int + }{ + { + n: 1, f: 0, + q: 1, + }, + { + n: 2, f: 0, + q: 2, + }, + { + n: 3, f: 0, + q: 2, + }, + { + n: 4, f: 1, + q: 3, + }, + { + n: 5, f: 1, + q: 4, + }, + { + n: 6, f: 1, + q: 4, + }, + { + n: 7, f: 2, + q: 5, + }, + { + n: 8, f: 2, + q: 6, + }, + { + n: 9, f: 2, + q: 6, + }, + { + n: 10, f: 3, + q: 7, + }, + { + n: 11, f: 3, + q: 8, + }, + { + n: 12, f: 3, + q: 8, + }, + } { + t.Run(fmt.Sprintf("%d", testCase.n), func(t *testing.T) { + require.Equal(t, testCase.q, Quorum(testCase.n)) + }) + } +}