diff --git a/apitest/src/test/java/bisq/apitest/method/trade/BsqSwapTradeTestLoop.java b/apitest/src/test/java/bisq/apitest/method/trade/BsqSwapTradeTestLoop.java new file mode 100644 index 0000000000..d5cf9b7dd7 --- /dev/null +++ b/apitest/src/test/java/bisq/apitest/method/trade/BsqSwapTradeTestLoop.java @@ -0,0 +1,65 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.apitest.method.trade; + +import lombok.extern.slf4j.Slf4j; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; + + + +import bisq.apitest.method.offer.AbstractOfferTest; + +// @Disabled +@Slf4j +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class BsqSwapTradeTestLoop extends AbstractOfferTest { + + @BeforeAll + public static void setUp() { + AbstractOfferTest.setUp(); + createBsqSwapBsqPaymentAccounts(); + } + + @Test + @Order(1) + public void testGetBalancesBeforeTrade() { + BsqSwapTradeTest test = new BsqSwapTradeTest(); + runTradeLoop(test); + } + + private void runTradeLoop(BsqSwapTradeTest test) { + // TODO Fix wallet inconsistency bugs after 2nd trades. + for (int tradeCount = 1; tradeCount <= 2; tradeCount++) { + log.warn("================================ Trade # {} ================================", tradeCount); + test.testGetBalancesBeforeTrade(); + + test.testAliceCreateBsqSwapBuyOffer(); + genBtcBlocksThenWait(1, 8000); + + test.testBobTakesBsqSwapOffer(); + genBtcBlocksThenWait(1, 8000); + + test.testGetBalancesAfterTrade(); + } + } +} diff --git a/common/src/main/java/bisq/common/persistence/PersistenceManager.java b/common/src/main/java/bisq/common/persistence/PersistenceManager.java index 823d417c59..ba9d62625b 100644 --- a/common/src/main/java/bisq/common/persistence/PersistenceManager.java +++ b/common/src/main/java/bisq/common/persistence/PersistenceManager.java @@ -501,8 +501,6 @@ public class PersistenceManager { if (completeHandler != null) { UserThread.execute(completeHandler); } - - GcUtil.maybeReleaseMemory(); } } diff --git a/common/src/main/java/bisq/common/util/GcUtil.java b/common/src/main/java/bisq/common/util/GcUtil.java index 62dbcb7db3..65227f2d91 100644 --- a/common/src/main/java/bisq/common/util/GcUtil.java +++ b/common/src/main/java/bisq/common/util/GcUtil.java @@ -18,6 +18,7 @@ package bisq.common.util; import bisq.common.UserThread; +import bisq.common.app.DevEnv; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -47,18 +48,31 @@ public class GcUtil { * @param trigger Threshold for free memory in MB when we invoke the garbage collector */ private static void autoReleaseMemory(long trigger) { - UserThread.runPeriodically(() -> maybeReleaseMemory(trigger), 60); + UserThread.runPeriodically(() -> maybeReleaseMemory(trigger), 120); } /** * @param trigger Threshold for free memory in MB when we invoke the garbage collector */ private static void maybeReleaseMemory(long trigger) { - long totalMemory = Runtime.getRuntime().totalMemory(); - if (totalMemory > trigger * 1024 * 1024) { - log.info("Invoke garbage collector. Total memory: {} {} {}", Utilities.readableFileSize(totalMemory), totalMemory, trigger * 1024 * 1024); + long ts = System.currentTimeMillis(); + long preGcMemory = Runtime.getRuntime().totalMemory(); + if (preGcMemory > trigger * 1024 * 1024) { System.gc(); - log.info("Total memory after gc() call: {}", Utilities.readableFileSize(Runtime.getRuntime().totalMemory())); + long postGcMemory = Runtime.getRuntime().totalMemory(); + log.info("GC reduced memory by {}. Total memory before/after: {}/{}. Took {} ms.", + Utilities.readableFileSize(preGcMemory - postGcMemory), + Utilities.readableFileSize(preGcMemory), + Utilities.readableFileSize(postGcMemory), + System.currentTimeMillis() - ts); + if (DevEnv.isDevMode()) { + try { + // To see from where we got called + throw new RuntimeException("Dummy Exception for print stacktrace at maybeReleaseMemory"); + } catch (Throwable t) { + t.printStackTrace(); + } + } } } } diff --git a/core/src/main/java/bisq/core/app/misc/AppSetupWithP2PAndDAO.java b/core/src/main/java/bisq/core/app/misc/AppSetupWithP2PAndDAO.java index 95751832c9..1c77a6529e 100644 --- a/core/src/main/java/bisq/core/app/misc/AppSetupWithP2PAndDAO.java +++ b/core/src/main/java/bisq/core/app/misc/AppSetupWithP2PAndDAO.java @@ -28,6 +28,7 @@ import bisq.core.dao.governance.proofofburn.MyProofOfBurnListService; import bisq.core.dao.governance.proposal.MyProposalListService; import bisq.core.filter.FilterManager; import bisq.core.trade.statistics.TradeStatisticsManager; +import bisq.core.user.Preferences; import bisq.network.p2p.P2PService; import bisq.network.p2p.peers.PeerManager; @@ -42,6 +43,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class AppSetupWithP2PAndDAO extends AppSetupWithP2P { private final DaoSetup daoSetup; + private final Preferences preferences; @Inject public AppSetupWithP2PAndDAO(P2PService p2PService, @@ -58,6 +60,7 @@ public class AppSetupWithP2PAndDAO extends AppSetupWithP2P { MyProposalListService myProposalListService, MyReputationListService myReputationListService, MyProofOfBurnListService myProofOfBurnListService, + Preferences preferences, Config config) { super(p2PService, p2PDataStorage, @@ -69,6 +72,7 @@ public class AppSetupWithP2PAndDAO extends AppSetupWithP2P { config); this.daoSetup = daoSetup; + this.preferences = preferences; // TODO Should be refactored/removed. In the meantime keep in sync with CorePersistedDataHost if (config.daoActivated) { @@ -86,5 +90,8 @@ public class AppSetupWithP2PAndDAO extends AppSetupWithP2P { super.onBasicServicesInitialized(); daoSetup.onAllServicesInitialized(log::error, log::warn); + + // For seed nodes we need to set default value to true + preferences.setUseFullModeDaoMonitor(true); } } diff --git a/core/src/main/java/bisq/core/dao/DaoEventCoordinator.java b/core/src/main/java/bisq/core/dao/DaoEventCoordinator.java deleted file mode 100644 index 4c8cc0f409..0000000000 --- a/core/src/main/java/bisq/core/dao/DaoEventCoordinator.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.core.dao; - -import bisq.core.dao.monitoring.DaoStateMonitoringService; -import bisq.core.dao.state.DaoStateListener; -import bisq.core.dao.state.DaoStateService; -import bisq.core.dao.state.DaoStateSnapshotService; -import bisq.core.dao.state.model.blockchain.Block; - -import javax.inject.Inject; - -public class DaoEventCoordinator implements DaoSetupService, DaoStateListener { - private final DaoStateService daoStateService; - private final DaoStateSnapshotService daoStateSnapshotService; - private final DaoStateMonitoringService daoStateMonitoringService; - - @Inject - public DaoEventCoordinator(DaoStateService daoStateService, - DaoStateSnapshotService daoStateSnapshotService, - DaoStateMonitoringService daoStateMonitoringService) { - this.daoStateService = daoStateService; - this.daoStateSnapshotService = daoStateSnapshotService; - this.daoStateMonitoringService = daoStateMonitoringService; - } - - - /////////////////////////////////////////////////////////////////////////////////////////// - // DaoSetupService - /////////////////////////////////////////////////////////////////////////////////////////// - - @Override - public void addListeners() { - this.daoStateService.addDaoStateListener(this); - } - - @Override - public void start() { - } - - - /////////////////////////////////////////////////////////////////////////////////////////// - // DaoStateListener - /////////////////////////////////////////////////////////////////////////////////////////// - - // We listen onDaoStateChanged to ensure the dao state has been processed from listener clients after parsing. - // We need to listen during batch processing as well to write snapshots during that process. - @Override - public void onDaoStateChanged(Block block) { - // We need to execute first the daoStateMonitoringService - daoStateMonitoringService.createHashFromBlock(block); - daoStateSnapshotService.maybeCreateSnapshot(block); - } -} diff --git a/core/src/main/java/bisq/core/dao/DaoModule.java b/core/src/main/java/bisq/core/dao/DaoModule.java index 87d5f9e7a0..ed2f0efef3 100644 --- a/core/src/main/java/bisq/core/dao/DaoModule.java +++ b/core/src/main/java/bisq/core/dao/DaoModule.java @@ -102,7 +102,6 @@ public class DaoModule extends AppModule { protected void configure() { bind(DaoSetup.class).in(Singleton.class); bind(DaoFacade.class).in(Singleton.class); - bind(DaoEventCoordinator.class).in(Singleton.class); bind(DaoKillSwitch.class).in(Singleton.class); // Node, parser diff --git a/core/src/main/java/bisq/core/dao/DaoSetup.java b/core/src/main/java/bisq/core/dao/DaoSetup.java index b8c13582f7..728f94108a 100644 --- a/core/src/main/java/bisq/core/dao/DaoSetup.java +++ b/core/src/main/java/bisq/core/dao/DaoSetup.java @@ -39,6 +39,7 @@ import bisq.core.dao.node.BsqNode; import bisq.core.dao.node.BsqNodeProvider; import bisq.core.dao.node.explorer.ExportJsonFilesService; import bisq.core.dao.state.DaoStateService; +import bisq.core.dao.state.DaoStateSnapshotService; import com.google.inject.Inject; @@ -78,16 +79,11 @@ public class DaoSetup { DaoStateMonitoringService daoStateMonitoringService, ProposalStateMonitoringService proposalStateMonitoringService, BlindVoteStateMonitoringService blindVoteStateMonitoringService, - DaoEventCoordinator daoEventCoordinator) { + DaoStateSnapshotService daoStateSnapshotService) { bsqNode = bsqNodeProvider.getBsqNode(); // We need to take care of order of execution. - - // For order critical event flow we use the daoEventCoordinator to delegate the calls from anonymous listeners - // to concrete clients. - daoSetupServices.add(daoEventCoordinator); - daoSetupServices.add(daoStateService); daoSetupServices.add(cycleService); daoSetupServices.add(ballotListService); @@ -110,6 +106,7 @@ public class DaoSetup { daoSetupServices.add(daoStateMonitoringService); daoSetupServices.add(proposalStateMonitoringService); daoSetupServices.add(blindVoteStateMonitoringService); + daoSetupServices.add(daoStateSnapshotService); daoSetupServices.add(bsqNodeProvider.getBsqNode()); } diff --git a/core/src/main/java/bisq/core/dao/governance/period/CycleService.java b/core/src/main/java/bisq/core/dao/governance/period/CycleService.java index a52fb127af..af1db2fcb1 100644 --- a/core/src/main/java/bisq/core/dao/governance/period/CycleService.java +++ b/core/src/main/java/bisq/core/dao/governance/period/CycleService.java @@ -90,8 +90,7 @@ public class CycleService implements DaoStateListener, DaoSetupService { } public int getCycleIndex(Cycle cycle) { - Optional previousCycle = getCycle(cycle.getHeightOfFirstBlock() - 1, daoStateService.getCycles()); - return previousCycle.map(cycle1 -> getCycleIndex(cycle1) + 1).orElse(0); + return daoStateService.getCycles().indexOf(cycle); } public boolean isTxInCycle(Cycle cycle, String txId) { diff --git a/core/src/main/java/bisq/core/dao/monitoring/BlindVoteStateMonitoringService.java b/core/src/main/java/bisq/core/dao/monitoring/BlindVoteStateMonitoringService.java index a3a5162994..f803060a53 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/BlindVoteStateMonitoringService.java +++ b/core/src/main/java/bisq/core/dao/monitoring/BlindVoteStateMonitoringService.java @@ -274,7 +274,7 @@ public class BlindVoteStateMonitoringService implements DaoSetupService, DaoStat byte[] combined = ArrayUtils.addAll(prevHash, serializedBlindVotes); byte[] hash = Hash.getSha256Ripemd160hash(combined); - BlindVoteStateHash myBlindVoteStateHash = new BlindVoteStateHash(blockHeight, hash, prevHash, blindVotes.size()); + BlindVoteStateHash myBlindVoteStateHash = new BlindVoteStateHash(blockHeight, hash, blindVotes.size()); BlindVoteStateBlock blindVoteStateBlock = new BlindVoteStateBlock(myBlindVoteStateHash); blindVoteStateBlockChain.add(blindVoteStateBlock); blindVoteStateHashChain.add(myBlindVoteStateHash); diff --git a/core/src/main/java/bisq/core/dao/monitoring/DaoStateMonitoringService.java b/core/src/main/java/bisq/core/dao/monitoring/DaoStateMonitoringService.java index 35c7ebad0b..cb8d0b9533 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/DaoStateMonitoringService.java +++ b/core/src/main/java/bisq/core/dao/monitoring/DaoStateMonitoringService.java @@ -31,6 +31,7 @@ import bisq.core.dao.state.GenesisTxInfo; import bisq.core.dao.state.model.blockchain.BaseTxOutput; import bisq.core.dao.state.model.blockchain.Block; import bisq.core.dao.state.model.governance.IssuanceType; +import bisq.core.user.Preferences; import bisq.network.p2p.NodeAddress; import bisq.network.p2p.network.Connection; @@ -40,7 +41,6 @@ import bisq.common.UserThread; import bisq.common.config.Config; import bisq.common.crypto.Hash; import bisq.common.file.FileUtil; -import bisq.common.util.GcUtil; import bisq.common.util.Utilities; import javax.inject.Inject; @@ -54,19 +54,20 @@ import javafx.collections.ObservableList; import java.io.File; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import static com.google.common.base.Preconditions.checkArgument; +import javax.annotation.Nullable; /** * Monitors the DaoState by using a hash for the complete daoState and make it accessible to the network @@ -88,7 +89,7 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe DaoStateNetworkService.Listener { public interface Listener { - void onChangeAfterBatchProcessing(); + void onDaoStateHashesChanged(); void onCheckpointFail(); } @@ -98,7 +99,6 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe private final GenesisTxInfo genesisTxInfo; private final Set seedNodeAddresses; - @Getter private final LinkedList daoStateBlockChain = new LinkedList<>(); @Getter @@ -110,6 +110,8 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe @Getter private boolean isInConflictWithSeedNode; @Getter + private boolean daoStateBlockChainNotConnecting; + @Getter private final ObservableList utxoMismatches = FXCollections.observableArrayList(); private final List checkpoints = Arrays.asList( @@ -120,7 +122,13 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe private int numCalls; private long accumulatedDuration; + private final Preferences preferences; private final File storageDir; + @Nullable + private Runnable createSnapshotHandler; + // Lookup map + private Map daoStateBlockByHeight = new HashMap<>(); + /////////////////////////////////////////////////////////////////////////////////////////// // Constructor @@ -131,11 +139,13 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe DaoStateNetworkService daoStateNetworkService, GenesisTxInfo genesisTxInfo, SeedNodeRepository seedNodeRepository, + Preferences preferences, @Named(Config.STORAGE_DIR) File storageDir, @Named(Config.IGNORE_DEV_MSG) boolean ignoreDevMsg) { this.daoStateService = daoStateService; this.daoStateNetworkService = daoStateNetworkService; this.genesisTxInfo = genesisTxInfo; + this.preferences = preferences; this.storageDir = storageDir; this.ignoreDevMsg = ignoreDevMsg; seedNodeAddresses = seedNodeRepository.getSeedNodeAddresses().stream() @@ -163,16 +173,19 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe // DaoStateListener /////////////////////////////////////////////////////////////////////////////////////////// - // We do not use onDaoStateChanged but let the DaoEventCoordinator call createHashFromBlock to ensure the - // correct order of execution. - @Override public void onParseBlockChainComplete() { parseBlockChainComplete = true; + daoStateService.getLastBlock().ifPresent(this::checkUtxos); + daoStateNetworkService.addListeners(); - // We wait for processing messages until we have completed batch processing - int fromHeight = daoStateService.getChainHeight() - 10; + // We take either the height of the previous hashBlock we have or 10 blocks below the chain tip. + int nextBlockHeight = daoStateBlockChain.isEmpty() ? + genesisTxInfo.getGenesisBlockHeight() : + daoStateBlockChain.getLast().getHeight() + 1; + int past10 = daoStateService.getChainHeight() - 10; + int fromHeight = Math.min(nextBlockHeight, past10); daoStateNetworkService.requestHashesFromAllConnectedSeedNodes(fromHeight); if (!ignoreDevMsg) { @@ -188,16 +201,9 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe @Override public void onDaoStateChanged(Block block) { - long genesisTotalSupply = daoStateService.getGenesisTotalSupply().value; - long compensationIssuance = daoStateService.getTotalIssuedAmount(IssuanceType.COMPENSATION); - long reimbursementIssuance = daoStateService.getTotalIssuedAmount(IssuanceType.REIMBURSEMENT); - long totalAmountOfBurntBsq = daoStateService.getTotalAmountOfBurntBsq(); - // confiscated funds are still in the utxo set - long sumUtxo = daoStateService.getUnspentTxOutputMap().values().stream().mapToLong(BaseTxOutput::getValue).sum(); - long sumBsq = genesisTotalSupply + compensationIssuance + reimbursementIssuance - totalAmountOfBurntBsq; - - if (sumBsq != sumUtxo) { - utxoMismatches.add(new UtxoMismatch(block.getHeight(), sumUtxo, sumBsq)); + // During syncing we do not call checkUtxos as its a bit slow (about 4 ms) + if (parseBlockChainComplete) { + checkUtxos(block); } } @@ -208,44 +214,47 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe @Override public void onNewStateHashMessage(NewDaoStateHashMessage newStateHashMessage, Connection connection) { - if (newStateHashMessage.getStateHash().getHeight() <= daoStateService.getChainHeight()) { - processPeersDaoStateHash(newStateHashMessage.getStateHash(), connection.getPeersNodeAddressOptional(), true); + // Called when receiving NewDaoStateHashMessages from peers after a new block + DaoStateHash peersDaoStateHash = newStateHashMessage.getStateHash(); + if (peersDaoStateHash.getHeight() <= daoStateService.getChainHeight()) { + putInPeersMapAndCheckForConflicts(getPeersAddress(connection.getPeersNodeAddressOptional()), peersDaoStateHash); + listeners.forEach(Listener::onDaoStateHashesChanged); + } + } + + @Override + public void onPeersStateHashes(List stateHashes, Optional peersNodeAddress) { + // Called when receiving GetDaoStateHashesResponse from seed nodes + processPeersDaoStateHashes(stateHashes, peersNodeAddress); + listeners.forEach(Listener::onDaoStateHashesChanged); + if (createSnapshotHandler != null) { + createSnapshotHandler.run(); + // As we get called multiple times from hashes of diff. seed nodes we want to avoid to + // call our handler multiple times. + createSnapshotHandler = null; } } @Override public void onGetStateHashRequest(Connection connection, GetDaoStateHashesRequest getStateHashRequest) { int fromHeight = getStateHashRequest.getHeight(); - List daoStateHashes = daoStateBlockChain.stream() + List daoStateHashes = daoStateHashChain.stream() .filter(e -> e.getHeight() >= fromHeight) - .map(DaoStateBlock::getMyStateHash) .collect(Collectors.toList()); daoStateNetworkService.sendGetStateHashesResponse(connection, getStateHashRequest.getNonce(), daoStateHashes); } - @Override - public void onPeersStateHashes(List stateHashes, Optional peersNodeAddress) { - AtomicBoolean hasChanged = new AtomicBoolean(false); - - stateHashes.forEach(daoStateHash -> { - boolean changed = processPeersDaoStateHash(daoStateHash, peersNodeAddress, false); - if (changed) { - hasChanged.set(true); - } - }); - - if (hasChanged.get()) { - listeners.forEach(Listener::onChangeAfterBatchProcessing); - } - } - /////////////////////////////////////////////////////////////////////////////////////////// // API /////////////////////////////////////////////////////////////////////////////////////////// public void createHashFromBlock(Block block) { - updateHashChain(block); + createDaoStateBlock(block); + if (parseBlockChainComplete) { + // We notify listeners only after batch processing to avoid performance issues at UI code + listeners.forEach(Listener::onDaoStateHashesChanged); + } } public void requestHashesFromGenesisBlockHeight(String peersAddress) { @@ -256,6 +265,7 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe // We could get a reset from a reorg, so we clear all and start over from the genesis block. daoStateHashChain.clear(); daoStateBlockChain.clear(); + daoStateBlockByHeight.clear(); daoStateNetworkService.reset(); if (!persistedDaoStateHashChain.isEmpty()) { @@ -263,7 +273,15 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe persistedDaoStateHashChain.size(), persistedDaoStateHashChain.getLast()); } daoStateHashChain.addAll(persistedDaoStateHashChain); - daoStateHashChain.forEach(e -> daoStateBlockChain.add(new DaoStateBlock(e))); + daoStateHashChain.forEach(daoStateHash -> { + DaoStateBlock daoStateBlock = new DaoStateBlock(daoStateHash); + daoStateBlockChain.add(daoStateBlock); + daoStateBlockByHeight.put(daoStateHash.getHeight(), daoStateBlock); + }); + } + + public void setCreateSnapshotHandler(Runnable handler) { + createSnapshotHandler = handler; } @@ -284,7 +302,7 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe // Private /////////////////////////////////////////////////////////////////////////////////////////// - private void updateHashChain(Block block) { + private Optional createDaoStateBlock(Block block) { long ts = System.currentTimeMillis(); byte[] prevHash; int height = block.getHeight(); @@ -295,34 +313,45 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe } else { log.warn("DaoStateBlockchain is empty but we received the block which was not the genesis block. " + "We stop execution here."); - return; + daoStateBlockChainNotConnecting = true; + listeners.forEach(Listener::onDaoStateHashesChanged); + return Optional.empty(); } } else { - checkArgument(height == daoStateBlockChain.getLast().getHeight() + 1, - "New block must be 1 block above previous block. height={}, " + - "daoStateBlockChain.getLast().getHeight()={}", - height, daoStateBlockChain.getLast().getHeight()); - prevHash = daoStateBlockChain.getLast().getHash(); + DaoStateBlock last = daoStateBlockChain.getLast(); + int heightOfLastBlock = last.getHeight(); + if (height == heightOfLastBlock + 1) { + prevHash = last.getHash(); + } else { + log.warn("New block must be 1 block above previous block. height={}, " + + "daoStateBlockChain.getLast().getHeight()={}", + height, heightOfLastBlock); + daoStateBlockChainNotConnecting = true; + listeners.forEach(Listener::onDaoStateHashesChanged); + return Optional.empty(); + } } + byte[] stateAsBytes = daoStateService.getSerializedStateForHashChain(); // We include the prev. hash in our new hash so we can be sure that if one hash is matching all the past would // match as well. byte[] combined = ArrayUtils.addAll(prevHash, stateAsBytes); byte[] hash = Hash.getSha256Ripemd160hash(combined); - DaoStateHash myDaoStateHash = new DaoStateHash(height, hash, prevHash); + DaoStateHash myDaoStateHash = new DaoStateHash(height, hash, true); DaoStateBlock daoStateBlock = new DaoStateBlock(myDaoStateHash); daoStateBlockChain.add(daoStateBlock); + daoStateBlockByHeight.put(height, daoStateBlock); daoStateHashChain.add(myDaoStateHash); // We only broadcast after parsing of blockchain is complete if (parseBlockChainComplete) { - // We notify listeners only after batch processing to avoid performance issues at UI code - listeners.forEach(Listener::onChangeAfterBatchProcessing); - // We delay broadcast to give peers enough time to have received the block. // Otherwise they would ignore our data if received block is in future to their local blockchain. int delayInSec = 5 + new Random().nextInt(10); + if (Config.baseCurrencyNetwork().isRegtest()) { + delayInSec = 1; + } UserThread.runAfter(() -> daoStateNetworkService.broadcastMyStateHash(myDaoStateHash), delayInSec); } long duration = System.currentTimeMillis() - ts; @@ -332,59 +361,93 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe duration); accumulatedDuration += duration; numCalls++; + return Optional.of(daoStateBlock); } - private boolean processPeersDaoStateHash(DaoStateHash daoStateHash, - Optional peersNodeAddress, - boolean notifyListeners) { - GcUtil.maybeReleaseMemory(); + private void processPeersDaoStateHashes(List stateHashes, Optional peersNodeAddress) { + boolean useDaoMonitor = preferences.isUseFullModeDaoMonitor(); + stateHashes.forEach(peersHash -> { + Optional optionalDaoStateBlock; + // If we do not add own hashes during initial parsing we fill the missing hashes from the peer and create + // at the last block our own hash. + int height = peersHash.getHeight(); + if (!useDaoMonitor && + !findDaoStateBlock(height).isPresent()) { + if (daoStateService.getChainHeight() == height) { + // At the most recent block we create our own hash + optionalDaoStateBlock = daoStateService.getLastBlock() + .map(this::createDaoStateBlock) + .orElse(findDaoStateBlock(height)); + } else { + // Otherwise we create a block from the peers daoStateHash + DaoStateHash daoStateHash = new DaoStateHash(height, peersHash.getHash(), false); + DaoStateBlock daoStateBlock = new DaoStateBlock(daoStateHash); + daoStateBlockChain.add(daoStateBlock); + daoStateBlockByHeight.put(height, daoStateBlock); + daoStateHashChain.add(daoStateHash); + optionalDaoStateBlock = Optional.of(daoStateBlock); + } + } else { + optionalDaoStateBlock = findDaoStateBlock(height); + } - AtomicBoolean changed = new AtomicBoolean(false); - AtomicBoolean inConflictWithNonSeedNode = new AtomicBoolean(this.isInConflictWithNonSeedNode); - AtomicBoolean inConflictWithSeedNode = new AtomicBoolean(this.isInConflictWithSeedNode); - StringBuilder sb = new StringBuilder(); - daoStateBlockChain.stream() - .filter(e -> e.getHeight() == daoStateHash.getHeight()).findAny() - .ifPresent(daoStateBlock -> { - String peersNodeAddressAsString = peersNodeAddress.map(NodeAddress::getFullAddress) - .orElseGet(() -> "Unknown peer " + new Random().nextInt(10000)); - daoStateBlock.putInPeersMap(peersNodeAddressAsString, daoStateHash); - if (!daoStateBlock.getMyStateHash().hasEqualHash(daoStateHash)) { - daoStateBlock.putInConflictMap(peersNodeAddressAsString, daoStateHash); - if (seedNodeAddresses.contains(peersNodeAddressAsString)) { - inConflictWithSeedNode.set(true); - } else { - inConflictWithNonSeedNode.set(true); - } - sb.append("We received a block hash from peer ") - .append(peersNodeAddressAsString) - .append(" which conflicts with our block hash.\n") - .append("my daoStateHash=") - .append(daoStateBlock.getMyStateHash()) - .append("\npeers daoStateHash=") - .append(daoStateHash); - } - changed.set(true); - }); + // In any case we add the peer to our peersMap and check for conflicts on the relevant daoStateBlock + putInPeersMapAndCheckForConflicts(optionalDaoStateBlock, getPeersAddress(peersNodeAddress), peersHash); + }); + } - this.isInConflictWithNonSeedNode = inConflictWithNonSeedNode.get(); - this.isInConflictWithSeedNode = inConflictWithSeedNode.get(); + private void putInPeersMapAndCheckForConflicts(String peersAddress, DaoStateHash peersHash) { + putInPeersMapAndCheckForConflicts(findDaoStateBlock(peersHash.getHeight()), peersAddress, peersHash); + } - String conflictMsg = sb.toString(); - if (!conflictMsg.isEmpty()) { - if (this.isInConflictWithSeedNode) - log.warn("Conflict with seed nodes: {}", conflictMsg); - else if (this.isInConflictWithNonSeedNode) - log.debug("Conflict with non-seed nodes: {}", conflictMsg); + private void putInPeersMapAndCheckForConflicts(Optional optionalDaoStateBlock, + String peersAddress, + DaoStateHash peersHash) { + optionalDaoStateBlock.ifPresent(daoStateBlock -> { + daoStateBlock.putInPeersMap(peersAddress, peersHash); + checkForHashConflicts(peersHash, peersAddress, daoStateBlock); + }); + } + + private void checkForHashConflicts(DaoStateHash peersDaoStateHash, + String peersNodeAddress, + DaoStateBlock daoStateBlock) { + if (daoStateBlock.getMyStateHash().hasEqualHash(peersDaoStateHash)) { + return; } - if (notifyListeners && changed.get()) { - listeners.forEach(Listener::onChangeAfterBatchProcessing); + daoStateBlock.putInConflictMap(peersNodeAddress, peersDaoStateHash); + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("We received a block hash from peer ") + .append(peersNodeAddress) + .append(" which conflicts with our block hash.\n") + .append("my peersDaoStateHash=") + .append(daoStateBlock.getMyStateHash()) + .append("\npeers peersDaoStateHash=") + .append(peersDaoStateHash); + String conflictMsg = stringBuilder.toString(); + + if (isSeedNode(peersNodeAddress)) { + isInConflictWithSeedNode = true; + log.warn("Conflict with seed nodes: {}", conflictMsg); + } else { + isInConflictWithNonSeedNode = true; + log.debug("Conflict with non-seed nodes: {}", conflictMsg); } + } - GcUtil.maybeReleaseMemory(); + private void checkUtxos(Block block) { + long genesisTotalSupply = daoStateService.getGenesisTotalSupply().value; + long compensationIssuance = daoStateService.getTotalIssuedAmount(IssuanceType.COMPENSATION); + long reimbursementIssuance = daoStateService.getTotalIssuedAmount(IssuanceType.REIMBURSEMENT); + long totalAmountOfBurntBsq = daoStateService.getTotalAmountOfBurntBsq(); + // confiscated funds are still in the utxo set + long sumUtxo = daoStateService.getUnspentTxOutputMap().values().stream().mapToLong(BaseTxOutput::getValue).sum(); + long sumBsq = genesisTotalSupply + compensationIssuance + reimbursementIssuance - totalAmountOfBurntBsq; - return changed.get(); + if (sumBsq != sumUtxo) { + utxoMismatches.add(new UtxoMismatch(block.getHeight(), sumUtxo, sumBsq)); + } } private void verifyCheckpoints() { @@ -431,4 +494,17 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe log.error(t.toString()); } } + + private boolean isSeedNode(String peersNodeAddress) { + return seedNodeAddresses.contains(peersNodeAddress); + } + + private String getPeersAddress(Optional peersNodeAddress) { + return peersNodeAddress.map(NodeAddress::getFullAddress) + .orElseGet(() -> "Unknown peer " + new Random().nextInt(10000)); + } + + private Optional findDaoStateBlock(int height) { + return Optional.ofNullable(daoStateBlockByHeight.get(height)); + } } diff --git a/core/src/main/java/bisq/core/dao/monitoring/ProposalStateMonitoringService.java b/core/src/main/java/bisq/core/dao/monitoring/ProposalStateMonitoringService.java index 81d40496d8..c28f33a235 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/ProposalStateMonitoringService.java +++ b/core/src/main/java/bisq/core/dao/monitoring/ProposalStateMonitoringService.java @@ -275,7 +275,7 @@ public class ProposalStateMonitoringService implements DaoSetupService, DaoState } byte[] combined = ArrayUtils.addAll(prevHash, serializedProposals); byte[] hash = Hash.getSha256Ripemd160hash(combined); - ProposalStateHash myProposalStateHash = new ProposalStateHash(blockHeight, hash, prevHash, proposals.size()); + ProposalStateHash myProposalStateHash = new ProposalStateHash(blockHeight, hash, proposals.size()); ProposalStateBlock proposalStateBlock = new ProposalStateBlock(myProposalStateHash); proposalStateBlockChain.add(proposalStateBlock); proposalStateHashChain.add(myProposalStateHash); diff --git a/core/src/main/java/bisq/core/dao/monitoring/model/BlindVoteStateHash.java b/core/src/main/java/bisq/core/dao/monitoring/model/BlindVoteStateHash.java index 79a59032c7..ce08528d95 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/model/BlindVoteStateHash.java +++ b/core/src/main/java/bisq/core/dao/monitoring/model/BlindVoteStateHash.java @@ -29,8 +29,8 @@ public final class BlindVoteStateHash extends StateHash { @Getter private final int numBlindVotes; - public BlindVoteStateHash(int cycleStartBlockHeight, byte[] hash, byte[] prevHash, int numBlindVotes) { - super(cycleStartBlockHeight, hash, prevHash); + public BlindVoteStateHash(int cycleStartBlockHeight, byte[] hash, int numBlindVotes) { + super(cycleStartBlockHeight, hash); this.numBlindVotes = numBlindVotes; } @@ -43,14 +43,12 @@ public final class BlindVoteStateHash extends StateHash { return protobuf.BlindVoteStateHash.newBuilder() .setHeight(height) .setHash(ByteString.copyFrom(hash)) - .setPrevHash(ByteString.copyFrom(prevHash)) .setNumBlindVotes(numBlindVotes).build(); } public static BlindVoteStateHash fromProto(protobuf.BlindVoteStateHash proto) { return new BlindVoteStateHash(proto.getHeight(), proto.getHash().toByteArray(), - proto.getPrevHash().toByteArray(), proto.getNumBlindVotes()); } diff --git a/core/src/main/java/bisq/core/dao/monitoring/model/DaoStateBlock.java b/core/src/main/java/bisq/core/dao/monitoring/model/DaoStateBlock.java index 8bd91e67fa..04e24b4d45 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/model/DaoStateBlock.java +++ b/core/src/main/java/bisq/core/dao/monitoring/model/DaoStateBlock.java @@ -18,12 +18,14 @@ package bisq.core.dao.monitoring.model; import lombok.EqualsAndHashCode; -import lombok.Getter; -@Getter @EqualsAndHashCode(callSuper = true) public class DaoStateBlock extends StateBlock { public DaoStateBlock(DaoStateHash myDaoStateHash) { super(myDaoStateHash); } + + public boolean isSelfCreated() { + return myStateHash.isSelfCreated(); + } } diff --git a/core/src/main/java/bisq/core/dao/monitoring/model/DaoStateHash.java b/core/src/main/java/bisq/core/dao/monitoring/model/DaoStateHash.java index e0e7f5e403..e2a624576c 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/model/DaoStateHash.java +++ b/core/src/main/java/bisq/core/dao/monitoring/model/DaoStateHash.java @@ -21,11 +21,17 @@ package bisq.core.dao.monitoring.model; import com.google.protobuf.ByteString; import lombok.EqualsAndHashCode; +import lombok.Getter; +@Getter @EqualsAndHashCode(callSuper = true) public final class DaoStateHash extends StateHash { - public DaoStateHash(int height, byte[] hash, byte[] prevHash) { - super(height, hash, prevHash); + // If we have built the hash by ourself opposed to that we got delivered the hash from seed nodes or resources + private final boolean isSelfCreated; + + public DaoStateHash(int height, byte[] hash, boolean isSelfCreated) { + super(height, hash); + this.isSelfCreated = isSelfCreated; } @@ -38,12 +44,18 @@ public final class DaoStateHash extends StateHash { return protobuf.DaoStateHash.newBuilder() .setHeight(height) .setHash(ByteString.copyFrom(hash)) - .setPrevHash(ByteString.copyFrom(prevHash)).build(); + .setIsSelfCreated(isSelfCreated) + .build(); } public static DaoStateHash fromProto(protobuf.DaoStateHash proto) { - return new DaoStateHash(proto.getHeight(), - proto.getHash().toByteArray(), - proto.getPrevHash().toByteArray()); + return new DaoStateHash(proto.getHeight(), proto.getHash().toByteArray(), proto.getIsSelfCreated()); + } + + @Override + public String toString() { + return "DaoStateHash{" + + "\r\n isSelfCreated=" + isSelfCreated + + "\r\n} " + super.toString(); } } diff --git a/core/src/main/java/bisq/core/dao/monitoring/model/ProposalStateHash.java b/core/src/main/java/bisq/core/dao/monitoring/model/ProposalStateHash.java index dc2ccdeabf..995b28cefc 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/model/ProposalStateHash.java +++ b/core/src/main/java/bisq/core/dao/monitoring/model/ProposalStateHash.java @@ -29,8 +29,8 @@ public final class ProposalStateHash extends StateHash { @Getter private final int numProposals; - public ProposalStateHash(int cycleStartBlockHeight, byte[] hash, byte[] prevHash, int numProposals) { - super(cycleStartBlockHeight, hash, prevHash); + public ProposalStateHash(int cycleStartBlockHeight, byte[] hash, int numProposals) { + super(cycleStartBlockHeight, hash); this.numProposals = numProposals; } @@ -43,14 +43,12 @@ public final class ProposalStateHash extends StateHash { return protobuf.ProposalStateHash.newBuilder() .setHeight(height) .setHash(ByteString.copyFrom(hash)) - .setPrevHash(ByteString.copyFrom(prevHash)) .setNumProposals(numProposals).build(); } public static ProposalStateHash fromProto(protobuf.ProposalStateHash proto) { return new ProposalStateHash(proto.getHeight(), proto.getHash().toByteArray(), - proto.getPrevHash().toByteArray(), proto.getNumProposals()); } diff --git a/core/src/main/java/bisq/core/dao/monitoring/model/StateBlock.java b/core/src/main/java/bisq/core/dao/monitoring/model/StateBlock.java index b41ddc2296..8f5a27b78f 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/model/StateBlock.java +++ b/core/src/main/java/bisq/core/dao/monitoring/model/StateBlock.java @@ -56,10 +56,6 @@ public abstract class StateBlock { return myStateHash.getHash(); } - public byte[] getPrevHash() { - return myStateHash.getPrevHash(); - } - @Override public String toString() { return "StateBlock{" + diff --git a/core/src/main/java/bisq/core/dao/monitoring/model/StateHash.java b/core/src/main/java/bisq/core/dao/monitoring/model/StateHash.java index 12cb299a11..63771d83ee 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/model/StateHash.java +++ b/core/src/main/java/bisq/core/dao/monitoring/model/StateHash.java @@ -40,13 +40,10 @@ import lombok.extern.slf4j.Slf4j; public abstract class StateHash implements PersistablePayload, NetworkPayload { protected final int height; protected final byte[] hash; - // For first block the prevHash is an empty byte array - protected final byte[] prevHash; - StateHash(int height, byte[] hash, byte[] prevHash) { + StateHash(int height, byte[] hash) { this.height = height; this.hash = hash; - this.prevHash = prevHash; } /////////////////////////////////////////////////////////////////////////////////////////// @@ -67,7 +64,6 @@ public abstract class StateHash implements PersistablePayload, NetworkPayload { return "StateHash{" + "\n height=" + height + ",\n hash=" + Utilities.bytesAsHexString(hash) + - ",\n prevHash=" + Utilities.bytesAsHexString(prevHash) + "\n}"; } } diff --git a/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java b/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java index 8300c76b23..92e110c9c1 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java +++ b/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java @@ -102,7 +102,8 @@ public abstract class StateNetworkService listener); + protected abstract Han getRequestStateHashesHandler(NodeAddress nodeAddress, + RequestStateHashesHandler.Listener listener); /////////////////////////////////////////////////////////////////////////////////////////// @@ -155,8 +156,7 @@ public abstract class StateNetworkService(blockList), () -> { - log.info("runDelayedBatchProcessing Parsing {} blocks took {} seconds.", blockList.size(), + log.info("Parsing {} blocks took {} seconds.", blockList.size(), (System.currentTimeMillis() - ts) / 1000d); // We only request again if wallet is synced, otherwise we would get repeated calls we want to avoid. // We deal with that case at the setupWalletBestBlockListener method above. diff --git a/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java b/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java index 189afea334..9c46f8220a 100644 --- a/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java +++ b/core/src/main/java/bisq/core/dao/node/lite/network/LiteNodeNetworkService.java @@ -200,6 +200,7 @@ public class LiteNodeNetworkService implements MessageListener, ConnectionListen @Override public void onAllConnectionsLost() { + log.info("onAllConnectionsLost"); closeAllHandlers(); stopRetryTimer(); stopped = true; @@ -208,6 +209,7 @@ public class LiteNodeNetworkService implements MessageListener, ConnectionListen @Override public void onNewConnectionAfterAllConnectionsLost() { + log.info("onNewConnectionAfterAllConnectionsLost"); closeAllHandlers(); stopped = false; tryWithNewSeedNode(lastRequestedBlockHeight); diff --git a/core/src/main/java/bisq/core/dao/node/parser/BlockParser.java b/core/src/main/java/bisq/core/dao/node/parser/BlockParser.java index 01ca2c34e8..92157f2b7b 100644 --- a/core/src/main/java/bisq/core/dao/node/parser/BlockParser.java +++ b/core/src/main/java/bisq/core/dao/node/parser/BlockParser.java @@ -24,7 +24,6 @@ import bisq.core.dao.state.DaoStateService; import bisq.core.dao.state.model.blockchain.Block; import bisq.common.app.DevEnv; -import bisq.common.util.GcUtil; import org.bitcoinj.core.Coin; @@ -113,10 +112,12 @@ public class BlockParser { .ifPresent(tx -> daoStateService.onNewTxForLastBlock(block, tx))); daoStateService.onParseBlockComplete(block); - log.info("Parsing {} transactions at block height {} took {} ms", rawBlock.getRawTxs().size(), - blockHeight, System.currentTimeMillis() - startTs); + long duration = System.currentTimeMillis() - startTs; + if (duration > 10) { + log.info("Parsing {} transactions at block height {} took {} ms", rawBlock.getRawTxs().size(), + blockHeight, duration); + } - GcUtil.maybeReleaseMemory(); return block; } diff --git a/core/src/main/java/bisq/core/dao/state/DaoStateSnapshotService.java b/core/src/main/java/bisq/core/dao/state/DaoStateSnapshotService.java index 3b179551d3..7a724a3f20 100644 --- a/core/src/main/java/bisq/core/dao/state/DaoStateSnapshotService.java +++ b/core/src/main/java/bisq/core/dao/state/DaoStateSnapshotService.java @@ -17,11 +17,13 @@ package bisq.core.dao.state; +import bisq.core.dao.DaoSetupService; import bisq.core.dao.monitoring.DaoStateMonitoringService; import bisq.core.dao.monitoring.model.DaoStateHash; import bisq.core.dao.state.model.DaoState; import bisq.core.dao.state.model.blockchain.Block; import bisq.core.dao.state.storage.DaoStateStorageService; +import bisq.core.user.Preferences; import bisq.common.config.Config; import bisq.common.util.GcUtil; @@ -49,13 +51,14 @@ import javax.annotation.Nullable; * SNAPSHOT_GRID old not less than 2 times the SNAPSHOT_GRID old. */ @Slf4j -public class DaoStateSnapshotService { +public class DaoStateSnapshotService implements DaoSetupService, DaoStateListener { private static final int SNAPSHOT_GRID = 20; private final DaoStateService daoStateService; private final GenesisTxInfo genesisTxInfo; private final DaoStateStorageService daoStateStorageService; private final DaoStateMonitoringService daoStateMonitoringService; + private final Preferences preferences; private final File storageDir; private DaoState daoStateSnapshotCandidate; @@ -64,7 +67,8 @@ public class DaoStateSnapshotService { @Setter @Nullable private Runnable daoRequiresRestartHandler; - private boolean requestPersistenceCalled; + private boolean readyForPersisting = true; + private boolean isParseBlockChainComplete; /////////////////////////////////////////////////////////////////////////////////////////// @@ -76,21 +80,86 @@ public class DaoStateSnapshotService { GenesisTxInfo genesisTxInfo, DaoStateStorageService daoStateStorageService, DaoStateMonitoringService daoStateMonitoringService, + Preferences preferences, @Named(Config.STORAGE_DIR) File storageDir) { this.daoStateService = daoStateService; this.genesisTxInfo = genesisTxInfo; this.daoStateStorageService = daoStateStorageService; this.daoStateMonitoringService = daoStateMonitoringService; + this.preferences = preferences; this.storageDir = storageDir; } + /////////////////////////////////////////////////////////////////////////////////////////// + // DaoSetupService + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public void addListeners() { + daoStateService.addDaoStateListener(this); + } + + @Override + public void start() { + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // DaoStateListener + /////////////////////////////////////////////////////////////////////////////////////////// + + // We listen onDaoStateChanged to ensure the dao state has been processed from listener clients after parsing. + // We need to listen during batch processing as well to write snapshots during that process. + @Override + public void onDaoStateChanged(Block block) { + // If we have isUseDaoMonitor activated we apply the hash and snapshots at each new block during initial parsing. + // Otherwise we do it only after the initial blockchain parsing is completed to not delay the parsing. + // In that case we get the missing hashes from the seed nodes. At any new block we do the hash calculation + // ourself and therefore get back confidence that our DAO state is in sync with the network. + if (preferences.isUseFullModeDaoMonitor() || isParseBlockChainComplete) { + // We need to execute first the daoStateMonitoringService.createHashFromBlock to get the hash created + daoStateMonitoringService.createHashFromBlock(block); + maybeCreateSnapshot(block); + } + } + + @Override + public void onParseBlockChainComplete() { + isParseBlockChainComplete = true; + + // In case we have dao monitoring deactivated we create the snapshot after we are completed with parsing + // and we got called back from daoStateMonitoringService once the hashes are created from peers data. + if (!preferences.isUseFullModeDaoMonitor()) { + // We register a callback handler once the daoStateMonitoringService has received the missing hashes from + // the seed node and applied the latest hash. After that we are ready to make a snapshot and persist it. + daoStateMonitoringService.setCreateSnapshotHandler(() -> { + // As we did not have created any snapshots during initial parsing we create it now. We cannot use the past + // snapshot height as we have not cloned a candidate (that would cause quite some delay during parsing). + // The next snapshots will be created again according to the snapshot height grid (each 20 blocks). + // This also comes with the improvement that the user does not need to load the past blocks back to the last + // snapshot height. Though it comes also with the small risk that in case of re-orgs the user need to do + // a resync in case the dao state would have been affected by that reorg. + long ts = System.currentTimeMillis(); + // We do not keep a copy of the clone as we use it immediately for persistence. + GcUtil.maybeReleaseMemory(); + log.info("Create snapshot at height {}", daoStateService.getChainHeight()); + daoStateStorageService.requestPersistence(daoStateService.getClone(), + new LinkedList<>(daoStateMonitoringService.getDaoStateHashChain()), + () -> { + log.info("Persisted daoState after parsing completed at height {}. Took {} ms", + daoStateService.getChainHeight(), System.currentTimeMillis() - ts); + }); + GcUtil.maybeReleaseMemory(); + }); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////// // API /////////////////////////////////////////////////////////////////////////////////////////// - // We do not use DaoStateListener.onDaoStateChanged but let the DaoEventCoordinator call maybeCreateSnapshot to ensure the - // correct order of execution. // We need to process during batch processing as well to write snapshots during that process. public void maybeCreateSnapshot(Block block) { int chainHeight = block.getHeight(); @@ -107,42 +176,68 @@ public class DaoStateSnapshotService { // We protect to get called while we are not completed with persisting the daoState. This can take about // 20 seconds and it is not expected that we get triggered another snapshot event in that period, but this // check guards that we would skip such calls.. - if (requestPersistenceCalled) { - log.warn("We try to persist a daoState but the previous call has not completed yet. " + - "We ignore that call and skip that snapshot. " + - "Snapshot will be created at next snapshot height again. This is not to be expected with live " + - "blockchain data."); + if (!readyForPersisting) { + if (preferences.isUseFullModeDaoMonitor()) { + // In case we dont use isUseFullModeDaoMonitor we might called here too often as the parsing is much + // faster than the persistence and we likely create only 1 snapshot during initial parsing, so + // we log only if isUseFullModeDaoMonitor is true as then parsing is likely slower and we would + // expect that we do a snapshot at each trigger block. + log.info("We try to persist a daoState but the previous call has not completed yet. " + + "We ignore that call and skip that snapshot. " + + "Snapshot will be created at next snapshot height again. This is not to be expected with live " + + "blockchain data."); + } return; } - GcUtil.maybeReleaseMemory(); - - // At trigger event we store the latest snapshotCandidate to disc - long ts = System.currentTimeMillis(); - requestPersistenceCalled = true; - daoStateStorageService.requestPersistence(daoStateSnapshotCandidate, - daoStateHashChainSnapshotCandidate, - () -> { - log.info("Serializing snapshotCandidate for writing to Disc with height {} at height {} took {} ms", - daoStateSnapshotCandidate != null ? daoStateSnapshotCandidate.getChainHeight() : "N/A", - chainHeight, - System.currentTimeMillis() - ts); - - long ts2 = System.currentTimeMillis(); - - GcUtil.maybeReleaseMemory(); - - // Now we clone and keep it in memory for the next trigger event - daoStateSnapshotCandidate = daoStateService.getClone(); - daoStateHashChainSnapshotCandidate = new LinkedList<>(daoStateMonitoringService.getDaoStateHashChain()); - - log.info("Cloned new snapshotCandidate at height {} took {} ms", chainHeight, System.currentTimeMillis() - ts2); - requestPersistenceCalled = false; - GcUtil.maybeReleaseMemory(); - }); + if (daoStateSnapshotCandidate != null) { + persist(); + } else { + createClones(); + } } } + private void persist() { + long ts = System.currentTimeMillis(); + readyForPersisting = false; + daoStateStorageService.requestPersistence(daoStateSnapshotCandidate, + daoStateHashChainSnapshotCandidate, + () -> { + log.info("Serializing snapshotCandidate for writing to Disc at chainHeight {} took {} ms.\n" + + "daoStateSnapshotCandidate.height={};\n" + + "daoStateHashChainSnapshotCandidate.height={}", + daoStateService.getChainHeight(), + System.currentTimeMillis() - ts, + daoStateSnapshotCandidate != null ? daoStateSnapshotCandidate.getChainHeight() : "N/A", + daoStateHashChainSnapshotCandidate != null && !daoStateHashChainSnapshotCandidate.isEmpty() ? + daoStateHashChainSnapshotCandidate.getLast().getHeight() : "N/A"); + + createClones(); + readyForPersisting = true; + }); + } + + private void createClones() { + long ts = System.currentTimeMillis(); + // Now we clone and keep it in memory for the next trigger event + // We do not fit into the target grid of 20 blocks as we get called here once persistence is + // done from the write thread (mapped back to user thread). + // As we want to prevent to maintain 2 clones we prefer that strategy. If we would do the clone + // after the persist call we would keep an additional copy in memory. + daoStateSnapshotCandidate = daoStateService.getClone(); + daoStateHashChainSnapshotCandidate = new LinkedList<>(daoStateMonitoringService.getDaoStateHashChain()); + GcUtil.maybeReleaseMemory(); + + log.info("Cloned new snapshotCandidate at chainHeight {} took {} ms.\n" + + "daoStateSnapshotCandidate.height={};\n" + + "daoStateHashChainSnapshotCandidate.height={}", + daoStateService.getChainHeight(), System.currentTimeMillis() - ts, + daoStateSnapshotCandidate != null ? daoStateSnapshotCandidate.getChainHeight() : "N/A", + daoStateHashChainSnapshotCandidate != null && !daoStateHashChainSnapshotCandidate.isEmpty() ? + daoStateHashChainSnapshotCandidate.getLast().getHeight() : "N/A"); + } + public void applySnapshot(boolean fromReorg) { DaoState persistedBsqState = daoStateStorageService.getPersistedBsqState(); LinkedList persistedDaoStateHashChain = daoStateStorageService.getPersistedDaoStateHashChain(); @@ -156,6 +251,7 @@ public class DaoStateSnapshotService { chainHeightOfLastApplySnapshot = chainHeightOfPersisted; daoStateService.applySnapshot(persistedBsqState); daoStateMonitoringService.applySnapshot(persistedDaoStateHashChain); + daoStateStorageService.pruneStore(); } else { // The reorg might have been caused by the previous parsing which might contains a range of // blocks. diff --git a/core/src/main/java/bisq/core/dao/state/storage/DaoStateStorageService.java b/core/src/main/java/bisq/core/dao/state/storage/DaoStateStorageService.java index 2abf8bafe5..84625a0f7a 100644 --- a/core/src/main/java/bisq/core/dao/state/storage/DaoStateStorageService.java +++ b/core/src/main/java/bisq/core/dao/state/storage/DaoStateStorageService.java @@ -27,6 +27,7 @@ import bisq.network.p2p.storage.persistence.StoreService; import bisq.common.config.Config; import bisq.common.file.FileUtil; import bisq.common.persistence.PersistenceManager; +import bisq.common.util.GcUtil; import javax.inject.Inject; import javax.inject.Named; @@ -92,15 +93,20 @@ public class DaoStateStorageService extends StoreService { new Thread(() -> { Thread.currentThread().setName("Serialize and write DaoState"); persistenceManager.persistNow(() -> { - // After we have written to disk we remove the the daoState in the store to avoid that it stays in + // After we have written to disk we remove the daoState in the store to avoid that it stays in // memory there until the next persist call. - store.setDaoState(null); - + pruneStore(); completeHandler.run(); }); }).start(); } + public void pruneStore() { + store.setDaoState(null); + store.setDaoStateHashChain(null); + GcUtil.maybeReleaseMemory(); + } + public DaoState getPersistedBsqState() { return store.getDaoState(); } diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java index 6049e67e38..e94ed5d683 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatisticsManager.java @@ -82,7 +82,6 @@ public class TradeStatisticsManager { this.storageDir = storageDir; this.dumpStatistics = dumpStatistics; - appendOnlyDataStoreService.addService(tradeStatistics3StorageService); } diff --git a/core/src/main/java/bisq/core/trade/txproof/xmr/XmrTxProofRequestsPerTrade.java b/core/src/main/java/bisq/core/trade/txproof/xmr/XmrTxProofRequestsPerTrade.java index 2560d71a11..cbd20b8738 100644 --- a/core/src/main/java/bisq/core/trade/txproof/xmr/XmrTxProofRequestsPerTrade.java +++ b/core/src/main/java/bisq/core/trade/txproof/xmr/XmrTxProofRequestsPerTrade.java @@ -138,7 +138,7 @@ class XmrTxProofRequestsPerTrade implements AssetTxProofRequestsPerTrade { // We set serviceAddresses at request time. If user changes AutoConfirmSettings after request has started // it will have no impact on serviceAddresses and numRequiredSuccessResults. - // Thought numRequiredConfirmations can be changed during request process and will be read from + // Though numRequiredConfirmations can be changed during request process and will be read from // autoConfirmSettings at result parsing. List serviceAddresses = autoConfirmSettings.getServiceAddresses(); numRequiredSuccessResults = serviceAddresses.size(); diff --git a/core/src/main/java/bisq/core/user/Preferences.java b/core/src/main/java/bisq/core/user/Preferences.java index 30e93dc800..70eef439fb 100644 --- a/core/src/main/java/bisq/core/user/Preferences.java +++ b/core/src/main/java/bisq/core/user/Preferences.java @@ -800,6 +800,11 @@ public final class Preferences implements PersistedDataHost, BridgeAddressProvid requestPersistence(); } + public void setUseFullModeDaoMonitor(boolean value) { + prefPayload.setUseFullModeDaoMonitor(value); + requestPersistence(); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Getter @@ -1115,5 +1120,7 @@ public final class Preferences implements PersistedDataHost, BridgeAddressProvid void setDenyApiTaker(boolean value); void setNotifyOnPreRelease(boolean value); + + void setUseFullModeDaoMonitor(boolean value); } } diff --git a/core/src/main/java/bisq/core/user/PreferencesPayload.java b/core/src/main/java/bisq/core/user/PreferencesPayload.java index 3eb6ecfa7b..ffa943f8ca 100644 --- a/core/src/main/java/bisq/core/user/PreferencesPayload.java +++ b/core/src/main/java/bisq/core/user/PreferencesPayload.java @@ -134,6 +134,7 @@ public final class PreferencesPayload implements PersistableEnvelope { private boolean showOffersMatchingMyAccounts; private boolean denyApiTaker; private boolean notifyOnPreRelease; + private boolean useFullModeDaoMonitor; /////////////////////////////////////////////////////////////////////////////////////////// // Constructor @@ -201,7 +202,8 @@ public final class PreferencesPayload implements PersistableEnvelope { .setHideNonAccountPaymentMethods(hideNonAccountPaymentMethods) .setShowOffersMatchingMyAccounts(showOffersMatchingMyAccounts) .setDenyApiTaker(denyApiTaker) - .setNotifyOnPreRelease(notifyOnPreRelease); + .setNotifyOnPreRelease(notifyOnPreRelease) + .setUseFullModeDaoMonitor(useFullModeDaoMonitor); Optional.ofNullable(backupDirectory).ifPresent(builder::setBackupDirectory); Optional.ofNullable(preferredTradeCurrency).ifPresent(e -> builder.setPreferredTradeCurrency((protobuf.TradeCurrency) e.toProtoMessage())); @@ -299,7 +301,8 @@ public final class PreferencesPayload implements PersistableEnvelope { proto.getHideNonAccountPaymentMethods(), proto.getShowOffersMatchingMyAccounts(), proto.getDenyApiTaker(), - proto.getNotifyOnPreRelease() + proto.getNotifyOnPreRelease(), + proto.getUseFullModeDaoMonitor() ); } } diff --git a/core/src/main/resources/i18n/displayStrings.properties b/core/src/main/resources/i18n/displayStrings.properties index 0fbc9aa99a..e88e5d77df 100644 --- a/core/src/main/resources/i18n/displayStrings.properties +++ b/core/src/main/resources/i18n/displayStrings.properties @@ -1351,6 +1351,18 @@ setting.preferences.dao.resyncFromGenesis.resync=Resync from genesis and shutdow setting.preferences.dao.isDaoFullNode=Run Bisq as DAO full node setting.preferences.dao.activated=DAO activated setting.preferences.dao.activated.popup=The change will be applied after a restart + +setting.preferences.dao.fullModeDaoMonitor=Full-mode DAO state monitoring +setting.preferences.dao.fullModeDaoMonitor.popup=If full-mode DAO state monitoring is activated the DAO state \ + hashes are created during parsing the BSQ blocks. This comes with considerable performance costs at the initial DAO sync.\n\n\ + For users who are regularily using Bisq this should not be an issue as there are not many blocks pending for parsing, though for \ + users who only use Bisq casually creating the DAO state hashes for 100s or 1000s of blocks degrades heavily the user experience.\n\n\ + In case full-mode is deactivated (default) the missing DAO state hashes are requested from network nodes and \ + the DAO state hash based on the most recent block will be created by the user. As all hashes are connected by \ + reference to the previous hash a correct hash at the chain tip means that all past hashes are correct as well. The \ + main functionality of the DAO state monitoring - to detect if the local DAO state is out of sync with the rest of the network - \ + is therefore still fulfilled. + setting.preferences.dao.rpcUser=RPC username setting.preferences.dao.rpcPw=RPC password setting.preferences.dao.blockNotifyPort=Block notify port @@ -2499,6 +2511,9 @@ dao.monitor.proposals=Proposals state dao.monitor.blindVotes=Blind votes state dao.monitor.table.peers=Peers +dao.monitor.table.hashCreator=Hash creator +dao.monitor.table.hashCreator.self=Self +dao.monitor.table.hashCreator.peer=Peer dao.monitor.table.conflicts=Conflicts dao.monitor.state=Status dao.monitor.requestAlHashes=Request all hashes @@ -2532,6 +2547,8 @@ dao.monitor.isInConflictWithSeedNode=Your local data is not in consensus with at Please resync the DAO state. dao.monitor.isInConflictWithNonSeedNode=One of your peers is not in consensus with the network but your node \ is in sync with the seed nodes. +dao.monitor.isDaoStateBlockChainNotConnecting=Your DAO state chain is not connecting with the new data. \ + Please resync the DAO state. dao.monitor.daoStateInSync=Your local node is in consensus with the network dao.monitor.blindVote.headline=Blind votes state diff --git a/core/src/test/java/bisq/core/dao/state/DaoStateSnapshotServiceTest.java b/core/src/test/java/bisq/core/dao/state/DaoStateSnapshotServiceTest.java index a9e773878a..5e8ccefe05 100644 --- a/core/src/test/java/bisq/core/dao/state/DaoStateSnapshotServiceTest.java +++ b/core/src/test/java/bisq/core/dao/state/DaoStateSnapshotServiceTest.java @@ -38,6 +38,7 @@ public class DaoStateSnapshotServiceTest { mock(GenesisTxInfo.class), mock(DaoStateStorageService.class), mock(DaoStateMonitoringService.class), + null, null); } diff --git a/desktop/src/main/java/bisq/desktop/main/MainView.java b/desktop/src/main/java/bisq/desktop/main/MainView.java index 259f1439a7..72d6a36263 100644 --- a/desktop/src/main/java/bisq/desktop/main/MainView.java +++ b/desktop/src/main/java/bisq/desktop/main/MainView.java @@ -416,7 +416,7 @@ public class MainView extends InitializableView /////////////////////////////////////////////////////////////////////////////////////////// @Override - public void onChangeAfterBatchProcessing() { + public void onDaoStateHashesChanged() { } @Override diff --git a/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateBlockListItem.java b/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateBlockListItem.java index eff55bb186..b7093b8cea 100644 --- a/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateBlockListItem.java +++ b/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateBlockListItem.java @@ -36,10 +36,9 @@ import lombok.extern.slf4j.Slf4j; @Getter @EqualsAndHashCode public abstract class StateBlockListItem> { - private final StateBlock stateBlock; + protected final StateBlock stateBlock; private final Supplier height; private final String hash; - private final String prevHash; private final String numNetworkMessages; private final String numMisMatches; private final boolean isInSync; @@ -58,7 +57,6 @@ public abstract class StateBlockListItem 0 ? Utilities.bytesAsHexString(stateBlock.getPrevHash()) : "-"; numNetworkMessages = String.valueOf(stateBlock.getPeersMap().size()); int size = stateBlock.getInConflictMap().size(); numMisMatches = String.valueOf(size); diff --git a/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateInConflictListItem.java b/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateInConflictListItem.java index cf51c261c2..3ca42f1474 100644 --- a/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateInConflictListItem.java +++ b/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateInConflictListItem.java @@ -38,7 +38,6 @@ public abstract class StateInConflictListItem { private final String peerAddressString; private final String height; private final String hash; - private final String prevHash; private final T stateHash; protected StateInConflictListItem(String peerAddress, T stateHash, int cycleIndex, @@ -52,7 +51,5 @@ public abstract class StateInConflictListItem { cycleIndex + 1, String.valueOf(stateHash.getHeight())); hash = Utilities.bytesAsHexString(stateHash.getHash()); - prevHash = stateHash.getPrevHash().length > 0 ? - Utilities.bytesAsHexString(stateHash.getPrevHash()) : "-"; } } diff --git a/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateMonitorView.java b/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateMonitorView.java index 962ca6eaf2..8816f0b95e 100644 --- a/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateMonitorView.java +++ b/desktop/src/main/java/bisq/desktop/main/dao/monitor/StateMonitorView.java @@ -99,6 +99,7 @@ public abstract class StateMonitorView resyncDaoState()); @@ -177,8 +180,6 @@ public abstract class StateMonitorView(getHashTableHeader()); column.setMinWidth(120); column.setCellValueFactory((item) -> new ReadOnlyObjectWrapper<>(item.getValue())); @@ -353,31 +356,6 @@ public abstract class StateMonitorView(getPrevHashTableHeader()); - column.setMinWidth(120); - column.setCellValueFactory((item) -> new ReadOnlyObjectWrapper<>(item.getValue())); - column.setCellFactory( - new Callback<>() { - - @Override - public TableCell call(TableColumn column) { - return new TableCell<>() { - @Override - public void updateItem(final BLI item, boolean empty) { - super.updateItem(item, empty); - if (item != null) - setText(item.getPrevHash()); - else - setText(""); - } - }; - } - }); - column.setComparator(Comparator.comparing(BLI::getPrevHash)); - tableView.getColumns().add(column); - - column = new AutoTooltipTableColumn<>(getPeersTableHeader()); column.setMinWidth(80); column.setMaxWidth(column.getMinWidth()); @@ -543,30 +521,6 @@ public abstract class StateMonitorView(getPrevHashTableHeader()); - column.setMinWidth(120); - column.setCellValueFactory((item) -> new ReadOnlyObjectWrapper<>(item.getValue())); - column.setCellFactory( - new Callback<>() { - @Override - public TableCell call( - TableColumn column) { - return new TableCell<>() { - @Override - public void updateItem(final CLI item, boolean empty) { - super.updateItem(item, empty); - if (item != null) - setText(item.getPrevHash()); - else - setText(""); - } - }; - } - }); - column.setComparator(Comparator.comparing(CLI::getPrevHash)); - conflictTableView.getColumns().add(column); - - column = new AutoTooltipTableColumn<>(""); column.setMinWidth(120); column.setCellValueFactory((item) -> new ReadOnlyObjectWrapper<>(item.getValue())); diff --git a/desktop/src/main/java/bisq/desktop/main/dao/monitor/blindvotes/BlindVoteStateMonitorView.java b/desktop/src/main/java/bisq/desktop/main/dao/monitor/blindvotes/BlindVoteStateMonitorView.java index 6949fa8b02..5f470109bf 100644 --- a/desktop/src/main/java/bisq/desktop/main/dao/monitor/blindvotes/BlindVoteStateMonitorView.java +++ b/desktop/src/main/java/bisq/desktop/main/dao/monitor/blindvotes/BlindVoteStateMonitorView.java @@ -149,11 +149,6 @@ public class BlindVoteStateMonitorView extends StateMonitorView column = new AutoTooltipTableColumn<>(Res.get("dao.monitor.table.hashCreator")); + column.setMinWidth(90); + column.setMaxWidth(column.getMinWidth()); + column.setCellValueFactory((item) -> new ReadOnlyObjectWrapper<>(item.getValue())); + column.setCellFactory( + new Callback<>() { + @Override + public TableCell call( + TableColumn column) { + return new TableCell<>() { + @Override + public void updateItem(final DaoStateBlockListItem item, boolean empty) { + super.updateItem(item, empty); + if (item != null) + setText(item.hashCreator()); + else + setText(""); + } + }; + } + }); + column.setComparator(Comparator.comparing(e -> e.getStateBlock().getPeersMap().size())); + tableView.getColumns().add(2, column); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Private diff --git a/desktop/src/main/java/bisq/desktop/main/dao/monitor/proposals/ProposalStateMonitorView.java b/desktop/src/main/java/bisq/desktop/main/dao/monitor/proposals/ProposalStateMonitorView.java index f12e574694..d46264c70b 100644 --- a/desktop/src/main/java/bisq/desktop/main/dao/monitor/proposals/ProposalStateMonitorView.java +++ b/desktop/src/main/java/bisq/desktop/main/dao/monitor/proposals/ProposalStateMonitorView.java @@ -147,11 +147,6 @@ public class ProposalStateMonitorView extends StateMonitorView { + preferences.setUseFullModeDaoMonitor(fullModeDaoMonitorToggleButton.isSelected()); + if (fullModeDaoMonitorToggleButton.isSelected()) { + String key = "fullModeDaoMonitor"; + if (DontShowAgainLookup.showAgain(key)) { + new Popup().information(Res.get("setting.preferences.dao.fullModeDaoMonitor.popup")) + .width(1000) + .dontShowAgainId(key) + .closeButtonText(Res.get("shared.iUnderstand")) + .show(); + } + } + }); + boolean daoFullNode = preferences.isDaoFullNode(); isDaoFullNodeToggleButton.setSelected(daoFullNode); @@ -1173,6 +1191,7 @@ public class PreferencesView extends ActivatableViewAndModel