diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 474ecf1c59..df6a83c8fc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,13 +18,13 @@ jobs: java: [ '11', '11.0.3', '15', '15.0.5'] name: Test Java ${{ matrix.Java }}, ${{ matrix.os }} steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 + - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Set up JDK - uses: actions/setup-java@19eeec562b37d29a1ad055b7de9c280bd0906d8d + uses: actions/setup-java@1df8dbefe2a8cbc99770194893dd902763bee34b with: java-version: ${{ matrix.java }} distribution: 'zulu' diff --git a/build.gradle b/build.gradle index d6a75ca027..c0c0d04d37 100644 --- a/build.gradle +++ b/build.gradle @@ -128,47 +128,6 @@ configure([project(':cli'), } } -configure(project(':proto')) { - apply plugin: 'com.google.protobuf' - - dependencies { - annotationProcessor libs.lombok - compileOnly libs.javax.annotation - compileOnly libs.lombok - implementation libs.logback.classic - implementation libs.logback.core - implementation libs.google.guava - implementation libs.protobuf.java - implementation libs.slf4j.api - implementation(libs.grpc.protobuf) { - exclude(module: 'animal-sniffer-annotations') - exclude(module: 'guava') - } - implementation(libs.grpc.stub) { - exclude(module: 'animal-sniffer-annotations') - exclude(module: 'guava') - } - } - - sourceSets.main.java.srcDirs += [ - 'build/generated/source/proto/main/grpc', - 'build/generated/source/proto/main/java' - ] - - protobuf { - protoc { - artifact = "com.google.protobuf:protoc:${protocVersion}" - } - plugins { - grpc { - artifact = "io.grpc:protoc-gen-grpc-java:${grpcVersion}" - } - } - generateProtoTasks { - all()*.plugins { grpc {} } - } - } -} configure(project(':assets')) { dependencies { diff --git a/common/src/main/java/bisq/common/setup/CommonSetup.java b/common/src/main/java/bisq/common/setup/CommonSetup.java index ae92f30af5..d46d933433 100644 --- a/common/src/main/java/bisq/common/setup/CommonSetup.java +++ b/common/src/main/java/bisq/common/setup/CommonSetup.java @@ -76,6 +76,11 @@ public class CommonSetup { if (throwable.getCause() != null && throwable.getCause().getCause() != null && throwable.getCause().getCause() instanceof BlockStoreException) { log.error("Uncaught BlockStoreException ", throwable); + } else if (throwable instanceof OutOfMemoryError) { + Profiler.printSystemLoad(); + log.error("OutOfMemoryError occurred. We shut down.", throwable); + // Leave it to the handleUncaughtException to shut down or not. + UserThread.execute(() -> uncaughtExceptionHandler.handleUncaughtException(throwable, false)); } else if (throwable instanceof ClassCastException && "sun.awt.image.BufImgSurfaceData cannot be cast to sun.java2d.xr.XRSurfaceData".equals(throwable.getMessage())) { log.warn(throwable.getMessage()); diff --git a/common/src/main/java/bisq/common/util/CompletableFutureUtil.java b/common/src/main/java/bisq/common/util/CompletableFutureUtil.java deleted file mode 100644 index 7c885ab4db..0000000000 --- a/common/src/main/java/bisq/common/util/CompletableFutureUtil.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * This file is part of Bisq. - * - * Bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * Bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Bisq. If not, see . - */ - -package bisq.common.util; - -import java.util.Collection; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -//todo -public class CompletableFutureUtil { - public static CompletableFuture> allOf(Collection> collection) { - //noinspection unchecked - return allOf(collection.toArray(new CompletableFuture[0])); - } - - public static CompletableFuture> allOf(Stream> stream) { - return allOf(stream.collect(Collectors.toList())); - } - - public static CompletableFuture> allOf(CompletableFuture... list) { - CompletableFuture> result = CompletableFuture.allOf(list).thenApply(v -> - Stream.of(list) - .map(future -> { - // We want to return the results in list, not the futures. Once allOf call is complete - // we know that all futures have completed (normally, exceptional or cancelled). - // For exceptional and canceled cases we throw an exception. - T res = future.join(); - if (future.isCompletedExceptionally()) { - throw new RuntimeException((future.handle((r, throwable) -> throwable).join())); - } - if (future.isCancelled()) { - throw new RuntimeException("Future got canceled"); - } - return res; - }) - .collect(Collectors.toList()) - ); - return result; - } -} diff --git a/core/src/main/java/bisq/core/app/BisqExecutable.java b/core/src/main/java/bisq/core/app/BisqExecutable.java index e861d15b51..a92134dceb 100644 --- a/core/src/main/java/bisq/core/app/BisqExecutable.java +++ b/core/src/main/java/bisq/core/app/BisqExecutable.java @@ -74,7 +74,7 @@ public abstract class BisqExecutable implements GracefulShutDownHandler, BisqSet protected Injector injector; protected AppModule module; protected Config config; - private boolean isShutdownInProgress; + protected volatile boolean isShutdownInProgress; private boolean hasDowngraded; public BisqExecutable(String fullName, String scriptName, String appName, String version) { @@ -281,7 +281,7 @@ public abstract class BisqExecutable implements GracefulShutDownHandler, BisqSet } } - private void flushAndExit(ResultHandler resultHandler, int status) { + protected void flushAndExit(ResultHandler resultHandler, int status) { if (!hasDowngraded) { // If user tried to downgrade we do not write the persistable data to avoid data corruption log.info("PersistenceManager flushAllDataToDiskAtShutdown started"); diff --git a/core/src/main/java/bisq/core/dao/burningman/accounting/node/AccountingNodeProvider.java b/core/src/main/java/bisq/core/dao/burningman/accounting/node/AccountingNodeProvider.java index 0022aa7b77..9efff49e2c 100644 --- a/core/src/main/java/bisq/core/dao/burningman/accounting/node/AccountingNodeProvider.java +++ b/core/src/main/java/bisq/core/dao/burningman/accounting/node/AccountingNodeProvider.java @@ -48,10 +48,10 @@ public class AccountingNodeProvider { && preferences.getRpcPw() != null && !preferences.getRpcPw().isEmpty() && preferences.getBlockNotifyPort() > 0; - if (BurningManService.isActivated()) { - accountingNode = isBmFullNode && rpcDataSet ? fullNode : liteNode; + if (isBmFullNode && rpcDataSet) { + accountingNode = fullNode; } else { - accountingNode = inActiveAccountingNode; + accountingNode = BurningManService.isActivated() ? liteNode : inActiveAccountingNode; } } } diff --git a/core/src/main/java/bisq/core/dao/monitoring/BlindVoteStateMonitoringService.java b/core/src/main/java/bisq/core/dao/monitoring/BlindVoteStateMonitoringService.java index f803060a53..e6915a0c92 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/BlindVoteStateMonitoringService.java +++ b/core/src/main/java/bisq/core/dao/monitoring/BlindVoteStateMonitoringService.java @@ -25,6 +25,7 @@ import bisq.core.dao.governance.period.PeriodService; import bisq.core.dao.monitoring.model.BlindVoteStateBlock; import bisq.core.dao.monitoring.model.BlindVoteStateHash; import bisq.core.dao.monitoring.network.BlindVoteStateNetworkService; +import bisq.core.dao.monitoring.network.StateNetworkService; import bisq.core.dao.monitoring.network.messages.GetBlindVoteStateHashesRequest; import bisq.core.dao.monitoring.network.messages.NewBlindVoteStateHashMessage; import bisq.core.dao.state.DaoStateListener; @@ -230,6 +231,10 @@ public class BlindVoteStateMonitoringService implements DaoSetupService, DaoStat blindVoteStateNetworkService.requestHashes(genesisTxInfo.getGenesisBlockHeight(), peersAddress); } + public void addResponseListener(StateNetworkService.ResponseListener responseListener) { + blindVoteStateNetworkService.addResponseListener(responseListener); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Listeners diff --git a/core/src/main/java/bisq/core/dao/monitoring/DaoStateMonitoringService.java b/core/src/main/java/bisq/core/dao/monitoring/DaoStateMonitoringService.java index 1d35f3b237..ff6205ec71 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/DaoStateMonitoringService.java +++ b/core/src/main/java/bisq/core/dao/monitoring/DaoStateMonitoringService.java @@ -23,6 +23,7 @@ import bisq.core.dao.monitoring.model.DaoStateHash; import bisq.core.dao.monitoring.model.UtxoMismatch; import bisq.core.dao.monitoring.network.Checkpoint; import bisq.core.dao.monitoring.network.DaoStateNetworkService; +import bisq.core.dao.monitoring.network.StateNetworkService; import bisq.core.dao.monitoring.network.messages.GetDaoStateHashesRequest; import bisq.core.dao.monitoring.network.messages.NewDaoStateHashMessage; import bisq.core.dao.state.DaoStateListener; @@ -289,6 +290,10 @@ public class DaoStateMonitoringService implements DaoSetupService, DaoStateListe createSnapshotHandler = handler; } + public void addResponseListener(StateNetworkService.ResponseListener responseListener) { + daoStateNetworkService.addResponseListener(responseListener); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Listeners diff --git a/core/src/main/java/bisq/core/dao/monitoring/ProposalStateMonitoringService.java b/core/src/main/java/bisq/core/dao/monitoring/ProposalStateMonitoringService.java index c28f33a235..a87c1b1518 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/ProposalStateMonitoringService.java +++ b/core/src/main/java/bisq/core/dao/monitoring/ProposalStateMonitoringService.java @@ -24,6 +24,7 @@ import bisq.core.dao.governance.proposal.ProposalService; import bisq.core.dao.monitoring.model.ProposalStateBlock; import bisq.core.dao.monitoring.model.ProposalStateHash; import bisq.core.dao.monitoring.network.ProposalStateNetworkService; +import bisq.core.dao.monitoring.network.StateNetworkService; import bisq.core.dao.monitoring.network.messages.GetProposalStateHashesRequest; import bisq.core.dao.monitoring.network.messages.NewProposalStateHashMessage; import bisq.core.dao.state.DaoStateListener; @@ -232,6 +233,10 @@ public class ProposalStateMonitoringService implements DaoSetupService, DaoState proposalStateNetworkService.requestHashes(genesisTxInfo.getGenesisBlockHeight(), peersAddress); } + public void addResponseListener(StateNetworkService.ResponseListener responseListener) { + proposalStateNetworkService.addResponseListener(responseListener); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Listeners @@ -294,7 +299,9 @@ public class ProposalStateMonitoringService implements DaoSetupService, DaoState return true; } - private boolean processPeersProposalStateHash(ProposalStateHash proposalStateHash, Optional peersNodeAddress, boolean notifyListeners) { + private boolean processPeersProposalStateHash(ProposalStateHash proposalStateHash, + Optional peersNodeAddress, + boolean notifyListeners) { AtomicBoolean changed = new AtomicBoolean(false); AtomicBoolean inConflictWithNonSeedNode = new AtomicBoolean(this.isInConflictWithNonSeedNode); AtomicBoolean inConflictWithSeedNode = new AtomicBoolean(this.isInConflictWithSeedNode); diff --git a/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java b/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java index 2df00fc94f..af3e4eedae 100644 --- a/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java +++ b/core/src/main/java/bisq/core/dao/monitoring/network/StateNetworkService.java @@ -29,10 +29,16 @@ import bisq.network.p2p.network.NetworkNode; import bisq.network.p2p.peers.Broadcaster; import bisq.network.p2p.peers.PeerManager; +import bisq.common.UserThread; import bisq.common.proto.network.NetworkEnvelope; import javax.inject.Inject; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -42,6 +48,8 @@ import java.util.concurrent.CopyOnWriteArrayList; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + import javax.annotation.Nullable; @Slf4j @@ -59,6 +67,12 @@ public abstract class StateNetworkService stateHashes, Optional peersNodeAddress); } + public interface ResponseListener { + void onSuccess(int serializedSize); + + void onFault(); + } + protected final NetworkNode networkNode; protected final PeerManager peerManager; private final Broadcaster broadcaster; @@ -67,6 +81,7 @@ public abstract class StateNetworkService requestStateHashHandlerMap = new HashMap<>(); private final List> listeners = new CopyOnWriteArrayList<>(); private boolean messageListenerAdded; + private final List responseListeners = new CopyOnWriteArrayList<>(); /////////////////////////////////////////////////////////////////////////////////////////// @@ -145,7 +160,20 @@ public abstract class StateNetworkService future = networkNode.sendMessage(connection, getStateHashesResponse); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(Connection connection) { + UserThread.execute(() -> responseListeners.forEach(listeners -> listeners.onSuccess(getStateHashesResponse.toProtoMessage().getSerializedSize())) + ); + } + + @Override + public void onFailure(@NotNull Throwable throwable) { + UserThread.execute(() -> responseListeners.forEach(StateNetworkService.ResponseListener::onFault) + ); + } + }, MoreExecutors.directExecutor()); } public void requestHashesFromAllConnectedSeedNodes(int fromHeight) { @@ -171,6 +199,10 @@ public abstract class StateNetworkService doParseBlock(RawBlock rawBlock) throws RequiredReorgFromSnapshotException { + if (shutdownInProgress) { + return Optional.empty(); + } + // We check if we have a block with that height. If so we return. We do not use the chainHeight as with genesis // height we have no block but chainHeight is initially set to genesis height (bad design ;-( but a bit tricky // to change now as it used in many areas.) diff --git a/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java b/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java index 1175a26446..8ff811b440 100644 --- a/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java +++ b/core/src/main/java/bisq/core/dao/node/explorer/ExportJsonFilesService.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -63,8 +64,7 @@ public class ExportJsonFilesService implements DaoSetupService { private final File storageDir; private final boolean dumpBlockchainData; - private final ListeningExecutorService executor = Utilities.getListeningExecutorService("JsonExporter", - 1, 1, 1200); + private final ListeningExecutorService executor; private JsonFileManager txFileManager, txOutputFileManager, bsqStateFileManager; @Inject @@ -74,6 +74,9 @@ public class ExportJsonFilesService implements DaoSetupService { this.daoStateService = daoStateService; this.storageDir = storageDir; this.dumpBlockchainData = dumpBlockchainData; + + ThreadPoolExecutor threadPoolExecutor = Utilities.getThreadPoolExecutor("JsonExporter", 1, 1, 20, 60); + executor = MoreExecutors.listeningDecorator(threadPoolExecutor); } diff --git a/core/src/main/java/bisq/core/dao/node/full/FullNode.java b/core/src/main/java/bisq/core/dao/node/full/FullNode.java index 93c18a9a67..469f19b608 100644 --- a/core/src/main/java/bisq/core/dao/node/full/FullNode.java +++ b/core/src/main/java/bisq/core/dao/node/full/FullNode.java @@ -97,6 +97,7 @@ public class FullNode extends BsqNode { public void shutDown() { super.shutDown(); + rpcService.shutDown(); fullNodeNetworkService.shutDown(); } @@ -239,6 +240,9 @@ public class FullNode extends BsqNode { Consumer newBlockHandler, ResultHandler resultHandler, Consumer errorHandler) { + if (shutdownInProgress) { + return; + } rpcService.requestDtoBlock(blockHeight, rawBlock -> { try { diff --git a/core/src/main/java/bisq/core/dao/node/full/RpcService.java b/core/src/main/java/bisq/core/dao/node/full/RpcService.java index d7e557db38..2412e4e356 100644 --- a/core/src/main/java/bisq/core/dao/node/full/RpcService.java +++ b/core/src/main/java/bisq/core/dao/node/full/RpcService.java @@ -92,7 +92,7 @@ public class RpcService { // We could use multiple threads, but then we need to support ordering of results in a queue // Keep that for optimization after measuring performance differences private final ListeningExecutorService executor = Utilities.getSingleThreadListeningExecutor("RpcService"); - private volatile boolean isShutDown; + private volatile boolean shutdownInProgress; private final Set setupResultHandlers = new CopyOnWriteArraySet<>(); private final Set> setupErrorHandlers = new CopyOnWriteArraySet<>(); private volatile boolean setupComplete; @@ -139,14 +139,17 @@ public class RpcService { /////////////////////////////////////////////////////////////////////////////////////////// public void shutDown() { - isShutDown = true; + if (shutdownInProgress) { + return; + } + shutdownInProgress = true; if (daemon != null) { daemon.shutdown(); log.info("daemon shut down"); } // A hard shutdown is justified for the RPC service. - executor.shutdown(); + executor.shutdownNow(); } public void setup(ResultHandler resultHandler, Consumer errorHandler) { @@ -216,11 +219,14 @@ public class RpcService { }); } }, MoreExecutors.directExecutor()); - } catch (Exception e) { - if (!isShutDown || !(e instanceof RejectedExecutionException)) { - log.warn(e.toString(), e); + } catch (RejectedExecutionException e) { + if (!shutdownInProgress) { + log.error(e.toString(), e); throw e; } + } catch (Exception e) { + log.error(e.toString(), e); + throw e; } } @@ -310,11 +316,14 @@ public class RpcService { UserThread.execute(() -> errorHandler.accept(throwable)); } }, MoreExecutors.directExecutor()); - } catch (Exception e) { - if (!isShutDown || !(e instanceof RejectedExecutionException)) { + } catch (RejectedExecutionException e) { + if (!shutdownInProgress) { log.error("Exception at requestChainHeadHeight", e); throw e; } + } catch (Exception e) { + log.error("Exception at requestChainHeadHeight", e); + throw e; } } @@ -344,12 +353,14 @@ public class RpcService { UserThread.execute(() -> errorHandler.accept(throwable)); } }, MoreExecutors.directExecutor()); - } catch (Exception e) { - log.error("Exception at requestDtoBlock", e); - if (!isShutDown || !(e instanceof RejectedExecutionException)) { - log.warn(e.toString(), e); + } catch (RejectedExecutionException e) { + if (!shutdownInProgress) { + log.error("Exception at requestDtoBlock", e); throw e; } + } catch (Exception e) { + log.error("Exception at requestDtoBlock", e); + throw e; } } @@ -380,11 +391,14 @@ public class RpcService { UserThread.execute(() -> errorHandler.accept(throwable)); } }, MoreExecutors.directExecutor()); - } catch (Exception e) { - if (!isShutDown || !(e instanceof RejectedExecutionException)) { - log.warn(e.toString(), e); + } catch (RejectedExecutionException e) { + if (!shutdownInProgress) { + log.error(e.toString(), e); throw e; } + } catch (Exception e) { + log.error(e.toString(), e); + throw e; } } diff --git a/core/src/main/java/bisq/core/dao/node/full/network/FullNodeNetworkService.java b/core/src/main/java/bisq/core/dao/node/full/network/FullNodeNetworkService.java index c59bed70e6..430426dba5 100644 --- a/core/src/main/java/bisq/core/dao/node/full/network/FullNodeNetworkService.java +++ b/core/src/main/java/bisq/core/dao/node/full/network/FullNodeNetworkService.java @@ -37,7 +37,9 @@ import bisq.common.proto.network.NetworkEnvelope; import javax.inject.Inject; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; import lombok.extern.slf4j.Slf4j; @@ -51,6 +53,12 @@ public class FullNodeNetworkService implements MessageListener, PeerManager.List private static final long CLEANUP_TIMER = 120; + public interface ResponseListener { + void onSuccess(int serializedSize); + + void onFault(); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Class fields @@ -65,6 +73,7 @@ public class FullNodeNetworkService implements MessageListener, PeerManager.List // Key is connection UID private final Map getBlocksRequestHandlers = new HashMap<>(); private boolean stopped; + private final List responseListeners = new CopyOnWriteArrayList<>(); /////////////////////////////////////////////////////////////////////////////////////////// @@ -107,6 +116,10 @@ public class FullNodeNetworkService implements MessageListener, PeerManager.List broadcaster.broadcast(newBlockBroadcastMessage, networkNode.getNodeAddress()); } + public void addResponseListener(ResponseListener responseListener) { + responseListeners.add(responseListener); + } + /////////////////////////////////////////////////////////////////////////////////////////// // PeerManager.Listener implementation @@ -166,8 +179,10 @@ public class FullNodeNetworkService implements MessageListener, PeerManager.List daoStateService, new GetBlocksRequestHandler.Listener() { @Override - public void onComplete() { + public void onComplete(int serializedSize) { getBlocksRequestHandlers.remove(uid); + + responseListeners.forEach(listener -> listener.onSuccess(serializedSize)); } @Override @@ -179,6 +194,8 @@ public class FullNodeNetworkService implements MessageListener, PeerManager.List if (connection != null) { peerManager.handleConnectionFault(connection); } + + responseListeners.forEach(ResponseListener::onFault); } else { log.warn("We have stopped already. We ignore that getDataRequestHandler.handle.onFault call."); } diff --git a/core/src/main/java/bisq/core/dao/node/full/network/GetBlocksRequestHandler.java b/core/src/main/java/bisq/core/dao/node/full/network/GetBlocksRequestHandler.java index de8fad9f8b..411d49de4f 100644 --- a/core/src/main/java/bisq/core/dao/node/full/network/GetBlocksRequestHandler.java +++ b/core/src/main/java/bisq/core/dao/node/full/network/GetBlocksRequestHandler.java @@ -57,7 +57,7 @@ class GetBlocksRequestHandler { /////////////////////////////////////////////////////////////////////////////////////////// public interface Listener { - void onComplete(); + void onComplete(int serializedSize); void onFault(String errorMessage, Connection connection); } @@ -120,7 +120,7 @@ class GetBlocksRequestHandler { log.info("Send DataResponse to {} succeeded. getBlocksResponse.getBlocks().size()={}", connection.getPeersNodeAddressOptional(), getBlocksResponse.getBlocks().size()); cleanup(); - listener.onComplete(); + listener.onComplete(getBlocksResponse.toProtoNetworkEnvelope().getSerializedSize()); } else { log.trace("We have stopped already. We ignore that networkNode.sendMessage.onSuccess call."); } diff --git a/core/src/main/java/bisq/core/dao/node/lite/LiteNode.java b/core/src/main/java/bisq/core/dao/node/lite/LiteNode.java index 6d4fa40f3c..68e2769e14 100644 --- a/core/src/main/java/bisq/core/dao/node/lite/LiteNode.java +++ b/core/src/main/java/bisq/core/dao/node/lite/LiteNode.java @@ -248,6 +248,9 @@ public class LiteNode extends BsqNode { } private void runDelayedBatchProcessing(List blocks, Runnable resultHandler) { + if (shutdownInProgress) { + return; + } UserThread.execute(() -> { if (blocks.isEmpty()) { resultHandler.run(); diff --git a/core/src/main/java/bisq/core/provider/mempool/MempoolService.java b/core/src/main/java/bisq/core/provider/mempool/MempoolService.java index d0dcf806e3..dde464e27c 100644 --- a/core/src/main/java/bisq/core/provider/mempool/MempoolService.java +++ b/core/src/main/java/bisq/core/provider/mempool/MempoolService.java @@ -18,6 +18,7 @@ package bisq.core.provider.mempool; import bisq.core.dao.DaoFacade; +import bisq.core.dao.burningman.BurningManPresentationService; import bisq.core.dao.state.DaoStateService; import bisq.core.filter.FilterManager; import bisq.core.offer.bisq_v1.OfferPayload; @@ -43,8 +44,10 @@ import com.google.common.util.concurrent.SettableFuture; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; +import java.util.stream.Collectors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -60,6 +63,7 @@ public class MempoolService { private final FilterManager filterManager; private final DaoFacade daoFacade; private final DaoStateService daoStateService; + private final BurningManPresentationService burningManPresentationService; @Getter private int outstandingRequests = 0; @@ -69,13 +73,15 @@ public class MempoolService { Preferences preferences, FilterManager filterManager, DaoFacade daoFacade, - DaoStateService daoStateService) { + DaoStateService daoStateService, + BurningManPresentationService burningManPresentationService) { this.socks5ProxyProvider = socks5ProxyProvider; this.config = config; this.preferences = preferences; this.filterManager = filterManager; this.daoFacade = daoFacade; this.daoStateService = daoStateService; + this.burningManPresentationService = burningManPresentationService; } public void onAllServicesInitialized() { @@ -96,26 +102,36 @@ public class MempoolService { } public void validateOfferMakerTx(TxValidator txValidator, Consumer resultHandler) { - if (!isServiceSupported()) { - UserThread.runAfter(() -> resultHandler.accept(txValidator.endResult("mempool request not supported, bypassing", true)), 1); - return; + if (txValidator.getIsFeeCurrencyBtc() != null && txValidator.getIsFeeCurrencyBtc()) { + if (!isServiceSupported()) { + UserThread.runAfter(() -> resultHandler.accept(txValidator.endResult("mempool request not supported, bypassing", true)), 1); + return; + } + MempoolRequest mempoolRequest = new MempoolRequest(preferences, socks5ProxyProvider); + validateOfferMakerTx(mempoolRequest, txValidator, resultHandler); + } else { + // using BSQ for fees + UserThread.runAfter(() -> resultHandler.accept(txValidator.validateBsqFeeTx(true)), 1); } - MempoolRequest mempoolRequest = new MempoolRequest(preferences, socks5ProxyProvider); - validateOfferMakerTx(mempoolRequest, txValidator, resultHandler); } public void validateOfferTakerTx(Trade trade, Consumer resultHandler) { validateOfferTakerTx(new TxValidator(daoStateService, trade.getTakerFeeTxId(), trade.getAmount(), - trade.isCurrencyForTakerFeeBtc(), filterManager), resultHandler); + trade.isCurrencyForTakerFeeBtc(), trade.getLockTime(), filterManager), resultHandler); } public void validateOfferTakerTx(TxValidator txValidator, Consumer resultHandler) { - if (!isServiceSupported()) { - UserThread.runAfter(() -> resultHandler.accept(txValidator.endResult("mempool request not supported, bypassing", true)), 1); - return; + if (txValidator.getIsFeeCurrencyBtc() != null && txValidator.getIsFeeCurrencyBtc()) { + if (!isServiceSupported()) { + UserThread.runAfter(() -> resultHandler.accept(txValidator.endResult("mempool request not supported, bypassing", true)), 1); + return; + } + MempoolRequest mempoolRequest = new MempoolRequest(preferences, socks5ProxyProvider); + validateOfferTakerTx(mempoolRequest, txValidator, resultHandler); + } else { + // using BSQ for fees + resultHandler.accept(txValidator.validateBsqFeeTx(false)); } - MempoolRequest mempoolRequest = new MempoolRequest(preferences, socks5ProxyProvider); - validateOfferTakerTx(mempoolRequest, txValidator, resultHandler); } public void checkTxIsConfirmed(String txId, Consumer resultHandler) { @@ -256,7 +272,17 @@ public class MempoolService { } }); btcFeeReceivers.addAll(daoFacade.getAllDonationAddresses()); - log.debug("Known BTC fee receivers: {}", btcFeeReceivers.toString()); + + // We use all BM who had ever had burned BSQ to avoid if a BM just got "deactivated" due decayed burn amounts + // that it would trigger a failure here. There is still a small risk that new BM used for the trade fee payment + // is not yet visible to the other peer, but that should be very unlikely. + // We also get all addresses related to comp. requests, so this list is still rather long, but much shorter + // than if we would use all addresses of all BM. + Set distributedBMAddresses = burningManPresentationService.getBurningManCandidatesByName().values().stream() + .filter(burningManCandidate -> burningManCandidate.getAccumulatedBurnAmount() > 0) + .flatMap(burningManCandidate -> burningManCandidate.getAllAddresses().stream()) + .collect(Collectors.toSet()); + btcFeeReceivers.addAll(distributedBMAddresses); return btcFeeReceivers; } diff --git a/core/src/main/java/bisq/core/provider/mempool/TxValidator.java b/core/src/main/java/bisq/core/provider/mempool/TxValidator.java index 566253eddc..0d71fd55f1 100644 --- a/core/src/main/java/bisq/core/provider/mempool/TxValidator.java +++ b/core/src/main/java/bisq/core/provider/mempool/TxValidator.java @@ -19,6 +19,7 @@ package bisq.core.provider.mempool; import bisq.core.dao.governance.param.Param; import bisq.core.dao.state.DaoStateService; +import bisq.core.dao.state.model.blockchain.Tx; import bisq.core.filter.FilterManager; import bisq.common.util.Tuple2; @@ -56,12 +57,12 @@ public class TxValidator { private final DaoStateService daoStateService; private final FilterManager filterManager; - private long blockHeightAtOfferCreation; // Only set for maker. + private long feePaymentBlockHeight; // applicable to maker and taker fees private final List errorList; private final String txId; private Coin amount; @Nullable - private Boolean isFeeCurrencyBtc = null; + private Boolean isFeeCurrencyBtc; @Nullable private Long chainHeight; @Setter @@ -70,28 +71,14 @@ public class TxValidator { public TxValidator(DaoStateService daoStateService, String txId, Coin amount, - @Nullable Boolean isFeeCurrencyBtc, + boolean isFeeCurrencyBtc, + long feePaymentBlockHeight, FilterManager filterManager) { this.daoStateService = daoStateService; this.txId = txId; this.amount = amount; this.isFeeCurrencyBtc = isFeeCurrencyBtc; - this.filterManager = filterManager; - this.errorList = new ArrayList<>(); - this.jsonTxt = ""; - } - - public TxValidator(DaoStateService daoStateService, - String txId, - Coin amount, - @Nullable Boolean isFeeCurrencyBtc, - long blockHeightAtOfferCreation, - FilterManager filterManager) { - this.daoStateService = daoStateService; - this.txId = txId; - this.amount = amount; - this.isFeeCurrencyBtc = isFeeCurrencyBtc; - this.blockHeightAtOfferCreation = blockHeightAtOfferCreation; + this.feePaymentBlockHeight = feePaymentBlockHeight; this.filterManager = filterManager; this.errorList = new ArrayList<>(); this.jsonTxt = ""; @@ -119,8 +106,6 @@ public class TxValidator { if (checkNotNull(isFeeCurrencyBtc)) { status = checkFeeAddressBTC(jsonTxt, btcFeeReceivers) && checkFeeAmountBTC(jsonTxt, amount, true, getBlockHeightForFeeCalculation(jsonTxt)); - } else { - status = checkFeeAmountBSQ(jsonTxt, amount, true, getBlockHeightForFeeCalculation(jsonTxt)); } } } catch (JsonSyntaxException e) { @@ -132,6 +117,17 @@ public class TxValidator { return endResult("Maker tx validation", status); } + public TxValidator validateBsqFeeTx(boolean isMaker) { + Optional tx = daoStateService.getTx(txId); + String statusStr = isMaker ? "Maker" : "Taker" + " tx validation"; + if (tx.isEmpty()) { + log.info("DAO does not yet have the tx {}, bypassing check of burnt BSQ amount.", txId); + return endResult(statusStr, true); + } else { + return endResult(statusStr, checkFeeAmountBSQ(tx.get(), amount, isMaker, feePaymentBlockHeight)); + } + } + public TxValidator parseJsonValidateTakerFeeTx(String jsonTxt, List btcFeeReceivers) { this.jsonTxt = jsonTxt; boolean status = initialSanityChecks(txId, jsonTxt); @@ -143,8 +139,6 @@ public class TxValidator { if (isFeeCurrencyBtc) { status = checkFeeAddressBTC(jsonTxt, btcFeeReceivers) && checkFeeAmountBTC(jsonTxt, amount, false, getBlockHeightForFeeCalculation(jsonTxt)); - } else { - status = checkFeeAmountBSQ(jsonTxt, amount, false, getBlockHeightForFeeCalculation(jsonTxt)); } } } catch (JsonSyntaxException e) { @@ -250,28 +244,16 @@ public class TxValidator { return false; } - // I think its better to postpone BSQ fee check once the BSQ trade fee tx is confirmed and then use the BSQ explorer to request the - // BSQ fee to check if it is correct. - // Otherwise the requirements here become very complicated and potentially impossible to verify as we don't know - // if inputs and outputs are valid BSQ without the BSQ parser and confirmed transactions. - private boolean checkFeeAmountBSQ(String jsonTxt, Coin tradeAmount, boolean isMaker, long blockHeight) { - JsonArray jsonVin = getVinAndVout(jsonTxt).first; - JsonArray jsonVout = getVinAndVout(jsonTxt).second; - JsonObject jsonVin0 = jsonVin.get(0).getAsJsonObject(); - JsonObject jsonVout0 = jsonVout.get(0).getAsJsonObject(); - JsonElement jsonVIn0Value = jsonVin0.getAsJsonObject("prevout").get("value"); - JsonElement jsonVOut0Value = jsonVout0.getAsJsonObject().get("value"); - if (jsonVIn0Value == null || jsonVOut0Value == null) { - throw new JsonSyntaxException("vin/vout missing data"); - } + private boolean checkFeeAmountBSQ(Tx bsqTx, Coin tradeAmount, boolean isMaker, long blockHeight) { Param minFeeParam = isMaker ? Param.MIN_MAKER_FEE_BSQ : Param.MIN_TAKER_FEE_BSQ; long expectedFeeAsLong = calculateFee(tradeAmount, isMaker ? getMakerFeeRateBsq(blockHeight) : getTakerFeeRateBsq(blockHeight), minFeeParam).getValue(); - long feeValue = getBsqBurnt(jsonVin, jsonVOut0Value.getAsLong(), expectedFeeAsLong); + + long feeValue = bsqTx.getBurntBsq(); log.debug("BURNT BSQ maker fee: {} BSQ ({} sats)", (double) feeValue / 100.0, feeValue); - String description = String.format("Expected fee: %.2f BSQ, actual fee paid: %.2f BSQ", - (double) expectedFeeAsLong / 100.0, (double) feeValue / 100.0); + String description = String.format("Expected fee: %.2f BSQ, actual fee paid: %.2f BSQ, Trade amount: %s", + (double) expectedFeeAsLong / 100.0, (double) feeValue / 100.0, tradeAmount.toPlainString()); if (expectedFeeAsLong == feeValue) { log.debug("The fee matched. " + description); @@ -279,7 +261,7 @@ public class TxValidator { } if (expectedFeeAsLong < feeValue) { - log.info("The fee was more than what we expected. " + description); + log.info("The fee was more than what we expected. " + description + " Tx:" + bsqTx.getId()); return true; } @@ -350,39 +332,6 @@ public class TxValidator { // we don't care if it is confirmed or not, just that it exists. } - // a BSQ maker/taker fee transaction looks like this: - // BSQ INPUT 1 BSQ OUTPUT - // BSQ INPUT 2 BTC OUTPUT FOR RESERVED AMOUNT - // BSQ INPUT n BTC OUTPUT FOR CHANGE - // BTC INPUT 1 - // BTC INPUT 2 - // BTC INPUT n - // there can be any number of BSQ inputs and BTC inputs - // BSQ inputs always come first in the tx, followed by BTC for the collateral. - // the sum of all BSQ inputs minus the BSQ output is the burnt amount, or trading fee. - long getBsqBurnt(JsonArray jsonVin, long bsqOutValue, long expectedFee) { - // sum consecutive inputs until we have accumulated enough to cover the output + burnt amount - long bsqInValue = 0; - for (int txIndex = 0; txIndex < jsonVin.size() - 1; txIndex++) { - bsqInValue += jsonVin.get(txIndex).getAsJsonObject().getAsJsonObject("prevout").get("value").getAsLong(); - if (bsqInValue - expectedFee >= bsqOutValue) { - break; // target reached - bsq input exceeds the output and expected burn amount - } - } - // guard against negative burn amount (i.e. only 1 tx input, or first in < first out) - long burntAmount = Math.max(0, bsqInValue - bsqOutValue); - // since we do not know which of the first 'n' are definitively BSQ inputs, sanity-check that the burnt amount - // is not too ridiculously high, as that would imply that we counted a BTC input. - if (burntAmount > 10 * expectedFee) { - log.error("The apparent BSQ fee burnt seems ridiculously high ({}) compared to expected ({})", burntAmount, expectedFee); - burntAmount = 0; // returning zero will flag the trade for manual review - } - if (burntAmount == 0) { - log.error("Could not obtain the burnt BSQ amount, trade will be flagged for manual review."); - } - return burntAmount; - } - private static long getTxConfirms(String jsonTxt, long chainHeight) { long blockHeight = getTxBlockHeight(jsonTxt); if (blockHeight > 0) { @@ -395,8 +344,8 @@ public class TxValidator { // if the tx is not yet confirmed, use current block tip, if tx is confirmed use the block it was confirmed at. private long getBlockHeightForFeeCalculation(String jsonTxt) { // For the maker we set the blockHeightAtOfferCreation from the offer - if (blockHeightAtOfferCreation > 0) { - return blockHeightAtOfferCreation; + if (feePaymentBlockHeight > 0) { + return feePaymentBlockHeight; } long txBlockHeight = getTxBlockHeight(jsonTxt); diff --git a/core/src/main/java/bisq/core/trade/TradeManager.java b/core/src/main/java/bisq/core/trade/TradeManager.java index 0384187bf1..507e1934e8 100644 --- a/core/src/main/java/bisq/core/trade/TradeManager.java +++ b/core/src/main/java/bisq/core/trade/TradeManager.java @@ -156,7 +156,18 @@ public class TradeManager implements PersistedDataHost, DecryptedDirectMessageLi private final Provider provider; private final ClockWatcher clockWatcher; - private final Map tradeProtocolByTradeId = new HashMap<>(); + // We use uid for that map not the trade ID + private final Map tradeProtocolByTradeUid = new HashMap<>(); + + // We maintain a map with trade (offer) ID to reset a pending trade protocol for the same offer. + // Pending trade protocol could happen in edge cases when an early error did not cause a removal of the + // offer and the same peer takes the offer later again. Usually it is prevented for the taker to take again after a + // failure but that is only based on failed trades state and it can be that either the taker deletes the failed trades + // file or it was not persisted. Such rare cases could lead to a pending protocol and when taker takes again the + // offer the message listener from the old pending protocol gets invoked and processes the messages based on + // potentially outdated model data (e.g. old inputs). + private final Map pendingTradeProtocolByTradeId = new HashMap<>(); + private final PersistenceManager> persistenceManager; private final TradableList tradableList = new TradableList<>(); @Getter @@ -408,15 +419,20 @@ public class TradeManager implements PersistedDataHost, DecryptedDirectMessageLi public TradeProtocol getTradeProtocol(TradeModel trade) { String uid = trade.getUid(); - if (tradeProtocolByTradeId.containsKey(uid)) { - return tradeProtocolByTradeId.get(uid); + if (tradeProtocolByTradeUid.containsKey(uid)) { + return tradeProtocolByTradeUid.get(uid); } else { TradeProtocol tradeProtocol = TradeProtocolFactory.getNewTradeProtocol(trade); - TradeProtocol prev = tradeProtocolByTradeId.put(uid, tradeProtocol); + TradeProtocol prev = tradeProtocolByTradeUid.put(uid, tradeProtocol); if (prev != null) { log.error("We had already an entry with uid {}", trade.getUid()); } + TradeProtocol pending = pendingTradeProtocolByTradeId.put(trade.getId(), tradeProtocol); + if (pending != null) { + pending.reset(); + } + return tradeProtocol; } } @@ -618,7 +634,7 @@ public class TradeManager implements PersistedDataHost, DecryptedDirectMessageLi private TradeProtocol createTradeProtocol(TradeModel tradeModel) { TradeProtocol tradeProtocol = TradeProtocolFactory.getNewTradeProtocol(tradeModel); - TradeProtocol prev = tradeProtocolByTradeId.put(tradeModel.getUid(), tradeProtocol); + TradeProtocol prev = tradeProtocolByTradeUid.put(tradeModel.getUid(), tradeProtocol); if (prev != null) { log.error("We had already an entry with uid {}", tradeModel.getUid()); } @@ -626,6 +642,11 @@ public class TradeManager implements PersistedDataHost, DecryptedDirectMessageLi tradableList.add((Trade) tradeModel); } + TradeProtocol pending = pendingTradeProtocolByTradeId.put(tradeModel.getId(), tradeProtocol); + if (pending != null) { + pending.reset(); + } + // For BsqTrades we only store the trade at completion return tradeProtocol; diff --git a/core/src/main/java/bisq/core/trade/model/bisq_v1/Trade.java b/core/src/main/java/bisq/core/trade/model/bisq_v1/Trade.java index 97048c035d..8c2ada636b 100644 --- a/core/src/main/java/bisq/core/trade/model/bisq_v1/Trade.java +++ b/core/src/main/java/bisq/core/trade/model/bisq_v1/Trade.java @@ -897,6 +897,10 @@ public abstract class Trade extends TradeModel { return new Date(getTradeStartTime() + getMaxTradePeriod()); } + public long getTradeAge() { + return System.currentTimeMillis() - getTradeStartTime(); + } + private long getMaxTradePeriod() { return offer.getPaymentMethod().getMaxTradePeriod(); } diff --git a/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java b/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java index 549439e61b..933ba70ceb 100644 --- a/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java +++ b/core/src/main/java/bisq/core/trade/protocol/TradeProtocol.java @@ -96,6 +96,12 @@ public abstract class TradeProtocol implements DecryptedDirectMessageListener, D cleanup(); } + // Resets a potentially pending protocol + public void reset() { + tradeModel.setErrorMessage("Outdated pending protocol got reset."); + protocolModel.getP2PService().removeDecryptedDirectMessageListener(this); + } + protected void onMailboxMessage(TradeMessage message, NodeAddress peerNodeAddress) { log.info("Received {} as MailboxMessage from {} with tradeId {} and uid {}", message.getClass().getSimpleName(), peerNodeAddress, message.getTradeId(), message.getUid()); diff --git a/core/src/main/java/bisq/core/trade/protocol/bisq_v1/SellerProtocol.java b/core/src/main/java/bisq/core/trade/protocol/bisq_v1/SellerProtocol.java index a3e4fd90d6..46a4a74fd1 100644 --- a/core/src/main/java/bisq/core/trade/protocol/bisq_v1/SellerProtocol.java +++ b/core/src/main/java/bisq/core/trade/protocol/bisq_v1/SellerProtocol.java @@ -167,11 +167,11 @@ public abstract class SellerProtocol extends DisputeProtocol { @Override protected void onTradeMessage(TradeMessage message, NodeAddress peer) { - super.onTradeMessage(message, peer); - log.info("Received {} from {} with tradeId {} and uid {}", message.getClass().getSimpleName(), peer, message.getTradeId(), message.getUid()); + super.onTradeMessage(message, peer); + if (message instanceof DelayedPayoutTxSignatureResponse) { handle((DelayedPayoutTxSignatureResponse) message, peer); } else if (message instanceof ShareBuyerPaymentAccountMessage) { diff --git a/core/src/test/java/bisq/core/provider/mempool/TxValidatorTest.java b/core/src/test/java/bisq/core/provider/mempool/TxValidatorTest.java index 412e282478..6c99f1dd06 100644 --- a/core/src/test/java/bisq/core/provider/mempool/TxValidatorTest.java +++ b/core/src/test/java/bisq/core/provider/mempool/TxValidatorTest.java @@ -19,6 +19,7 @@ package bisq.core.provider.mempool; import bisq.core.dao.governance.param.Param; import bisq.core.dao.state.DaoStateService; +import bisq.core.dao.state.model.blockchain.Tx; import bisq.core.filter.Filter; import bisq.core.filter.FilterManager; import bisq.core.trade.DelayedPayoutAddressProvider; @@ -38,6 +39,7 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.slf4j.Logger; @@ -76,30 +78,35 @@ public class TxValidatorTest { public void testMakerTx() { String mempoolData, offerData; + log.info("checking issue from user 2022-10-07"); + offerData = "1322804,5bec4007de1cb8cf18a5fa859d80d66031b8c78cfd99674e09ffd65cf23b50fc,9630000,137,0,757500"; + mempoolData = "{\"txid\":\"5bec4007de1cb8cf18a5fa859d80d66031b8c78cfd99674e09ffd65cf23b50fc\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":8921}},{\"vout\":1,\"prevout\":{\"value\":12155000}},{\"vout\":1,\"prevout\":{\"value\":2967000}}],\"vout\":[{\"scriptpubkey_address\":\"bc1qtyl6dququ2amxtsh4f3kx5rk9f5w9cuscz7ugm\",\"value\":8784},{\"scriptpubkey_address\":\"bc1qwj0jktuyjwj2ecwp9wgcrztxhve0hwn7n5lnxg\",\"value\":12519000},{\"scriptpubkey_address\":\"bc1qn3rd52mzkp6mgduz5wxprjw4rk9xpft6kga2mk\",\"value\":2600037}],\"size\":551,\"weight\":1229,\"fee\":3100,\"status\":{\"confirmed\":true,\"block_height\":757528,\"block_hash\":\"00000000000000000006b4426a0d2688a7e933e563e4d4fd80f572d935b12ae9\",\"block_time\":1665150690}}"; + Assert.assertTrue(createTxValidator(offerData).parseJsonValidateMakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + log.info("expected: paid the correct amount of BSQ fees"); offerData = "msimscqb,0636bafb14890edfb95465e66e2b1e15915f7fb595f9b653b9129c15ef4c1c4b,1000000,10,0,662390"; mempoolData = "{\"txid\":\"0636bafb14890edfb95465e66e2b1e15915f7fb595f9b653b9129c15ef4c1c4b\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":7899}},{\"vout\":2,\"prevout\":{\"value\":54877439}}],\"vout\":[{\"scriptpubkey_address\":\"1FCUu7hqKCSsGhVJaLbGEoCWdZRJRNqq8w\",\"value\":7889},{\"scriptpubkey_address\":\"bc1qkj5l4wxl00ufdx6ygcnrck9fz5u927gkwqcgey\",\"value\":1600000},{\"scriptpubkey_address\":\"bc1qkw4a8u9l5w9fhdh3ue9v7e7celk4jyudzg5gk5\",\"value\":53276799}],\"size\":405,\"weight\":1287,\"fee\":650,\"status\":{\"confirmed\":true,\"block_height\":663140}}"; Assert.assertTrue(createTxValidator(offerData).parseJsonValidateMakerFeeTx(mempoolData, btcFeeReceivers).getResult()); log.info("expected: paid the correct amount of BSQ fees with two UTXOs"); - offerData = "qmmtead,94b2589f3270caa0df63437707d4442cae34498ee5b0285090deed9c0ce8584d,800000,10,0,705301"; + offerData = "qmmtead,94b2589f3270caa0df63437707d4442cae34498ee5b0285090deed9c0ce8584d,800000,11,0,705301"; mempoolData = "{\"txid\":\"94b2589f3270caa0df63437707d4442cae34498ee5b0285090deed9c0ce8584d\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":577}},{\"vout\":0,\"prevout\":{\"value\":19989}},{\"vout\":2,\"prevout\":{\"value\":3008189}}],\"vout\":[{\"scriptpubkey_address\":\"bc1q48p2nvqf3tepjy7x33c5sfx3tp89e8c05z46cs\",\"value\":20555},{\"scriptpubkey_address\":\"bc1q9h69k8l0vy2yv3c72lw2cgn95sd7hlwjjzul05\",\"value\":920000},{\"scriptpubkey_address\":\"bc1qxmwscy2krw7zzfryw5g8868dexfy6pnq9yx3rv\",\"value\":2085750}],\"size\":550,\"weight\":1228,\"fee\":2450,\"status\":{\"confirmed\":true,\"block_height\":705301}}"; Assert.assertTrue(createTxValidator(offerData).parseJsonValidateMakerFeeTx(mempoolData, btcFeeReceivers).getResult()); log.info("expected: UNDERPAID expected 1.01 BSQ, actual fee paid 0.80 BSQ (USED 8.00 RATE INSTEAD OF 10.06 RATE"); offerData = "48067552,3b6009da764b71d79a4df8e2d8960b6919cae2e9bdccd5ef281e261fa9cd31b3,10000000,80,0,667656"; - mempoolData = "{\"txid\":\"3b6009da764b71d79a4df8e2d8960b6919cae2e9bdccd5ef281e261fa9cd31b3\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":9717}},{\"vout\":0,\"prevout\":{\"value\":4434912}},{\"vout\":2,\"prevout\":{\"value\":12809932}}],\"vout\":[{\"scriptpubkey_address\":\"1Nzqa4J7ck5bgz7QNXKtcjZExAvReozFo4\",\"value\":9637},{\"scriptpubkey_address\":\"bc1qhmmulf5prreqhccqy2wqpxxn6dcye7ame9dd57\",\"value\":11500000},{\"scriptpubkey_address\":\"bc1qx6hg8km2jdjc5ukhuedmkseu9wlsjtd8zeajpj\",\"value\":5721894}],\"size\":553,\"weight\":1879,\"fee\":23030,\"status\":{\"confirmed\":true,\"block_height\":667660}}"; - Assert.assertFalse(createTxValidator(offerData).parseJsonValidateMakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + //mempoolData = "{\"txid\":\"3b6009da764b71d79a4df8e2d8960b6919cae2e9bdccd5ef281e261fa9cd31b3\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":9717}},{\"vout\":0,\"prevout\":{\"value\":4434912}},{\"vout\":2,\"prevout\":{\"value\":12809932}}],\"vout\":[{\"scriptpubkey_address\":\"1Nzqa4J7ck5bgz7QNXKtcjZExAvReozFo4\",\"value\":9637},{\"scriptpubkey_address\":\"bc1qhmmulf5prreqhccqy2wqpxxn6dcye7ame9dd57\",\"value\":11500000},{\"scriptpubkey_address\":\"bc1qx6hg8km2jdjc5ukhuedmkseu9wlsjtd8zeajpj\",\"value\":5721894}],\"size\":553,\"weight\":1879,\"fee\":23030,\"status\":{\"confirmed\":true,\"block_height\":667660}}"; + Assert.assertFalse(createTxValidator(offerData).validateBsqFeeTx(true).getResult()); log.info("expected: UNDERPAID Expected fee: 0.61 BSQ, actual fee paid: 0.35 BSQ (USED 5.75 RATE INSTEAD OF 10.06 RATE"); offerData = "am7DzIv,4cdea8872a7d96210f378e0221dc1aae8ee9abb282582afa7546890fb39b7189,6100000,35,0,668195"; - mempoolData = "{\"txid\":\"4cdea8872a7d96210f378e0221dc1aae8ee9abb282582afa7546890fb39b7189\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":23893}},{\"vout\":1,\"prevout\":{\"value\":1440000}},{\"vout\":2,\"prevout\":{\"value\":16390881}}],\"vout\":[{\"scriptpubkey_address\":\"1Kmrzq3WGCQsZw5kroEphuk1KgsEr65yB7\",\"value\":23858},{\"scriptpubkey_address\":\"bc1qyw5qql9m7rkse9mhcun225nrjpwycszsa5dpjg\",\"value\":7015000},{\"scriptpubkey_address\":\"bc1q90y3p6mg0pe3rvvzfeudq4mfxafgpc9rulruff\",\"value\":10774186}],\"size\":554,\"weight\":1559,\"fee\":41730,\"status\":{\"confirmed\":true,\"block_height\":668198}}"; - Assert.assertFalse(createTxValidator(offerData).parseJsonValidateMakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + //mempoolData = "{\"txid\":\"4cdea8872a7d96210f378e0221dc1aae8ee9abb282582afa7546890fb39b7189\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":23893}},{\"vout\":1,\"prevout\":{\"value\":1440000}},{\"vout\":2,\"prevout\":{\"value\":16390881}}],\"vout\":[{\"scriptpubkey_address\":\"1Kmrzq3WGCQsZw5kroEphuk1KgsEr65yB7\",\"value\":23858},{\"scriptpubkey_address\":\"bc1qyw5qql9m7rkse9mhcun225nrjpwycszsa5dpjg\",\"value\":7015000},{\"scriptpubkey_address\":\"bc1q90y3p6mg0pe3rvvzfeudq4mfxafgpc9rulruff\",\"value\":10774186}],\"size\":554,\"weight\":1559,\"fee\":41730,\"status\":{\"confirmed\":true,\"block_height\":668198}}"; + Assert.assertFalse(createTxValidator(offerData).validateBsqFeeTx(true).getResult()); log.info("expected: UNDERPAID expected 0.11 BSQ, actual fee paid 0.08 BSQ (USED 5.75 RATE INSTEAD OF 7.53"); offerData = "F1dzaFNQ,f72e263947c9dee6fbe7093fc85be34a149ef5bcfdd49b59b9cc3322fea8967b,1440000,8,0,670822, bsq paid too little"; - mempoolData = "{\"txid\":\"f72e263947c9dee6fbe7093fc85be34a149ef5bcfdd49b59b9cc3322fea8967b\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":15163}},{\"vout\":2,\"prevout\":{\"value\":6100000}}],\"vout\":[{\"scriptpubkey_address\":\"1MEsc2m4MSomNJWSr1p6fhnUQMyA3DRGrN\",\"value\":15155},{\"scriptpubkey_address\":\"bc1qztgwe9ry9a9puchjuscqdnv4v9lsm2ut0jtfec\",\"value\":2040000},{\"scriptpubkey_address\":\"bc1q0nstwxc0vqkj4x000xt328mfjapvlsd56nn70h\",\"value\":4048308}],\"size\":406,\"weight\":1291,\"fee\":11700,\"status\":{\"confirmed\":true,\"block_height\":670823}}"; - Assert.assertFalse(createTxValidator(offerData).parseJsonValidateMakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + //mempoolData = "{\"txid\":\"f72e263947c9dee6fbe7093fc85be34a149ef5bcfdd49b59b9cc3322fea8967b\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":15163}},{\"vout\":2,\"prevout\":{\"value\":6100000}}],\"vout\":[{\"scriptpubkey_address\":\"1MEsc2m4MSomNJWSr1p6fhnUQMyA3DRGrN\",\"value\":15155},{\"scriptpubkey_address\":\"bc1qztgwe9ry9a9puchjuscqdnv4v9lsm2ut0jtfec\",\"value\":2040000},{\"scriptpubkey_address\":\"bc1q0nstwxc0vqkj4x000xt328mfjapvlsd56nn70h\",\"value\":4048308}],\"size\":406,\"weight\":1291,\"fee\":11700,\"status\":{\"confirmed\":true,\"block_height\":670823}}"; + Assert.assertFalse(createTxValidator(offerData).validateBsqFeeTx(true).getResult()); } @Test @@ -114,22 +121,17 @@ public class TxValidatorTest { log.info("========== test case: The fee matched what we expected (BSQ)"); offerData = "00072328,12f658954890d38ce698355be0b27fdd68d092c7b1b7475381918db060f46166,6250000,188,0,615955"; mempoolData = "{\"txid\":\"12f658954890d38ce698355be0b27fdd68d092c7b1b7475381918db060f46166\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":19980}},{\"vout\":2,\"prevout\":{\"value\":2086015}},{\"vout\":0,\"prevout\":{\"value\":1100000}},{\"vout\":2,\"prevout\":{\"value\":938200}}],\"vout\":[{\"scriptpubkey_address\":\"17qiF1TYgT1YvsCPJyXQoKMtBZ7YJBW9GH\",\"value\":19792},{\"scriptpubkey_address\":\"16aFKD5hvEjJgPme5yRNJT2rAPdTXzdQc2\",\"value\":3768432},{\"scriptpubkey_address\":\"1D5V3QW8f5n4PhwfPgNkW9eWZwNJFyVU8n\",\"value\":346755}],\"size\":701,\"weight\":2804,\"fee\":9216,\"status\":{\"confirmed\":true,\"block_height\":615955}}"; - Assert.assertTrue(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + Assert.assertTrue(createTxValidator(offerData).validateBsqFeeTx(false).getResult()); log.info("========== test case: No BSQ was burnt (error)"); - offerData = "NOBURN,12f658954890d38ce698355be0b27fdd68d092c7b1b7475381918db060f46166,6250000,188,0,615955"; + offerData = "NOBURN,12f658954890d38ce698355be0b27fdd68d092c7b1b7475381918db060f46166,6250000,0,0,615955"; mempoolData = "{\"txid\":\"12f658954890d38ce698355be0b27fdd68d092c7b1b7475381918db060f46166\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":19980}},{\"vout\":2,\"prevout\":{\"value\":2086015}},{\"vout\":0,\"prevout\":{\"value\":1100000}},{\"vout\":2,\"prevout\":{\"value\":938200}}],\"vout\":[{\"scriptpubkey_address\":\"17qiF1TYgT1YvsCPJyXQoKMtBZ7YJBW9GH\",\"value\":19980},{\"scriptpubkey_address\":\"16aFKD5hvEjJgPme5yRNJT2rAPdTXzdQc2\",\"value\":3768432},{\"scriptpubkey_address\":\"1D5V3QW8f5n4PhwfPgNkW9eWZwNJFyVU8n\",\"value\":346755}],\"size\":701,\"weight\":2804,\"fee\":9216,\"status\":{\"confirmed\":true,\"block_height\":615955}}"; - Assert.assertFalse(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + Assert.assertFalse(createTxValidator(offerData).validateBsqFeeTx(false).getResult()); log.info("========== test case: No BSQ input (error)"); - offerData = "NOBSQ,12f658954890d38ce698355be0b27fdd68d092c7b1b7475381918db060f46166,6250000,188,0,615955"; + offerData = "NOBSQ,12f658954890d38ce698355be0b27fdd68d092c7b1b7475381918db060f46166,6250000,0,0,615955"; mempoolData = "{\"txid\":\"12f658954890d38ce698355be0b27fdd68d092c7b1b7475381918db060f46166\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":2,\"prevout\":{\"value\":2086015}},{\"vout\":0,\"prevout\":{\"value\":1100000}},{\"vout\":2,\"prevout\":{\"value\":938200}}],\"vout\":[{\"scriptpubkey_address\":\"16aFKD5hvEjJgPme5yRNJT2rAPdTXzdQc2\",\"value\":3768432},{\"scriptpubkey_address\":\"1D5V3QW8f5n4PhwfPgNkW9eWZwNJFyVU8n\",\"value\":346755}],\"size\":701,\"weight\":2804,\"fee\":9216,\"status\":{\"confirmed\":true,\"block_height\":615955}}"; - Assert.assertFalse(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); - - log.info("========== test case: only one input (error)"); - offerData = "1INPUT,12f658954890d38ce698355be0b27fdd68d092c7b1b7475381918db060f46166,6250000,188,0,615955"; - mempoolData = "{\"txid\":\"12f658954890d38ce698355be0b27fdd68d092c7b1b7475381918db060f46166\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":2,\"prevout\":{\"value\":4186015}}],\"vout\":[{\"scriptpubkey_address\":\"16aFKD5hvEjJgPme5yRNJT2rAPdTXzdQc2\",\"value\":3768432},{\"scriptpubkey_address\":\"1D5V3QW8f5n4PhwfPgNkW9eWZwNJFyVU8n\",\"value\":346755}],\"size\":701,\"weight\":2804,\"fee\":9216,\"status\":{\"confirmed\":true,\"block_height\":615955}}"; - Assert.assertFalse(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + Assert.assertFalse(createTxValidator(offerData).validateBsqFeeTx(false).getResult()); log.info("========== test case: The fee was what we expected: (7000 sats)"); offerData = "bsqtrade,dfa4555ab78c657cad073e3f29c38c563d9dafc53afaa8c6af28510c734305c4,1000000,10,1,662390"; @@ -137,14 +139,14 @@ public class TxValidatorTest { Assert.assertTrue(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); log.info("========== test case: The fee matched what we expected"); - offerData = "89284,e1269aad63b3d894f5133ad658960971ef5c0fce6a13ad10544dc50fa3360588,900000,9,0,666473"; + offerData = "89284,e1269aad63b3d894f5133ad658960971ef5c0fce6a13ad10544dc50fa3360588,900000,47,0,666473"; mempoolData = "{\"txid\":\"e1269aad63b3d894f5133ad658960971ef5c0fce6a13ad10544dc50fa3360588\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":72738}},{\"vout\":0,\"prevout\":{\"value\":1600000}}],\"vout\":[{\"scriptpubkey_address\":\"17Kh5Ype9yNomqRrqu2k1mdV5c6FcKfGwQ\",\"value\":72691},{\"scriptpubkey_address\":\"bc1qdr9zcw7gf2sehxkux4fmqujm5uguhaqz7l9lca\",\"value\":629016},{\"scriptpubkey_address\":\"bc1qgqrrqv8q6l5d3t52fe28ghuhz4xqrsyxlwn03z\",\"value\":956523}],\"size\":404,\"weight\":1286,\"fee\":14508,\"status\":{\"confirmed\":true,\"block_height\":672388}}"; Assert.assertTrue(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); log.info("========== test case for UNDERPAID: Expected fee: 7.04 BSQ, actual fee paid: 1.01 BSQ"); offerData = "VOxRS,e99ea06aefc824fd45031447f7a0b56efb8117a09f9b8982e2c4da480a3a0e91,10000000,101,0,669129"; mempoolData = "{\"txid\":\"e99ea06aefc824fd45031447f7a0b56efb8117a09f9b8982e2c4da480a3a0e91\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":16739}},{\"vout\":2,\"prevout\":{\"value\":113293809}}],\"vout\":[{\"scriptpubkey_address\":\"1F14nF6zoUfJkqZrFgdmK5VX5QVwEpAnKW\",\"value\":16638},{\"scriptpubkey_address\":\"bc1q80y688ev7u43vqy964yf7feqddvt2mkm8977cm\",\"value\":11500000},{\"scriptpubkey_address\":\"bc1q9whgyc2du9mrgnxz0nl0shwpw8ugrcae0j0w8p\",\"value\":101784485}],\"size\":406,\"weight\":1291,\"fee\":9425,\"status\":{\"confirmed\":true,\"block_height\":669134}}"; - Assert.assertFalse(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + Assert.assertFalse(createTxValidator(offerData).validateBsqFeeTx(false).getResult()); log.info("========== test case for UNDERPAID: Expected fee: 1029000 sats BTC, actual fee paid: 441000 sats BTC because they used the default rate of 0.003 should have been 0.007 per BTC"); // after 1.6.0 we introduce additional leniency to allow the default rate (which is not stored in the DAO param change list) @@ -154,29 +156,29 @@ public class TxValidatorTest { Assert.assertFalse(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); log.info("========== test case for UNDERPAID: Expected fee: 2.12 BSQ, actual fee paid: 0.03 BSQ -- this is the example from the BSQ fee scammer Oct 2021"); - offerData = "957500,26e1a5e1f842cb7baa18bd197bd084e7f043d07720b9853e947158eb0a32677d,2000000,101,0,709426"; + offerData = "957500,26e1a5e1f842cb7baa18bd197bd084e7f043d07720b9853e947158eb0a32677d,2000000,101,3,709426"; mempoolData = "{\"txid\":\"26e1a5e1f842cb7baa18bd197bd084e7f043d07720b9853e947158eb0a32677d\",\"version\":1,\"locktime\":0,\"vin\":[{\"txid\":\"\",\"vout\":0,\"prevout\":{\"scriptpubkey\":\"\",\"scriptpubkey_asm\":\"\",\"scriptpubkey_type\":\"v0_p2wpkh\",\"scriptpubkey_address\":\"\",\"value\":3688},\"scriptsig\":\"\",\"scriptsig_asm\":\"\",\"witness\":[\"\",\"\"],\"is_coinbase\":false,\"sequence\":4294967295},{\"txid\":\"\",\"vout\":2,\"prevout\":{\"scriptpubkey\":\"\",\"scriptpubkey_asm\":\"\",\"scriptpubkey_type\":\"v0_p2wpkh\",\"scriptpubkey_address\":\"\",\"value\":796203},\"scriptsig\":\"\",\"scriptsig_asm\":\"\",\"witness\":[\"\",\"\"],\"is_coinbase\":false,\"sequence\":4294967295}],\"vout\":[{\"scriptpubkey\":\"\",\"scriptpubkey_asm\":\"\",\"scriptpubkey_type\":\"v0_p2wpkh\",\"scriptpubkey_address\":\"bc1qydcyfe7kp6968hywcp0uek2xvgem3nlx0x0hfy\",\"value\":3685},{\"scriptpubkey\":\"\",\"scriptpubkey_asm\":\"\",\"scriptpubkey_type\":\"v0_p2wpkh\",\"scriptpubkey_address\":\"bc1qc4amk6sd3c4gzxjgd5sdlaegt0r5juq54vnrll\",\"value\":503346},{\"scriptpubkey\":\"\",\"scriptpubkey_asm\":\"\",\"scriptpubkey_type\":\"v0_p2wpkh\",\"scriptpubkey_address\":\"bc1q66e7m8y5lzfk5smg2a80xeaqzhslgeavg9y70t\",\"value\":291187}],\"size\":403,\"weight\":958,\"fee\":1673,\"status\":{\"confirmed\":true,\"block_height\":709426,\"block_hash\":\"\",\"block_time\":1636751288}}"; - Assert.assertFalse(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + Assert.assertFalse(createTxValidator(offerData).validateBsqFeeTx(false).getResult()); log.info("========== test case: expected fee paid using two BSQ UTXOs"); - offerData = "ZHNYCAE,a91c6f1cb62721a7943678547aa814d6f29125ed63ad076073eb5ae7f16a76e9,83000000,101,0,717000"; + offerData = "ZHNYCAE,a91c6f1cb62721a7943678547aa814d6f29125ed63ad076073eb5ae7f16a76e9,83000000,8796,0,717000"; mempoolData = "{\"txid\":\"a91c6f1cb62721a7943678547aa814d6f29125ed63ad076073eb5ae7f16a76e9\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":3510}},{\"vout\":0,\"prevout\":{\"value\":6190}},{\"vout\":0,\"prevout\":{\"value\":46000000}}],\"vout\":[{\"scriptpubkey_address\":\"bc1qmqphx028eu4tzdvgccf5re52qtv6pmjanrpq29\",\"value\":904},{\"scriptpubkey_address\":\"bc1qtkvu4zeh0g0pce452335tgnswxd8ayxlktfj2s\",\"value\":30007648},{\"scriptpubkey_address\":\"bc1qdatwgzrrntp2m53tpzmax4dxu6md2c0c9vj8ut\",\"value\":15997324}],\"size\":549,\"weight\":1227,\"fee\":3824,\"status\":{\"confirmed\":true,\"block_height\":716444}}"; - Assert.assertTrue(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + Assert.assertTrue(createTxValidator(offerData).validateBsqFeeTx(false).getResult()); log.info("========== test case: expected fee paid using three BSQ UTXOs"); - offerData = "3UTXOGOOD,c7dddc267a366fa1d87840eeb0dcd89918a886ccb9aabee80f667635a5d4e262,200000000,101,0,733715"; + offerData = "3UTXOGOOD,c7dddc267a366fa1d87840eeb0dcd89918a886ccb9aabee80f667635a5d4e262,200000000,17888,0,733715"; mempoolData = "{\"txid\":\"c7dddc267a366fa1d87840eeb0dcd89918a886ccb9aabee80f667635a5d4e262\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":9833}},{\"vout\":0,\"prevout\":{\"value\":1362}},{vout\":0,\"prevout\":{\"value\":17488}},{\"vout\":2,\"prevout\":{\"value\":573360131}}],\"vout\":[{\"scriptpubkey_address\":\"bc1qvwpm87kmrlgave9srxk6nfwleehll0kxetu5j0\",\"value\":10795},{\"scriptpubkey_address\":\"bc1qz5n83ppfpdznnzff4e7tjep5c6f6jce9mqnrzh\",\"value\":230004780},{\"scriptpubkey_address\":\"bc1qcfyjajhuv55fyu6g5ug664r57u9a7qg55cgt5p\",\"value\":343370849}],\"size\":699,\"weight\":1500,\"fee\":2390,\"status\":{\"confirmed\":true,\"block_height\":733715}}"; - Assert.assertTrue(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + Assert.assertTrue(createTxValidator(offerData).validateBsqFeeTx(false).getResult()); log.info("========== test case: expected fee paid using four BSQ UTXOs"); - offerData = "4UTXOGOOD,c7dddc267a366fa1d87840eeb0dcd89918a886ccb9aabee80f667635a5d4e262,200000000,101,0,733715"; + offerData = "4UTXOGOOD,c7dddc267a366fa1d87840eeb0dcd89918a886ccb9aabee80f667635a5d4e262,200000000,17888,0,733715"; mempoolData = "{\"txid\":\"c7dddc267a366fa1d87840eeb0dcd89918a886ccb9aabee80f667635a5d4e262\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":4833}},{\"vout\":0,\"prevout\":{\"value\":5000}},{\"vout\":0,\"prevout\":{\"value\":1362}},{vout\":0,\"prevout\":{\"value\":17488}},{\"vout\":2,\"prevout\":{\"value\":573360131}}],\"vout\":[{\"scriptpubkey_address\":\"bc1qvwpm87kmrlgave9srxk6nfwleehll0kxetu5j0\",\"value\":10795},{\"scriptpubkey_address\":\"bc1qz5n83ppfpdznnzff4e7tjep5c6f6jce9mqnrzh\",\"value\":230004780},{\"scriptpubkey_address\":\"bc1qcfyjajhuv55fyu6g5ug664r57u9a7qg55cgt5p\",\"value\":343370849}],\"size\":699,\"weight\":1500,\"fee\":2390,\"status\":{\"confirmed\":true,\"block_height\":733715}}"; - Assert.assertTrue(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + Assert.assertTrue(createTxValidator(offerData).validateBsqFeeTx(false).getResult()); log.info("========== test case: three BSQ UTXOs, but fee paid is too low"); offerData = "3UTXOLOWFEE,c7dddc267a366fa1d87840eeb0dcd89918a886ccb9aabee80f667635a5d4e262,200000000,101,0,733715"; mempoolData = "{\"txid\":\"c7dddc267a366fa1d87840eeb0dcd89918a886ccb9aabee80f667635a5d4e262\",\"version\":1,\"locktime\":0,\"vin\":[{\"vout\":0,\"prevout\":{\"value\":9833}},{\"vout\":0,\"prevout\":{\"value\":1362}},{vout\":0,\"prevout\":{\"value\":1362}},{\"vout\":2,\"prevout\":{\"value\":573360131}}],\"vout\":[{\"scriptpubkey_address\":\"bc1qvwpm87kmrlgave9srxk6nfwleehll0kxetu5j0\",\"value\":10795},{\"scriptpubkey_address\":\"bc1qz5n83ppfpdznnzff4e7tjep5c6f6jce9mqnrzh\",\"value\":230004780},{\"scriptpubkey_address\":\"bc1qcfyjajhuv55fyu6g5ug664r57u9a7qg55cgt5p\",\"value\":343370849}],\"size\":699,\"weight\":1500,\"fee\":2390,\"status\":{\"confirmed\":true,\"block_height\":733715}}"; - Assert.assertFalse(createTxValidator(offerData).parseJsonValidateTakerFeeTx(mempoolData, btcFeeReceivers).getResult()); + Assert.assertFalse(createTxValidator(offerData).validateBsqFeeTx(false).getResult()); } @@ -206,12 +208,17 @@ public class TxValidatorTest { knownValuesList.forEach(offerData -> { TxValidator txValidator = createTxValidator(offerData); log.warn("TESTING {}", txValidator.getTxId()); - String jsonTxt = mempoolData.get(txValidator.getTxId()); - if (jsonTxt == null || jsonTxt.isEmpty()) { - log.warn("{} was not found in the mempool", txValidator.getTxId()); - Assert.assertFalse(expectedResult); // tx was not found in explorer + if (txValidator.getIsFeeCurrencyBtc() != null && txValidator.getIsFeeCurrencyBtc()) { + String jsonTxt = mempoolData.get(txValidator.getTxId()); + if (jsonTxt == null || jsonTxt.isEmpty()) { + log.warn("{} was not found in the mempool", txValidator.getTxId()); + Assert.assertFalse(expectedResult); // tx was not found in explorer + } else { + txValidator.parseJsonValidateMakerFeeTx(jsonTxt, btcFeeReceivers); + Assert.assertTrue(expectedResult == txValidator.getResult()); + } } else { - txValidator.parseJsonValidateMakerFeeTx(jsonTxt, btcFeeReceivers); + txValidator.validateBsqFeeTx(true); Assert.assertTrue(expectedResult == txValidator.getResult()); } }); @@ -235,8 +242,11 @@ public class TxValidatorTest { String[] y = offerData.split(","); String txId = y[1]; long amount = Long.parseLong(y[2]); + long feePaid = Long.parseLong(y[3]); boolean isCurrencyForMakerFeeBtc = Long.parseLong(y[4]) > 0; + long feePaymentBlockHeight = Long.parseLong(y[5]); DaoStateService mockedDaoStateService = mock(DaoStateService.class); + Tx mockedTx = mock(Tx.class); Answer mockGetFeeRate = invocation -> { return mockedLookupFeeRate(invocation.getArgument(0), invocation.getArgument(1)); @@ -247,9 +257,17 @@ public class TxValidatorTest { Answer> mockGetParamChangeList = invocation -> { return mockedGetParamChangeList(invocation.getArgument(0)); }; + Answer> mockGetBsqTx = invocation -> { + return Optional.of(mockedTx); + }; + Answer mockGetBurntBsq = invocation -> { + return feePaid; + }; when(mockedDaoStateService.getParamValueAsCoin(Mockito.any(Param.class), Mockito.anyInt())).thenAnswer(mockGetFeeRate); when(mockedDaoStateService.getParamValueAsCoin(Mockito.any(Param.class), Mockito.anyString())).thenAnswer(mockGetParamValueAsCoin); when(mockedDaoStateService.getParamChangeList(Mockito.any())).thenAnswer(mockGetParamChangeList); + when(mockedDaoStateService.getTx(Mockito.any())).thenAnswer(mockGetBsqTx); + when(mockedTx.getBurntBsq()).thenAnswer(mockGetBurntBsq); Answer getMakerFeeBsq = invocation -> 1514L; Answer getTakerFeeBsq = invocation -> 10597L; @@ -262,7 +280,7 @@ public class TxValidatorTest { when(mockedFilter.getTakerFeeBtc()).thenAnswer(getTakerFeeBtc); FilterManager filterManager = mock(FilterManager.class); when(filterManager.getFilter()).thenReturn(mockedFilter); - TxValidator txValidator = new TxValidator(mockedDaoStateService, txId, Coin.valueOf(amount), isCurrencyForMakerFeeBtc, filterManager); + TxValidator txValidator = new TxValidator(mockedDaoStateService, txId, Coin.valueOf(amount), isCurrencyForMakerFeeBtc, feePaymentBlockHeight, filterManager); return txValidator; } catch (RuntimeException ignore) { // If input format is not as expected we ignore entry @@ -290,6 +308,8 @@ public class TxValidatorTest { private LinkedHashMap mockedGetFeeRateMap(Param param) { LinkedHashMap feeMap = new LinkedHashMap<>(); if (param == Param.DEFAULT_MAKER_FEE_BSQ) { + feeMap.put(754620L, "14.21"); // https://github.com/bisq-network/proposals/issues/380 + feeMap.put(750000L, "17.19"); // https://github.com/bisq-network/proposals/issues/379 feeMap.put(721063L, "12.78"); // https://github.com/bisq-network/proposals/issues/357 feeMap.put(706305L, "15.14"); // https://github.com/bisq-network/proposals/issues/345 feeMap.put(697011L, "13.16"); // https://github.com/bisq-network/proposals/issues/339 @@ -307,8 +327,10 @@ public class TxValidatorTest { feeMap.put(585787L, "8.0"); feeMap.put(581107L, "1.6"); } else if (param == Param.DEFAULT_TAKER_FEE_BSQ) { + feeMap.put(754620L, "104.24"); // https://github.com/bisq-network/proposals/issues/380 + feeMap.put(750000L, "126.02"); // https://github.com/bisq-network/proposals/issues/379 feeMap.put(721063L, "89.44"); // https://github.com/bisq-network/proposals/issues/357 - feeMap.put(706305L, "105.97"); // https://github.com/bisq-network/proposals/issues/345 + feeMap.put(706305L, "105.97"); // https://github.com/bisq-network/proposals/issues/345 feeMap.put(697011L, "92.15"); // https://github.com/bisq-network/proposals/issues/339 feeMap.put(682901L, "80.13"); // https://github.com/bisq-network/proposals/issues/333 feeMap.put(677862L, "69.68"); // https://github.com/bisq-network/proposals/issues/325 diff --git a/core/src/test/resources/bisq/core/provider/mempool/badOfferTestData.json b/core/src/test/resources/bisq/core/provider/mempool/badOfferTestData.json index c48788399a..836b471104 100644 --- a/core/src/test/resources/bisq/core/provider/mempool/badOfferTestData.json +++ b/core/src/test/resources/bisq/core/provider/mempool/badOfferTestData.json @@ -2,9 +2,7 @@ "ef1ea38b46402deb7df08c13a6dc379a65542a6940ac9d4ba436641ffd4bcb6e": "FQ0A7G,ef1ea38b46402deb7df08c13a6dc379a65542a6940ac9d4ba436641ffd4bcb6e,15970000,92,0,640438, underpaid but accepted due to use of different DAO parameter", "4cdea8872a7d96210f378e0221dc1aae8ee9abb282582afa7546890fb39b7189": "am7DzIv,4cdea8872a7d96210f378e0221dc1aae8ee9abb282582afa7546890fb39b7189,6100000,35,0,668195, underpaid but accepted due to use of different DAO parameter", "051770f8d7f43a9b6ca10fefa6cdf4cb124a81eed26dc8af2e40f57d2589107b": "046698,051770f8d7f43a9b6ca10fefa6cdf4cb124a81eed26dc8af2e40f57d2589107b,15970000,92,0,667927, bsq fee underpaid using 5.75 rate for some weird reason", - "37fba8bf119c289481eef031c0a35e126376f71d13d7cce35eb0d5e05799b5da": "hUWPf,37fba8bf119c289481eef031c0a35e126376f71d13d7cce35eb0d5e05799b5da,19910000,200,0,668994, tx_missing_from_blockchain_for_4_days", "b3bc726aa2aa6533cb1e61901ce351eecde234378fe650aee267388886aa6e4b": "ebdttmzh,b3bc726aa2aa6533cb1e61901ce351eecde234378fe650aee267388886aa6e4b,4000000,5000,1,669137, tx_missing_from_blockchain_for_2_days", - "10f32fe53081466f003185a9ef0324d6cbe3f59334ee9ccb2f7155cbfad9c1de": "kmbyoexc,10f32fe53081466f003185a9ef0324d6cbe3f59334ee9ccb2f7155cbfad9c1de,33000000,332,0,668954, tx_not_found", "cd99836ac4246c3e3980edf95773060481ce52271b74dadeb41e18c42ed21188": "nlaIlAvE,cd99836ac4246c3e3980edf95773060481ce52271b74dadeb41e18c42ed21188,5000000,546,1,669262, invalid_missing_fee_address", "fc3cb16293895fea8ea5d2d8ab4e39d1b27f583e2c160468b586789a861efa74": "feescammer,fc3cb16293895fea8ea5d2d8ab4e39d1b27f583e2c160468b586789a861efa74,1000000,546,1,669442, invalid_missing_fee_address", "72cabb5c323c923b43c7f6551974f591dcee148778ee34f9131011ea0ca82813": "PBFICEAS,72cabb5c323c923b43c7f6551974f591dcee148778ee34f9131011ea0ca82813,2000000,546,1,672969, dust_fee_scammer", diff --git a/desktop/src/main/java/bisq/desktop/main/offer/bisq_v1/MutableOfferView.java b/desktop/src/main/java/bisq/desktop/main/offer/bisq_v1/MutableOfferView.java index ebdcbd437b..ae0a2471b7 100644 --- a/desktop/src/main/java/bisq/desktop/main/offer/bisq_v1/MutableOfferView.java +++ b/desktop/src/main/java/bisq/desktop/main/offer/bisq_v1/MutableOfferView.java @@ -296,7 +296,7 @@ public abstract class MutableOfferView> exten /////////////////////////////////////////////////////////////////////////////////////////// public void onTabSelected(boolean isSelected) { - if (isSelected && !model.getDataModel().isTabSelected) { + if (isSelected) { doActivate(); } else { deactivate(); diff --git a/desktop/src/main/java/bisq/desktop/main/portfolio/pendingtrades/PendingTradesViewModel.java b/desktop/src/main/java/bisq/desktop/main/portfolio/pendingtrades/PendingTradesViewModel.java index 6237fa1450..176dd14c4f 100644 --- a/desktop/src/main/java/bisq/desktop/main/portfolio/pendingtrades/PendingTradesViewModel.java +++ b/desktop/src/main/java/bisq/desktop/main/portfolio/pendingtrades/PendingTradesViewModel.java @@ -211,26 +211,29 @@ public class PendingTradesViewModel extends ActivatableWithDataModel { - mempoolStatus.setValue(txValidator.isFail() ? 0 : 1); - if (txValidator.isFail()) { - String errorMessage = "Validation of Taker Tx returned: " + txValidator.toString(); - log.warn(errorMessage); - // prompt user to open mediation - if (trade.getDisputeState() == Trade.DisputeState.NO_DISPUTE) { - UserThread.runAfter(() -> { - Popup popup = new Popup(); - popup.headLine(Res.get("portfolio.pending.openSupportTicket.headline")) - .message(Res.get("portfolio.pending.invalidTx", errorMessage)) - .actionButtonText(Res.get("portfolio.pending.openSupportTicket.headline")) - .onAction(dataModel::onOpenSupportTicket) - .closeButtonText(Res.get("shared.cancel")) - .onClose(popup::hide) - .show(); - }, 100, TimeUnit.MILLISECONDS); + UserThread.runAfter(() -> { + mempoolService.validateOfferTakerTx(trade, (txValidator -> { + mempoolStatus.setValue(txValidator.isFail() ? 0 : 1); + if (txValidator.isFail()) { + String errorMessage = "Validation of Taker Tx returned: " + txValidator.toString(); + log.warn(errorMessage); + // prompt user to open mediation + if (trade.getDisputeState() == Trade.DisputeState.NO_DISPUTE) { + UserThread.runAfter(() -> { + Popup popup = new Popup(); + popup.headLine(Res.get("portfolio.pending.openSupportTicket.headline")) + .message(Res.get("portfolio.pending.invalidTx", errorMessage)) + .actionButtonText(Res.get("portfolio.pending.openSupportTicket.headline")) + .onAction(dataModel::onOpenSupportTicket) + .closeButtonText(Res.get("shared.cancel")) + .onClose(popup::hide) + .show(); + }, 100, TimeUnit.MILLISECONDS); + } } - } - })); + })); + }, Math.max(5000 - trade.getTradeAge(), 100), TimeUnit.MILLISECONDS); + // we wait until the trade has confirmed for at least 5 seconds to allow for DAO to process the block } /////////////////////////////////////////////////////////////////////////////////////////// diff --git a/p2p/src/main/java/bisq/network/p2p/network/Connection.java b/p2p/src/main/java/bisq/network/p2p/network/Connection.java index be0a3390d6..049c690e04 100644 --- a/p2p/src/main/java/bisq/network/p2p/network/Connection.java +++ b/p2p/src/main/java/bisq/network/p2p/network/Connection.java @@ -595,7 +595,7 @@ public class Connection implements HasCapabilities, Runnable, MessageListener { public boolean reportInvalidRequest(RuleViolation ruleViolation) { - log.warn("We got reported the ruleViolation {} at connection {}", ruleViolation, this); + log.info("We got reported the ruleViolation {} at connection with address{} and uid {}", ruleViolation, this.getPeersNodeAddressProperty(), this.getUid()); int numRuleViolations; numRuleViolations = ruleViolations.getOrDefault(ruleViolation, 0); @@ -603,11 +603,11 @@ public class Connection implements HasCapabilities, Runnable, MessageListener { ruleViolations.put(ruleViolation, numRuleViolations); if (numRuleViolations >= ruleViolation.maxTolerance) { - log.warn("We close connection as we received too many corrupt requests.\n" + - "numRuleViolations={}\n\t" + - "corruptRequest={}\n\t" + - "corruptRequests={}\n\t" + - "connection={}", numRuleViolations, ruleViolation, ruleViolations, this); + log.warn("We close connection as we received too many corrupt requests. " + + "numRuleViolations={} " + + "corruptRequest={} " + + "corruptRequests={} " + + "connection with address{} and uid {}", numRuleViolations, ruleViolation, ruleViolations, this.getPeersNodeAddressProperty(), this.getUid()); this.ruleViolation = ruleViolation; if (ruleViolation == RuleViolation.PEER_BANNED) { log.warn("We close connection due RuleViolation.PEER_BANNED. peersNodeAddress={}", getPeersNodeAddressOptional()); diff --git a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java index ebc581bbe2..7d9e649b89 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/PeerManager.java @@ -84,6 +84,7 @@ public final class PeerManager implements ConnectionListener, PersistedDataHost private static final boolean PRINT_REPORTED_PEERS_DETAILS = true; private Timer printStatisticsTimer; private boolean shutDownRequested; + private int numOnConnections; /////////////////////////////////////////////////////////////////////////////////////////// @@ -216,6 +217,8 @@ public final class PeerManager implements ConnectionListener, PersistedDataHost doHouseKeeping(); + numOnConnections++; + if (lostAllConnections) { lostAllConnections = false; stopped = false; @@ -238,7 +241,8 @@ public final class PeerManager implements ConnectionListener, PersistedDataHost boolean previousLostAllConnections = lostAllConnections; lostAllConnections = networkNode.getAllConnections().isEmpty(); - if (lostAllConnections) { + // At start-up we ignore if we would lose a connection and would fall back to no connections + if (lostAllConnections && numOnConnections > 2) { stopped = true; if (!shutDownRequested) { @@ -562,7 +566,7 @@ public final class PeerManager implements ConnectionListener, PersistedDataHost if (!candidates.isEmpty()) { Connection connection = candidates.remove(0); - log.info("checkMaxConnections: Num candidates for shut down={}. We close oldest connection to peer {}", + log.info("checkMaxConnections: Num candidates (inbound/peer) for shut down={}. We close oldest connection to peer {}", candidates.size(), connection.getPeersNodeAddressOptional()); if (!connection.isStopped()) { connection.shutDown(CloseConnectionReason.TOO_MANY_CONNECTIONS_OPEN, diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java index 02aab8e164..c000449b90 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java @@ -50,7 +50,7 @@ public class GetDataRequestHandler { /////////////////////////////////////////////////////////////////////////////////////////// public interface Listener { - void onComplete(); + void onComplete(int serializedSize); void onFault(String errorMessage, Connection connection); } @@ -126,8 +126,8 @@ public class GetDataRequestHandler { if (!stopped) { log.trace("Send DataResponse to {} succeeded. getDataResponse={}", connection.getPeersNodeAddressOptional(), getDataResponse); + listener.onComplete(getDataResponse.toProtoNetworkEnvelope().getSerializedSize()); cleanup(); - listener.onComplete(); } else { log.trace("We have stopped already. We ignore that networkNode.sendMessage.onSuccess call."); } @@ -136,7 +136,7 @@ public class GetDataRequestHandler { @Override public void onFailure(@NotNull Throwable throwable) { if (!stopped) { - String errorMessage = "Sending getDataRequest to " + connection + + String errorMessage = "Sending getDataResponse to " + connection + " failed. That is expected if the peer is offline. getDataResponse=" + getDataResponse + "." + "Exception: " + throwable.getMessage(); handleFault(errorMessage, CloseConnectionReason.SEND_MSG_FAILURE, connection); diff --git a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java index 8225c5cb07..53fd181dd8 100644 --- a/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java +++ b/p2p/src/main/java/bisq/network/p2p/peers/getdata/RequestDataManager.java @@ -43,6 +43,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -63,6 +64,7 @@ public class RequestDataManager implements MessageListener, ConnectionListener, private static int NUM_ADDITIONAL_SEEDS_FOR_UPDATE_REQUEST = 1; private boolean isPreliminaryDataRequest = true; + /////////////////////////////////////////////////////////////////////////////////////////// // Listener /////////////////////////////////////////////////////////////////////////////////////////// @@ -81,6 +83,12 @@ public class RequestDataManager implements MessageListener, ConnectionListener, } } + public interface ResponseListener { + void onSuccess(int serializedSize); + + void onFault(); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Class fields @@ -90,6 +98,7 @@ public class RequestDataManager implements MessageListener, ConnectionListener, private final P2PDataStorage dataStorage; private final PeerManager peerManager; private final List seedNodeAddresses; + private final List responseListeners = new CopyOnWriteArrayList<>(); // As we use Guice injection we cannot set the listener in our constructor but the P2PService calls the setListener // in it's constructor so we can guarantee it is not null. @@ -205,6 +214,10 @@ public class RequestDataManager implements MessageListener, ConnectionListener, return nodeAddressOfPreliminaryDataRequest; } + public void addResponseListener(ResponseListener responseListener) { + responseListeners.add(responseListener); + } + /////////////////////////////////////////////////////////////////////////////////////////// // ConnectionListener implementation @@ -276,9 +289,11 @@ public class RequestDataManager implements MessageListener, ConnectionListener, GetDataRequestHandler getDataRequestHandler = new GetDataRequestHandler(networkNode, dataStorage, new GetDataRequestHandler.Listener() { @Override - public void onComplete() { + public void onComplete(int serializedSize) { getDataRequestHandlers.remove(uid); log.trace("requestDataHandshake completed.\n\tConnection={}", connection); + + responseListeners.forEach(listener -> listener.onSuccess(serializedSize)); } @Override @@ -288,6 +303,8 @@ public class RequestDataManager implements MessageListener, ConnectionListener, log.trace("GetDataRequestHandler failed.\n\tConnection={}\n\t" + "ErrorMessage={}", connection, errorMessage); peerManager.handleConnectionFault(connection); + + responseListeners.forEach(ResponseListener::onFault); } else { log.warn("We have stopped already. We ignore that getDataRequestHandler.handle.onFault call."); } diff --git a/proto/build.gradle b/proto/build.gradle new file mode 100644 index 0000000000..4336f76d77 --- /dev/null +++ b/proto/build.gradle @@ -0,0 +1,39 @@ +apply plugin: 'com.google.protobuf' + +dependencies { + annotationProcessor libs.lombok + compileOnly libs.javax.annotation + compileOnly libs.lombok + implementation libs.logback.classic + implementation libs.logback.core + implementation libs.google.guava + implementation libs.protobuf.java + implementation libs.slf4j.api + implementation(libs.grpc.protobuf) { + exclude(module: 'animal-sniffer-annotations') + exclude(module: 'guava') + } + implementation(libs.grpc.stub) { + exclude(module: 'animal-sniffer-annotations') + exclude(module: 'guava') + } +} + +sourceSets.main.java.srcDirs += [ + 'build/generated/source/proto/main/grpc', + 'build/generated/source/proto/main/java' +] + +protobuf { + protoc { + artifact = "com.google.protobuf:protoc:${protocVersion}" + } + plugins { + grpc { + artifact = "io.grpc:protoc-gen-grpc-java:${grpcVersion}" + } + } + generateProtoTasks { + all()*.plugins { grpc {} } + } +} diff --git a/seednode/src/main/java/bisq/seednode/SeedNodeMain.java b/seednode/src/main/java/bisq/seednode/SeedNodeMain.java index 12a836fd13..a4d66a5850 100644 --- a/seednode/src/main/java/bisq/seednode/SeedNodeMain.java +++ b/seednode/src/main/java/bisq/seednode/SeedNodeMain.java @@ -95,6 +95,19 @@ public class SeedNodeMain extends ExecutableForAppWithP2p { } + /////////////////////////////////////////////////////////////////////////////////////////// + // UncaughtExceptionHandler implementation + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public void handleUncaughtException(Throwable throwable, boolean doShutDown) { + if (throwable instanceof OutOfMemoryError || doShutDown) { + log.error("We got an OutOfMemoryError and shut down"); + gracefulShutDown(() -> log.info("gracefulShutDown complete")); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////// // We continue with a series of synchronous execution tasks /////////////////////////////////////////////////////////////////////////////////////////// diff --git a/seednode/src/main/java/bisq/seednode/reporting/DoubleValueReportingItem.java b/seednode/src/main/java/bisq/seednode/reporting/DoubleValueReportingItem.java index b4a7486127..d666a82340 100644 --- a/seednode/src/main/java/bisq/seednode/reporting/DoubleValueReportingItem.java +++ b/seednode/src/main/java/bisq/seednode/reporting/DoubleValueReportingItem.java @@ -17,12 +17,14 @@ package bisq.seednode.reporting; +import java.util.Optional; + import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; - +@Slf4j public enum DoubleValueReportingItem implements ReportingItem { - Unspecified("", "Unspecified"), sentBytesPerSec("network", "sentBytesPerSec"), receivedBytesPerSec("network", "receivedBytesPerSec"), receivedMessagesPerSec("network", "receivedMessagesPerSec"), @@ -47,16 +49,15 @@ public enum DoubleValueReportingItem implements ReportingItem { return this; } - public static DoubleValueReportingItem from(String key, double value) { - DoubleValueReportingItem item; + public static Optional from(String key, double value) { try { - item = DoubleValueReportingItem.valueOf(key); + DoubleValueReportingItem item = DoubleValueReportingItem.valueOf(key); + item.setValue(value); + return Optional.of(item); } catch (Throwable t) { - item = Unspecified; + log.warn("No enum value with {}", key); + return Optional.empty(); } - - item.setValue(value); - return item; } @Override @@ -66,8 +67,8 @@ public enum DoubleValueReportingItem implements ReportingItem { .build(); } - public static DoubleValueReportingItem fromProto(protobuf.ReportingItem baseProto, - protobuf.DoubleValueReportingItem proto) { + public static Optional fromProto(protobuf.ReportingItem baseProto, + protobuf.DoubleValueReportingItem proto) { return DoubleValueReportingItem.from(baseProto.getKey(), proto.getValue()); } diff --git a/seednode/src/main/java/bisq/seednode/reporting/LongValueReportingItem.java b/seednode/src/main/java/bisq/seednode/reporting/LongValueReportingItem.java index a62a2d381c..59de27e5ac 100644 --- a/seednode/src/main/java/bisq/seednode/reporting/LongValueReportingItem.java +++ b/seednode/src/main/java/bisq/seednode/reporting/LongValueReportingItem.java @@ -17,13 +17,16 @@ package bisq.seednode.reporting; +import java.util.Optional; + import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; - +@Slf4j public enum LongValueReportingItem implements ReportingItem { - Unspecified("", "Unspecified"), OfferPayload("data", "OfferPayload"), + BsqSwapOfferPayload("data", "BsqSwapOfferPayload"), MailboxStoragePayload("data", "MailboxStoragePayload"), TradeStatistics3("data", "TradeStatistics3"), AccountAgeWitness("data", "AccountAgeWitness"), @@ -47,6 +50,21 @@ public enum LongValueReportingItem implements ReportingItem { sentBytes("network", "sentBytes"), receivedBytes("network", "receivedBytes"), + PreliminaryGetDataRequest("network", "PreliminaryGetDataRequest"), + GetUpdatedDataRequest("network", "GetUpdatedDataRequest"), + GetBlocksRequest("network", "GetBlocksRequest"), + GetDaoStateHashesRequest("network", "GetDaoStateHashesRequest"), + GetProposalStateHashesRequest("network", "GetProposalStateHashesRequest"), + GetBlindVoteStateHashesRequest("network", "GetBlindVoteStateHashesRequest"), + + GetDataResponse("network", "GetDataResponse"), + GetBlocksResponse("network", "GetBlocksResponse"), + GetDaoStateHashesResponse("network", "GetDaoStateHashesResponse"), + GetProposalStateHashesResponse("network", "GetProposalStateHashesResponse"), + GetBlindVoteStateHashesResponse("network", "GetBlindVoteStateHashesResponse"), + + failedResponseClassName("network", "failedResponseClassName"), + usedMemoryInMB("node", "usedMemoryInMB"), totalMemoryInMB("node", "totalMemoryInMB"), jvmStartTimeInSec("node", "jvmStartTimeInSec"); @@ -69,16 +87,15 @@ public enum LongValueReportingItem implements ReportingItem { return this; } - public static LongValueReportingItem from(String key, long value) { - LongValueReportingItem item; + public static Optional from(String key, long value) { try { - item = LongValueReportingItem.valueOf(key); + LongValueReportingItem item = LongValueReportingItem.valueOf(key); + item.setValue(value); + return Optional.of(item); } catch (Throwable t) { - item = Unspecified; + log.warn("No enum value with {}", key); + return Optional.empty(); } - - item.setValue(value); - return item; } @Override @@ -88,8 +105,8 @@ public enum LongValueReportingItem implements ReportingItem { .build(); } - public static LongValueReportingItem fromProto(protobuf.ReportingItem baseProto, - protobuf.LongValueReportingItem proto) { + public static Optional fromProto(protobuf.ReportingItem baseProto, + protobuf.LongValueReportingItem proto) { return LongValueReportingItem.from(baseProto.getKey(), proto.getValue()); } diff --git a/seednode/src/main/java/bisq/seednode/reporting/ReportingItem.java b/seednode/src/main/java/bisq/seednode/reporting/ReportingItem.java index 828b658542..57e0020aa0 100644 --- a/seednode/src/main/java/bisq/seednode/reporting/ReportingItem.java +++ b/seednode/src/main/java/bisq/seednode/reporting/ReportingItem.java @@ -20,6 +20,8 @@ package bisq.seednode.reporting; import bisq.common.proto.ProtobufferRuntimeException; import bisq.common.proto.network.NetworkPayload; +import java.util.Optional; + public interface ReportingItem extends NetworkPayload { String getKey(); @@ -35,7 +37,7 @@ public interface ReportingItem extends NetworkPayload { protobuf.ReportingItem toProtoMessage(); - static ReportingItem fromProto(protobuf.ReportingItem proto) { + static Optional fromProto(protobuf.ReportingItem proto) { switch (proto.getMessageCase()) { case STRING_VALUE_REPORTING_ITEM: return StringValueReportingItem.fromProto(proto, proto.getStringValueReportingItem()); diff --git a/seednode/src/main/java/bisq/seednode/reporting/ReportingItems.java b/seednode/src/main/java/bisq/seednode/reporting/ReportingItems.java index d5cb51057d..66afbc6b54 100644 --- a/seednode/src/main/java/bisq/seednode/reporting/ReportingItems.java +++ b/seednode/src/main/java/bisq/seednode/reporting/ReportingItems.java @@ -23,6 +23,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.util.ArrayList; +import java.util.Optional; import java.util.stream.Collectors; import lombok.Getter; @@ -50,7 +51,10 @@ public class ReportingItems extends ArrayList implements NetworkP public static ReportingItems fromProto(protobuf.ReportingItems proto) { ReportingItems reportingItems = new ReportingItems(proto.getAddress()); reportingItems.addAll(proto.getReportingItemList().stream() - .map(ReportingItem::fromProto).collect(Collectors.toList())); + .map(ReportingItem::fromProto) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList())); return reportingItems; } diff --git a/seednode/src/main/java/bisq/seednode/reporting/SeedNodeReportingService.java b/seednode/src/main/java/bisq/seednode/reporting/SeedNodeReportingService.java index de03faecb3..c775d11bd7 100644 --- a/seednode/src/main/java/bisq/seednode/reporting/SeedNodeReportingService.java +++ b/seednode/src/main/java/bisq/seednode/reporting/SeedNodeReportingService.java @@ -24,6 +24,12 @@ import bisq.core.dao.monitoring.ProposalStateMonitoringService; import bisq.core.dao.monitoring.model.BlindVoteStateBlock; import bisq.core.dao.monitoring.model.DaoStateBlock; import bisq.core.dao.monitoring.model.ProposalStateBlock; +import bisq.core.dao.monitoring.network.StateNetworkService; +import bisq.core.dao.monitoring.network.messages.GetBlindVoteStateHashesRequest; +import bisq.core.dao.monitoring.network.messages.GetDaoStateHashesRequest; +import bisq.core.dao.monitoring.network.messages.GetProposalStateHashesRequest; +import bisq.core.dao.node.full.network.FullNodeNetworkService; +import bisq.core.dao.node.messages.GetBlocksRequest; import bisq.core.dao.state.DaoStateListener; import bisq.core.dao.state.DaoStateService; @@ -31,6 +37,9 @@ import bisq.network.p2p.P2PService; import bisq.network.p2p.network.NetworkNode; import bisq.network.p2p.network.Statistic; import bisq.network.p2p.peers.PeerManager; +import bisq.network.p2p.peers.getdata.RequestDataManager; +import bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; import bisq.network.p2p.storage.P2PDataStorage; import bisq.network.p2p.storage.payload.ProtectedStorageEntry; @@ -103,6 +112,8 @@ public class SeedNodeReportingService { DaoStateMonitoringService daoStateMonitoringService, ProposalStateMonitoringService proposalStateMonitoringService, BlindVoteStateMonitoringService blindVoteStateMonitoringService, + RequestDataManager requestDataManager, + FullNodeNetworkService fullNodeNetworkService, @Named(Config.MAX_CONNECTIONS) int maxConnections, @Named(Config.SEED_NODE_REPORTING_SERVER_URL) String seedNodeReportingServerUrl) { this.p2PService = p2PService; @@ -142,6 +153,106 @@ public class SeedNodeReportingService { } }; daoFacade.addBsqStateListener(daoStateListener); + + p2PService.getNetworkNode().addMessageListener((networkEnvelope, connection) -> { + if (networkEnvelope instanceof PreliminaryGetDataRequest || + networkEnvelope instanceof GetUpdatedDataRequest || + networkEnvelope instanceof GetBlocksRequest || + networkEnvelope instanceof GetDaoStateHashesRequest || + networkEnvelope instanceof GetProposalStateHashesRequest || + networkEnvelope instanceof GetBlindVoteStateHashesRequest) { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + int serializedSize = networkEnvelope.toProtoNetworkEnvelope().getSerializedSize(); + String simpleName = networkEnvelope.getClass().getSimpleName(); + try { + LongValueReportingItem reportingItem = LongValueReportingItem.valueOf(simpleName); + reportingItems.add(reportingItem.withValue(serializedSize)); + sendReportingItems(reportingItems); + } catch (Throwable t) { + log.warn("Could not find enum for {}. Error={}", simpleName, t); + } + } + }); + + requestDataManager.addResponseListener(new RequestDataManager.ResponseListener() { + @Override + public void onSuccess(int serializedSize) { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + reportingItems.add(LongValueReportingItem.GetDataResponse.withValue(serializedSize)); + sendReportingItems(reportingItems); + } + + @Override + public void onFault() { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + reportingItems.add(LongValueReportingItem.GetDataResponse.withValue(-1)); + sendReportingItems(reportingItems); + } + }); + + fullNodeNetworkService.addResponseListener(new FullNodeNetworkService.ResponseListener() { + @Override + public void onSuccess(int serializedSize) { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + reportingItems.add(LongValueReportingItem.GetBlocksResponse.withValue(serializedSize)); + sendReportingItems(reportingItems); + } + + @Override + public void onFault() { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + reportingItems.add(LongValueReportingItem.GetBlocksResponse.withValue(-1)); + sendReportingItems(reportingItems); + } + }); + + daoStateMonitoringService.addResponseListener(new StateNetworkService.ResponseListener() { + @Override + public void onSuccess(int serializedSize) { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + reportingItems.add(LongValueReportingItem.GetDaoStateHashesResponse.withValue(serializedSize)); + sendReportingItems(reportingItems); + } + + @Override + public void onFault() { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + reportingItems.add(LongValueReportingItem.GetDaoStateHashesResponse.withValue(-1)); + sendReportingItems(reportingItems); + } + }); + + proposalStateMonitoringService.addResponseListener(new StateNetworkService.ResponseListener() { + @Override + public void onSuccess(int serializedSize) { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + reportingItems.add(LongValueReportingItem.GetProposalStateHashesResponse.withValue(serializedSize)); + sendReportingItems(reportingItems); + } + + @Override + public void onFault() { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + reportingItems.add(LongValueReportingItem.GetProposalStateHashesResponse.withValue(-1)); + sendReportingItems(reportingItems); + } + }); + + blindVoteStateMonitoringService.addResponseListener(new StateNetworkService.ResponseListener() { + @Override + public void onSuccess(int serializedSize) { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + reportingItems.add(LongValueReportingItem.GetBlindVoteStateHashesResponse.withValue(serializedSize)); + sendReportingItems(reportingItems); + } + + @Override + public void onFault() { + ReportingItems reportingItems = new ReportingItems(getMyAddress()); + reportingItems.add(LongValueReportingItem.GetBlindVoteStateHashesResponse.withValue(-1)); + sendReportingItems(reportingItems); + } + }); } public void shutDown() { @@ -213,7 +324,7 @@ public class SeedNodeReportingService { numItemsByType.putIfAbsent(className, 0); numItemsByType.put(className, numItemsByType.get(className) + 1); }); - numItemsByType.forEach((key, value) -> reportingItems.add(LongValueReportingItem.from(key, value))); + numItemsByType.forEach((key, value) -> LongValueReportingItem.from(key, value).ifPresent(reportingItems::add)); // Network reportingItems.add(LongValueReportingItem.numConnections.withValue(networkNode.getAllConnections().size())); @@ -233,16 +344,15 @@ public class SeedNodeReportingService { reportingItems.add(LongValueReportingItem.maxConnections.withValue(maxConnections)); reportingItems.add(StringValueReportingItem.version.withValue(Version.VERSION)); - // If no commit hash is found we use 0 in hex format - String commitHash = Version.findCommitHash().orElse("00"); - reportingItems.add(StringValueReportingItem.commitHash.withValue(commitHash)); + Version.findCommitHash().ifPresent(commitHash -> reportingItems.add(StringValueReportingItem.commitHash.withValue(commitHash))); sendReportingItems(reportingItems); } private void sendReportingItems(ReportingItems reportingItems) { + String truncated = Utilities.toTruncatedString(reportingItems.toString()); try { - log.info("Send report to monitor server: {}", reportingItems.toString()); + log.info("Going to send report to monitor server: {}", truncated); // We send the data as hex encoded protobuf data. We do not use the envelope as it is not part of the p2p system. byte[] protoMessageAsBytes = reportingItems.toProtoMessageAsBytes(); HttpRequest request = HttpRequest.newBuilder() @@ -253,14 +363,16 @@ public class SeedNodeReportingService { httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofString()).whenComplete((response, throwable) -> { if (throwable != null) { log.warn("Exception at sending reporting data. {}", throwable.getMessage()); - } else if (response.statusCode() != 200) { - log.error("Response error message: {}", response); + } else if (response.statusCode() == 200) { + log.info("Sent successfully report to monitor server with {} items", reportingItems.size()); + } else { + log.warn("Server responded with error. Response={}", response); } }); } catch (RejectedExecutionException t) { - log.warn("Did not send reportingItems {} because of RejectedExecutionException {}", reportingItems, t.toString()); + log.warn("Did not send reportingItems {} because of RejectedExecutionException {}", truncated, t.toString()); } catch (Throwable t) { - log.warn("Did not send reportingItems {} because of exception {}", reportingItems, t.toString()); + log.warn("Did not send reportingItems {} because of exception {}", truncated, t.toString()); } } diff --git a/seednode/src/main/java/bisq/seednode/reporting/StringValueReportingItem.java b/seednode/src/main/java/bisq/seednode/reporting/StringValueReportingItem.java index 65f9edbf1f..e814fb42f6 100644 --- a/seednode/src/main/java/bisq/seednode/reporting/StringValueReportingItem.java +++ b/seednode/src/main/java/bisq/seednode/reporting/StringValueReportingItem.java @@ -17,13 +17,14 @@ package bisq.seednode.reporting; +import java.util.Optional; + import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; - +@Slf4j public enum StringValueReportingItem implements ReportingItem { - Unspecified("", "Unspecified"), - daoStateHash("dao", "daoStateHash"), proposalHash("dao", "proposalHash"), blindVoteHash("dao", "blindVoteHash"), @@ -49,16 +50,15 @@ public enum StringValueReportingItem implements ReportingItem { return this; } - public static StringValueReportingItem from(String key, String value) { - StringValueReportingItem item; + public static Optional from(String key, String value) { try { - item = StringValueReportingItem.valueOf(key); + StringValueReportingItem item = StringValueReportingItem.valueOf(key); + item.setValue(value); + return Optional.of(item); } catch (Throwable t) { - item = Unspecified; + log.warn("No enum value with {}", key); + return Optional.empty(); } - - item.setValue(value); - return item; } @Override @@ -73,8 +73,8 @@ public enum StringValueReportingItem implements ReportingItem { .build(); } - public static StringValueReportingItem fromProto(protobuf.ReportingItem baseProto, - protobuf.StringValueReportingItem proto) { + public static Optional fromProto(protobuf.ReportingItem baseProto, + protobuf.StringValueReportingItem proto) { return StringValueReportingItem.from(baseProto.getKey(), proto.getValue()); }