Use new Bitcoind(Client|Daemon) & remove btcd-cli4j

Migrate RpcService over to the new block notification daemon and RPC
client based on jsonrpc4j. Drop in own DTO classes in place of the ones
defined by btcd-cli4j and rename requestBtcBlock & addNewBtcBlockHandler
to requestDtoBlock & addNewDtoBlockHandler respectively.

Also remove now redundant filtering from the logback config and update
grade-witness.
This commit is contained in:
Steven Barclay 2020-12-21 17:46:01 +00:00
parent ac78639656
commit 8104301b52
No known key found for this signature in database
GPG Key ID: 9FED6BF1176D500B
11 changed files with 91 additions and 164 deletions

View File

@ -16,6 +16,5 @@
<appender-ref ref="CONSOLE_APPENDER"/>
</root>
<logger name="com.neemre.btcdcli4j" level="WARN"/>
<logger name="io.grpc.netty" level="WARN"/>
</configuration>

View File

@ -29,7 +29,6 @@ configure(subprojects) {
ext { // in alphabetical order
bcVersion = '1.63'
bitcoinjVersion = '2a80db4'
btcdCli4jVersion = '27b94333'
codecVersion = '1.13'
easybindVersion = '1.0.3'
easyVersion = '4.0.1'
@ -315,25 +314,9 @@ configure(project(':core')) {
exclude(module: 'commons-codec')
}
compile "com.google.guava:guava:$guavaVersion"
compile("network.bisq.btcd-cli4j:btcd-cli4j-core:$btcdCli4jVersion") {
exclude(module: 'guava')
exclude(module: 'slf4j-api')
exclude(module: 'httpclient')
exclude(module: 'commons-lang3')
exclude(module: 'jackson-core')
exclude(module: 'jackson-annotations')
exclude(module: 'jackson-databind')
compile("com.github.briandilley.jsonrpc4j:jsonrpc4j:$jsonrpc4jVersion") {
exclude(module: 'base64')
}
compile("network.bisq.btcd-cli4j:btcd-cli4j-daemon:$btcdCli4jVersion") {
exclude(module: 'guava')
exclude(module: 'slf4j-api')
exclude(module: 'httpclient')
exclude(module: 'commons-lang3')
exclude(module: 'jackson-core')
exclude(module: 'jackson-annotations')
exclude(module: 'jackson-databind')
}
compile "com.github.briandilley.jsonrpc4j:jsonrpc4j:$jsonrpc4jVersion"
compile "com.fasterxml.jackson.core:jackson-core:$jacksonVersion"
compile "com.fasterxml.jackson.core:jackson-annotations:$jacksonVersion"
compile("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion") {

View File

@ -20,6 +20,7 @@ package bisq.core.dao.node.full;
import bisq.core.dao.node.BsqNode;
import bisq.core.dao.node.explorer.ExportJsonFilesService;
import bisq.core.dao.node.full.network.FullNodeNetworkService;
import bisq.core.dao.node.full.rpc.NotificationHandlerException;
import bisq.core.dao.node.parser.BlockParser;
import bisq.core.dao.node.parser.exceptions.BlockHashNotConnectingException;
import bisq.core.dao.node.parser.exceptions.BlockHeightNotConnectingException;
@ -34,11 +35,10 @@ import bisq.network.p2p.network.ConnectionState;
import bisq.common.UserThread;
import bisq.common.handlers.ResultHandler;
import com.neemre.btcdcli4j.core.http.HttpLayerException;
import com.neemre.btcdcli4j.daemon.NotificationHandlerException;
import javax.inject.Inject;
import java.net.ConnectException;
import java.util.function.Consumer;
import lombok.extern.slf4j.Slf4j;
@ -64,9 +64,8 @@ public class FullNode extends BsqNode {
// Constructor
///////////////////////////////////////////////////////////////////////////////////////////
@SuppressWarnings("WeakerAccess")
@Inject
public FullNode(BlockParser blockParser,
private FullNode(BlockParser blockParser,
DaoStateService daoStateService,
DaoStateSnapshotService daoStateSnapshotService,
P2PService p2PService,
@ -150,7 +149,7 @@ public class FullNode extends BsqNode {
private void addBlockHandler() {
if (!addBlockHandlerAdded) {
addBlockHandlerAdded = true;
rpcService.addNewBtcBlockHandler(rawBlock -> {
rpcService.addNewDtoBlockHandler(rawBlock -> {
try {
// We need to call that before parsing to have set the chain tip correctly for clients
// which might listen for new blocks on daoStateService. DaoStateListener.onNewBlockHeight
@ -238,7 +237,7 @@ public class FullNode extends BsqNode {
Consumer<Block> newBlockHandler,
ResultHandler resultHandler,
Consumer<Throwable> errorHandler) {
rpcService.requestBtcBlock(blockHeight,
rpcService.requestDtoBlock(blockHeight,
rawBlock -> {
try {
doParseBlock(rawBlock).ifPresent(newBlockHandler);
@ -270,20 +269,18 @@ public class FullNode extends BsqNode {
if (throwable instanceof RpcException) {
Throwable cause = throwable.getCause();
if (cause != null) {
if (cause instanceof HttpLayerException) {
if (((HttpLayerException) cause).getCode() == 1004004) {
if (cause instanceof ConnectException) {
if (warnMessageHandler != null)
warnMessageHandler.accept("You have configured Bisq to run as DAO full node but there is no " +
"localhost Bitcoin Core node detected. You need to have Bitcoin Core started and synced before " +
"starting Bisq. Please restart Bisq with proper DAO full node setup or switch to lite node mode.");
return;
}
} else if (cause instanceof NotificationHandlerException) {
// Maybe we need to react specifically to errors as in NotificationHandlerException.getError()
// So far only IO_UNKNOWN was observed
log.error("Error type of NotificationHandlerException: " + ((NotificationHandlerException) cause).getError().toString());
log.error("Error from within block notification daemon: {}", cause.getCause().toString());
startReOrgFromLastSnapshot();
return;
} else if (cause instanceof Error) {
throw (Error) cause;
}
}
}

View File

@ -17,7 +17,11 @@
package bisq.core.dao.node.full;
import bisq.core.dao.node.full.rpc.BitcoindClient;
import bisq.core.dao.node.full.rpc.BitcoindDaemon;
import bisq.core.dao.node.full.rpc.dto.RawTransaction;
import bisq.core.dao.state.model.blockchain.PubKeyScript;
import bisq.core.dao.state.model.blockchain.ScriptType;
import bisq.core.dao.state.model.blockchain.TxInput;
import bisq.core.user.Preferences;
@ -28,21 +32,6 @@ import bisq.common.util.Utilities;
import org.bitcoinj.core.Utils;
import com.neemre.btcdcli4j.core.BitcoindException;
import com.neemre.btcdcli4j.core.BtcdCli4jVersion;
import com.neemre.btcdcli4j.core.CommunicationException;
import com.neemre.btcdcli4j.core.client.BtcdClient;
import com.neemre.btcdcli4j.core.client.BtcdClientImpl;
import com.neemre.btcdcli4j.core.domain.RawTransaction;
import com.neemre.btcdcli4j.core.domain.enums.ScriptTypes;
import com.neemre.btcdcli4j.daemon.BtcdDaemon;
import com.neemre.btcdcli4j.daemon.BtcdDaemonImpl;
import com.neemre.btcdcli4j.daemon.event.BlockListener;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import com.google.inject.Inject;
import javax.inject.Named;
@ -54,8 +43,9 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import java.math.BigDecimal;
import java.util.List;
import java.util.Properties;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@ -77,8 +67,8 @@ public class RpcService {
private final int rpcBlockPort;
private final String rpcBlockHost;
private BtcdClient client;
private BtcdDaemon daemon;
private BitcoindClient client;
private BitcoindDaemon daemon;
// We could use multiple threads but then we need to support ordering of results in a queue
// Keep that for optimization after measuring performance differences
@ -89,9 +79,8 @@ public class RpcService {
// Constructor
///////////////////////////////////////////////////////////////////////////////////////////
@SuppressWarnings("WeakerAccess")
@Inject
public RpcService(Preferences preferences,
private RpcService(Preferences preferences,
@Named(Config.RPC_HOST) String rpcHost,
@Named(Config.RPC_PORT) int rpcPort,
@Named(Config.RPC_BLOCK_NOTIFICATION_PORT) int rpcBlockPort,
@ -127,53 +116,31 @@ public class RpcService {
log.info("daemon shut down");
}
if (client != null) {
client.close();
log.info("client closed");
}
executor.shutdown();
}
void setup(ResultHandler resultHandler, Consumer<Throwable> errorHandler) {
ListenableFuture<Void> future = executor.submit(() -> {
try {
log.info("Starting RPCService with btcd-cli4j version {} on {}:{} with user {}, " +
"listening for blocknotify on port {} from {}",
BtcdCli4jVersion.VERSION, this.rpcHost, this.rpcPort, this.rpcUser, this.rpcBlockPort,
this.rpcBlockHost);
log.info("Starting RpcService on {}:{} with user {}, listening for blocknotify on port {} from {}",
this.rpcHost, this.rpcPort, this.rpcUser, this.rpcBlockPort, this.rpcBlockHost);
long startTs = System.currentTimeMillis();
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
CloseableHttpClient httpProvider = HttpClients.custom().setConnectionManager(cm).build();
Properties nodeConfig = new Properties();
nodeConfig.setProperty("node.bitcoind.rpc.protocol", "http");
nodeConfig.setProperty("node.bitcoind.rpc.host", rpcHost);
nodeConfig.setProperty("node.bitcoind.rpc.auth_scheme", "Basic");
nodeConfig.setProperty("node.bitcoind.rpc.user", rpcUser);
nodeConfig.setProperty("node.bitcoind.rpc.password", rpcPassword);
nodeConfig.setProperty("node.bitcoind.rpc.port", Integer.toString(rpcPort));
nodeConfig.setProperty("node.bitcoind.notification.block.port", Integer.toString(rpcBlockPort));
nodeConfig.setProperty("node.bitcoind.notification.block.host", rpcBlockHost);
nodeConfig.setProperty("node.bitcoind.notification.alert.port", Integer.toString(bisq.network.utils.Utils.findFreeSystemPort()));
nodeConfig.setProperty("node.bitcoind.notification.wallet.port", Integer.toString(bisq.network.utils.Utils.findFreeSystemPort()));
nodeConfig.setProperty("node.bitcoind.http.auth_scheme", "Basic");
BtcdClientImpl client = new BtcdClientImpl(httpProvider, nodeConfig);
daemon = new BtcdDaemonImpl(client, throwable -> {
client = BitcoindClient.builder()
.rpcHost(rpcHost)
.rpcPort(rpcPort)
.rpcUser(rpcUser)
.rpcPassword(rpcPassword)
.build();
daemon = new BitcoindDaemon(rpcBlockHost, rpcBlockPort, throwable -> {
log.error(throwable.toString());
throwable.printStackTrace();
UserThread.execute(() -> errorHandler.accept(new RpcException(throwable)));
});
// TODO: Client should ping or request network info from bitcoind to make sure it is up.
log.info("Setup took {} ms", System.currentTimeMillis() - startTs);
this.client = client;
} catch (BitcoindException | CommunicationException e) {
if (e instanceof CommunicationException)
log.error("Probably Bitcoin Core is not running or the rpc port is not set correctly. rpcPort=" + rpcPort);
log.error(e.toString());
e.printStackTrace();
log.error(e.getCause() != null ? e.getCause().toString() : "e.getCause()=null");
throw new RpcException(e.getMessage(), e);
} catch (Throwable e) {
log.error(e.toString());
e.printStackTrace();
@ -193,32 +160,24 @@ public class RpcService {
}, MoreExecutors.directExecutor());
}
void addNewBtcBlockHandler(Consumer<RawBlock> btcBlockHandler,
void addNewDtoBlockHandler(Consumer<RawBlock> dtoBlockHandler,
Consumer<Throwable> errorHandler) {
daemon.addBlockListener(new BlockListener() {
@Override
public void blockDetected(com.neemre.btcdcli4j.core.domain.RawBlock rawBtcBlock) {
if (rawBtcBlock.getHeight() == null || rawBtcBlock.getHeight() == 0) {
log.warn("We received a RawBlock with no data. blockHash={}", rawBtcBlock.getHash());
return;
}
daemon.setBlockListener(blockHash -> {
try {
log.info("New block received: height={}, id={}", rawBtcBlock.getHeight(), rawBtcBlock.getHash());
List<RawTx> txList = rawBtcBlock.getTx().stream()
.map(e -> getTxFromRawTransaction(e, rawBtcBlock))
var rawDtoBlock = client.getBlock(blockHash, 2);
log.info("New block received: height={}, id={}", rawDtoBlock.getHeight(), rawDtoBlock.getHash());
List<RawTx> txList = rawDtoBlock.getTx().stream()
.map(e -> getTxFromRawTransaction(e, rawDtoBlock))
.collect(Collectors.toList());
UserThread.execute(() -> {
btcBlockHandler.accept(new RawBlock(rawBtcBlock.getHeight(),
rawBtcBlock.getTime() * 1000, // rawBtcBlock.getTime() is in sec but we want ms
rawBtcBlock.getHash(),
rawBtcBlock.getPreviousBlockHash(),
ImmutableList.copyOf(txList)));
});
UserThread.execute(() -> dtoBlockHandler.accept(new RawBlock(rawDtoBlock.getHeight(),
rawDtoBlock.getTime() * 1000, // rawDtoBlock.getTime() is in sec but we want ms
rawDtoBlock.getHash(),
rawDtoBlock.getPreviousBlockHash(),
ImmutableList.copyOf(txList))));
} catch (Throwable t) {
errorHandler.accept(t);
}
}
});
}
@ -235,22 +194,22 @@ public class RpcService {
}, MoreExecutors.directExecutor());
}
void requestBtcBlock(int blockHeight,
void requestDtoBlock(int blockHeight,
Consumer<RawBlock> resultHandler,
Consumer<Throwable> errorHandler) {
ListenableFuture<RawBlock> future = executor.submit(() -> {
long startTs = System.currentTimeMillis();
String blockHash = client.getBlockHash(blockHeight);
com.neemre.btcdcli4j.core.domain.RawBlock rawBtcBlock = client.getBlock(blockHash, 2);
List<RawTx> txList = rawBtcBlock.getTx().stream()
.map(e -> getTxFromRawTransaction(e, rawBtcBlock))
var rawDtoBlock = client.getBlock(blockHash, 2);
List<RawTx> txList = rawDtoBlock.getTx().stream()
.map(e -> getTxFromRawTransaction(e, rawDtoBlock))
.collect(Collectors.toList());
log.info("requestBtcBlock from bitcoind at blockHeight {} with {} txs took {} ms",
log.info("requestDtoBlock from bitcoind at blockHeight {} with {} txs took {} ms",
blockHeight, txList.size(), System.currentTimeMillis() - startTs);
return new RawBlock(rawBtcBlock.getHeight(),
rawBtcBlock.getTime() * 1000, // rawBtcBlock.getTime() is in sec but we want ms
rawBtcBlock.getHash(),
rawBtcBlock.getPreviousBlockHash(),
return new RawBlock(rawDtoBlock.getHeight(),
rawDtoBlock.getTime() * 1000, // rawDtoBlock.getTime() is in sec but we want ms
rawDtoBlock.getHash(),
rawDtoBlock.getPreviousBlockHash(),
ImmutableList.copyOf(txList));
});
@ -262,7 +221,7 @@ public class RpcService {
@Override
public void onFailure(@NotNull Throwable throwable) {
log.error("Error at requestBtcBlock: blockHeight={}", blockHeight);
log.error("Error at requestDtoBlock: blockHeight={}", blockHeight);
UserThread.execute(() -> errorHandler.accept(throwable));
}
}, MoreExecutors.directExecutor());
@ -273,13 +232,13 @@ public class RpcService {
// Private
///////////////////////////////////////////////////////////////////////////////////////////
private RawTx getTxFromRawTransaction(RawTransaction rawBtcTx,
com.neemre.btcdcli4j.core.domain.RawBlock rawBtcBlock) {
String txId = rawBtcTx.getTxId();
long blockTime = rawBtcBlock.getTime() * 1000; // We convert block time from sec to ms
int blockHeight = rawBtcBlock.getHeight();
String blockHash = rawBtcBlock.getHash();
final List<TxInput> txInputs = rawBtcTx.getVIn()
private RawTx getTxFromRawTransaction(RawTransaction rawDtoTx,
bisq.core.dao.node.full.rpc.dto.RawBlock rawDtoBlock) {
String txId = rawDtoTx.getTxId();
long blockTime = rawDtoBlock.getTime() * 1000; // We convert block time from sec to ms
int blockHeight = rawDtoBlock.getHeight();
String blockHash = rawDtoBlock.getHash();
final List<TxInput> txInputs = rawDtoTx.getVIn()
.stream()
.filter(rawInput -> rawInput != null && rawInput.getVOut() != null && rawInput.getTxId() != null)
.map(rawInput -> {
@ -295,19 +254,19 @@ public class RpcService {
pubKeyAsHex = null;
log.debug("pubKeyAsHex is not set as we received a not supported sigScript " +
"(segWit or payToPubKey tx). txId={}, asm={}",
rawBtcTx.getTxId(), rawInput.getScriptSig().getAsm());
rawDtoTx.getTxId(), rawInput.getScriptSig().getAsm());
}
return new TxInput(rawInput.getTxId(), rawInput.getVOut(), pubKeyAsHex);
})
.collect(Collectors.toList());
final List<RawTxOutput> txOutputs = rawBtcTx.getVOut()
final List<RawTxOutput> txOutputs = rawDtoTx.getVOut()
.stream()
.filter(e -> e != null && e.getN() != null && e.getValue() != null && e.getScriptPubKey() != null)
.map(rawBtcTxOutput -> {
.map(rawDtoTxOutput -> {
byte[] opReturnData = null;
com.neemre.btcdcli4j.core.domain.PubKeyScript scriptPubKey = rawBtcTxOutput.getScriptPubKey();
if (ScriptTypes.NULL_DATA.equals(scriptPubKey.getType()) && scriptPubKey.getAsm() != null) {
bisq.core.dao.node.full.rpc.dto.PubKeyScript scriptPubKey = rawDtoTxOutput.getScriptPubKey();
if (ScriptType.NULL_DATA.equals(scriptPubKey.getType()) && scriptPubKey.getAsm() != null) {
String[] chunks = scriptPubKey.getAsm().split(" ");
// We get on testnet a lot of "OP_RETURN 0" data, so we filter those away
if (chunks.length == 2 && "OP_RETURN".equals(chunks[0]) && !"0".equals(chunks[1])) {
@ -327,9 +286,9 @@ public class RpcService {
String address = scriptPubKey.getAddresses() != null &&
scriptPubKey.getAddresses().size() == 1 ? scriptPubKey.getAddresses().get(0) : null;
PubKeyScript pubKeyScript = new PubKeyScript(scriptPubKey);
return new RawTxOutput(rawBtcTxOutput.getN(),
rawBtcTxOutput.getValue().movePointRight(8).longValue(),
rawBtcTx.getTxId(),
return new RawTxOutput(rawDtoTxOutput.getN(),
BigDecimal.valueOf(rawDtoTxOutput.getValue()).movePointRight(8).longValueExact(),
rawDtoTx.getTxId(),
pubKeyScript,
address,
opReturnData,

View File

@ -43,9 +43,9 @@ public class PubKeyScript implements PersistablePayload, ImmutableDaoStateModel
private final String asm;
private final String hex;
public PubKeyScript(com.neemre.btcdcli4j.core.domain.PubKeyScript scriptPubKey) {
public PubKeyScript(bisq.core.dao.node.full.rpc.dto.PubKeyScript scriptPubKey) {
this(scriptPubKey.getReqSigs() != null ? scriptPubKey.getReqSigs() : 0,
ScriptType.forName(scriptPubKey.getType().getName()),
scriptPubKey.getType(),
scriptPubKey.getAddresses() != null ? ImmutableList.copyOf(scriptPubKey.getAddresses()) : null,
scriptPubKey.getAsm(),
scriptPubKey.getHex());

View File

@ -10,7 +10,6 @@
<appender-ref ref="CONSOLE_APPENDER"/>
</root>
<logger name="com.neemre.btcdcli4j" level="WARN"/>
<logger name="io.grpc.netty" level="WARN"/>
</configuration>

View File

@ -10,7 +10,4 @@
<appender-ref ref="CONSOLE_APPENDER"/>
</root>
<logger name="com.neemre.btcdcli4j" level="WARN"/>
<logger name="com.neemre.btcdcli4j.core.client.ClientConfigurator" level="ERROR"/>
</configuration>

View File

@ -24,6 +24,7 @@ dependencyVerification {
'com.github.bisq-network.netlayer:tor.native:b15aba7fe987185037791c7ec7c529cb001b90d723d047d54aab87aceb3b3d45',
'com.github.bisq-network.netlayer:tor:a974190aa3a031067ccd1dda28a3ae58cad14060792299d86ea38a05fb21afc5',
'com.github.bisq-network:bitcoinj:65ed08fa5777ea4a08599bdd575e7dc1f4ba2d4d5835472551439d6f6252e68a',
'com.github.briandilley.jsonrpc4j:jsonrpc4j:c9078f037b3b5f45a30a4296f10e4a8de29aa6ace54607fe68e3693b5feeb314',
'com.github.cd2357.tor-binary:tor-binary-geoip:ae27b6aca1a3a50a046eb11e38202b6d21c2fcd2b8643bbeb5ea85e065fbc1be',
'com.github.cd2357.tor-binary:tor-binary-linux32:7b5d6770aa442ef6d235e8a9bfbaa7c62560690f9fe69ff03c7a752eae84f7dc',
'com.github.cd2357.tor-binary:tor-binary-linux64:24111fa35027599a750b0176392dc1e9417d919414396d1b221ac2e707eaba76',
@ -68,8 +69,6 @@ dependencyVerification {
'net.glxn:qrgen:c85d9d8512d91e8ad11fe56259a7825bd50ce0245447e236cf168d1b17591882',
'net.jcip:jcip-annotations:be5805392060c71474bf6c9a67a099471274d30b83eef84bfc4e0889a4f1dcc0',
'net.sf.jopt-simple:jopt-simple:df26cc58f235f477db07f753ba5a3ab243ebe5789d9f89ecf68dd62ea9a66c28',
'network.bisq.btcd-cli4j:btcd-cli4j-core:4634b39de93764c4609e295e254e8c3b1427ba24febf43352f4f315029c5b1b3',
'network.bisq.btcd-cli4j:btcd-cli4j-daemon:fa3580d2f309e220b9c4f67d0437461fa10cfec75f4468a038b58bdbc36caaee',
'org.apache.commons:commons-compress:5f2df1e467825e4cac5996d44890c4201c000b43c0b23cffc0782d28a0beb9b0',
'org.apache.commons:commons-lang3:9375aad1000cdd5bd3068e832de9802094fac1f145655251e141d5d0072fab9a',
'org.apache.httpcomponents:httpclient:bc5f065aba5dd815ee559dd24d9bcb797fb102ff9cfa036f5091ebc529bd3b93',

View File

@ -10,7 +10,4 @@
<appender-ref ref="CONSOLE_APPENDER"/>
</root>
<logger name="com.neemre.btcdcli4j" level="WARN"/>
<logger name="com.neemre.btcdcli4j.core.client.ClientConfigurator" level="ERROR"/>
</configuration>

View File

@ -10,8 +10,6 @@
<appender-ref ref="CONSOLE_APPENDER"/>
</root>
<logger name="com.neemre.btcdcli4j" level="WARN"/>
<logger name="com.msopentech.thali.toronionproxy.OnionProxyManagerEventHandler" level="INFO"/>
</configuration>

View File

@ -11,6 +11,5 @@
</root>
<logger name="com.msopentech.thali.toronionproxy.OnionProxyManagerEventHandler" level="INFO"/>
<logger name="com.neemre.btcdcli4j" level="WARN"/>
</configuration>