diff --git a/monitor/build.gradle b/monitor/build.gradle new file mode 100644 index 0000000000..b00b4f7b43 --- /dev/null +++ b/monitor/build.gradle @@ -0,0 +1,29 @@ +plugins { + id 'java' + id 'application' +} + +group = 'network.bisq' +version = '0.8.0-SNAPSHOT' + +sourceCompatibility = 1.8 + +mainClassName = 'bisq.monitor.MonitorMain' + +repositories { + jcenter() + maven { url "https://jitpack.io" } + maven { url 'https://raw.githubusercontent.com/JesusMcCloud/tor-binary/master/release/' } +} + +dependencies { + compile project(':core') + compile 'com.sparkjava:spark-core:2.5.2' + compile 'net.gpedro.integrations.slack:slack-webhook:1.1.1' + compileOnly 'org.projectlombok:lombok:1.16.16' + annotationProcessor 'org.projectlombok:lombok:1.16.16' +} + +build.dependsOn installDist +installDist.destinationDir = file('build/app') +distZip.enabled = false diff --git a/monitor/src/main/java/bisq/monitor/Monitor.java b/monitor/src/main/java/bisq/monitor/Monitor.java new file mode 100644 index 0000000000..5a51d00c15 --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/Monitor.java @@ -0,0 +1,44 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor; + +import bisq.monitor.metrics.MetricsModel; + +import com.google.inject.Injector; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class Monitor { + @Setter + private Injector injector; + @Getter + private MetricsModel metricsModel; + + public Monitor() { + } + + public void startApplication() { + metricsModel = injector.getInstance(MetricsModel.class); + + MonitorAppSetup appSetup = injector.getInstance(MonitorAppSetup.class); + appSetup.start(); + } +} diff --git a/monitor/src/main/java/bisq/monitor/MonitorAppSetup.java b/monitor/src/main/java/bisq/monitor/MonitorAppSetup.java new file mode 100644 index 0000000000..947bf554c7 --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/MonitorAppSetup.java @@ -0,0 +1,119 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.monitor; + +import bisq.monitor.metrics.p2p.MonitorP2PService; + +import bisq.core.app.BisqEnvironment; +import bisq.core.app.SetupUtils; +import bisq.core.btc.wallet.WalletsSetup; + +import bisq.network.crypto.EncryptionService; +import bisq.network.p2p.network.SetupListener; +import bisq.network.p2p.peers.PeerManager; + +import bisq.common.app.Version; +import bisq.common.crypto.KeyRing; +import bisq.common.proto.persistable.PersistedDataHost; + +import javax.inject.Inject; + +import java.util.ArrayList; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MonitorAppSetup { + private MonitorP2PService seedNodeMonitorP2PService; + private final WalletsSetup walletsSetup; + private PeerManager peerManager; + private final KeyRing keyRing; + private final EncryptionService encryptionService; + + @Inject + public MonitorAppSetup(MonitorP2PService seedNodeMonitorP2PService, + WalletsSetup walletsSetup, + PeerManager peerManager, + KeyRing keyRing, + EncryptionService encryptionService) { + this.seedNodeMonitorP2PService = seedNodeMonitorP2PService; + this.walletsSetup = walletsSetup; + this.peerManager = peerManager; + this.keyRing = keyRing; + this.encryptionService = encryptionService; + Version.setBaseCryptoNetworkId(BisqEnvironment.getBaseCurrencyNetwork().ordinal()); + Version.printVersion(); + } + + public void start() { + SetupUtils.checkCryptoSetup(keyRing, encryptionService, () -> { + initPersistedDataHosts(); + initBasicServices(); + }, throwable -> { + log.error(throwable.getMessage()); + throwable.printStackTrace(); + System.exit(1); + }); + } + + public void initPersistedDataHosts() { + ArrayList persistedDataHosts = new ArrayList<>(); + persistedDataHosts.add(seedNodeMonitorP2PService); + persistedDataHosts.add(peerManager); + + // we apply at startup the reading of persisted data but don't want to get it triggered in the constructor + persistedDataHosts.forEach(e -> { + try { + log.info("call readPersisted at " + e.getClass().getSimpleName()); + e.readPersisted(); + } catch (Throwable e1) { + log.error("readPersisted error", e1); + } + }); + } + + protected void initBasicServices() { + SetupUtils.readFromResources(seedNodeMonitorP2PService.getP2PDataStorage()).addListener((observable, oldValue, newValue) -> { + if (newValue) { + seedNodeMonitorP2PService.start(new SetupListener() { + + + @Override + public void onTorNodeReady() { + walletsSetup.initialize(null, + () -> log.info("walletsSetup completed"), + throwable -> log.error(throwable.toString())); + } + + @Override + public void onHiddenServicePublished() { + } + + @Override + public void onSetupFailed(Throwable throwable) { + } + + @Override + public void onRequestCustomBridges() { + } + }); + + } + }); + } +} diff --git a/monitor/src/main/java/bisq/monitor/MonitorEnvironment.java b/monitor/src/main/java/bisq/monitor/MonitorEnvironment.java new file mode 100644 index 0000000000..9428c2cd1b --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/MonitorEnvironment.java @@ -0,0 +1,129 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.monitor; + +import bisq.core.app.AppOptionKeys; +import bisq.core.app.BisqEnvironment; +import bisq.core.btc.BtcOptionKeys; +import bisq.core.btc.UserAgent; +import bisq.core.dao.DaoOptionKeys; + +import bisq.network.NetworkOptionKeys; + +import bisq.common.CommonOptionKeys; +import bisq.common.app.Version; +import bisq.common.crypto.KeyStorage; +import bisq.common.storage.Storage; + +import org.springframework.core.env.JOptCommandLinePropertySource; +import org.springframework.core.env.PropertiesPropertySource; +import org.springframework.core.env.PropertySource; + +import joptsimple.OptionSet; + +import java.nio.file.Paths; + +import java.util.Properties; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class MonitorEnvironment extends BisqEnvironment { + + private String slackUrlSeedChannel = ""; + private String slackUrlBtcChannel = ""; + private String slackUrlProviderChannel = ""; + private String port; + + public MonitorEnvironment(OptionSet options) { + this(new JOptCommandLinePropertySource(BISQ_COMMANDLINE_PROPERTY_SOURCE_NAME, checkNotNull(options))); + } + + public MonitorEnvironment(PropertySource commandLineProperties) { + super(commandLineProperties); + + slackUrlSeedChannel = commandLineProperties.containsProperty(MonitorOptionKeys.SLACK_URL_SEED_CHANNEL) ? + (String) commandLineProperties.getProperty(MonitorOptionKeys.SLACK_URL_SEED_CHANNEL) : + ""; + + slackUrlBtcChannel = commandLineProperties.containsProperty(MonitorOptionKeys.SLACK_BTC_SEED_CHANNEL) ? + (String) commandLineProperties.getProperty(MonitorOptionKeys.SLACK_BTC_SEED_CHANNEL) : + ""; + + slackUrlProviderChannel = commandLineProperties.containsProperty(MonitorOptionKeys.SLACK_PROVIDER_SEED_CHANNEL) ? + (String) commandLineProperties.getProperty(MonitorOptionKeys.SLACK_PROVIDER_SEED_CHANNEL) : + ""; + + port = commandLineProperties.containsProperty(MonitorOptionKeys.PORT) ? + (String) commandLineProperties.getProperty(MonitorOptionKeys.PORT) : + "80"; + + // hack because defaultProperties() is called from constructor and slackUrlSeedChannel would be null there + getPropertySources().remove("bisqDefaultProperties"); + getPropertySources().addLast(defaultPropertiesMonitor()); + } + + protected PropertySource defaultPropertiesMonitor() { + return new PropertiesPropertySource(BISQ_DEFAULT_PROPERTY_SOURCE_NAME, new Properties() { + { + setProperty(CommonOptionKeys.LOG_LEVEL_KEY, logLevel); + setProperty(MonitorOptionKeys.SLACK_URL_SEED_CHANNEL, slackUrlSeedChannel); + setProperty(MonitorOptionKeys.SLACK_BTC_SEED_CHANNEL, slackUrlBtcChannel); + setProperty(MonitorOptionKeys.SLACK_PROVIDER_SEED_CHANNEL, slackUrlProviderChannel); + setProperty(MonitorOptionKeys.PORT, port); + + setProperty(NetworkOptionKeys.SEED_NODES_KEY, seedNodes); + setProperty(NetworkOptionKeys.MY_ADDRESS, myAddress); + setProperty(NetworkOptionKeys.BAN_LIST, banList); + setProperty(NetworkOptionKeys.TOR_DIR, Paths.get(btcNetworkDir, "tor").toString()); + setProperty(NetworkOptionKeys.NETWORK_ID, String.valueOf(baseCurrencyNetwork.ordinal())); + setProperty(NetworkOptionKeys.SOCKS_5_PROXY_BTC_ADDRESS, socks5ProxyBtcAddress); + setProperty(NetworkOptionKeys.SOCKS_5_PROXY_HTTP_ADDRESS, socks5ProxyHttpAddress); + + setProperty(AppOptionKeys.APP_DATA_DIR_KEY, appDataDir); + setProperty(AppOptionKeys.IGNORE_DEV_MSG_KEY, ignoreDevMsg); + setProperty(AppOptionKeys.DUMP_STATISTICS, dumpStatistics); + setProperty(AppOptionKeys.APP_NAME_KEY, appName); + setProperty(AppOptionKeys.MAX_MEMORY, maxMemory); + setProperty(AppOptionKeys.USER_DATA_DIR_KEY, userDataDir); + setProperty(AppOptionKeys.PROVIDERS, providers); + + setProperty(DaoOptionKeys.RPC_USER, rpcUser); + setProperty(DaoOptionKeys.RPC_PASSWORD, rpcPassword); + setProperty(DaoOptionKeys.RPC_PORT, rpcPort); + setProperty(DaoOptionKeys.RPC_BLOCK_NOTIFICATION_PORT, rpcBlockNotificationPort); + setProperty(DaoOptionKeys.DUMP_BLOCKCHAIN_DATA, dumpBlockchainData); + setProperty(DaoOptionKeys.FULL_DAO_NODE, fullDaoNode); + setProperty(DaoOptionKeys.GENESIS_TX_ID, genesisTxId); + setProperty(DaoOptionKeys.GENESIS_BLOCK_HEIGHT, genesisBlockHeight); + + setProperty(BtcOptionKeys.BTC_NODES, btcNodes); + setProperty(BtcOptionKeys.USE_TOR_FOR_BTC, useTorForBtc); + setProperty(BtcOptionKeys.WALLET_DIR, btcNetworkDir); + setProperty(BtcOptionKeys.USER_AGENT, userAgent); + setProperty(BtcOptionKeys.USE_ALL_PROVIDED_NODES, useAllProvidedNodes); + setProperty(BtcOptionKeys.NUM_CONNECTIONS_FOR_BTC, numConnectionForBtc); + + setProperty(UserAgent.NAME_KEY, appName); + setProperty(UserAgent.VERSION_KEY, Version.VERSION); + + setProperty(Storage.STORAGE_DIR, Paths.get(btcNetworkDir, "db").toString()); + setProperty(KeyStorage.KEY_STORAGE_DIR, Paths.get(btcNetworkDir, "keys").toString()); + } + }); + } +} diff --git a/monitor/src/main/java/bisq/monitor/MonitorMain.java b/monitor/src/main/java/bisq/monitor/MonitorMain.java new file mode 100644 index 0000000000..5c038d1897 --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/MonitorMain.java @@ -0,0 +1,134 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor; + +import bisq.core.app.BisqEnvironment; +import bisq.core.app.BisqExecutable; +import bisq.core.app.misc.ExecutableForAppWithP2p; + +import bisq.common.UserThread; +import bisq.common.app.AppModule; +import bisq.common.setup.CommonSetup; + +import joptsimple.OptionParser; +import joptsimple.OptionSet; + +import lombok.extern.slf4j.Slf4j; + +import static com.google.common.base.Preconditions.checkNotNull; +import static spark.Spark.port; + + + +import spark.Spark; + +@Slf4j +public class MonitorMain extends ExecutableForAppWithP2p { + private static final String VERSION = "1.0.1"; + private Monitor monitor; + + public static void main(String[] args) throws Exception { + log.info("Monitor.VERSION: " + VERSION); + BisqEnvironment.setDefaultAppName("bisq_monitor"); + if (BisqExecutable.setupInitialOptionParser(args)) + new MonitorMain().execute(args); + } + + @Override + protected void doExecute(OptionSet options) { + super.doExecute(options); + + CommonSetup.setup(this); + checkMemory(bisqEnvironment, this); + + startHttpServer(bisqEnvironment.getProperty(MonitorOptionKeys.PORT)); + + keepRunning(); + } + + @Override + protected void setupEnvironment(OptionSet options) { + bisqEnvironment = new MonitorEnvironment(checkNotNull(options)); + } + + @Override + protected void launchApplication() { + UserThread.execute(() -> { + try { + monitor = new Monitor(); + UserThread.execute(this::onApplicationLaunched); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + + @Override + protected void onApplicationLaunched() { + super.onApplicationLaunched(); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // We continue with a series of synchronous execution tasks + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + protected AppModule getModule() { + return new MonitorModule(bisqEnvironment); + } + + @Override + protected void applyInjector() { + super.applyInjector(); + + monitor.setInjector(injector); + } + + @Override + protected void startApplication() { + monitor.startApplication(); + } + + private void startHttpServer(String port) { + port(Integer.parseInt(port)); + Spark.get("/", (req, res) -> { + log.info("Incoming request from: " + req.userAgent()); + final String resultAsHtml = monitor.getMetricsModel().getResultAsHtml(); + return resultAsHtml == null ? "Still starting up..." : resultAsHtml; + }); + } + + @Override + protected void customizeOptionParsing(OptionParser parser) { + super.customizeOptionParsing(parser); + + parser.accepts(MonitorOptionKeys.SLACK_URL_SEED_CHANNEL, + description("Set slack secret for seed node monitor", "")) + .withRequiredArg(); + parser.accepts(MonitorOptionKeys.SLACK_BTC_SEED_CHANNEL, + description("Set slack secret for Btc node monitor", "")) + .withRequiredArg(); + parser.accepts(MonitorOptionKeys.SLACK_PROVIDER_SEED_CHANNEL, + description("Set slack secret for provider node monitor", "")) + .withRequiredArg(); + parser.accepts(MonitorOptionKeys.PORT, + description("Set port to listen on", "80")) + .withRequiredArg(); + } +} diff --git a/monitor/src/main/java/bisq/monitor/MonitorModule.java b/monitor/src/main/java/bisq/monitor/MonitorModule.java new file mode 100644 index 0000000000..32cd9e280b --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/MonitorModule.java @@ -0,0 +1,56 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor; + +import bisq.monitor.metrics.p2p.MonitorP2PModule; + +import bisq.core.app.BisqEnvironment; +import bisq.core.app.misc.ModuleForAppWithP2p; + +import bisq.network.p2p.P2PModule; + +import org.springframework.core.env.Environment; + +import static com.google.inject.name.Names.named; + +class MonitorModule extends ModuleForAppWithP2p { + + public MonitorModule(Environment environment) { + super(environment); + } + + @Override + protected void configure() { + super.configure(); + + bindConstant().annotatedWith(named(MonitorOptionKeys.SLACK_URL_SEED_CHANNEL)).to(environment.getRequiredProperty(MonitorOptionKeys.SLACK_URL_SEED_CHANNEL)); + bindConstant().annotatedWith(named(MonitorOptionKeys.SLACK_BTC_SEED_CHANNEL)).to(environment.getRequiredProperty(MonitorOptionKeys.SLACK_BTC_SEED_CHANNEL)); + bindConstant().annotatedWith(named(MonitorOptionKeys.SLACK_PROVIDER_SEED_CHANNEL)).to(environment.getRequiredProperty(MonitorOptionKeys.SLACK_PROVIDER_SEED_CHANNEL)); + bindConstant().annotatedWith(named(MonitorOptionKeys.PORT)).to(environment.getRequiredProperty(MonitorOptionKeys.PORT)); + } + + @Override + protected void configEnvironment() { + bind(BisqEnvironment.class).toInstance((MonitorEnvironment) environment); + } + + @Override + protected P2PModule p2pModule() { + return new MonitorP2PModule(environment); + } +} diff --git a/monitor/src/main/java/bisq/monitor/MonitorOptionKeys.java b/monitor/src/main/java/bisq/monitor/MonitorOptionKeys.java new file mode 100644 index 0000000000..eb09ff272f --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/MonitorOptionKeys.java @@ -0,0 +1,26 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.monitor; + +public class MonitorOptionKeys { + + public static final String SLACK_URL_SEED_CHANNEL = "slackUrlSeedChannel"; + public static final String SLACK_BTC_SEED_CHANNEL = "slackUrlBtcChannel"; + public static final String SLACK_PROVIDER_SEED_CHANNEL = "slackUrlProviderChannel"; + public static final String PORT = "port"; +} diff --git a/monitor/src/main/java/bisq/monitor/metrics/Metrics.java b/monitor/src/main/java/bisq/monitor/metrics/Metrics.java new file mode 100644 index 0000000000..054a8bb39e --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/metrics/Metrics.java @@ -0,0 +1,38 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.monitor.metrics; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import lombok.Getter; +import lombok.Setter; + +@Getter +public class Metrics { + List requestDurations = new ArrayList<>(); + List errorMessages = new ArrayList<>(); + List> receivedObjectsList = new ArrayList<>(); + @Setter + long lastDataRequestTs; + @Setter + long lastDataResponseTs; + @Setter + long numRequestAttempts; +} diff --git a/monitor/src/main/java/bisq/monitor/metrics/MetricsModel.java b/monitor/src/main/java/bisq/monitor/metrics/MetricsModel.java new file mode 100644 index 0000000000..ddac2e4243 --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/metrics/MetricsModel.java @@ -0,0 +1,453 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.monitor.metrics; + +import bisq.monitor.MonitorOptionKeys; + +import bisq.core.btc.BitcoinNodes; +import bisq.core.btc.wallet.WalletsSetup; +import bisq.core.locale.Res; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.seed.SeedNodeRepository; + +import bisq.common.util.MathUtils; +import bisq.common.util.Tuple2; + +import net.gpedro.integrations.slack.SlackApi; +import net.gpedro.integrations.slack.SlackMessage; + +import org.bitcoinj.core.Peer; + +import javax.inject.Inject; +import javax.inject.Named; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DurationFormatUtils; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; + +import java.net.InetAddress; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Comparator; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.OptionalDouble; +import java.util.Set; +import java.util.TimeZone; +import java.util.stream.Collectors; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MetricsModel { + private final DateFormat dateFormat = new SimpleDateFormat("MMMMM dd, HH:mm:ss"); + @Getter + private String resultAsString; + @Getter + private String resultAsHtml; + private SeedNodeRepository seedNodeRepository; + private SlackApi slackSeedApi, slackBtcApi, slackProviderApi; + private BitcoinNodes bitcoinNodes; + @Setter + private long lastCheckTs; + private long btcNodeUptimeTs; + private int totalErrors = 0; + private HashMap map = new HashMap<>(); + private List connectedPeers; + private Map, Integer> btcNodeDownTimeMap = new HashMap<>(); + private Map, Integer> btcNodeUpTimeMap = new HashMap<>(); + @Getter + private Set nodesInError = new HashSet<>(); + + @Inject + public MetricsModel(SeedNodeRepository seedNodeRepository, + BitcoinNodes bitcoinNodes, + WalletsSetup walletsSetup, + @Named(MonitorOptionKeys.SLACK_URL_SEED_CHANNEL) String slackUrlSeedChannel, + @Named(MonitorOptionKeys.SLACK_BTC_SEED_CHANNEL) String slackUrlBtcChannel, + @Named(MonitorOptionKeys.SLACK_PROVIDER_SEED_CHANNEL) String slackUrlProviderChannel) { + this.seedNodeRepository = seedNodeRepository; + this.bitcoinNodes = bitcoinNodes; + if (!slackUrlSeedChannel.isEmpty()) + slackSeedApi = new SlackApi(slackUrlSeedChannel); + if (!slackUrlBtcChannel.isEmpty()) + slackBtcApi = new SlackApi(slackUrlBtcChannel); + if (!slackUrlProviderChannel.isEmpty()) + slackProviderApi = new SlackApi(slackUrlProviderChannel); + + walletsSetup.connectedPeersProperty().addListener((observable, oldValue, newValue) -> { + connectedPeers = newValue; + }); + } + + public void addToMap(NodeAddress nodeAddress, Metrics metrics) { + map.put(nodeAddress, metrics); + } + + public Metrics getMetrics(NodeAddress nodeAddress) { + return map.get(nodeAddress); + } + + public void updateReport() { + if (btcNodeUptimeTs == 0) + btcNodeUptimeTs = new Date().getTime(); + + Map accumulatedValues = new HashMap<>(); + final double[] items = {0}; + List> entryList = map.entrySet().stream() + .sorted(Comparator.comparing(entrySet -> seedNodeRepository.getOperator(entrySet.getKey()))) + .collect(Collectors.toList()); + + totalErrors = 0; + entryList.stream().forEach(e -> { + totalErrors += e.getValue().errorMessages.stream().filter(s -> !s.isEmpty()).count(); + final List> receivedObjectsList = e.getValue().getReceivedObjectsList(); + if (!receivedObjectsList.isEmpty()) { + items[0] += 1; + Map last = receivedObjectsList.get(receivedObjectsList.size() - 1); + last.entrySet().stream().forEach(e2 -> { + int accuValue = e2.getValue(); + if (accumulatedValues.containsKey(e2.getKey())) + accuValue += accumulatedValues.get(e2.getKey()); + + accumulatedValues.put(e2.getKey(), (double) accuValue); + }); + } + }); + + Map averageValues = new HashMap<>(); + accumulatedValues.entrySet().stream().forEach(e -> { + averageValues.put(e.getKey(), e.getValue() / items[0]); + }); + + Calendar calendar = new GregorianCalendar(); + calendar.setTimeZone(TimeZone.getTimeZone("CET")); + calendar.setTimeInMillis(lastCheckTs); + final String time = calendar.getTime().toString(); + + StringBuilder html = new StringBuilder(); + html.append("" + + "" + + "" + + "" + + "" + + "

") + .append("Seed nodes in error: " + totalErrors + "
" + + "Last check started at: " + time + "

" + + "" + + "" + + "" + + "" + + "" + + "" + + "" + + "" + + "" + + "" + + "" + + "" + + "" + + ""); + + StringBuilder sb = new StringBuilder(); + sb.append("Seed nodes in error:" + totalErrors); + sb.append("\nLast check started at: " + time + "\n"); + + entryList.forEach(e -> { + final Metrics metrics = e.getValue(); + final List allDurations = metrics.getRequestDurations(); + final String allDurationsString = allDurations.stream().map(Object::toString).collect(Collectors.joining("
")); + final OptionalDouble averageOptional = allDurations.stream().mapToLong(value -> value).average(); + double durationAverage = 0; + if (averageOptional.isPresent()) + durationAverage = averageOptional.getAsDouble() / 1000; + final NodeAddress nodeAddress = e.getKey(); + final String operator = seedNodeRepository.getOperator(nodeAddress); + final List errorMessages = metrics.getErrorMessages(); + final int numErrors = (int) errorMessages.stream().filter(s -> !s.isEmpty()).count(); + int numRequests = allDurations.size(); + String lastErrorMsg = ""; + int lastIndexOfError = 0; + for (int i = 0; i < errorMessages.size(); i++) { + final String msg = errorMessages.get(i); + if (!msg.isEmpty()) { + lastIndexOfError = i; + lastErrorMsg = "Error at request " + lastIndexOfError + ":" + msg; + } + } + // String lastErrorMsg = numErrors > 0 ? errorMessages.get(errorMessages.size() - 1) : ""; + final List> allReceivedData = metrics.getReceivedObjectsList(); + Map lastReceivedData = !allReceivedData.isEmpty() ? allReceivedData.get(allReceivedData.size() - 1) : new HashMap<>(); + final String lastReceivedDataString = lastReceivedData.entrySet().stream().map(Object::toString).collect(Collectors.joining("
")); + final String allReceivedDataString = allReceivedData.stream().map(Object::toString).collect(Collectors.joining("
")); + final String requestTs = metrics.getLastDataRequestTs() > 0 ? dateFormat.format(new Date(metrics.getLastDataRequestTs())) : "" + "
"; + final String responseTs = metrics.getLastDataResponseTs() > 0 ? dateFormat.format(new Date(metrics.getLastDataResponseTs())) : "" + "
"; + final String numRequestAttempts = metrics.getNumRequestAttempts() + "
"; + + sb.append("\nOperator: ").append(operator) + .append("\nNode address: ").append(nodeAddress) + .append("\nTotal num requests: ").append(numRequests) + .append("\nTotal num errors: ").append(numErrors) + .append("\nLast request: ").append(requestTs) + .append("\nLast response: ").append(responseTs) + .append("\nRRT average: ").append(durationAverage) + .append("\nNum requests (retries): ").append(numRequestAttempts) + .append("\nLast error message: ").append(lastErrorMsg) + .append("\nLast data: ").append(lastReceivedDataString); + + String colorNumErrors = lastIndexOfError == numErrors ? "black" : "red"; + String colorDurationAverage = durationAverage < 30 ? "black" : "red"; + html.append("") + .append("") + .append("") + .append("") + .append("") + .append("") + .append("") + .append("") + .append("") + .append("") + .append(""); + } + }); + html.append("
OperatorNode addressTotal num requestsTotal num errorsLast requestLast responseRRT averageNum requests (retries)Last error messageLast dataData deviation last request
").append("" + operator + " ").append("").append("" + nodeAddress + " ").append("").append("" + numRequests + " ").append("").append("" + numErrors + " ").append("").append("" + requestTs + " ").append("").append("" + responseTs + " ").append("").append("" + durationAverage + " ").append("").append("" + numRequestAttempts + " ").append("").append("" + lastErrorMsg + " ").append("").append(lastReceivedDataString).append(""); + + if (!allReceivedData.isEmpty()) { + sb.append("\nData deviation last request:\n"); + lastReceivedData.entrySet().stream().forEach(e2 -> { + final String dataItem = e2.getKey(); + double deviation = MathUtils.roundDouble((double) e2.getValue() / averageValues.get(dataItem) * 100, 2); + String str = dataItem + ": " + deviation + "%"; + sb.append(str).append("\n"); + String color; + final double devAbs = Math.abs(deviation - 100); + if (devAbs < 5) + color = "black"; + else if (devAbs < 10) + color = "blue"; + else + color = "red"; + + html.append("" + str + "").append("
"); + + if (devAbs >= 20) { + if (slackSeedApi != null) + slackSeedApi.call(new SlackMessage("Warning: " + nodeAddress.getFullAddress(), + "<" + seedNodeRepository.getOperator(nodeAddress) + ">" + " Your seed node delivers diverging results for " + dataItem + ". " + + "Please check the monitoring status page at http://seedmonitor.0-2-1.net:8080/")); + } + }); + sb.append("Duration all requests: ").append(allDurationsString) + .append("\nAll data: ").append(allReceivedDataString).append("\n"); + + html.append("
"); + + // btc nodes + sb.append("\n\n####################################\n\nBitcoin nodes\n"); + final long elapsed = new Date().getTime() - btcNodeUptimeTs; + Set connectedBtcPeers = connectedPeers.stream() + .map(e -> { + String hostname = e.getAddress().getHostname(); + InetAddress inetAddress = e.getAddress().getAddr(); + int port = e.getAddress().getPort(); + if (hostname != null) + return hostname + ":" + port; + else if (inetAddress != null) + return inetAddress.getHostAddress() + ":" + port; + else + return ""; + }) + .collect(Collectors.toSet()); + + List onionBtcNodes = new ArrayList<>(bitcoinNodes.getProvidedBtcNodes().stream() + .filter(BitcoinNodes.BtcNode::hasOnionAddress) + .collect(Collectors.toSet())); + onionBtcNodes.sort((o1, o2) -> o1.getOperator() != null && o2.getOperator() != null ? + o1.getOperator().compareTo(o2.getOperator()) : 0); + + printTableHeader(html, "Onion"); + printTable(html, sb, onionBtcNodes, connectedBtcPeers, elapsed, true); + html.append(""); + + List clearNetNodes = new ArrayList<>(bitcoinNodes.getProvidedBtcNodes().stream() + .filter(BitcoinNodes.BtcNode::hasClearNetAddress) + .collect(Collectors.toSet())); + clearNetNodes.sort((o1, o2) -> o1.getOperator() != null && o2.getOperator() != null ? + o1.getOperator().compareTo(o2.getOperator()) : 0); + + printTableHeader(html, "Clear net"); + printTable(html, sb, clearNetNodes, connectedBtcPeers, elapsed, false); + sb.append("\nConnected Bitcoin nodes: " + connectedBtcPeers + "\n"); + html.append(""); + html.append("
Connected Bitcoin nodes: " + connectedBtcPeers + "
"); + btcNodeUptimeTs = new Date().getTime(); + + html.append(""); + + resultAsString = sb.toString(); + resultAsHtml = html.toString(); + } + + private void printTableHeader(StringBuilder html, String type) { + html.append("

Bitcoin " + type + " nodes

" + + "" + + "" + + "" + + "" + + "" + + "" + + "" + + ""); + } + + private void printTable(StringBuilder html, StringBuilder sb, List allBtcNodes, Set connectedBtcPeers, long elapsed, boolean isOnion) { + allBtcNodes.stream().forEach(node -> { + int upTime = 0; + int downTime = 0; + Tuple2 key = new Tuple2<>(node, isOnion); + if (btcNodeUpTimeMap.containsKey(key)) + upTime = btcNodeUpTimeMap.get(key); + + key = new Tuple2<>(node, isOnion); + if (btcNodeDownTimeMap.containsKey(key)) + downTime = btcNodeDownTimeMap.get(key); + + boolean isConnected = false; + // return !connectedBtcPeers.contains(host); + if (node.hasOnionAddress() && connectedBtcPeers.contains(node.getOnionAddress() + ":" + node.getPort())) + isConnected = true; + + final String clearNetHost = node.getAddress() != null ? node.getAddress() + ":" + node.getPort() : node.getHostName() + ":" + node.getPort(); + if (node.hasClearNetAddress() && connectedBtcPeers.contains(clearNetHost)) + isConnected = true; + + if (isConnected) { + upTime += elapsed; + btcNodeUpTimeMap.put(key, upTime); + } else { + downTime += elapsed; + btcNodeDownTimeMap.put(key, downTime); + } + + String upTimeString = formatDurationAsWords(upTime, true); + String downTimeString = formatDurationAsWords(downTime, true); + String colorNumErrors = isConnected ? "black" : "red"; + html.append("") + .append("") + .append("") + .append("") + .append("") + .append("") + .append(""); + + sb.append("\nOperator: ").append(node.getOperator()).append("\n"); + sb.append("Domain name: ").append(node.getHostName()).append("\n"); + sb.append("IP address: ").append(node.getAddress()).append("\n"); + sb.append("Btc node onion address: ").append(node.getOnionAddress()).append("\n"); + sb.append("UpTime: ").append(upTimeString).append("\n"); + sb.append("DownTime: ").append(downTimeString).append("\n"); + }); + } + + public void log() { + log.info("\n\n#################################################################\n" + + resultAsString + + "#################################################################\n\n"); + } + + public static String formatDurationAsWords(long durationMillis, boolean showSeconds) { + String format; + String second = Res.get("time.second"); + String minute = Res.get("time.minute"); + String hour = Res.get("time.hour").toLowerCase(); + String day = Res.get("time.day").toLowerCase(); + String days = Res.get("time.days"); + String hours = Res.get("time.hours"); + String minutes = Res.get("time.minutes"); + String seconds = Res.get("time.seconds"); + if (showSeconds) { + format = "d\' " + days + ", \'H\' " + hours + ", \'m\' " + minutes + ", \'s\' " + seconds + "\'"; + } else + format = "d\' " + days + ", \'H\' " + hours + ", \'m\' " + minutes + "\'"; + String duration = DurationFormatUtils.formatDuration(durationMillis, format); + String tmp; + duration = " " + duration; + tmp = StringUtils.replaceOnce(duration, " 0 " + days, ""); + if (tmp.length() != duration.length()) { + duration = tmp; + tmp = StringUtils.replaceOnce(tmp, " 0 " + hours, ""); + if (tmp.length() != duration.length()) { + tmp = StringUtils.replaceOnce(tmp, " 0 " + minutes, ""); + duration = tmp; + if (tmp.length() != tmp.length()) { + duration = StringUtils.replaceOnce(tmp, " 0 " + seconds, ""); + } + } + } + + if (duration.length() != 0) { + duration = duration.substring(1); + } + + tmp = StringUtils.replaceOnce(duration, " 0 " + seconds, ""); + + if (tmp.length() != duration.length()) { + duration = tmp; + tmp = StringUtils.replaceOnce(tmp, " 0 " + minutes, ""); + if (tmp.length() != duration.length()) { + duration = tmp; + tmp = StringUtils.replaceOnce(tmp, " 0 " + hours, ""); + if (tmp.length() != duration.length()) { + duration = StringUtils.replaceOnce(tmp, " 0 " + days, ""); + } + } + } + + duration = " " + duration; + duration = StringUtils.replaceOnce(duration, " 1 " + seconds, " 1 " + second); + duration = StringUtils.replaceOnce(duration, " 1 " + minutes, " 1 " + minute); + duration = StringUtils.replaceOnce(duration, " 1 " + hours, " 1 " + hour); + duration = StringUtils.replaceOnce(duration, " 1 " + days, " 1 " + day); + duration = duration.trim(); + if (duration.equals(",")) + duration = duration.replace(",", ""); + if (duration.startsWith(" ,")) + duration = duration.replace(" ,", ""); + else if (duration.startsWith(", ")) + duration = duration.replace(", ", ""); + return duration; + } + + public void addNodesInError(NodeAddress nodeAddress) { + nodesInError.add(nodeAddress); + } + + public void removeNodesInError(NodeAddress nodeAddress) { + nodesInError.remove(nodeAddress); + } +} diff --git a/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorP2PModule.java b/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorP2PModule.java new file mode 100644 index 0000000000..a8b10acc3a --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorP2PModule.java @@ -0,0 +1,89 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor.metrics.p2p; + +import bisq.monitor.metrics.MetricsModel; + +import bisq.network.NetworkOptionKeys; +import bisq.network.Socks5ProxyProvider; +import bisq.network.p2p.NetworkNodeProvider; +import bisq.network.p2p.P2PModule; +import bisq.network.p2p.P2PService; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.peers.BanList; +import bisq.network.p2p.peers.Broadcaster; +import bisq.network.p2p.peers.PeerManager; +import bisq.network.p2p.peers.getdata.RequestDataManager; +import bisq.network.p2p.peers.keepalive.KeepAliveManager; +import bisq.network.p2p.peers.peerexchange.PeerExchangeManager; +import bisq.network.p2p.storage.P2PDataStorage; + +import org.springframework.core.env.Environment; + +import com.google.inject.Singleton; +import com.google.inject.name.Names; + +import java.io.File; + +import static com.google.inject.name.Names.named; + + +public class MonitorP2PModule extends P2PModule { + + public MonitorP2PModule(Environment environment) { + super(environment); + } + + @Override + protected void configure() { + bind(MetricsModel.class).in(Singleton.class); + bind(MonitorP2PService.class).in(Singleton.class); + + bind(PeerManager.class).in(Singleton.class); + bind(P2PDataStorage.class).in(Singleton.class); + bind(RequestDataManager.class).in(Singleton.class); + bind(PeerExchangeManager.class).in(Singleton.class); + bind(KeepAliveManager.class).in(Singleton.class); + bind(Broadcaster.class).in(Singleton.class); + bind(BanList.class).in(Singleton.class); + bind(NetworkNode.class).toProvider(NetworkNodeProvider.class).in(Singleton.class); + + bind(Socks5ProxyProvider.class).in(Singleton.class); + + Boolean useLocalhostForP2P = environment.getProperty(NetworkOptionKeys.USE_LOCALHOST_FOR_P2P, boolean.class, false); + bind(boolean.class).annotatedWith(Names.named(NetworkOptionKeys.USE_LOCALHOST_FOR_P2P)).toInstance(useLocalhostForP2P); + + File torDir = new File(environment.getRequiredProperty(NetworkOptionKeys.TOR_DIR)); + bind(File.class).annotatedWith(named(NetworkOptionKeys.TOR_DIR)).toInstance(torDir); + + // use a fixed port as arbitrator use that for his ID + Integer port = environment.getProperty(NetworkOptionKeys.PORT_KEY, int.class, 9999); + bind(int.class).annotatedWith(Names.named(NetworkOptionKeys.PORT_KEY)).toInstance(port); + + Integer maxConnections = environment.getProperty(NetworkOptionKeys.MAX_CONNECTIONS, int.class, P2PService.MAX_CONNECTIONS_DEFAULT); + bind(int.class).annotatedWith(Names.named(NetworkOptionKeys.MAX_CONNECTIONS)).toInstance(maxConnections); + + Integer networkId = environment.getProperty(NetworkOptionKeys.NETWORK_ID, int.class, 1); + bind(int.class).annotatedWith(Names.named(NetworkOptionKeys.NETWORK_ID)).toInstance(networkId); + bindConstant().annotatedWith(named(NetworkOptionKeys.SEED_NODES_KEY)).to(environment.getRequiredProperty(NetworkOptionKeys.SEED_NODES_KEY)); + bindConstant().annotatedWith(named(NetworkOptionKeys.MY_ADDRESS)).to(environment.getRequiredProperty(NetworkOptionKeys.MY_ADDRESS)); + bindConstant().annotatedWith(named(NetworkOptionKeys.BAN_LIST)).to(environment.getRequiredProperty(NetworkOptionKeys.BAN_LIST)); + bindConstant().annotatedWith(named(NetworkOptionKeys.SOCKS_5_PROXY_BTC_ADDRESS)).to(environment.getRequiredProperty(NetworkOptionKeys.SOCKS_5_PROXY_BTC_ADDRESS)); + bindConstant().annotatedWith(named(NetworkOptionKeys.SOCKS_5_PROXY_HTTP_ADDRESS)).to(environment.getRequiredProperty(NetworkOptionKeys.SOCKS_5_PROXY_HTTP_ADDRESS)); + } +} diff --git a/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorP2PService.java b/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorP2PService.java new file mode 100644 index 0000000000..3f4a63973a --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorP2PService.java @@ -0,0 +1,129 @@ +/* + * This file is part of Bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package bisq.monitor.metrics.p2p; + +import bisq.network.Socks5ProxyProvider; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.network.SetupListener; +import bisq.network.p2p.storage.P2PDataStorage; + +import bisq.common.app.Log; +import bisq.common.proto.persistable.PersistedDataHost; + +import javax.inject.Inject; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import static com.google.common.base.Preconditions.checkArgument; + +@Slf4j +public class MonitorP2PService implements SetupListener, PersistedDataHost { + private final NetworkNode networkNode; + @Getter + private final P2PDataStorage p2PDataStorage; + private final MonitorRequestManager requestDataManager; + private final Socks5ProxyProvider socks5ProxyProvider; + + private SetupListener listener; + private volatile boolean shutDownInProgress; + private boolean shutDownComplete; + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Constructor + /////////////////////////////////////////////////////////////////////////////////////////// + + @Inject + public MonitorP2PService(NetworkNode networkNode, + P2PDataStorage p2PDataStorage, + MonitorRequestManager requestDataManager, + Socks5ProxyProvider socks5ProxyProvider) { + this.networkNode = networkNode; + this.p2PDataStorage = p2PDataStorage; + this.requestDataManager = requestDataManager; + this.socks5ProxyProvider = socks5ProxyProvider; + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // API + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public void readPersisted() { + p2PDataStorage.readPersisted(); + } + + public void start(SetupListener listener) { + this.listener = listener; + networkNode.start(this); + } + + public void shutDown(Runnable shutDownCompleteHandler) { + Log.traceCall(); + if (!shutDownInProgress) { + shutDownInProgress = true; + + if (requestDataManager != null) { + requestDataManager.shutDown(); + } + + if (networkNode != null) { + networkNode.shutDown(() -> { + shutDownComplete = true; + }); + } else { + shutDownComplete = true; + } + } else { + log.debug("shutDown already in progress"); + if (shutDownComplete) { + shutDownCompleteHandler.run(); + } + } + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // SetupListener implementation + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public void onTorNodeReady() { + socks5ProxyProvider.setSocks5ProxyInternal(networkNode.getSocksProxy()); + listener.onTorNodeReady(); + } + + @Override + public void onHiddenServicePublished() { + checkArgument(networkNode.getNodeAddress() != null, "Address must be set when we have the hidden service ready"); + requestDataManager.start(); + listener.onHiddenServicePublished(); + } + + @Override + public void onSetupFailed(Throwable throwable) { + listener.onSetupFailed(throwable); + } + + @Override + public void onRequestCustomBridges() { + listener.onRequestCustomBridges(); + } +} diff --git a/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorRequestHandler.java b/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorRequestHandler.java new file mode 100644 index 0000000000..676582fd1c --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorRequestHandler.java @@ -0,0 +1,300 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor.metrics.p2p; + +import bisq.monitor.metrics.Metrics; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.network.CloseConnectionReason; +import bisq.network.p2p.network.Connection; +import bisq.network.p2p.network.MessageListener; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.peers.getdata.messages.GetDataRequest; +import bisq.network.p2p.peers.getdata.messages.GetDataResponse; +import bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest; +import bisq.network.p2p.storage.P2PDataStorage; +import bisq.network.p2p.storage.payload.PersistableNetworkPayload; +import bisq.network.p2p.storage.payload.ProtectedStorageEntry; +import bisq.network.p2p.storage.payload.ProtectedStoragePayload; + +import bisq.common.Timer; +import bisq.common.UserThread; +import bisq.common.app.DevEnv; +import bisq.common.app.Log; +import bisq.common.proto.network.NetworkEnvelope; +import bisq.common.proto.network.NetworkPayload; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.SettableFuture; + +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +import lombok.extern.slf4j.Slf4j; + +import org.jetbrains.annotations.NotNull; + +@Slf4j +class MonitorRequestHandler implements MessageListener { + private static final long TIMEOUT = 120; + private NodeAddress peersNodeAddress; + private long requestTs; + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Listener + /////////////////////////////////////////////////////////////////////////////////////////// + + public interface Listener { + void onComplete(); + + @SuppressWarnings("UnusedParameters") + void onFault(String errorMessage, NodeAddress nodeAddress); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Class fields + /////////////////////////////////////////////////////////////////////////////////////////// + + private final NetworkNode networkNode; + private final P2PDataStorage dataStorage; + private final Metrics metrics; + private final Listener listener; + private Timer timeoutTimer; + private final int nonce = new Random().nextInt(); + private boolean stopped; + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Constructor + /////////////////////////////////////////////////////////////////////////////////////////// + + public MonitorRequestHandler(NetworkNode networkNode, P2PDataStorage dataStorage, Metrics metrics, Listener listener) { + this.networkNode = networkNode; + this.dataStorage = dataStorage; + this.metrics = metrics; + this.listener = listener; + } + + public void cancel() { + cleanup(); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // API + /////////////////////////////////////////////////////////////////////////////////////////// + + public void requestData(NodeAddress nodeAddress) { + Log.traceCall("nodeAddress=" + nodeAddress); + peersNodeAddress = nodeAddress; + requestTs = new Date().getTime(); + if (!stopped) { + Set excludedKeys = dataStorage.getAppendOnlyDataStoreMap().entrySet().stream() + .map(e -> e.getKey().bytes) + .collect(Collectors.toSet()); + + GetDataRequest getDataRequest = new PreliminaryGetDataRequest(nonce, excludedKeys); + metrics.setLastDataRequestTs(System.currentTimeMillis()); + + if (timeoutTimer != null) { + log.warn("timeoutTimer was already set. That must not happen."); + timeoutTimer.stop(); + + if (DevEnv.isDevMode()) + throw new RuntimeException("timeoutTimer was already set. That must not happen."); + } + timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions + if (!stopped) { + String errorMessage = "A timeout occurred at sending getDataRequest:" + getDataRequest + + " on nodeAddress:" + nodeAddress; + log.warn(errorMessage + " / RequestDataHandler=" + MonitorRequestHandler.this); + handleFault(errorMessage, nodeAddress, CloseConnectionReason.SEND_MSG_TIMEOUT); + } else { + log.trace("We have stopped already. We ignore that timeoutTimer.run call. " + + "Might be caused by an previous networkNode.sendMessage.onFailure."); + } + }, + TIMEOUT); + + log.info("We send a PreliminaryGetDataRequest to peer {}. ", nodeAddress); + networkNode.addMessageListener(this); + SettableFuture future = networkNode.sendMessage(nodeAddress, getDataRequest); + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(Connection connection) { + if (!stopped) { + log.info("Send PreliminaryGetDataRequest to " + nodeAddress + " has succeeded."); + } else { + log.trace("We have stopped already. We ignore that networkNode.sendMessage.onSuccess call." + + "Might be caused by an previous timeout."); + } + } + + @Override + public void onFailure(@NotNull Throwable throwable) { + if (!stopped) { + String errorMessage = "Sending getDataRequest to " + nodeAddress + + " failed.\n\t" + + "getDataRequest=" + getDataRequest + "." + + "\n\tException=" + throwable.getMessage(); + log.warn(errorMessage); + handleFault(errorMessage, nodeAddress, CloseConnectionReason.SEND_MSG_FAILURE); + } else { + log.trace("We have stopped already. We ignore that networkNode.sendMessage.onFailure call. " + + "Might be caused by an previous timeout."); + } + } + }); + } else { + log.warn("We have stopped already. We ignore that requestData call."); + } + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // MessageListener implementation + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public void onMessage(NetworkEnvelope networkEnvelop, Connection connection) { + if (networkEnvelop instanceof GetDataResponse && + connection.getPeersNodeAddressOptional().isPresent() && + connection.getPeersNodeAddressOptional().get().equals(peersNodeAddress)) { + Log.traceCall(networkEnvelop.toString() + "\n\tconnection=" + connection); + if (!stopped) { + GetDataResponse getDataResponse = (GetDataResponse) networkEnvelop; + if (getDataResponse.getRequestNonce() == nonce) { + stopTimeoutTimer(); + + Map> payloadByClassName = new HashMap<>(); + final Set dataSet = getDataResponse.getDataSet(); + dataSet.stream().forEach(e -> { + final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload(); + if (protectedStoragePayload == null) { + log.warn("StoragePayload was null: {}", networkEnvelop.toString()); + return; + } + + // For logging different data types + String className = protectedStoragePayload.getClass().getSimpleName(); + if (!payloadByClassName.containsKey(className)) + payloadByClassName.put(className, new HashSet<>()); + + payloadByClassName.get(className).add(protectedStoragePayload); + }); + + + Set persistableNetworkPayloadSet = getDataResponse.getPersistableNetworkPayloadSet(); + if (persistableNetworkPayloadSet != null) { + persistableNetworkPayloadSet.stream().forEach(persistableNetworkPayload -> { + // For logging different data types + String className = persistableNetworkPayload.getClass().getSimpleName(); + if (!payloadByClassName.containsKey(className)) + payloadByClassName.put(className, new HashSet<>()); + + payloadByClassName.get(className).add(persistableNetworkPayload); + }); + } + + // Log different data types + StringBuilder sb = new StringBuilder(); + sb.append("\n#################################################################\n"); + sb.append("Connected to node: ").append(peersNodeAddress.getFullAddress()).append("\n"); + final int items = dataSet.size() + + (persistableNetworkPayloadSet != null ? persistableNetworkPayloadSet.size() : 0); + sb.append("Received ").append(items).append(" instances\n"); + Map receivedObjects = new HashMap<>(); + final boolean[] arbitratorReceived = new boolean[1]; + payloadByClassName.entrySet().stream().forEach(e -> { + final String dataItemName = e.getKey(); + // We expect always at least an Arbitrator + if (!arbitratorReceived[0] && dataItemName.equals("Arbitrator")) + arbitratorReceived[0] = true; + + sb.append(dataItemName) + .append(": ") + .append(e.getValue().size()) + .append("\n"); + receivedObjects.put(dataItemName, e.getValue().size()); + }); + sb.append("#################################################################"); + log.info(sb.toString()); + metrics.getReceivedObjectsList().add(receivedObjects); + + final long duration = new Date().getTime() - requestTs; + log.info("Requesting data took {} ms", duration); + metrics.getRequestDurations().add(duration); + metrics.getErrorMessages().add(arbitratorReceived[0] ? "" : "No Arbitrator objects received! Seed node need to be restarted!"); + metrics.setLastDataResponseTs(System.currentTimeMillis()); + + cleanup(); + connection.shutDown(CloseConnectionReason.CLOSE_REQUESTED_BY_PEER, listener::onComplete); + } else { + log.debug("Nonce not matching. That can happen rarely if we get a response after a canceled " + + "handshake (timeout causes connection close but peer might have sent a msg before " + + "connection was closed).\n\t" + + "We drop that message. nonce={} / requestNonce={}", + nonce, getDataResponse.getRequestNonce()); + } + } else { + log.warn("We have stopped already. We ignore that onDataRequest call."); + } + } + } + + public void stop() { + cleanup(); + } + + /////////////////////////////////////////////////////////////////////////////////////////// + // Private + /////////////////////////////////////////////////////////////////////////////////////////// + + + private void handleFault(String errorMessage, NodeAddress nodeAddress, CloseConnectionReason closeConnectionReason) { + cleanup(); + // We do not log every error only if it fails several times in a row. + + // We do not close the connection as it might be we have opened a new connection for that peer and + // we don't want to close that. We do not know the connection at fault as the fault handler does not contain that, + // so we could only search for connections for that nodeAddress but that would close an new connection attempt. + listener.onFault(errorMessage, nodeAddress); + } + + private void cleanup() { + Log.traceCall(); + stopped = true; + networkNode.removeMessageListener(this); + stopTimeoutTimer(); + } + + private void stopTimeoutTimer() { + if (timeoutTimer != null) { + timeoutTimer.stop(); + timeoutTimer = null; + } + } +} diff --git a/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorRequestManager.java b/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorRequestManager.java new file mode 100644 index 0000000000..513406894f --- /dev/null +++ b/monitor/src/main/java/bisq/monitor/metrics/p2p/MonitorRequestManager.java @@ -0,0 +1,283 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.monitor.metrics.p2p; + +import bisq.monitor.MonitorOptionKeys; +import bisq.monitor.metrics.Metrics; +import bisq.monitor.metrics.MetricsModel; + +import bisq.network.p2p.NodeAddress; +import bisq.network.p2p.network.CloseConnectionReason; +import bisq.network.p2p.network.Connection; +import bisq.network.p2p.network.ConnectionListener; +import bisq.network.p2p.network.NetworkNode; +import bisq.network.p2p.seed.SeedNodeRepository; +import bisq.network.p2p.storage.P2PDataStorage; + +import bisq.common.Timer; +import bisq.common.UserThread; + +import net.gpedro.integrations.slack.SlackApi; +import net.gpedro.integrations.slack.SlackMessage; + +import javax.inject.Inject; +import javax.inject.Named; + +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MonitorRequestManager implements ConnectionListener { + private static final long RETRY_DELAY_SEC = 30; + private static final long CLEANUP_TIMER = 60; + private static final long REQUEST_PERIOD_MIN = 10; + private static final int MAX_RETRIES = 5; + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Class fields + /////////////////////////////////////////////////////////////////////////////////////////// + + private final NetworkNode networkNode; + private final int numNodes; + + private SlackApi slackApi; + private P2PDataStorage dataStorage; + private SeedNodeRepository seedNodeRepository; + private MetricsModel metricsModel; + private final Set seedNodeAddresses; + + private final Map handlerMap = new HashMap<>(); + private Map retryTimerMap = new HashMap<>(); + private Map retryCounterMap = new HashMap<>(); + private boolean stopped; + private int completedRequestIndex; + + /////////////////////////////////////////////////////////////////////////////////////////// + // Constructor + /////////////////////////////////////////////////////////////////////////////////////////// + + @Inject + public MonitorRequestManager(NetworkNode networkNode, + P2PDataStorage dataStorage, + SeedNodeRepository seedNodeRepository, + MetricsModel metricsModel, + @Named(MonitorOptionKeys.SLACK_URL_SEED_CHANNEL) String slackUrlSeedChannel) { + this.networkNode = networkNode; + this.dataStorage = dataStorage; + this.seedNodeRepository = seedNodeRepository; + this.metricsModel = metricsModel; + + if (!slackUrlSeedChannel.isEmpty()) + slackApi = new SlackApi(slackUrlSeedChannel); + this.networkNode.addConnectionListener(this); + + seedNodeAddresses = new HashSet<>(seedNodeRepository.getSeedNodeAddresses()); + seedNodeAddresses.stream().forEach(nodeAddress -> metricsModel.addToMap(nodeAddress, new Metrics())); + numNodes = seedNodeAddresses.size(); + } + + public void shutDown() { + stopped = true; + stopAllRetryTimers(); + networkNode.removeConnectionListener(this); + closeAllHandlers(); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // API + /////////////////////////////////////////////////////////////////////////////////////////// + + public void start() { + requestAllNodes(); + UserThread.runPeriodically(this::requestAllNodes, REQUEST_PERIOD_MIN, TimeUnit.MINUTES); + + // We want to update the data for the btc nodes more frequently + UserThread.runPeriodically(metricsModel::updateReport, 10); + } + + private void requestAllNodes() { + stopAllRetryTimers(); + closeAllConnections(); + // we give 1 sec. for all connection shutdown + final int[] delay = {1000}; + metricsModel.setLastCheckTs(System.currentTimeMillis()); + + seedNodeAddresses.stream().forEach(nodeAddress -> { + UserThread.runAfter(() -> requestFromNode(nodeAddress), delay[0], TimeUnit.MILLISECONDS); + delay[0] += 100; + }); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // ConnectionListener implementation + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + public void onConnection(Connection connection) { + } + + @Override + public void onDisconnect(CloseConnectionReason closeConnectionReason, Connection connection) { + closeHandler(connection); + } + + @Override + public void onError(Throwable throwable) { + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // RequestData + /////////////////////////////////////////////////////////////////////////////////////////// + + private void requestFromNode(NodeAddress nodeAddress) { + if (!stopped) { + if (!handlerMap.containsKey(nodeAddress)) { + final Metrics metrics = metricsModel.getMetrics(nodeAddress); + MonitorRequestHandler requestDataHandler = new MonitorRequestHandler(networkNode, + dataStorage, + metrics, + new MonitorRequestHandler.Listener() { + @Override + public void onComplete() { + log.trace("RequestDataHandshake of outbound connection complete. nodeAddress={}", + nodeAddress); + stopRetryTimer(nodeAddress); + retryCounterMap.remove(nodeAddress); + metrics.setNumRequestAttempts(retryCounterMap.getOrDefault(nodeAddress, 1)); + + // need to remove before listeners are notified as they cause the update call + handlerMap.remove(nodeAddress); + + metricsModel.updateReport(); + completedRequestIndex++; + if (completedRequestIndex == numNodes) + metricsModel.log(); + + if (metricsModel.getNodesInError().contains(nodeAddress)) { + metricsModel.removeNodesInError(nodeAddress); + if (slackApi != null) + slackApi.call(new SlackMessage("Fixed: " + nodeAddress.getFullAddress(), + "<" + seedNodeRepository.getOperator(nodeAddress) + ">" + " Your seed node is recovered.")); + } + } + + @Override + public void onFault(String errorMessage, NodeAddress nodeAddress) { + handlerMap.remove(nodeAddress); + stopRetryTimer(nodeAddress); + + int retryCounter = retryCounterMap.getOrDefault(nodeAddress, 0); + metrics.setNumRequestAttempts(retryCounter); + if (retryCounter < MAX_RETRIES) { + log.info("We got an error at peer={}. We will try again after a delay of {} sec. error={} ", + nodeAddress, RETRY_DELAY_SEC, errorMessage); + final Timer timer = UserThread.runAfter(() -> requestFromNode(nodeAddress), RETRY_DELAY_SEC); + retryTimerMap.put(nodeAddress, timer); + retryCounterMap.put(nodeAddress, ++retryCounter); + } else { + log.warn("We got repeated errors at peer={}. error={} ", + nodeAddress, errorMessage); + + metricsModel.addNodesInError(nodeAddress); + metrics.getErrorMessages().add(errorMessage + " (" + new Date().toString() + ")"); + + metricsModel.updateReport(); + completedRequestIndex++; + if (completedRequestIndex == numNodes) + metricsModel.log(); + + retryCounterMap.remove(nodeAddress); + + if (slackApi != null) + slackApi.call(new SlackMessage("Error: " + nodeAddress.getFullAddress(), + "<" + seedNodeRepository.getOperator(nodeAddress) + ">" + " Your seed node failed " + RETRY_DELAY_SEC + " times with error message: " + errorMessage)); + } + } + }); + handlerMap.put(nodeAddress, requestDataHandler); + requestDataHandler.requestData(nodeAddress); + } else { + log.warn("We have started already a requestDataHandshake to peer. nodeAddress=" + nodeAddress + "\n" + + "We start a cleanup timer if the handler has not closed by itself in between 2 minutes."); + + UserThread.runAfter(() -> { + if (handlerMap.containsKey(nodeAddress)) { + MonitorRequestHandler handler = handlerMap.get(nodeAddress); + handler.stop(); + handlerMap.remove(nodeAddress); + } + }, CLEANUP_TIMER); + } + } else { + log.warn("We have stopped already. We ignore that requestData call."); + } + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Utils + /////////////////////////////////////////////////////////////////////////////////////////// + + private void closeAllConnections() { + networkNode.getAllConnections().stream().forEach(connection -> connection.shutDown(CloseConnectionReason.CLOSE_REQUESTED_BY_PEER)); + } + + private void stopAllRetryTimers() { + retryTimerMap.values().stream().forEach(Timer::stop); + retryTimerMap.clear(); + + retryCounterMap.clear(); + } + + private void stopRetryTimer(NodeAddress nodeAddress) { + retryTimerMap.entrySet().stream() + .filter(e -> e.getKey().equals(nodeAddress)) + .forEach(e -> e.getValue().stop()); + retryTimerMap.remove(nodeAddress); + } + + private void closeHandler(Connection connection) { + Optional peersNodeAddressOptional = connection.getPeersNodeAddressOptional(); + if (peersNodeAddressOptional.isPresent()) { + NodeAddress nodeAddress = peersNodeAddressOptional.get(); + if (handlerMap.containsKey(nodeAddress)) { + handlerMap.get(nodeAddress).cancel(); + handlerMap.remove(nodeAddress); + } + } else { + log.trace("closeRequestDataHandler: nodeAddress not set in connection " + connection); + } + } + + private void closeAllHandlers() { + handlerMap.values().stream().forEach(MonitorRequestHandler::cancel); + handlerMap.clear(); + } + +} diff --git a/monitor/src/main/resources/logback.xml b/monitor/src/main/resources/logback.xml new file mode 100644 index 0000000000..0d084c25ca --- /dev/null +++ b/monitor/src/main/resources/logback.xml @@ -0,0 +1,19 @@ + + + + + %highlight(%d{MMM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{15}: %msg %xEx%n) + + + + + + + + + + + + + + diff --git a/pricenode/Procfile b/pricenode/Procfile new file mode 100644 index 0000000000..ff15a5d03d --- /dev/null +++ b/pricenode/Procfile @@ -0,0 +1 @@ +web: if [ "$HIDDEN" == true ]; then ./tor/bin/run_tor java -jar -Dserver.port=$PORT build/libs/bisq-pricenode.jar; else java -jar -Dserver.port=$PORT build/libs/bisq-pricenode.jar; fi diff --git a/pricenode/README-HEROKU.md b/pricenode/README-HEROKU.md new file mode 100644 index 0000000000..19365dc9ef --- /dev/null +++ b/pricenode/README-HEROKU.md @@ -0,0 +1,43 @@ +Deploy on Heroku +-------- + +Run the following commands: + + heroku create + heroku buildpacks:add heroku/gradle + heroku config:set BITCOIN_AVG_PUBKEY=[your pubkey] BITCOIN_AVG_PRIVKEY=[your privkey] + git push heroku master + curl https://your-app-123456.herokuapp.com/getAllMarketPrices + +To register the node as a Tor hidden service, first install the Heroku Tor buildpack: + + heroku buildpacks:add https://github.com/cbeams/heroku-buildpack-tor.git + git push heroku master + +> NOTE: this deployment will take a while, because the new buildpack must download and build Tor from source. + +Next, generate your Tor hidden service private key and .onion address: + + heroku run bash + ./tor/bin/tor -f torrc + +When the process reports that it is "100% bootstrapped", kill it, then copy the generated private key and .onion hostname values: + + cat build/tor-hidden-service/hostname + cat build/tor-hidden-service/private_key + exit + +> IMPORTANT: Save the private key value in a secure location so that this node can be re-created elsewhere with the same .onion address in the future if necessary. + +Now configure the hostname and private key values as environment variables for your Heroku app: + + heroku config:set HIDDEN=true HIDDEN_DOT_ONION=[your .onion] HIDDEN_PRIVATE_KEY="[your tor privkey]" + git push heroku master + +When the application finishes restarting, you should still be able to access it via the clearnet, e.g. with: + + curl https://your-app-123456.herokuapp.com/getAllMarketPrices + +And via your Tor Browser at: + + http://$YOUR_ONION/getAllMarketPrices diff --git a/pricenode/TODO.md b/pricenode/TODO.md new file mode 100644 index 0000000000..6ce8d074a8 --- /dev/null +++ b/pricenode/TODO.md @@ -0,0 +1,20 @@ +# Refactorings + +The list of stuff remaining to complete the PR at https://github.com/bisq-network/pricenode/pull/7 + + - Document provider implementations w/ links to API docs, etc + - Add integration tests + - Document / discuss how operators should (ideally) operate their pricenodes on a push-to-deploy model, e.g. how it's done on Heroku + +## Non-refactorings + +Most or all of these will become individual issues / PRs. Just capturing them here for convenience now. Not all may make sense. + + - Deprecate existing get* endpoints (e.g. /getAllMarketPrices) in favor of '/exchange-rates', '/fee-estimate; + - Eliminate dependency on bisq-core (only real need now is CurrencyUtil for list of supported coins) + - Remove command line args for fee estimation params; hard-code these values and update them via commits, not via one-off changes by each operator + - Remove 'getParams' in favor of Boot actuator endpoint + - Update bisq-network/exchange to refer to 'provider' as 'pricenode' + - Invert the dependency arrangement. Move 'ProviderRepository' et al from bisq-network/exchange here into + bisq-network/pricenode and have bisq-network/exchange depend on it as a client lib + - Save bandwidth and be idiomatic by not pretty-printing json returned from /getAllMarketPrices et al diff --git a/pricenode/build.gradle b/pricenode/build.gradle new file mode 100644 index 0000000000..c39959467c --- /dev/null +++ b/pricenode/build.gradle @@ -0,0 +1,34 @@ +plugins { + id "java" + id "org.springframework.boot" version "1.5.10.RELEASE" +} + +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + +version = file("src/main/resources/version.txt").text + +jar.manifest.attributes( + "Implementation-Title": rootProject.name, + "Implementation-Version": version) + +jar.archiveName "${rootProject.name}.jar" + +repositories { + jcenter() + maven { url "https://jitpack.io" } + maven { url "https://raw.githubusercontent.com/JesusMcCloud/tor-binary/master/release/" } +} + +dependencies { + compile project(":core") + compile("org.knowm.xchange:xchange-bitcoinaverage:4.3.3") + compile("org.knowm.xchange:xchange-coinmarketcap:4.3.3") + compile("org.knowm.xchange:xchange-poloniex:4.3.3") + compile("org.springframework.boot:spring-boot-starter-web:1.5.10.RELEASE") + compile("org.springframework.boot:spring-boot-starter-actuator") +} + +task stage { + dependsOn assemble +} diff --git a/pricenode/docker/Dockerfile b/pricenode/docker/Dockerfile new file mode 100644 index 0000000000..cc195cd63b --- /dev/null +++ b/pricenode/docker/Dockerfile @@ -0,0 +1,26 @@ +### +# The directory of the Dockerfile should contain your 'hostname' and 'private_key' files. +# In the docker-compose.yml file you can pass the ONION_ADDRESS referenced below. +### + +# pull base image +FROM openjdk:8-jdk + +RUN apt-get update && apt-get install -y --no-install-recommends \ + vim \ + tor \ + fakeroot \ + sudo \ + openjfx && rm -rf /var/lib/apt/lists/* + +RUN git clone https://github.com/bisq-network/pricenode.git +WORKDIR /pricenode/ +RUN ./gradlew assemble + +COPY loop.sh start_node.sh start_tor.sh ./ +COPY hostname private_key /var/lib/tor/ +COPY torrc /etc/tor/ +RUN chmod +x *.sh && chown debian-tor:debian-tor /etc/tor/torrc /var/lib/tor/hostname /var/lib/tor/private_key + +CMD ./start_tor.sh && ./start_node.sh +#CMD tail -f /dev/null diff --git a/pricenode/docker/README.md b/pricenode/docker/README.md new file mode 100644 index 0000000000..5f7a4e6d11 --- /dev/null +++ b/pricenode/docker/README.md @@ -0,0 +1,43 @@ +Needed information to start a pricenode +== + +Copy to this directory: +-- + +* a tor `hostname` file, containing your onion address +* a tor `private_key` file, containing the private key for your tor hidden service + +Edit docker-compose.yml: +-- + +* fill in your public and private api keys (needs a btcaverage developer subscription) + +Needed software to start a pricenode +== + +* docker +* docker-compose + +How to start +== + +`docker-compose up -d` + + +How to monitor +== + +See if it's running: `docker ps` + +Check the logs: `docker-compose logs` + + +Notes when using CoreOs +== + +Using CoreOs as host OS is entirely optional! + +* the cloudconfig.yml file is a configuration file for starting a coreos machine +from scratch. +* when installing a Coreos server, docker-compose needs to be additionally installed next to the +already provided docker installation diff --git a/pricenode/docker/cloudconfig.yml b/pricenode/docker/cloudconfig.yml new file mode 100644 index 0000000000..d6450b70f7 --- /dev/null +++ b/pricenode/docker/cloudconfig.yml @@ -0,0 +1,103 @@ +#cloud-config + +coreos: + update: + reboot-strategy: off + units: + - name: iptables-restore.service + enable: true + command: start + - name: create-swap.service + command: start + runtime: true + content: | + [Unit] + Description=Create swap file + Before=swap.service + + [Service] + Type=oneshot + Environment="SWAPFILE=/2GiB.swap" + ExecStart=/usr/bin/touch ${SWAPFILE} + ExecStart=/usr/bin/chattr +C ${SWAPFILE} + ExecStart=/usr/bin/fallocate -l 2048m ${SWAPFILE} + ExecStart=/usr/bin/chmod 600 ${SWAPFILE} + ExecStart=/usr/sbin/mkswap ${SWAPFILE} + + [Install] + WantedBy=multi-user.target + - name: swap.service + command: start + content: | + [Unit] + Description=Turn on swap + + [Service] + Type=oneshot + Environment="SWAPFILE=/2GiB.swap" + RemainAfterExit=true + ExecStartPre=/usr/sbin/losetup -f ${SWAPFILE} + ExecStart=/usr/bin/sh -c "/sbin/swapon $(/usr/sbin/losetup -j ${SWAPFILE} | /usr/bin/cut -d : -f 1)" + ExecStop=/usr/bin/sh -c "/sbin/swapoff $(/usr/sbin/losetup -j ${SWAPFILE} | /usr/bin/cut -d : -f 1)" + ExecStopPost=/usr/bin/sh -c "/usr/sbin/losetup -d $(/usr/sbin/losetup -j ${SWAPFILE} | /usr/bin/cut -d : -f 1)" + + [Install] + WantedBy=multi-user.target + - name: restart.service + content: | + [Unit] + Description=Restart docker containers + + [Service] + Type=oneshot + ExecStart=/home/core/docker/restartContainers.sh + - name: restart.timer + command: start + content: | + [Unit] + Description=Restarts the app container 2 times a week + + [Timer] + OnCalendar=Mon,Thu *-*-* 6:0:0 + +write_files: + - path: /etc/sysctl.d/swap.conf + permissions: 0644 + owner: root + content: | + vm.swappiness=10 + vm.vfs_cache_pressure=50 + +write_files: + - path: /etc/ssh/sshd_config + permissions: 0600 + owner: root + content: | + # Use most defaults for sshd configuration. + UsePrivilegeSeparation sandbox + Subsystem sftp internal-sftp + UseDNS no + + PermitRootLogin no + AllowUsers core + AuthenticationMethods publickey + +write_files: + - path: /var/lib/iptables/rules-save + permissions: 0644 + owner: 'root:root' + content: | + *filter + :INPUT DROP [0:0] + :FORWARD DROP [0:0] + :OUTPUT ACCEPT [0:0] + -A INPUT -i lo -j ACCEPT + -A INPUT -i eth1 -j ACCEPT + -A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + -A INPUT -p tcp -m tcp --dport 22 -j ACCEPT + -A INPUT -p tcp -m tcp --dport 80 -j ACCEPT + -A INPUT -p icmp -m icmp --icmp-type 0 -j ACCEPT + -A INPUT -p icmp -m icmp --icmp-type 3 -j ACCEPT + -A INPUT -p icmp -m icmp --icmp-type 11 -j ACCEPT + COMMIT + # the last line of the file needs to be a blank line or a comment diff --git a/pricenode/docker/docker-compose.yml b/pricenode/docker/docker-compose.yml new file mode 100644 index 0000000000..9ecdb84688 --- /dev/null +++ b/pricenode/docker/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3' + +# Fill in your own BTCAVERAGE public and private keys + +services: + pricenode: + restart: unless-stopped + build: + context: . + image: bisq:pricenode + ports: + - 80:80 + - 8080:8080 + environment: + - BTCAVERAGE_PRIVKEY=!!!!!!!!!!!!!!!!!!!!!!!!! YOUR PRIVATE KEY !!!!!!!!!!!!!!!!!!!!!!!!!!! + - BTCAVERAGE_PUBKEY=!!!!!!!!!!!!!!!!!!!!!!!!!! YOUR PUBKEY !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + entropy: + restart: always + image: harbur/haveged:1.7c-1 + container_name: haveged-entropy + privileged: true diff --git a/pricenode/docker/installDockerCompose.sh b/pricenode/docker/installDockerCompose.sh new file mode 100644 index 0000000000..f41ef50307 --- /dev/null +++ b/pricenode/docker/installDockerCompose.sh @@ -0,0 +1,4 @@ +#!/bin/bash +mkdir -p /opt/bin +curl -L `curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r '.assets[].browser_download_url | select(contains("Linux") and contains("x86_64"))'` > /opt/bin/docker-compose +chmod +x /opt/bin/docker-compose diff --git a/pricenode/docker/loop.sh b/pricenode/docker/loop.sh new file mode 100644 index 0000000000..1720f6eed9 --- /dev/null +++ b/pricenode/docker/loop.sh @@ -0,0 +1,8 @@ +#!/bin/bash +while true +do +echo `date` "(Re)-starting node" +BITCOIN_AVG_PUBKEY=$BTCAVERAGE_PUBKEY BITCOIN_AVG_PRIVKEY=$BTCAVERAGE_PRIVKEY java -jar ./build/libs/bisq-pricenode.jar 2 2 +echo `date` "node terminated unexpectedly!!" +sleep 3 +done diff --git a/pricenode/docker/rebuildAndRestart.sh b/pricenode/docker/rebuildAndRestart.sh new file mode 100755 index 0000000000..a30d2b21db --- /dev/null +++ b/pricenode/docker/rebuildAndRestart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +docker-compose build --no-cache && docker-compose up -d +docker image prune -f +docker-compose logs -f diff --git a/pricenode/docker/start_node.sh b/pricenode/docker/start_node.sh new file mode 100644 index 0000000000..225994130a --- /dev/null +++ b/pricenode/docker/start_node.sh @@ -0,0 +1 @@ +nohup sh loop.sh diff --git a/pricenode/docker/start_tor.sh b/pricenode/docker/start_tor.sh new file mode 100644 index 0000000000..40fa1e9e0c --- /dev/null +++ b/pricenode/docker/start_tor.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# sudo -u debian-tor +nohup sudo -u debian-tor tor > /dev/null 2>errors_tor.log & diff --git a/pricenode/docker/torrc b/pricenode/docker/torrc new file mode 100644 index 0000000000..9de1a9166e --- /dev/null +++ b/pricenode/docker/torrc @@ -0,0 +1,2 @@ +HiddenServiceDir /var/lib/tor/ +HiddenServicePort 80 127.0.0.1:8080 diff --git a/pricenode/src/main/java/bisq/price/Main.java b/pricenode/src/main/java/bisq/price/Main.java new file mode 100644 index 0000000000..0bb12678d2 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/Main.java @@ -0,0 +1,51 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price; + +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.builder.SpringApplicationBuilder; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.UncheckedIOException; + +import java.util.Properties; + +@SpringBootApplication +public class Main { + + public static void main(String[] args) { + new SpringApplicationBuilder(Main.class) + .properties(bisqProperties()) + .run(args); + } + + private static Properties bisqProperties() { + Properties props = new Properties(); + File propsFile = new File(System.getenv("HOME"), ".config/bisq.properties"); + if (propsFile.exists()) { + try { + props.load(new FileInputStream(propsFile)); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + return props; + } +} diff --git a/pricenode/src/main/java/bisq/price/PriceController.java b/pricenode/src/main/java/bisq/price/PriceController.java new file mode 100644 index 0000000000..d3c98027de --- /dev/null +++ b/pricenode/src/main/java/bisq/price/PriceController.java @@ -0,0 +1,35 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price; + +import org.springframework.web.bind.annotation.ModelAttribute; + +import javax.servlet.http.HttpServletRequest; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class PriceController { + + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + @ModelAttribute + public void logRequest(HttpServletRequest request) { + log.info("Incoming {} request from: {}", request.getServletPath(), request.getHeader("User-Agent")); + } +} diff --git a/pricenode/src/main/java/bisq/price/PriceProvider.java b/pricenode/src/main/java/bisq/price/PriceProvider.java new file mode 100644 index 0000000000..13b1dc7b0a --- /dev/null +++ b/pricenode/src/main/java/bisq/price/PriceProvider.java @@ -0,0 +1,115 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price; + +import org.springframework.context.SmartLifecycle; + +import java.time.Duration; + +import java.util.Timer; +import java.util.TimerTask; +import java.util.function.Supplier; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class PriceProvider implements SmartLifecycle, Supplier { + + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final Timer timer = new Timer(true); + + protected final Duration refreshInterval; + + private T cachedResult; + + public PriceProvider(Duration refreshInterval) { + this.refreshInterval = refreshInterval; + log.info("will refresh every {}", refreshInterval); + } + + @Override + public final T get() { + if (!isRunning()) + throw new IllegalStateException("call start() before calling get()"); + + return cachedResult; + } + + @Override + public final void start() { + // we call refresh outside the context of a timer once at startup to ensure that + // any exceptions thrown get propagated and cause the application to halt + refresh(); + + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + try { + refresh(); + } catch (Throwable t) { + // we only log scheduled calls to refresh that fail to ensure that + // the application does *not* halt, assuming the failure is temporary + // and on the side of the upstream price provider, eg. BitcoinAverage + log.warn("refresh failed", t); + } + } + }, refreshInterval.toMillis(), refreshInterval.toMillis()); + } + + private void refresh() { + long ts = System.currentTimeMillis(); + + cachedResult = doGet(); + + log.info("refresh took {} ms.", (System.currentTimeMillis() - ts)); + + onRefresh(); + } + + protected abstract T doGet(); + + protected void onRefresh() { + } + + @Override + public void stop() { + timer.cancel(); + } + + @Override + public void stop(Runnable callback) { + stop(); + callback.run(); + } + + @Override + public boolean isAutoStartup() { + return true; + } + + @Override + public boolean isRunning() { + return cachedResult != null; + } + + @Override + public int getPhase() { + return 0; + } +} diff --git a/pricenode/src/main/java/bisq/price/mining/FeeRate.java b/pricenode/src/main/java/bisq/price/mining/FeeRate.java new file mode 100644 index 0000000000..19a548cfa4 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/mining/FeeRate.java @@ -0,0 +1,46 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.mining; + +/** + * A value object representing the mining fee rate for a given base currency. + */ +public class FeeRate { + + private final String currency; + private final long price; + private final long timestamp; + + public FeeRate(String currency, long price, long timestamp) { + this.currency = currency; + this.price = price; + this.timestamp = timestamp; + } + + public String getCurrency() { + return currency; + } + + public long getPrice() { + return price; + } + + public long getTimestamp() { + return timestamp; + } +} diff --git a/pricenode/src/main/java/bisq/price/mining/FeeRateController.java b/pricenode/src/main/java/bisq/price/mining/FeeRateController.java new file mode 100644 index 0000000000..b0ac4710b5 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/mining/FeeRateController.java @@ -0,0 +1,40 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.mining; + +import bisq.price.PriceController; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +@RestController +class FeeRateController extends PriceController { + + private final FeeRateService feeRateService; + + public FeeRateController(FeeRateService feeRateService) { + this.feeRateService = feeRateService; + } + + @GetMapping(path = "/getFees") + public Map getFees() { + return feeRateService.getFees(); + } +} diff --git a/pricenode/src/main/java/bisq/price/mining/FeeRateProvider.java b/pricenode/src/main/java/bisq/price/mining/FeeRateProvider.java new file mode 100644 index 0000000000..00002af019 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/mining/FeeRateProvider.java @@ -0,0 +1,32 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.mining; + +import bisq.price.PriceProvider; + +import java.time.Duration; + +/** + * Abstract base class for providers of mining {@link FeeRate} data. + */ +public abstract class FeeRateProvider extends PriceProvider { + + public FeeRateProvider(Duration refreshInterval) { + super(refreshInterval); + } +} diff --git a/pricenode/src/main/java/bisq/price/mining/FeeRateService.java b/pricenode/src/main/java/bisq/price/mining/FeeRateService.java new file mode 100644 index 0000000000..bc6634630c --- /dev/null +++ b/pricenode/src/main/java/bisq/price/mining/FeeRateService.java @@ -0,0 +1,56 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.mining; + +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * High-level mining {@link FeeRate} operations. + */ +@Service +class FeeRateService { + + private final Set providers; + + public FeeRateService(Set providers) { + this.providers = providers; + } + + public Map getFees() { + Map metadata = new HashMap<>(); + Map allFeeRates = new HashMap<>(); + + providers.forEach(p -> { + FeeRate feeRate = p.get(); + String currency = feeRate.getCurrency(); + if ("BTC".equals(currency)) { + metadata.put("bitcoinFeesTs", feeRate.getTimestamp()); + } + allFeeRates.put(currency.toLowerCase() + "TxFee", feeRate.getPrice()); + }); + + return new HashMap() {{ + putAll(metadata); + put("dataMap", allFeeRates); + }}; + } +} diff --git a/pricenode/src/main/java/bisq/price/mining/providers/BitcoinFeeRateProvider.java b/pricenode/src/main/java/bisq/price/mining/providers/BitcoinFeeRateProvider.java new file mode 100644 index 0000000000..9d4c9fa0be --- /dev/null +++ b/pricenode/src/main/java/bisq/price/mining/providers/BitcoinFeeRateProvider.java @@ -0,0 +1,121 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.mining.providers; + +import bisq.price.PriceController; +import bisq.price.mining.FeeRate; +import bisq.price.mining.FeeRateProvider; + +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.env.CommandLinePropertySource; +import org.springframework.core.env.Environment; +import org.springframework.http.RequestEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import java.time.Duration; +import java.time.Instant; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +@Component +class BitcoinFeeRateProvider extends FeeRateProvider { + + private static final long MIN_FEE_RATE = 10; // satoshi/byte + private static final long MAX_FEE_RATE = 1000; + + private static final int DEFAULT_MAX_BLOCKS = 2; + private static final int DEFAULT_REFRESH_INTERVAL = 2; + + private final RestTemplate restTemplate = new RestTemplate(); + + private final int maxBlocks; + + public BitcoinFeeRateProvider(Environment env) { + super(Duration.ofMinutes(refreshInterval(env))); + this.maxBlocks = maxBlocks(env); + } + + protected FeeRate doGet() { + return new FeeRate("BTC", getEstimatedFeeRate(), Instant.now().getEpochSecond()); + } + + private long getEstimatedFeeRate() { + return getFeeRatePredictions() + .filter(p -> p.get("maxDelay") <= maxBlocks) + .findFirst() + .map(p -> p.get("maxFee")) + .map(r -> { + log.info("latest fee rate prediction is {} sat/byte", r); + return r; + }) + .map(r -> Math.max(r, MIN_FEE_RATE)) + .map(r -> Math.min(r, MAX_FEE_RATE)) + .orElse(MIN_FEE_RATE); + } + + private Stream> getFeeRatePredictions() { + return restTemplate.exchange( + RequestEntity + .get(UriComponentsBuilder + // now using /fees/list because /fees/recommended estimates were too high + .fromUriString("https://bitcoinfees.earn.com/api/v1/fees/list") + .build().toUri()) + .header("User-Agent", "") // required to avoid 403 + .build(), + new ParameterizedTypeReference>>>() { + } + ).getBody().entrySet().stream() + .flatMap(e -> e.getValue().stream()); + } + + private static Optional args(Environment env) { + return Optional.ofNullable( + env.getProperty(CommandLinePropertySource.DEFAULT_NON_OPTION_ARGS_PROPERTY_NAME, String[].class)); + } + + private static int maxBlocks(Environment env) { + return args(env) + .filter(args -> args.length >= 1) + .map(args -> Integer.valueOf(args[0])) + .orElse(DEFAULT_MAX_BLOCKS); + } + + private static long refreshInterval(Environment env) { + return args(env) + .filter(args -> args.length >= 2) + .map(args -> Integer.valueOf(args[1])) + .orElse(DEFAULT_REFRESH_INTERVAL); + } + + + @RestController + class Controller extends PriceController { + + @GetMapping(path = "/getParams") + public String getParams() { + return String.format("%s;%s", maxBlocks, refreshInterval.toMillis()); + } + } +} diff --git a/pricenode/src/main/java/bisq/price/mining/providers/DashFeeRateProvider.java b/pricenode/src/main/java/bisq/price/mining/providers/DashFeeRateProvider.java new file mode 100644 index 0000000000..abedc627e2 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/mining/providers/DashFeeRateProvider.java @@ -0,0 +1,28 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.mining.providers; + +import org.springframework.stereotype.Component; + +@Component +class DashFeeRateProvider extends FixedFeeRateProvider { + + public DashFeeRateProvider() { + super("DASH", 50); + } +} diff --git a/pricenode/src/main/java/bisq/price/mining/providers/DogecoinFeeRateProvider.java b/pricenode/src/main/java/bisq/price/mining/providers/DogecoinFeeRateProvider.java new file mode 100644 index 0000000000..5edcf3c091 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/mining/providers/DogecoinFeeRateProvider.java @@ -0,0 +1,28 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.mining.providers; + +import org.springframework.stereotype.Component; + +@Component +class DogecoinFeeRateProvider extends FixedFeeRateProvider { + + public DogecoinFeeRateProvider() { + super("DOGE", 5_000_000); + } +} diff --git a/pricenode/src/main/java/bisq/price/mining/providers/FixedFeeRateProvider.java b/pricenode/src/main/java/bisq/price/mining/providers/FixedFeeRateProvider.java new file mode 100644 index 0000000000..2366100179 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/mining/providers/FixedFeeRateProvider.java @@ -0,0 +1,40 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.mining.providers; + +import bisq.price.mining.FeeRate; +import bisq.price.mining.FeeRateProvider; + +import java.time.Duration; +import java.time.Instant; + +abstract class FixedFeeRateProvider extends FeeRateProvider { + + private final String currency; + private final long price; + + public FixedFeeRateProvider(String currency, long price) { + super(Duration.ofDays(1)); + this.currency = currency; + this.price = price; + } + + protected final FeeRate doGet() { + return new FeeRate(currency, price, Instant.now().getEpochSecond()); + } +} diff --git a/pricenode/src/main/java/bisq/price/mining/providers/LitecoinFeeRateProvider.java b/pricenode/src/main/java/bisq/price/mining/providers/LitecoinFeeRateProvider.java new file mode 100644 index 0000000000..b6681d9a81 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/mining/providers/LitecoinFeeRateProvider.java @@ -0,0 +1,28 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.mining.providers; + +import org.springframework.stereotype.Component; + +@Component +class LitecoinFeeRateProvider extends FixedFeeRateProvider { + + public LitecoinFeeRateProvider() { + super("LTC", 500); + } +} diff --git a/pricenode/src/main/java/bisq/price/spot/ExchangeRate.java b/pricenode/src/main/java/bisq/price/spot/ExchangeRate.java new file mode 100644 index 0000000000..78bc5ff21b --- /dev/null +++ b/pricenode/src/main/java/bisq/price/spot/ExchangeRate.java @@ -0,0 +1,99 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.spot; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.math.BigDecimal; + +import java.util.Date; +import java.util.Objects; + +/** + * A value object representing the spot price in bitcoin for a given currency at a given + * time as reported by a given provider. + */ +public class ExchangeRate { + + private final String currency; + private final double price; + private final long timestamp; + private final String provider; + + public ExchangeRate(String currency, BigDecimal price, Date timestamp, String provider) { + this( + currency, + price.doubleValue(), + timestamp.getTime(), + provider + ); + } + + public ExchangeRate(String currency, double price, long timestamp, String provider) { + this.currency = currency; + this.price = price; + this.timestamp = timestamp; + this.provider = provider; + } + + @JsonProperty(value = "currencyCode", index = 1) + public String getCurrency() { + return currency; + } + + @JsonProperty(value = "price", index = 2) + public double getPrice() { + return this.price; + } + + @JsonProperty(value = "timestampSec", index = 3) + public long getTimestamp() { + return this.timestamp; + } + + @JsonProperty(value = "provider", index = 4) + public String getProvider() { + return provider; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExchangeRate exchangeRate = (ExchangeRate) o; + return Double.compare(exchangeRate.price, price) == 0 && + timestamp == exchangeRate.timestamp && + Objects.equals(currency, exchangeRate.currency) && + Objects.equals(provider, exchangeRate.provider); + } + + @Override + public int hashCode() { + return Objects.hash(currency, price, timestamp, provider); + } + + @Override + public String toString() { + return "ExchangeRate{" + + "currency='" + currency + '\'' + + ", price=" + price + + ", timestamp=" + timestamp + + ", provider=" + provider + + '}'; + } +} diff --git a/pricenode/src/main/java/bisq/price/spot/ExchangeRateController.java b/pricenode/src/main/java/bisq/price/spot/ExchangeRateController.java new file mode 100644 index 0000000000..dd99d22fce --- /dev/null +++ b/pricenode/src/main/java/bisq/price/spot/ExchangeRateController.java @@ -0,0 +1,40 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.spot; + +import bisq.price.PriceController; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +@RestController +class ExchangeRateController extends PriceController { + + private final ExchangeRateService exchangeRateService; + + public ExchangeRateController(ExchangeRateService exchangeRateService) { + this.exchangeRateService = exchangeRateService; + } + + @GetMapping(path = "/getAllMarketPrices") + public Map getAllMarketPrices() { + return exchangeRateService.getAllMarketPrices(); + } +} diff --git a/pricenode/src/main/java/bisq/price/spot/ExchangeRateProvider.java b/pricenode/src/main/java/bisq/price/spot/ExchangeRateProvider.java new file mode 100644 index 0000000000..fc3e75cbf4 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/spot/ExchangeRateProvider.java @@ -0,0 +1,63 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.spot; + +import bisq.price.PriceProvider; + +import java.time.Duration; + +import java.util.Set; + +/** + * Abstract base class for providers of bitcoin {@link ExchangeRate} data. Implementations + * are marked with the {@link org.springframework.stereotype.Component} annotation in + * order to be discovered via classpath scanning. Implementations are also marked with the + * {@link org.springframework.core.annotation.Order} annotation to determine their + * precedence over each other in the case of two or more services returning exchange rate + * data for the same currency pair. In such cases, results from the provider with the + * higher order value will taking precedence over the provider with a lower value, + * presuming that such providers are being iterated over in an ordered list. + * + * @see ExchangeRateService#ExchangeRateService(java.util.List) + */ +public abstract class ExchangeRateProvider extends PriceProvider> { + + private final String name; + private final String prefix; + + public ExchangeRateProvider(String name, String prefix, Duration refreshInterval) { + super(refreshInterval); + this.name = name; + this.prefix = prefix; + } + + public String getName() { + return name; + } + + public String getPrefix() { + return prefix; + } + + @Override + protected void onRefresh() { + get().stream() + .filter(e -> "USD".equals(e.getCurrency()) || "LTC".equals(e.getCurrency())) + .forEach(e -> log.info("BTC/{}: {}", e.getCurrency(), e.getPrice())); + } +} diff --git a/pricenode/src/main/java/bisq/price/spot/ExchangeRateService.java b/pricenode/src/main/java/bisq/price/spot/ExchangeRateService.java new file mode 100644 index 0000000000..2dec931462 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/spot/ExchangeRateService.java @@ -0,0 +1,108 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.spot; + +import bisq.price.spot.providers.BitcoinAverage; + +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * High-level {@link ExchangeRate} data operations. + */ +@Service +class ExchangeRateService { + protected final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final List providers; + + /** + * Construct an {@link ExchangeRateService} with a list of all + * {@link ExchangeRateProvider} implementations discovered via classpath scanning. + * + * @param providers all {@link ExchangeRateProvider} implementations in ascending + * order of precedence + */ + public ExchangeRateService(List providers) { + this.providers = providers; + } + + public Map getAllMarketPrices() { + Map metadata = new LinkedHashMap<>(); + Map allExchangeRates = new LinkedHashMap<>(); + + providers.forEach(p -> { + Set exchangeRates = p.get(); + metadata.putAll(getMetadata(p, exchangeRates)); + exchangeRates.forEach(e -> + allExchangeRates.put(e.getCurrency(), e) + ); + }); + + return new LinkedHashMap() {{ + putAll(metadata); + // Use a sorted list by currency code to make comparision of json data between different + // price nodes easier + List values = new ArrayList<>(allExchangeRates.values()); + values.sort(Comparator.comparing(ExchangeRate::getCurrency)); + put("data", values); + }}; + } + + private Map getMetadata(ExchangeRateProvider provider, Set exchangeRates) { + Map metadata = new LinkedHashMap<>(); + + // In case a provider is not available we still want to deliver the data of the other providers, so we catch + // a possible exception and leave timestamp at 0. The Bisq app will check if the timestamp is in a tolerance + // window and if it is too old it will show that the price is not available. + long timestamp = 0; + try { + timestamp = getTimestamp(provider, exchangeRates); + } catch (Throwable t) { + log.error(t.toString()); + t.printStackTrace(); + } + + if (provider instanceof BitcoinAverage.Local) { + metadata.put("btcAverageTs", timestamp); + } + + String prefix = provider.getPrefix(); + metadata.put(prefix + "Ts", timestamp); + metadata.put(prefix + "Count", exchangeRates.size()); + + return metadata; + } + + private long getTimestamp(ExchangeRateProvider provider, Set exchangeRates) { + return exchangeRates.stream() + .filter(e -> provider.getName().equals(e.getProvider())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No exchange rate data found for " + provider.getName())) + .getTimestamp(); + } +} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/BitcoinAverage.java b/pricenode/src/main/java/bisq/price/spot/providers/BitcoinAverage.java new file mode 100644 index 0000000000..de36b886ae --- /dev/null +++ b/pricenode/src/main/java/bisq/price/spot/providers/BitcoinAverage.java @@ -0,0 +1,158 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.spot.providers; + +import bisq.price.spot.ExchangeRate; +import bisq.price.spot.ExchangeRateProvider; + +import org.knowm.xchange.bitcoinaverage.dto.marketdata.BitcoinAverageTicker; +import org.knowm.xchange.bitcoinaverage.dto.marketdata.BitcoinAverageTickers; + +import org.springframework.core.annotation.Order; +import org.springframework.core.env.Environment; +import org.springframework.http.RequestEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import org.bouncycastle.util.encoders.Hex; + +import javax.crypto.Mac; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +import java.time.Duration; +import java.time.Instant; + +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * See the BitcoinAverage API documentation at https://apiv2.bitcoinaverage.com/#ticker-data-all + */ +public abstract class BitcoinAverage extends ExchangeRateProvider { + + /** + * Max number of requests allowed per month on the BitcoinAverage developer plan. + * Note the actual max value is 45,000; we use the more conservative value below to + * ensure we do not exceed it. See https://bitcoinaverage.com/en/plans. + */ + private static final double MAX_REQUESTS_PER_MONTH = 42_514; + + private final RestTemplate restTemplate = new RestTemplate(); + private final String symbolSet; + + private String pubKey; + private Mac mac; + + /** + * @param symbolSet "global" or "local"; see https://apiv2.bitcoinaverage.com/#supported-currencies + */ + public BitcoinAverage(String name, String prefix, double pctMaxRequests, String symbolSet, Environment env) { + super(name, prefix, refreshIntervalFor(pctMaxRequests)); + this.symbolSet = symbolSet; + this.pubKey = env.getRequiredProperty("BITCOIN_AVG_PUBKEY"); + this.mac = initMac(env.getRequiredProperty("BITCOIN_AVG_PRIVKEY")); + } + + @Override + public Set doGet() { + + return getTickersKeyedByCurrency().entrySet().stream() + .filter(e -> supportedCurrency(e.getKey())) + .map(e -> + new ExchangeRate( + e.getKey(), + e.getValue().getLast(), + e.getValue().getTimestamp(), + this.getName() + ) + ) + .collect(Collectors.toSet()); + } + + private boolean supportedCurrency(String currencyCode) { + // ignore Venezuelan bolivars as the "official" exchange rate is just wishful thinking + // we should use this API with a custom provider instead: http://api.bitcoinvenezuela.com/1 + return !"VEF".equals(currencyCode); + } + + private Map getTickersKeyedByCurrency() { + // go from a map with keys like "BTCUSD", "BTCVEF" + return getTickersKeyedByCurrencyPair().entrySet().stream() + // to a map with keys like "USD", "VEF" + .collect(Collectors.toMap(e -> e.getKey().substring(3), Map.Entry::getValue)); + } + + private Map getTickersKeyedByCurrencyPair() { + return restTemplate.exchange( + RequestEntity + .get(UriComponentsBuilder + .fromUriString("https://apiv2.bitcoinaverage.com/indices/{symbol-set}/ticker/all?crypto=BTC") + .buildAndExpand(symbolSet) + .toUri()) + .header("X-signature", getAuthSignature()) + .build(), + BitcoinAverageTickers.class + ).getBody().getTickers(); + } + + protected String getAuthSignature() { + String payload = String.format("%s.%s", Instant.now().getEpochSecond(), pubKey); + return String.format("%s.%s", payload, Hex.toHexString(mac.doFinal(payload.getBytes()))); + } + + private static Mac initMac(String privKey) { + String algorithm = "HmacSHA256"; + SecretKey secretKey = new SecretKeySpec(privKey.getBytes(), algorithm); + try { + Mac mac = Mac.getInstance(algorithm); + mac.init(secretKey); + return mac; + } catch (NoSuchAlgorithmException | InvalidKeyException ex) { + throw new RuntimeException(ex); + } + } + + private static Duration refreshIntervalFor(double pctMaxRequests) { + long requestsPerMonth = (long) (MAX_REQUESTS_PER_MONTH * pctMaxRequests); + return Duration.ofDays(31).dividedBy(requestsPerMonth); + } + + + @Component + @Order(1) + public static class Global extends BitcoinAverage { + public Global(Environment env) { + super("BTCA_G", "btcAverageG", 0.3, "global", env); + } + } + + + @Component + @Order(2) + public static class Local extends BitcoinAverage { + public Local(Environment env) { + super("BTCA_L", "btcAverageL", 0.7, "local", env); + } + } +} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/CoinMarketCap.java b/pricenode/src/main/java/bisq/price/spot/providers/CoinMarketCap.java new file mode 100644 index 0000000000..f5ce64f745 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/spot/providers/CoinMarketCap.java @@ -0,0 +1,77 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.spot.providers; + +import bisq.price.spot.ExchangeRate; +import bisq.price.spot.ExchangeRateProvider; +import bisq.price.util.Altcoins; + +import org.knowm.xchange.coinmarketcap.dto.marketdata.CoinMarketCapTicker; + +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.annotation.Order; +import org.springframework.http.RequestEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import java.time.Duration; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Component +@Order(3) +class CoinMarketCap extends ExchangeRateProvider { + + private final RestTemplate restTemplate = new RestTemplate(); + + public CoinMarketCap() { + super("CMC", "coinmarketcap", Duration.ofMinutes(5)); // large data structure, so don't request it too often + } + + @Override + public Set doGet() { + + return getTickers() + .filter(t -> Altcoins.ALL_SUPPORTED.contains(t.getIsoCode())) + .map(t -> + new ExchangeRate( + t.getIsoCode(), + t.getPriceBTC(), + t.getLastUpdated(), + this.getName() + ) + ) + .collect(Collectors.toSet()); + } + + private Stream getTickers() { + return restTemplate.exchange( + RequestEntity + .get(UriComponentsBuilder + .fromUriString("https://api.coinmarketcap.com/v1/ticker/?limit=200").build() + .toUri()) + .build(), + new ParameterizedTypeReference>() { + } + ).getBody().stream(); + } +} diff --git a/pricenode/src/main/java/bisq/price/spot/providers/Poloniex.java b/pricenode/src/main/java/bisq/price/spot/providers/Poloniex.java new file mode 100644 index 0000000000..684f867b25 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/spot/providers/Poloniex.java @@ -0,0 +1,94 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.spot.providers; + +import bisq.price.spot.ExchangeRate; +import bisq.price.spot.ExchangeRateProvider; +import bisq.price.util.Altcoins; + +import org.knowm.xchange.currency.Currency; +import org.knowm.xchange.currency.CurrencyPair; +import org.knowm.xchange.poloniex.dto.marketdata.PoloniexMarketData; +import org.knowm.xchange.poloniex.dto.marketdata.PoloniexTicker; + +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.annotation.Order; +import org.springframework.http.RequestEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import java.time.Duration; + +import java.util.Date; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Component +@Order(4) +class Poloniex extends ExchangeRateProvider { + + private final RestTemplate restTemplate = new RestTemplate(); + + public Poloniex() { + super("POLO", "poloniex", Duration.ofMinutes(1)); + } + + @Override + public Set doGet() { + Date timestamp = new Date(); // Poloniex tickers don't include their own timestamp + + return getTickers() + .filter(t -> t.getCurrencyPair().base.equals(Currency.BTC)) + .filter(t -> Altcoins.ALL_SUPPORTED.contains(t.getCurrencyPair().counter.getCurrencyCode())) + .map(t -> + new ExchangeRate( + t.getCurrencyPair().counter.getCurrencyCode(), + t.getPoloniexMarketData().getLast(), + timestamp, + this.getName() + ) + ) + .collect(Collectors.toSet()); + } + + private Stream getTickers() { + + return getTickersKeyedByCurrencyPair().entrySet().stream() + .map(e -> { + String pair = e.getKey(); + PoloniexMarketData data = e.getValue(); + String[] symbols = pair.split("_"); // e.g. BTC_USD => [BTC, USD] + return new PoloniexTicker(data, new CurrencyPair(symbols[0], symbols[1])); + }); + } + + private Map getTickersKeyedByCurrencyPair() { + return restTemplate.exchange( + RequestEntity + .get(UriComponentsBuilder + .fromUriString("https://poloniex.com/public?command=returnTicker").build() + .toUri()) + .build(), + new ParameterizedTypeReference>() { + } + ).getBody(); + } +} diff --git a/pricenode/src/main/java/bisq/price/util/Altcoins.java b/pricenode/src/main/java/bisq/price/util/Altcoins.java new file mode 100644 index 0000000000..e7917ce8c6 --- /dev/null +++ b/pricenode/src/main/java/bisq/price/util/Altcoins.java @@ -0,0 +1,32 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.util; + +import bisq.core.locale.CurrencyUtil; +import bisq.core.locale.TradeCurrency; + +import java.util.Set; +import java.util.stream.Collectors; + +public abstract class Altcoins { + + public static final Set ALL_SUPPORTED = + CurrencyUtil.getAllSortedCryptoCurrencies().stream() + .map(TradeCurrency::getCode) + .collect(Collectors.toSet()); +} diff --git a/pricenode/src/main/java/bisq/price/util/VersionController.java b/pricenode/src/main/java/bisq/price/util/VersionController.java new file mode 100644 index 0000000000..d754e551bc --- /dev/null +++ b/pricenode/src/main/java/bisq/price/util/VersionController.java @@ -0,0 +1,55 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.price.util; + +import bisq.price.PriceController; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.actuate.info.Info; +import org.springframework.boot.actuate.info.InfoContributor; +import org.springframework.core.io.Resource; +import org.springframework.util.FileCopyUtils; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; +import java.io.InputStreamReader; + +@RestController +class VersionController extends PriceController implements InfoContributor { + + private final String version; + + public VersionController(@Value("classpath:version.txt") Resource versionTxt) throws IOException { + this.version = FileCopyUtils.copyToString( + new InputStreamReader( + versionTxt.getInputStream() + ) + ).trim(); + } + + @GetMapping(path = "/getVersion") + public String getVersion() { + return version; + } + + @Override + public void contribute(Info.Builder builder) { + builder.withDetail("version", version); + } +} diff --git a/pricenode/src/main/resources/application.properties b/pricenode/src/main/resources/application.properties new file mode 100644 index 0000000000..d8f4060256 --- /dev/null +++ b/pricenode/src/main/resources/application.properties @@ -0,0 +1 @@ +spring.jackson.serialization.indent_output=true diff --git a/pricenode/src/main/resources/banner.txt b/pricenode/src/main/resources/banner.txt new file mode 100644 index 0000000000..08d29b7388 --- /dev/null +++ b/pricenode/src/main/resources/banner.txt @@ -0,0 +1,6 @@ + __ _ _ __ + / /_ (_)________ _ ____ _____(_)_______ ____ ____ ____/ /__ + / __ \/ / ___/ __ `/_____/ __ \/ ___/ / ___/ _ \/ __ \/ __ \/ __ / _ \ + / /_/ / (__ ) /_/ /_____/ /_/ / / / / /__/ __/ / / / /_/ / /_/ / __/ +/_.___/_/____/\__, / / .___/_/ /_/\___/\___/_/ /_/\____/\__,_/\___/ + /_/ /_/ ${application.formatted-version} diff --git a/pricenode/src/main/resources/logback.xml b/pricenode/src/main/resources/logback.xml new file mode 100644 index 0000000000..02acbae6f9 --- /dev/null +++ b/pricenode/src/main/resources/logback.xml @@ -0,0 +1,16 @@ + + + + + %highlight(%d{MMM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{15}: %msg %xEx%n) + + + + + + + + + + + diff --git a/pricenode/src/main/resources/version.txt b/pricenode/src/main/resources/version.txt new file mode 100644 index 0000000000..e87c17ec4a --- /dev/null +++ b/pricenode/src/main/resources/version.txt @@ -0,0 +1 @@ +0.7.2-SNAPSHOT diff --git a/pricenode/torrc b/pricenode/torrc new file mode 100644 index 0000000000..3bfc6285ee --- /dev/null +++ b/pricenode/torrc @@ -0,0 +1,2 @@ +HiddenServiceDir build/tor-hidden-service +HiddenServicePort 80 127.0.0.1:8080 diff --git a/relay/Procfile b/relay/Procfile new file mode 100644 index 0000000000..0429696422 --- /dev/null +++ b/relay/Procfile @@ -0,0 +1 @@ +web: if [ "$HIDDEN" == true ]; then ./tor/bin/run_tor java -jar -Dserver.port=$PORT build/libs/bisq-relay.jar; else java -jar -Dserver.port=$PORT build/libs/bisq-relay.jar; fi diff --git a/relay/build.gradle b/relay/build.gradle new file mode 100644 index 0000000000..9006d27cba --- /dev/null +++ b/relay/build.gradle @@ -0,0 +1,36 @@ +plugins { + id "java" + id "org.springframework.boot" version "1.5.10.RELEASE" +} + +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + +version = file("src/main/resources/version.txt").text + +jar.manifest.attributes( + "Implementation-Title": rootProject.name, + "Implementation-Version": version) + +jar.archiveName "${rootProject.name}.jar" + +repositories { + mavenLocal() + jcenter() + maven { url "https://jitpack.io" } + maven { url "https://raw.githubusercontent.com/JesusMcCloud/tor-binary/master/release/" } +} + +dependencies { + compile project(":common") + compile("com.sparkjava:spark-core:2.5.2") + compile("com.turo:pushy:0.13.2") + compile("com.google.firebase:firebase-admin:6.2.0") + + compileOnly 'org.projectlombok:lombok:1.16.16' + //annotationProcessor 'org.projectlombok:lombok:1.16.16' +} + +task stage { + dependsOn assemble +} diff --git a/relay/src/main/java/bisq/relay/RelayMain.java b/relay/src/main/java/bisq/relay/RelayMain.java new file mode 100644 index 0000000000..e7c1cd01e0 --- /dev/null +++ b/relay/src/main/java/bisq/relay/RelayMain.java @@ -0,0 +1,127 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.relay; + +import bisq.common.app.Log; +import bisq.common.util.Utilities; + +import org.apache.commons.codec.binary.Hex; + +import java.io.File; + +import java.util.Locale; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.Level; + +import static spark.Spark.get; +import static spark.Spark.port; + +public class RelayMain { + private static final Logger log = LoggerFactory.getLogger(RelayMain.class); + private static final String VERSION = "0.1.0"; + private static RelayService relayService; + + static { + // Need to set default locale initially otherwise we get problems at non-english OS + Locale.setDefault(new Locale("en", Locale.getDefault().getCountry())); + + Utilities.removeCryptographyRestrictions(); + } + + /** + * @param args Pass port as program argument if other port than default port 8080 is wanted. + */ + public static void main(String[] args) { + final String logPath = System.getProperty("user.home") + File.separator + "provider"; + Log.setup(logPath); + Log.setLevel(Level.INFO); + log.info("Log files under: " + logPath); + log.info("RelayVersion.VERSION: " + VERSION); + Utilities.printSysInfo(); + + + String appleCertPwPath; + if (args.length > 0) + appleCertPwPath = args[0]; + else + throw new RuntimeException("You need to set the path to the password text file for the Apple push certificate as first argument."); + + String appleCertPath; + if (args.length > 1) + appleCertPath = args[1]; + else + throw new RuntimeException("You need to set the path to the Apple push certificate as second argument."); + + String appleBundleId; + if (args.length > 2) + appleBundleId = args[2]; + else + throw new RuntimeException("You need to set the Apple bundle ID as third argument."); + + String androidCertPath; + if (args.length > 3) + androidCertPath = args[3]; + else + throw new RuntimeException("You need to set the Android certificate path as 4th argument."); + + + int port = 8080; + if (args.length > 4) + port = Integer.parseInt(args[4]); + + port(port); + + relayService = new RelayService(appleCertPwPath, appleCertPath, appleBundleId, androidCertPath); + + handleRelay(); + + keepRunning(); + } + + private static void handleRelay() { + get("/relay", (request, response) -> { + log.info("Incoming relay request from: " + request.userAgent()); + boolean isAndroid = request.queryParams("isAndroid").equalsIgnoreCase("true"); + boolean useSound = request.queryParams("snd").equalsIgnoreCase("true"); + String token = new String(Hex.decodeHex(request.queryParams("token").toCharArray()), "UTF-8"); + String encryptedMessage = new String(Hex.decodeHex(request.queryParams("msg").toCharArray()), "UTF-8"); + log.info("isAndroid={}\nuseSound={}\napsTokenHex={}\nencryptedMessage={}", isAndroid, useSound, token, + encryptedMessage); + if (isAndroid) { + return relayService.sendAndroidMessage(token, encryptedMessage, useSound); + } else { + boolean isProduction = request.queryParams("isProduction").equalsIgnoreCase("true"); + boolean isContentAvailable = request.queryParams("isContentAvailable").equalsIgnoreCase("true"); + return relayService.sendAppleMessage(isProduction, isContentAvailable, token, encryptedMessage, useSound); + } + }); + } + + private static void keepRunning() { + //noinspection InfiniteLoopStatement + while (true) { + try { + Thread.sleep(Long.MAX_VALUE); + } catch (InterruptedException ignore) { + } + } + } +} diff --git a/relay/src/main/java/bisq/relay/RelayService.java b/relay/src/main/java/bisq/relay/RelayService.java new file mode 100644 index 0000000000..4bb7910e85 --- /dev/null +++ b/relay/src/main/java/bisq/relay/RelayService.java @@ -0,0 +1,154 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.relay; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + +import java.util.Scanner; +import java.util.concurrent.ExecutionException; + +import lombok.extern.slf4j.Slf4j; + + + +import com.google.auth.oauth2.GoogleCredentials; +import com.google.firebase.FirebaseApp; +import com.google.firebase.FirebaseOptions; +import com.google.firebase.messaging.FirebaseMessaging; +import com.google.firebase.messaging.FirebaseMessagingException; +import com.google.firebase.messaging.Message; +import com.google.firebase.messaging.Notification; +import com.turo.pushy.apns.ApnsClient; +import com.turo.pushy.apns.ApnsClientBuilder; +import com.turo.pushy.apns.PushNotificationResponse; +import com.turo.pushy.apns.util.ApnsPayloadBuilder; +import com.turo.pushy.apns.util.SimpleApnsPushNotification; +import com.turo.pushy.apns.util.concurrent.PushNotificationFuture; + +@Slf4j +class RelayService { + private static final String ANDROID_DATABASE_URL = "https://bisqnotifications.firebaseio.com"; + // Used in Bisq app to check for success state. We won't want a code dependency just for that string so we keep it + // duplicated in core and here. Must not be changed. + private static final String SUCCESS = "success"; + + private final String appleBundleId; + + private ApnsClient productionApnsClient; + private ApnsClient devApnsClient; // used for iOS development in XCode + + RelayService(String appleCertPwPath, String appleCertPath, String appleBundleId, String androidCertPath) { + this.appleBundleId = appleBundleId; + + setupForAndroid(androidCertPath); + setupForApple(appleCertPwPath, appleCertPath); + } + + private void setupForAndroid(String androidCertPath) { + try { + InputStream androidCertStream = new FileInputStream(androidCertPath); + FirebaseOptions options = new FirebaseOptions.Builder() + .setCredentials(GoogleCredentials.fromStream(androidCertStream)) + .setDatabaseUrl(ANDROID_DATABASE_URL) + .build(); + FirebaseApp.initializeApp(options); + } catch (IOException e) { + log.error(e.toString()); + e.printStackTrace(); + } + } + + private void setupForApple(String appleCertPwPath, String appleCertPath) { + try { + InputStream certInputStream = new FileInputStream(appleCertPwPath); + Scanner scanner = new Scanner(certInputStream); + String password = scanner.next(); + productionApnsClient = new ApnsClientBuilder() + .setApnsServer(ApnsClientBuilder.PRODUCTION_APNS_HOST) + .setClientCredentials(new File(appleCertPath), password) + .build(); + devApnsClient = new ApnsClientBuilder() + .setApnsServer(ApnsClientBuilder.DEVELOPMENT_APNS_HOST) + .setClientCredentials(new File(appleCertPath), password) + .build(); + } catch (IOException e) { + log.error(e.toString()); + e.printStackTrace(); + } + } + + String sendAppleMessage(boolean isProduction, boolean isContentAvailable, String apsTokenHex, String encryptedMessage, boolean useSound) { + ApnsPayloadBuilder payloadBuilder = new ApnsPayloadBuilder(); + if (useSound) + payloadBuilder.setSoundFileName("default"); + payloadBuilder.setAlertBody("Bisq notification"); + payloadBuilder.setContentAvailable(isContentAvailable); + payloadBuilder.addCustomProperty("encrypted", encryptedMessage); + final String payload = payloadBuilder.buildWithDefaultMaximumLength(); + log.info("payload " + payload); + SimpleApnsPushNotification simpleApnsPushNotification = new SimpleApnsPushNotification(apsTokenHex, appleBundleId, payload); + + ApnsClient apnsClient = isProduction ? productionApnsClient : devApnsClient; + PushNotificationFuture> + notificationFuture = apnsClient.sendNotification(simpleApnsPushNotification); + try { + PushNotificationResponse pushNotificationResponse = notificationFuture.get(); + if (pushNotificationResponse.isAccepted()) { + log.info("Push notification accepted by APNs gateway."); + return SUCCESS; + } else { + String msg1 = "Notification rejected by the APNs gateway: " + + pushNotificationResponse.getRejectionReason(); + String msg2 = ""; + if (pushNotificationResponse.getTokenInvalidationTimestamp() != null) + msg2 = " and the token is invalid as of " + + pushNotificationResponse.getTokenInvalidationTimestamp(); + + log.info(msg1 + msg2); + return "Error: " + msg1 + msg2; + } + } catch (InterruptedException | ExecutionException e) { + log.error(e.toString()); + e.printStackTrace(); + return "Error: " + e.toString(); + } + } + + String sendAndroidMessage(String apsTokenHex, String encryptedMessage, boolean useSound) { + Message.Builder messageBuilder = Message.builder(); + Notification notification = new Notification("Bisq", "Notification"); + messageBuilder.setNotification(notification); + messageBuilder.putData("encrypted", encryptedMessage); + messageBuilder.setToken(apsTokenHex); + if (useSound) + messageBuilder.putData("sound", "default"); + Message message = messageBuilder.build(); + try { + FirebaseMessaging firebaseMessaging = FirebaseMessaging.getInstance(); + firebaseMessaging.send(message); + return SUCCESS; + } catch (FirebaseMessagingException e) { + log.error(e.toString()); + e.printStackTrace(); + return "Error: " + e.toString(); + } + } +} diff --git a/relay/src/main/resources/logback.xml b/relay/src/main/resources/logback.xml new file mode 100644 index 0000000000..bb9c8209f5 --- /dev/null +++ b/relay/src/main/resources/logback.xml @@ -0,0 +1,15 @@ + + + + + %highlight(%d{MMM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{15}: %msg %xEx%n) + + + + + + + + + + diff --git a/relay/src/main/resources/version.txt b/relay/src/main/resources/version.txt new file mode 100644 index 0000000000..a3df0a6959 --- /dev/null +++ b/relay/src/main/resources/version.txt @@ -0,0 +1 @@ +0.8.0 diff --git a/relay/torrc b/relay/torrc new file mode 100644 index 0000000000..3bfc6285ee --- /dev/null +++ b/relay/torrc @@ -0,0 +1,2 @@ +HiddenServiceDir build/tor-hidden-service +HiddenServicePort 80 127.0.0.1:8080 diff --git a/seednode/.dockerignore b/seednode/.dockerignore new file mode 100644 index 0000000000..d50ac5ff22 --- /dev/null +++ b/seednode/.dockerignore @@ -0,0 +1,23 @@ +docs/ +.git/ +.dockerignore +.editorconfig +.travis.yml +docker-compose.yml +docker/development/ +docker/prod/ +docker/README.md + +# Gradle +.gradle +build + +# IDEA +.idea +*.iml + +# macOS +.DS_Store + +# Vim +*.sw[op] diff --git a/seednode/build.gradle b/seednode/build.gradle new file mode 100644 index 0000000000..9e054eb4da --- /dev/null +++ b/seednode/build.gradle @@ -0,0 +1,39 @@ +buildscript { + repositories { + jcenter() + } + dependencies { + classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4' + } +} + +apply plugin: 'java' +apply plugin: 'application' +apply plugin: 'maven' +apply plugin: 'com.github.johnrengelman.shadow' + +group = 'network.bisq' +version = '0.8.0-SNAPSHOT' + +sourceCompatibility = 1.8 + +mainClassName = 'bisq.seednode.SeedNodeMain' + +sourceSets.main.resources.srcDirs += ['src/main/java'] // to copy fxml and css files + +repositories { + jcenter() + maven { url "https://jitpack.io" } + maven { url 'https://raw.githubusercontent.com/JesusMcCloud/tor-binary/master/release/' } +} + +dependencies { + compile project(':core') + runtime 'org.bouncycastle:bcprov-jdk15on:1.56' + compileOnly 'org.projectlombok:lombok:1.16.16' + annotationProcessor 'org.projectlombok:lombok:1.16.16' +} + +build.dependsOn installDist +installDist.destinationDir = file('build/app') +distZip.enabled = false diff --git a/seednode/create_jar.sh b/seednode/create_jar.sh new file mode 100755 index 0000000000..3027264a29 --- /dev/null +++ b/seednode/create_jar.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +./gradlew build -x test shadowJar diff --git a/seednode/docker-compose.yml b/seednode/docker-compose.yml new file mode 100644 index 0000000000..ff7ae6e744 --- /dev/null +++ b/seednode/docker-compose.yml @@ -0,0 +1,22 @@ +version: '2.1' + +services: + seednode: + build: + context: . + dockerfile: docker/development/Dockerfile + image: bisq-seednode + ports: + - 8000:8000 + environment: + - NODE_PORT=8000 + - BASE_CURRENCY_NETWORK=BTC_REGTEST + - SEED_NODES=seednode:8000 + - MY_ADDRESS=seednode:8000 + - USE_LOCALHOST_FOR_P2P=true + volumes: + - m2:/root/.m2 + +volumes: + m2: + name: m2 diff --git a/seednode/docker/README.md b/seednode/docker/README.md new file mode 100644 index 0000000000..887866ee94 --- /dev/null +++ b/seednode/docker/README.md @@ -0,0 +1,21 @@ +# bisq-seednode docker + +Both images use the same [startSeedNode.sh](startSeedNode.sh) script so inspect it to see what environment variables you can tweak. + +## Production image + +In order to build image: + + docker build . -f docker/prod/Dockerfile -t bisq/seednode + +Run: + + docker run bisq/seednode + +You might want to mount tor hidden service directory: + + docker run -v /your/tor/dir:/root/.local/share/seednode/btc_mainnet/tor/hiddenservice/ bisq/seednode + +## Development image + + docker-compose build diff --git a/seednode/docker/development/Dockerfile b/seednode/docker/development/Dockerfile new file mode 100644 index 0000000000..9973cc4daa --- /dev/null +++ b/seednode/docker/development/Dockerfile @@ -0,0 +1,18 @@ +### +# WARNING!!! THIS IMAGE IS FOR D E V E L O P M E N T USE ONLY! +### + +FROM openjdk:8-jdk + +RUN apt-get update && apt-get install -y --no-install-recommends \ + openjfx && rm -rf /var/lib/apt/lists/* + +WORKDIR /bisq-seednode +CMD ./docker/startSeedNode.sh + +ENV APP_NAME=seednode +ENV NODE_PORT=8000 + +EXPOSE 8000 + +COPY . . diff --git a/seednode/docker/prod/Dockerfile b/seednode/docker/prod/Dockerfile new file mode 100644 index 0000000000..2832948138 --- /dev/null +++ b/seednode/docker/prod/Dockerfile @@ -0,0 +1,16 @@ +FROM openjdk:8-jdk + +RUN apt-get update && apt-get install -y --no-install-recommends \ + openjfx && rm -rf /var/lib/apt/lists/* + +WORKDIR /bisq-seednode +CMD ./docker/startSeedNode.sh + +ENV APP_NAME=seednode +ENV NODE_PORT=8000 + +EXPOSE 8000 + +COPY . . +RUN ./docker/setup.sh +ENV SKIP_BUILD=true diff --git a/seednode/docker/setup.sh b/seednode/docker/setup.sh new file mode 100755 index 0000000000..2b1afc7605 --- /dev/null +++ b/seednode/docker/setup.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +if [ "$SKIP_BUILD" != "true" ]; then + ./gradlew build +fi diff --git a/seednode/docker/startSeedNode.sh b/seednode/docker/startSeedNode.sh new file mode 100755 index 0000000000..fc6278c767 --- /dev/null +++ b/seednode/docker/startSeedNode.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +SETUP_SCRIPT=${SCRIPT_DIR}/setup.sh +source ${SETUP_SCRIPT} + +ARGS="" + +if [ ! -z "$BASE_CURRENCY_NETWORK" ]; then + ARGS="$ARGS --baseCurrencyNetwork=$BASE_CURRENCY_NETWORK" +fi +if [ ! -z "$MAX_CONNECTIONS" ]; then + ARGS="$ARGS --maxConnections=$MAX_CONNECTIONS" +fi +if [ ! -z "$NODE_PORT" ]; then + ARGS="$ARGS --nodePort=$NODE_PORT" +fi +if [ ! -z "$APP_NAME" ]; then + ARGS="$ARGS --appName=$APP_NAME" +fi +if [ ! -z "$SEED_NODES" ]; then + ARGS="$ARGS --seedNodes=$SEED_NODES" +fi +if [ ! -z "$BTC_NODES" ]; then + ARGS="$ARGS --btcNodes=$BTC_NODES" +fi +if [ ! -z "$USE_LOCALHOST_FOR_P2P" ]; then + ARGS="$ARGS --useLocalhostForP2P=$USE_LOCALHOST_FOR_P2P" +fi +if [ ! -z "$MY_ADDRESS" ]; then + ARGS="$ARGS --myAddress=${MY_ADDRESS}" +elif [ ! -z "$ONION_ADDRESS" ]; then + ARGS="$ARGS --myAddress=${ONION_ADDRESS}.onion:$NODE_PORT" +fi + +JAVA_OPTS='-Xms1800m -Xmx1800m' ./build/app/bin/bisq-seednode $ARGS diff --git a/seednode/src/main/java/bisq/seednode/SeedNode.java b/seednode/src/main/java/bisq/seednode/SeedNode.java new file mode 100644 index 0000000000..b8b786a440 --- /dev/null +++ b/seednode/src/main/java/bisq/seednode/SeedNode.java @@ -0,0 +1,46 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.seednode; + +import bisq.core.app.misc.AppSetup; +import bisq.core.app.misc.AppSetupWithP2P; +import bisq.core.app.misc.AppSetupWithP2PAndDAO; +import bisq.core.dao.DaoOptionKeys; + +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.name.Names; + +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SeedNode { + @Setter + private Injector injector; + private AppSetup appSetup; + + public SeedNode() { + } + + public void startApplication() { + Boolean fullDaoNode = injector.getInstance(Key.get(Boolean.class, Names.named(DaoOptionKeys.FULL_DAO_NODE))); + appSetup = fullDaoNode ? injector.getInstance(AppSetupWithP2PAndDAO.class) : injector.getInstance(AppSetupWithP2P.class); + appSetup.start(); + } +} diff --git a/seednode/src/main/java/bisq/seednode/SeedNodeMain.java b/seednode/src/main/java/bisq/seednode/SeedNodeMain.java new file mode 100644 index 0000000000..f9a2cb1468 --- /dev/null +++ b/seednode/src/main/java/bisq/seednode/SeedNodeMain.java @@ -0,0 +1,100 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.seednode; + +import bisq.core.app.BisqEnvironment; +import bisq.core.app.BisqExecutable; +import bisq.core.app.misc.ExecutableForAppWithP2p; +import bisq.core.app.misc.ModuleForAppWithP2p; + +import bisq.common.UserThread; +import bisq.common.app.AppModule; +import bisq.common.app.Capabilities; +import bisq.common.setup.CommonSetup; + +import joptsimple.OptionSet; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SeedNodeMain extends ExecutableForAppWithP2p { + private static final String VERSION = "0.8.0"; + private SeedNode seedNode; + + public static void main(String[] args) throws Exception { + log.info("SeedNode.VERSION: " + VERSION); + BisqEnvironment.setDefaultAppName("bisq_seednode"); + + if (BisqExecutable.setupInitialOptionParser(args)) + new SeedNodeMain().execute(args); + } + + @Override + protected void doExecute(OptionSet options) { + super.doExecute(options); + + checkMemory(bisqEnvironment, this); + CommonSetup.setup(this); + + keepRunning(); + } + + @Override + protected void addCapabilities() { + Capabilities.addCapability(Capabilities.Capability.SEED_NODE.ordinal()); + } + + @Override + protected void launchApplication() { + UserThread.execute(() -> { + try { + seedNode = new SeedNode(); + UserThread.execute(this::onApplicationLaunched); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + + @Override + protected void onApplicationLaunched() { + super.onApplicationLaunched(); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // We continue with a series of synchronous execution tasks + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + protected AppModule getModule() { + return new ModuleForAppWithP2p(bisqEnvironment); + } + + @Override + protected void applyInjector() { + super.applyInjector(); + + seedNode.setInjector(injector); + } + + @Override + protected void startApplication() { + seedNode.startApplication(); + } +} diff --git a/seednode/src/main/resources/logback.xml b/seednode/src/main/resources/logback.xml new file mode 100644 index 0000000000..0d084c25ca --- /dev/null +++ b/seednode/src/main/resources/logback.xml @@ -0,0 +1,19 @@ + + + + + %highlight(%d{MMM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{15}: %msg %xEx%n) + + + + + + + + + + + + + + diff --git a/settings.gradle b/settings.gradle index eb44099e23..500b2803c4 100644 --- a/settings.gradle +++ b/settings.gradle @@ -3,5 +3,10 @@ include 'common' include 'p2p' include 'core' include 'desktop' +include 'monitor' +include 'pricenode' +include 'relay' +include 'seednode' +include 'statsnode' rootProject.name = 'bisq' diff --git a/statsnode/build.gradle b/statsnode/build.gradle new file mode 100644 index 0000000000..32d0e2786b --- /dev/null +++ b/statsnode/build.gradle @@ -0,0 +1,27 @@ +plugins { + id 'java' + id 'application' +} + +group = 'network.bisq' +version = '0.8.0-SNAPSHOT' + +sourceCompatibility = 1.8 + +mainClassName = 'bisq.statistics.StatisticsMain' + +repositories { + jcenter() + maven { url "https://jitpack.io" } + maven { url 'https://raw.githubusercontent.com/JesusMcCloud/tor-binary/master/release/' } +} + +dependencies { + compile project(':core') + compileOnly 'org.projectlombok:lombok:1.16.16' + annotationProcessor 'org.projectlombok:lombok:1.16.16' +} + +build.dependsOn installDist +installDist.destinationDir = file('build/app') +distZip.enabled = false diff --git a/statsnode/src/main/java/bisq/statistics/Statistics.java b/statsnode/src/main/java/bisq/statistics/Statistics.java new file mode 100644 index 0000000000..361323e91b --- /dev/null +++ b/statsnode/src/main/java/bisq/statistics/Statistics.java @@ -0,0 +1,71 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.statistics; + +import bisq.core.app.misc.AppSetup; +import bisq.core.app.misc.AppSetupWithP2P; +import bisq.core.offer.OfferBookService; +import bisq.core.provider.price.PriceFeedService; +import bisq.core.trade.statistics.TradeStatisticsManager; + +import bisq.network.p2p.BootstrapListener; +import bisq.network.p2p.P2PService; + +import com.google.inject.Injector; + +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class Statistics { + @Setter + private Injector injector; + + private OfferBookService offerBookService; // pin to not get GC'ed + private PriceFeedService priceFeedService; + private TradeStatisticsManager tradeStatisticsManager; + private P2PService p2pService; + private AppSetup appSetup; + + public Statistics() { + } + + public void startApplication() { + p2pService = injector.getInstance(P2PService.class); + offerBookService = injector.getInstance(OfferBookService.class); + priceFeedService = injector.getInstance(PriceFeedService.class); + tradeStatisticsManager = injector.getInstance(TradeStatisticsManager.class); + + // We need the price feed for market based offers + priceFeedService.setCurrencyCode("USD"); + p2pService.addP2PServiceListener(new BootstrapListener() { + @Override + public void onUpdatedDataReceived() { + // we need to have tor ready + log.info("onBootstrapComplete: we start requestPriceFeed"); + priceFeedService.requestPriceFeed(price -> log.info("requestPriceFeed. price=" + price), + (errorMessage, throwable) -> log.warn("Exception at requestPriceFeed: " + throwable.getMessage())); + + tradeStatisticsManager.onAllServicesInitialized(); + } + }); + + appSetup = injector.getInstance(AppSetupWithP2P.class); + appSetup.start(); + } +} diff --git a/statsnode/src/main/java/bisq/statistics/StatisticsMain.java b/statsnode/src/main/java/bisq/statistics/StatisticsMain.java new file mode 100644 index 0000000000..75f81a4687 --- /dev/null +++ b/statsnode/src/main/java/bisq/statistics/StatisticsMain.java @@ -0,0 +1,93 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.statistics; + +import bisq.core.app.BisqEnvironment; +import bisq.core.app.BisqExecutable; +import bisq.core.app.misc.ExecutableForAppWithP2p; +import bisq.core.app.misc.ModuleForAppWithP2p; + +import bisq.common.UserThread; +import bisq.common.app.AppModule; +import bisq.common.setup.CommonSetup; + +import joptsimple.OptionSet; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class StatisticsMain extends ExecutableForAppWithP2p { + private static final String VERSION = "0.6.1"; + private Statistics statistics; + + public static void main(String[] args) throws Exception { + log.info("Statistics.VERSION: " + VERSION); + BisqEnvironment.setDefaultAppName("bisq_statistics"); + if (BisqExecutable.setupInitialOptionParser(args)) + new StatisticsMain().execute(args); + } + + @Override + protected void doExecute(OptionSet options) { + super.doExecute(options); + + CommonSetup.setup(this); + checkMemory(bisqEnvironment, this); + + keepRunning(); + } + + @Override + protected void launchApplication() { + UserThread.execute(() -> { + try { + statistics = new Statistics(); + UserThread.execute(this::onApplicationLaunched); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + + @Override + protected void onApplicationLaunched() { + super.onApplicationLaunched(); + } + + + /////////////////////////////////////////////////////////////////////////////////////////// + // We continue with a series of synchronous execution tasks + /////////////////////////////////////////////////////////////////////////////////////////// + + @Override + protected AppModule getModule() { + return new ModuleForAppWithP2p(bisqEnvironment); + } + + @Override + protected void applyInjector() { + super.applyInjector(); + + statistics.setInjector(injector); + } + + @Override + protected void startApplication() { + statistics.startApplication(); + } +} diff --git a/statsnode/src/main/resources/logback.xml b/statsnode/src/main/resources/logback.xml new file mode 100644 index 0000000000..5cabc40449 --- /dev/null +++ b/statsnode/src/main/resources/logback.xml @@ -0,0 +1,18 @@ + + + + + %highlight(%d{MMM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{15}: %msg %xEx%n) + + + + + + + + + + + + +
OperatorDomain nameIP addressBtc node onion addressUpTimeDownTime
").append("" + node.getOperator() + " ").append("").append("" + node.getHostName() + " ").append("").append("" + node.getAddress() + " ").append("").append("" + node.getOnionAddress() + " ").append("").append("" + upTimeString + " ").append("").append("" + downTimeString + " ").append("