From a65a28855fffecc45d42c9a3d8c491d37f414fb3 Mon Sep 17 00:00:00 2001 From: Manfred Karrer Date: Wed, 19 Apr 2017 19:41:40 -0500 Subject: [PATCH] Add mapping for json files --- .../bisq/common/storage/JsonFileManager.java | 102 ++++++++ .../main/java/io/bisq/core/dao/DaoModule.java | 4 +- .../bisq/core/dao/blockchain/BsqFullNode.java | 10 +- .../dao/blockchain/json/DaoJsonExporter.java | 221 ++++++++++++++++++ .../dao/blockchain/json/JsonExporter.java | 152 ------------ .../blockchain/json/ScriptPubKeyForJson.java | 9 + .../dao/blockchain/json/SpentInfoForJson.java | 7 + .../core/dao/blockchain/json/TxForJson.java | 35 +++ .../dao/blockchain/json/TxInputForJson.java | 31 +++ .../dao/blockchain/json/TxOutputForJson.java | 18 +- .../dao/blockchain/json/TxTypeForJson.java | 37 +++ .../dao/blockchain/parse/BsqChainState.java | 74 +++++- .../core/dao/blockchain/parse/BsqParser.java | 2 +- .../bisq/core/dao/blockchain/vo/TxOutput.java | 30 +-- .../main/java/io/bisq/gui/app/BisqApp.java | 2 + 15 files changed, 532 insertions(+), 202 deletions(-) create mode 100644 common/src/main/java/io/bisq/common/storage/JsonFileManager.java create mode 100644 core/src/main/java/io/bisq/core/dao/blockchain/json/DaoJsonExporter.java delete mode 100644 core/src/main/java/io/bisq/core/dao/blockchain/json/JsonExporter.java create mode 100644 core/src/main/java/io/bisq/core/dao/blockchain/json/TxForJson.java create mode 100644 core/src/main/java/io/bisq/core/dao/blockchain/json/TxInputForJson.java create mode 100644 core/src/main/java/io/bisq/core/dao/blockchain/json/TxTypeForJson.java diff --git a/common/src/main/java/io/bisq/common/storage/JsonFileManager.java b/common/src/main/java/io/bisq/common/storage/JsonFileManager.java new file mode 100644 index 0000000000..d73e3f1a3c --- /dev/null +++ b/common/src/main/java/io/bisq/common/storage/JsonFileManager.java @@ -0,0 +1,102 @@ +/* + * This file is part of bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package io.bisq.common.storage; + +import io.bisq.common.UserThread; +import io.bisq.common.util.Utilities; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Paths; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class JsonFileManager { + private final ThreadPoolExecutor executor = Utilities.getThreadPoolExecutor("saveToDiscExecutor", 5, 50, 60); + private File dir; + + + /////////////////////////////////////////////////////////////////////////////////////////// + // Constructor + /////////////////////////////////////////////////////////////////////////////////////////// + + public JsonFileManager(File dir) { + this.dir = dir; + + if (!dir.exists()) + if (!dir.mkdir()) + log.warn("make dir failed"); + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + UserThread.execute(JsonFileManager.this::shutDown); + }, "WriteOnlyFileManager.ShutDownHook")); + } + + public void shutDown() { + executor.shutdown(); + try { + executor.awaitTermination(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + public void writeToDisc(String json, String fileName) { + executor.execute(() -> { + File jsonFile = new File(Paths.get(dir.getAbsolutePath(), fileName + ".json").toString()); + File tempFile = null; + PrintWriter printWriter = null; + try { + tempFile = File.createTempFile("temp", null, dir); + if (!executor.isShutdown() && !executor.isTerminated() && !executor.isTerminating()) + tempFile.deleteOnExit(); + + printWriter = new PrintWriter(tempFile); + printWriter.println(json); + + if (Utilities.isWindows()) { + // Work around an issue on Windows whereby you can't rename over existing files. + final File canonical = jsonFile.getCanonicalFile(); + if (canonical.exists() && !canonical.delete()) { + throw new IOException("Failed to delete canonical file for replacement with save"); + } + if (!tempFile.renameTo(canonical)) { + throw new IOException("Failed to rename " + tempFile + " to " + canonical); + } + } else if (!tempFile.renameTo(jsonFile)) { + throw new IOException("Failed to rename " + tempFile + " to " + jsonFile); + } + } catch (Throwable t) { + log.error("storageFile " + jsonFile.toString()); + t.printStackTrace(); + } finally { + if (tempFile != null && tempFile.exists()) { + log.warn("Temp file still exists after failed save. We will delete it now. storageFile=" + fileName); + if (!tempFile.delete()) + log.error("Cannot delete temp file."); + } + + if (printWriter != null) + printWriter.close(); + } + }); + } +} diff --git a/core/src/main/java/io/bisq/core/dao/DaoModule.java b/core/src/main/java/io/bisq/core/dao/DaoModule.java index 64118820dc..17123a84cd 100644 --- a/core/src/main/java/io/bisq/core/dao/DaoModule.java +++ b/core/src/main/java/io/bisq/core/dao/DaoModule.java @@ -22,7 +22,7 @@ import io.bisq.common.app.AppModule; import io.bisq.core.dao.blockchain.BsqBlockchainManager; import io.bisq.core.dao.blockchain.BsqFullNode; import io.bisq.core.dao.blockchain.BsqLiteNode; -import io.bisq.core.dao.blockchain.json.JsonExporter; +import io.bisq.core.dao.blockchain.json.DaoJsonExporter; import io.bisq.core.dao.blockchain.parse.*; import io.bisq.core.dao.compensation.CompensationRequestManager; import io.bisq.core.dao.compensation.CompensationRequestModel; @@ -60,7 +60,7 @@ public class DaoModule extends AppModule { bind(VotingVerification.class).in(Singleton.class); bind(IssuanceVerification.class).in(Singleton.class); - bind(JsonExporter.class).in(Singleton.class); + bind(DaoJsonExporter.class).in(Singleton.class); bind(DaoPeriodService.class).in(Singleton.class); bind(VotingService.class).in(Singleton.class); diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/BsqFullNode.java b/core/src/main/java/io/bisq/core/dao/blockchain/BsqFullNode.java index baa3541374..a240ec762c 100644 --- a/core/src/main/java/io/bisq/core/dao/blockchain/BsqFullNode.java +++ b/core/src/main/java/io/bisq/core/dao/blockchain/BsqFullNode.java @@ -26,7 +26,7 @@ import io.bisq.common.handlers.ErrorMessageHandler; import io.bisq.common.network.Msg; import io.bisq.common.util.Utilities; import io.bisq.core.dao.blockchain.exceptions.BlockNotConnectingException; -import io.bisq.core.dao.blockchain.json.JsonExporter; +import io.bisq.core.dao.blockchain.json.DaoJsonExporter; import io.bisq.core.dao.blockchain.p2p.GetBsqBlocksRequest; import io.bisq.core.dao.blockchain.p2p.GetBsqBlocksResponse; import io.bisq.core.dao.blockchain.p2p.NewBsqBlockBroadcastMsg; @@ -47,7 +47,7 @@ import org.jetbrains.annotations.NotNull; public class BsqFullNode extends BsqNode { private BsqFullNodeExecutor bsqFullNodeExecutor; - private final JsonExporter jsonExporter; + private final DaoJsonExporter daoJsonExporter; @Getter private boolean parseBlockchainComplete; @@ -62,14 +62,14 @@ public class BsqFullNode extends BsqNode { BsqParser bsqParser, BsqFullNodeExecutor bsqFullNodeExecutor, BsqChainState bsqChainState, - JsonExporter jsonExporter, + DaoJsonExporter daoJsonExporter, FeeService feeService) { super(p2PService, bsqParser, bsqChainState, feeService); this.bsqFullNodeExecutor = bsqFullNodeExecutor; - this.jsonExporter = jsonExporter; + this.daoJsonExporter = daoJsonExporter; } @@ -187,7 +187,7 @@ public class BsqFullNode extends BsqNode { @Override protected void onNewBsqBlock(BsqBlock bsqBlock) { super.onNewBsqBlock(bsqBlock); - jsonExporter.maybeExport(); + daoJsonExporter.maybeExport(); if (parseBlockchainComplete && p2pNetworkReady) publishNewBlock(bsqBlock); } diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/json/DaoJsonExporter.java b/core/src/main/java/io/bisq/core/dao/blockchain/json/DaoJsonExporter.java new file mode 100644 index 0000000000..174cb58ee2 --- /dev/null +++ b/core/src/main/java/io/bisq/core/dao/blockchain/json/DaoJsonExporter.java @@ -0,0 +1,221 @@ +/* + * This file is part of bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package io.bisq.core.dao.blockchain.json; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.inject.Inject; +import io.bisq.common.storage.JsonFileManager; +import io.bisq.common.storage.Storage; +import io.bisq.common.util.Utilities; +import io.bisq.core.dao.DaoOptionKeys; +import io.bisq.core.dao.blockchain.parse.BsqChainState; +import io.bisq.core.dao.blockchain.vo.*; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + +import javax.inject.Named; +import java.io.File; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class DaoJsonExporter { + private final boolean dumpBlockchainData; + + private final ListeningExecutorService executor = Utilities.getListeningExecutorService("JsonExporter", 1, 1, 1200); + private File txDir, txOutputDir, bsqChainStateDir; + private JsonFileManager txFileManager, txOutputFileManager, bsqChainStateFileManager; + private BsqChainState bsqChainState; + + @Inject + public DaoJsonExporter(BsqChainState bsqChainState, + @Named(Storage.STORAGE_DIR) File storageDir, + @Named(DaoOptionKeys.DUMP_BLOCKCHAIN_DATA) boolean dumpBlockchainData) { + this.bsqChainState = bsqChainState; + this.dumpBlockchainData = dumpBlockchainData; + + if (dumpBlockchainData) { + txDir = new File(Paths.get(storageDir.getAbsolutePath(), "tx").toString()); + if (!txDir.exists()) + if (!txDir.mkdir()) + log.warn("make txDir failed.\ntxDir=" + txDir.getAbsolutePath()); + + txOutputDir = new File(Paths.get(storageDir.getAbsolutePath(), "txo").toString()); + if (!txOutputDir.exists()) + if (!txOutputDir.mkdir()) + log.warn("make txOutputDir failed.\ntxOutputDir=" + txOutputDir.getAbsolutePath()); + + bsqChainStateDir = new File(Paths.get(storageDir.getAbsolutePath(), "all").toString()); + if (!bsqChainStateDir.exists()) + if (!bsqChainStateDir.mkdir()) + log.warn("make bsqChainStateDir failed.\nbsqChainStateDir=" + bsqChainStateDir.getAbsolutePath()); + + txFileManager = new JsonFileManager(txDir); + txOutputFileManager = new JsonFileManager(txOutputDir); + bsqChainStateFileManager = new JsonFileManager(bsqChainStateDir); + } + } + + public void shutDown() { + txFileManager.shutDown(); + txOutputFileManager.shutDown(); + bsqChainStateFileManager.shutDown(); + } + + public void maybeExport() { + if (dumpBlockchainData) { + /* List list = bsqChainState.getVerifiedTxOutputSet().stream() + .map(this::getTxOutputForJson) + .collect(Collectors.toList()); + + list.sort((o1, o2) -> (o1.getSortData().compareTo(o2.getSortData()))); + TxOutputForJson[] array = new TxOutputForJson[list.size()]; + list.toArray(array);*/ + //jsonStorage.queueUpForSave(new PlainTextWrapper(Utilities.objectToJson(array)), 5000); + + ListenableFuture future = executor.submit(() -> { + final BsqChainState bsqChainStateClone = bsqChainState.getClone(); + Map burntFeeByTxIdMap = bsqChainStateClone.getBurntFeeByTxIdMap(); + Map spentInfoByTxOutputMap = bsqChainStateClone.getSpentInfoByTxOutputMap(); + Set unspentTxOutputSet = bsqChainStateClone.getUnspentTxOutputSet(); + Set compensationRequestOpReturnTxOutputs = bsqChainStateClone.getCompensationRequestOpReturnTxOutputs(); + Set votingTxOutputs = bsqChainStateClone.getVotingTxOutputs(); + Set invalidTxOutputs = bsqChainStateClone.getInvalidatedTxOutputs(); + Set issuanceTxOutputSet = bsqChainStateClone.getIssuanceBtcTxOutputsByBtcAddressMap() + .values() + .stream() + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + String genesisTxId = bsqChainStateClone.getGenesisTxId(); + + // map all txs to json txs + List txs = new ArrayList<>(); + for (Tx tx : bsqChainStateClone.getTxMap().values()) { + String txId = tx.getId(); + int blockHeight = tx.getBlockHeight(); + String blockHash = tx.getBlockHash(); + boolean isGenesisTx = tx.isGenesisTx(); + + //boolean isUnspent = false; + boolean isVote = false; + boolean isCompensationRequest = false; + boolean hasBurnedFee = burntFeeByTxIdMap.containsKey(txId); + boolean isIssuance = false; + + String txType = TxTypeForJson.UNDEFINED.getDisplayString(); + boolean hasAnyInvalidTxOutput = false; + List outputs = new ArrayList<>(); + for (TxOutput txOutput : tx.getOutputs()) { + // isUnspent = isUnspent || unspentTxOutputSet.contains(txOutput); + isVote = isVote || votingTxOutputs.contains(txOutput); + isCompensationRequest = isCompensationRequest || compensationRequestOpReturnTxOutputs.contains(txOutput); + isIssuance = isIssuance || issuanceTxOutputSet.contains(txOutput); + + int outputIndex = txOutput.getIndex(); + final long bsqAmount = -1;//txOutput.getValue(); + //TODO + final long btcAmount = -1; + final int height = txOutput.getBlockHeight(); + + final boolean verified = !invalidTxOutputs.contains(txOutput); + hasAnyInvalidTxOutput = !verified || hasAnyInvalidTxOutput; + final long burntFee = burntFeeByTxIdMap.containsKey(txId) ? burntFeeByTxIdMap.get(txId) : 0; + final ScriptPubKeyForJson scriptPubKey = new ScriptPubKeyForJson(txOutput.getPubKeyScript()); + SpentInfoForJson spentInfoJson = spentInfoByTxOutputMap.containsKey(txOutput.getTxIdIndexTuple()) ? + new SpentInfoForJson(spentInfoByTxOutputMap.get(txOutput.getTxIdIndexTuple())) : null; + final long time = txOutput.getTime(); + outputs.add(new TxOutputForJson(txId, + outputIndex, + bsqAmount, + btcAmount, + height, + verified, + burntFee, + scriptPubKey, + spentInfoJson, + time + )); + } + // after iteration of all txOutputs we can evaluate the txType + if (txId.equals(genesisTxId)) + txType = TxTypeForJson.GENESIS.getDisplayString(); + else if (isVote) + txType = TxTypeForJson.VOTE.getDisplayString(); + else if (isCompensationRequest) + txType = TxTypeForJson.COMPENSATION_REQUEST.getDisplayString(); + else if (hasBurnedFee) // burned fee contains also vote and comp request but we detect those cases above + txType = TxTypeForJson.PAY_TRADE_FEE.getDisplayString(); + else if (isIssuance) + txType = TxTypeForJson.ISSUANCE.getDisplayString(); + else + txType = TxTypeForJson.SEND_BSQ.getDisplayString(); + + List inputs = new ArrayList<>(); + for (TxInput txInput : tx.getInputs()) { + int spendingTxOutputIndex = txInput.getSpendingTxOutputIndex(); + String spendingTxId = txInput.getSpendingTxId(); + //TODO + long bsqAmount = -1; + boolean isVerified = !hasAnyInvalidTxOutput; + inputs.add(new TxInputForJson(spendingTxOutputIndex, + spendingTxId, + bsqAmount, + isVerified)); + } + + // we evaluated during the tx loop so we apply txType here at the end again to have all + // txOutputs the same txType. + final String finalTxType = txType; + outputs.stream().forEach(txOutputForJson -> { + txOutputForJson.setTxType(finalTxType); + txOutputFileManager.writeToDisc(Utilities.objectToJson(txOutputForJson), txOutputForJson.getId()); + }); + + final TxForJson txForJson = new TxForJson(txId, + blockHeight, + blockHash, + inputs, + outputs, + isGenesisTx, + txType); + txs.add(txForJson); + + txFileManager.writeToDisc(Utilities.objectToJson(txForJson), txId); + } + + bsqChainStateFileManager.writeToDisc(Utilities.objectToJson(bsqChainStateClone), "bsqChainState"); + return null; + }); + + Futures.addCallback(future, new FutureCallback() { + public void onSuccess(Void ignore) { + log.trace("onSuccess"); + } + + public void onFailure(@NotNull Throwable throwable) { + log.error(throwable.toString()); + throwable.printStackTrace(); + } + }); + } + } +} diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/json/JsonExporter.java b/core/src/main/java/io/bisq/core/dao/blockchain/json/JsonExporter.java deleted file mode 100644 index 937ac300dd..0000000000 --- a/core/src/main/java/io/bisq/core/dao/blockchain/json/JsonExporter.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * This file is part of bisq. - * - * bisq is free software: you can redistribute it and/or modify it - * under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or (at - * your option) any later version. - * - * bisq is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public - * License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with bisq. If not, see . - */ - -package io.bisq.core.dao.blockchain.json; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.inject.Inject; -import io.bisq.common.storage.PlainTextWrapper; -import io.bisq.common.storage.Storage; -import io.bisq.common.util.Utilities; -import io.bisq.core.dao.DaoOptionKeys; -import io.bisq.core.dao.blockchain.btcd.PubKeyScript; -import io.bisq.core.dao.blockchain.parse.BsqChainState; -import io.bisq.core.dao.blockchain.vo.SpentInfo; -import io.bisq.core.dao.blockchain.vo.TxOutput; -import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; - -import javax.inject.Named; -import java.io.File; - -@Slf4j -public class JsonExporter { - private final Storage jsonStorage; - private final boolean dumpBlockchainData; - private final File storageDir; - private final ListeningExecutorService executor = Utilities.getListeningExecutorService("JsonExporter", 1, 1, 1200); - private BsqChainState bsqChainState; - - @Inject - public JsonExporter(Storage jsonStorage, - BsqChainState bsqChainState, - @Named(Storage.STORAGE_DIR) File storageDir, - @Named(DaoOptionKeys.DUMP_BLOCKCHAIN_DATA) boolean dumpBlockchainData) { - this.bsqChainState = bsqChainState; - this.storageDir = storageDir; - this.jsonStorage = jsonStorage; - this.dumpBlockchainData = dumpBlockchainData; - - if (dumpBlockchainData) - this.jsonStorage.initWithFileName("bsqChainState.json"); - } - - public void maybeExport() { - if (dumpBlockchainData) { - /* List list = bsqChainState.getVerifiedTxOutputSet().stream() - .map(this::getTxOutputForJson) - .collect(Collectors.toList()); - - list.sort((o1, o2) -> (o1.getSortData().compareTo(o2.getSortData()))); - TxOutputForJson[] array = new TxOutputForJson[list.size()]; - list.toArray(array);*/ - //jsonStorage.queueUpForSave(new PlainTextWrapper(Utilities.objectToJson(array)), 5000); - - ListenableFuture future = executor.submit(() -> { - jsonStorage.queueUpForSave(new PlainTextWrapper(Utilities.objectToJson(bsqChainState.getClone())), 5000); - return null; - }); - - Futures.addCallback(future, new FutureCallback() { - public void onSuccess(Void ignore) { - log.trace("onSuccess"); - } - - public void onFailure(@NotNull Throwable throwable) { - log.error(throwable.toString()); - throwable.printStackTrace(); - } - }); - - - // keep the individual file storage option as code as we dont know yet what we will use. - /* log.error("txOutputForJson " + txOutputForJson); - File txoDir = new File(Paths.get(storageDir.getAbsolutePath(), "txo").toString()); - if (!txoDir.exists()) - if (!txoDir.mkdir()) - log.warn("make txoDir failed.\ntxoDir=" + txoDir.getAbsolutePath()); - File txoFile = new File(Paths.get(txoDir.getAbsolutePath(), - txOutput.getTxId() + ":" + outputIndex + ".json").toString()); - - // Nr of write requests might be a bit heavy, consider write whole list to one file - FileManager fileManager = new FileManager<>(storageDir, txoFile, 1); - fileManager.saveLater(new PlainTextWrapper(Utilities.objectToJson(txOutputForJson)));*/ - } - } - - private TxOutputForJson getTxOutputForJson(TxOutput txOutput) { - String txId = txOutput.getTxId(); - int outputIndex = txOutput.getIndex(); - final long bsqAmount = txOutput.getValue(); - final int height = txOutput.getBlockHeight(); - - /* final boolean isBsqCoinBase = txOutput.isIssuanceOutput(); - final boolean verified = txOutput.isVerified(); - final long burntFee = txOutput.getBurntFee(); - final long btcTxFee = txOutput.getBtcTxFee();*/ - - // TODO - final boolean isBsqCoinBase = false; - final boolean verified = true; - final long burntFee = 0; - final long btcTxFee = 0; - - PubKeyScript pubKeyScript = txOutput.getPubKeyScript(); - final ScriptPubKeyForJson scriptPubKey = new ScriptPubKeyForJson(pubKeyScript.getAddresses(), - pubKeyScript.getAsm(), - pubKeyScript.getHex(), - pubKeyScript.getReqSigs(), - pubKeyScript.getType().toString()); - SpentInfoForJson spentInfoJson = null; - // SpentInfo spentInfo = txOutput.getSpentInfo(); - SpentInfo spentInfo = null; - if (spentInfo != null) - spentInfoJson = new SpentInfoForJson(spentInfo.getBlockHeight(), - spentInfo.getInputIndex(), - spentInfo.getTxId()); - - final long time = txOutput.getTime(); - final String txVersion = "";//txOutput.getTxVersion(); - return new TxOutputForJson(txId, - outputIndex, - bsqAmount, - height, - isBsqCoinBase, - verified, - burntFee, - btcTxFee, - scriptPubKey, - spentInfoJson, - time, - txVersion - ); - } - -} diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/json/ScriptPubKeyForJson.java b/core/src/main/java/io/bisq/core/dao/blockchain/json/ScriptPubKeyForJson.java index 99817177ff..1513aee11f 100644 --- a/core/src/main/java/io/bisq/core/dao/blockchain/json/ScriptPubKeyForJson.java +++ b/core/src/main/java/io/bisq/core/dao/blockchain/json/ScriptPubKeyForJson.java @@ -17,6 +17,7 @@ package io.bisq.core.dao.blockchain.json; +import io.bisq.core.dao.blockchain.btcd.PubKeyScript; import lombok.Value; import java.util.List; @@ -28,4 +29,12 @@ public class ScriptPubKeyForJson { private final String hex; private final int reqSigs; private final String type; + + public ScriptPubKeyForJson(PubKeyScript pubKeyScript) { + addresses = pubKeyScript.getAddresses(); + asm = pubKeyScript.getAsm(); + hex = pubKeyScript.getHex(); + reqSigs = pubKeyScript.getReqSigs(); + type = pubKeyScript.getType().toString(); + } } diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/json/SpentInfoForJson.java b/core/src/main/java/io/bisq/core/dao/blockchain/json/SpentInfoForJson.java index 1a05eae57e..84a3d2c022 100644 --- a/core/src/main/java/io/bisq/core/dao/blockchain/json/SpentInfoForJson.java +++ b/core/src/main/java/io/bisq/core/dao/blockchain/json/SpentInfoForJson.java @@ -17,6 +17,7 @@ package io.bisq.core.dao.blockchain.json; +import io.bisq.core.dao.blockchain.vo.SpentInfo; import lombok.Value; @Value @@ -24,4 +25,10 @@ public class SpentInfoForJson { private final long height; private final int inputIndex; private final String txId; + + public SpentInfoForJson(SpentInfo spentInfo) { + height = spentInfo.getBlockHeight(); + inputIndex = spentInfo.getInputIndex(); + txId = spentInfo.getTxId(); + } } \ No newline at end of file diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/json/TxForJson.java b/core/src/main/java/io/bisq/core/dao/blockchain/json/TxForJson.java new file mode 100644 index 0000000000..3914f1e72c --- /dev/null +++ b/core/src/main/java/io/bisq/core/dao/blockchain/json/TxForJson.java @@ -0,0 +1,35 @@ +/* + * This file is part of bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package io.bisq.core.dao.blockchain.json; + +import io.bisq.common.app.Version; +import lombok.Value; + +import java.util.List; + +@Value +public class TxForJson { + private final String txVersion = Version.BSQ_TX_VERSION; + private final String id; + private final int blockHeight; + private final String blockHash; + private final List inputs; + private final List outputs; + private final boolean isGenesisTx; + private final String txType; +} diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/json/TxInputForJson.java b/core/src/main/java/io/bisq/core/dao/blockchain/json/TxInputForJson.java new file mode 100644 index 0000000000..a32d1802b8 --- /dev/null +++ b/core/src/main/java/io/bisq/core/dao/blockchain/json/TxInputForJson.java @@ -0,0 +1,31 @@ +/* + * This file is part of bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package io.bisq.core.dao.blockchain.json; + +import lombok.Value; + +import javax.annotation.concurrent.Immutable; + +@Value +@Immutable +public class TxInputForJson { + private final int spendingTxOutputIndex; + private final String spendingTxId; + private final long bsqAmount; + private final boolean isVerified; +} diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/json/TxOutputForJson.java b/core/src/main/java/io/bisq/core/dao/blockchain/json/TxOutputForJson.java index 4cfb81656c..faaf8bf06d 100644 --- a/core/src/main/java/io/bisq/core/dao/blockchain/json/TxOutputForJson.java +++ b/core/src/main/java/io/bisq/core/dao/blockchain/json/TxOutputForJson.java @@ -17,25 +17,27 @@ package io.bisq.core.dao.blockchain.json; -import lombok.Value; +import io.bisq.common.app.Version; +import lombok.Data; +import lombok.Setter; -@Value +@Data public class TxOutputForJson { + private final String txVersion = Version.BSQ_TX_VERSION; private final String txId; private final int outputIndex; private final long bsqAmount; + private final long btcAmount; private final int height; - private final boolean isBsqCoinBase; private final boolean isVerified; private final long burntFee; - private final long btcTxFee; private final ScriptPubKeyForJson scriptPubKey; private final SpentInfoForJson spentInfo; private final long time; - private final String txVersion; + @Setter + private String txType; - public String getSortData() { - return height + txId + outputIndex; + public String getId() { + return txId + ":" + outputIndex; } - } diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/json/TxTypeForJson.java b/core/src/main/java/io/bisq/core/dao/blockchain/json/TxTypeForJson.java new file mode 100644 index 0000000000..a08fa4eb38 --- /dev/null +++ b/core/src/main/java/io/bisq/core/dao/blockchain/json/TxTypeForJson.java @@ -0,0 +1,37 @@ +/* + * This file is part of bisq. + * + * bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with bisq. If not, see . + */ + +package io.bisq.core.dao.blockchain.json; + +import lombok.Getter; + +public enum TxTypeForJson { + UNDEFINED("Undefined"), + GENESIS("Genesis"), + SEND_BSQ("Send BSQ"), + PAY_TRADE_FEE("Pay trade fee"), + COMPENSATION_REQUEST("Compensation request"), + VOTE("Vote"), + ISSUANCE("Issuance"); + + @Getter + private String displayString; + + TxTypeForJson(String displayString) { + this.displayString = displayString; + } +} diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/parse/BsqChainState.java b/core/src/main/java/io/bisq/core/dao/blockchain/parse/BsqChainState.java index 1e4c86a0b1..5ebe810685 100644 --- a/core/src/main/java/io/bisq/core/dao/blockchain/parse/BsqChainState.java +++ b/core/src/main/java/io/bisq/core/dao/blockchain/parse/BsqChainState.java @@ -35,6 +35,8 @@ import lombok.extern.slf4j.Slf4j; import javax.inject.Inject; import javax.inject.Named; import java.io.File; +import java.io.IOException; +import java.io.ObjectInputStream; import java.util.*; import java.util.stream.Collectors; @@ -100,8 +102,9 @@ public class BsqChainState implements Persistable { private final Set> compensationRequestFees = new HashSet<>(); private final Set> votingFees = new HashSet<>(); private final Set compensationRequestOpReturnTxOutputs = new HashSet<>(); - private final Set compensationRequestBtcAddresses = new HashSet<>(); private final Set votingTxOutputs = new HashSet<>(); + private final Set invalidatedTxOutputs = new HashSet<>(); + private final Set compensationRequestBtcAddresses = new HashSet<>(); private final Map> issuanceBtcTxOutputsByBtcAddressMap = new HashMap<>(); private final String genesisTxId; @@ -113,7 +116,7 @@ public class BsqChainState implements Persistable { transient private final boolean dumpBlockchainData; transient private final Storage snapshotBsqChainStateStorage; transient private BsqChainState snapshotCandidate; - transient private final FunctionalReadWriteLock lock; + transient private FunctionalReadWriteLock lock; /////////////////////////////////////////////////////////////////////////////////////////// @@ -144,6 +147,10 @@ public class BsqChainState implements Persistable { lock = new FunctionalReadWriteLock(true); } + private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException { + ois.defaultReadObject(); + lock = new FunctionalReadWriteLock(true); + } /////////////////////////////////////////////////////////////////////////////////////////// // Public write access @@ -263,9 +270,9 @@ public class BsqChainState implements Persistable { }); } - void addCompensationRequestOpReturnOutput(TxOutput opReturnTxOutput) { + void addCompensationRequestOpReturnOutput(TxOutput txOutput) { lock.write(() -> { - compensationRequestOpReturnTxOutputs.add(opReturnTxOutput); + compensationRequestOpReturnTxOutputs.add(txOutput); }); } @@ -275,9 +282,15 @@ public class BsqChainState implements Persistable { }); } - void addVotingOpReturnOutput(TxOutput opReturnTxOutput) { + void addVotingOpReturnOutput(TxOutput txOutput) { lock.write(() -> { - votingTxOutputs.add(opReturnTxOutput); + votingTxOutputs.add(txOutput); + }); + } + + void addInvalidatedTxOutputs(TxOutput txOutput) { + lock.write(() -> { + invalidatedTxOutputs.add(txOutput); }); } @@ -351,6 +364,55 @@ public class BsqChainState implements Persistable { }); } + // Only used for Json Exporter + public Map getTxMap() { + return lock.read(() -> { + return txMap; + }); + } + + public Set getVotingTxOutputs() { + return lock.read(() -> { + return votingTxOutputs; + }); + } + + public Set getInvalidatedTxOutputs() { + return lock.read(() -> { + return invalidatedTxOutputs; + }); + } + + public Map> getIssuanceBtcTxOutputsByBtcAddressMap() { + return lock.read(() -> { + return issuanceBtcTxOutputsByBtcAddressMap; + }); + } + + public Set getCompensationRequestOpReturnTxOutputs() { + return lock.read(() -> { + return compensationRequestOpReturnTxOutputs; + }); + } + + public Map getSpentInfoByTxOutputMap() { + return lock.read(() -> { + return spentInfoByTxOutputMap; + }); + } + + public Map getBurntFeeByTxIdMap() { + return lock.read(() -> { + return burntFeeByTxIdMap; + }); + } + + public Set getUnspentTxOutputSet() { + return lock.read(() -> { + return unspentTxOutputSet; + }); + } + /////////////////////////////////////////////////////////////////////////////////////////// // Package scope read access diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/parse/BsqParser.java b/core/src/main/java/io/bisq/core/dao/blockchain/parse/BsqParser.java index 5eb1f9f746..c03c222eb2 100644 --- a/core/src/main/java/io/bisq/core/dao/blockchain/parse/BsqParser.java +++ b/core/src/main/java/io/bisq/core/dao/blockchain/parse/BsqParser.java @@ -289,7 +289,7 @@ public class BsqParser { if (availableValue >= txOutputValue && txOutputValue != 0) { // We are spending available tokens bsqChainState.addUnspentTxOutput(txOutput); - + availableValue -= txOutputValue; if (availableValue == 0) { log.debug("We don't have anymore BSQ to spend"); diff --git a/core/src/main/java/io/bisq/core/dao/blockchain/vo/TxOutput.java b/core/src/main/java/io/bisq/core/dao/blockchain/vo/TxOutput.java index dea9c42860..e8a4c499c1 100644 --- a/core/src/main/java/io/bisq/core/dao/blockchain/vo/TxOutput.java +++ b/core/src/main/java/io/bisq/core/dao/blockchain/vo/TxOutput.java @@ -39,7 +39,6 @@ public class TxOutput implements Persistable { private final String txId; private final PubKeyScript pubKeyScript; @Nullable - @JsonExclude private final String address; @Nullable @JsonExclude @@ -47,36 +46,10 @@ public class TxOutput implements Persistable { private final int blockHeight; private final long time; -/* public String getAddress() { - String address = ""; - // Only at raw MS outputs addresses have more then 1 entry - // We do not support raw MS for BSQ but lets see if is needed anyway, might be removed - final List addresses = pubKeyScript.getAddresses(); - if (addresses.size() == 1) { - address = addresses.get(0); - } else if (addresses.size() > 1) { - final String msg = "We got a raw Multisig script. That is not supported for BSQ tokens."; - log.warn(msg); - address = addresses.toString(); - if (DevEnv.DEV_MODE) - throw new RuntimeException(msg); - } else { - final String msg = "We got no address. Unsupported pubKeyScript"; - log.warn(msg); - if (DevEnv.DEV_MODE) - throw new RuntimeException(msg); - } - return address; - }*/ - public String getId() { return txId + ":" + index; } - public String getSortString() { - return blockHeight + ":" + txId; - } - public TxIdIndexTuple getTxIdIndexTuple() { return new TxIdIndexTuple(txId, index); } @@ -88,8 +61,9 @@ public class TxOutput implements Persistable { ",\n value=" + value + ",\n txId='" + txId + '\'' + ",\n pubKeyScript=" + pubKeyScript + - ",\n blockHeight=" + blockHeight + + ",\n address='" + address + '\'' + ",\n opReturnData=" + (opReturnData != null ? Utils.HEX.encode(opReturnData) : "null") + + ",\n blockHeight=" + blockHeight + ",\n time=" + time + "\n}"; } diff --git a/gui/src/main/java/io/bisq/gui/app/BisqApp.java b/gui/src/main/java/io/bisq/gui/app/BisqApp.java index 041db7f6fd..bf865dcce9 100644 --- a/gui/src/main/java/io/bisq/gui/app/BisqApp.java +++ b/gui/src/main/java/io/bisq/gui/app/BisqApp.java @@ -37,6 +37,7 @@ import io.bisq.core.app.AppOptionKeys; import io.bisq.core.app.BisqEnvironment; import io.bisq.core.arbitration.ArbitratorManager; import io.bisq.core.btc.wallet.*; +import io.bisq.core.dao.blockchain.json.DaoJsonExporter; import io.bisq.core.filter.FilterManager; import io.bisq.core.offer.OpenOfferManager; import io.bisq.core.trade.TradeManager; @@ -409,6 +410,7 @@ public class BisqApp extends Application { if (injector != null) { injector.getInstance(ArbitratorManager.class).shutDown(); injector.getInstance(TradeManager.class).shutDown(); + injector.getInstance(DaoJsonExporter.class).shutDown(); //noinspection CodeBlock2Expr injector.getInstance(OpenOfferManager.class).shutDown(() -> { injector.getInstance(P2PService.class).shutDown(() -> {