mirrors/bisq
1
0
Fork 0
mirror of https://github.com/bisq-network/bisq.git synced 2024-11-19 18:03:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add new P2P network data structure for AccountAgeWitness without pubKey

Browse Source
This commit is contained in:
Manfred Karrer 2017-10-16 15:52:54 -05:00
parent 0f322ad37f
commit 851792d1f1
No known key found for this signature in database
GPG Key ID: 401250966A6B2C46
34 changed files with 658 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
common/src/main/java/io/bisq/common/locale/CurrencyUtil.java
View File

@ -95,7 +95,7 @@ public class CurrencyUtil {
result.add(new CryptoCurrency("DNET", "DarkNet"));
if (!baseCurrencyCode.equals("DASH"))
result.add(new CryptoCurrency("DASH", "Dash"));
result.add(new CryptoCurrency("DEC", "DECENT"));
result.add(new CryptoCurrency("DCT", "DECENT"));
result.add(new CryptoCurrency("DCR", "Decred"));
if (!baseCurrencyCode.equals("DOGE"))
result.add(new CryptoCurrency("DOGE", "Dogecoin"));

3
common/src/main/java/io/bisq/common/proto/ProtoResolver.java
View File

@ -18,8 +18,11 @@
package io.bisq.common.proto;
import io.bisq.common.Payload;
import io.bisq.common.proto.persistable.PersistableEnvelope;
import io.bisq.generated.protobuffer.PB;
public interface ProtoResolver {
Payload fromProto(PB.PaymentAccountPayload proto);
PersistableEnvelope fromProto(PB.PersistableNetworkPayload proto);
}

24
common/src/main/proto/pb.proto
View File

@ -55,6 +55,8 @@ message NetworkEnvelope {
GetBsqBlocksRequest get_bsq_blocks_request = 28;
GetBsqBlocksResponse get_bsq_blocks_response = 29;
NewBsqBlockBroadcastMessage new_bsq_block_broadcast_message = 30;
AddPersistableNetworkPayloadMessage add_persistable_network_payload_message = 31;
}
}
@ -69,6 +71,7 @@ message PreliminaryGetDataRequest {
int32 nonce = 21;
repeated bytes excluded_keys = 2;
repeated int32 supported_capabilities = 3;
repeated bytes excluded_pnp_keys = 4;
}
message GetDataResponse {
@ -76,12 +79,14 @@ message GetDataResponse {
bool is_get_updated_data_response = 2;
repeated StorageEntryWrapper data_set = 3;
repeated int32 supported_capabilities = 4;
repeated PersistableNetworkPayload persistable_network_payload_items = 5;
}
message GetUpdatedDataRequest {
NodeAddress sender_node_address = 1;
int32 nonce = 2;
repeated bytes excluded_keys = 3;
repeated bytes excluded_pnp_keys = 4;
}
@ -147,6 +152,10 @@ message RemoveMailboxDataMessage {
ProtectedMailboxStorageEntry protected_storage_entry = 1;
}
message AddPersistableNetworkPayloadMessage {
PersistableNetworkPayload payload = 1;
}
// misc
@ -345,6 +354,12 @@ message StoragePayload {
}
}
message PersistableNetworkPayload {
oneof message {
AccountAgeWitness account_age_witness = 1;
}
}
message ProtectedStorageEntry {
StoragePayload storagePayload = 1;
bytes owner_pub_key_bytes = 2;
@ -642,10 +657,9 @@ message CompensationRequestPayload {
message AccountAgeWitness {
bytes hash = 1;
bytes sig_pub_key = 2;
bytes sig_pub_key_hash = 2;
bytes signature = 3;
int64 date = 4;
map<string, string> extra_data = 5;
}
@ -922,7 +936,7 @@ message PersistableEnvelope {
VoteItemsList vote_items_list = 13;
BsqChainState bsq_chain_state = 14;
AccountAgeWitnessMap account_age_witness_map = 15;
PersistableNetworkPayloadList persistable_network_payload_list = 15;
}
}
@ -953,6 +967,10 @@ message PersistedEntryMap {
map<string, ProtectedStorageEntry> persisted_entry_map = 1;
}
message PersistableNetworkPayloadList {
repeated PersistableNetworkPayload items = 1;
}
message PeerList {
repeated Peer peer = 1;
}

17
core/src/main/java/io/bisq/core/app/AppSetupWithP2P.java
View File

@ -30,6 +30,9 @@ import io.bisq.network.p2p.network.ConnectionListener;
import javafx.beans.property.BooleanProperty;
import javafx.beans.property.SimpleBooleanProperty;
import lombok.extern.slf4j.Slf4j;
import org.fxmisc.easybind.EasyBind;
import org.fxmisc.easybind.Subscription;
import org.fxmisc.easybind.monadic.MonadicBinding;
import javax.inject.Inject;
import java.util.ArrayList;
@ -39,6 +42,8 @@ public class AppSetupWithP2P extends AppSetup {
protected final P2PService p2PService;
protected final AccountAgeWitnessService accountAgeWitnessService;
protected BooleanProperty p2pNetWorkReady;
private MonadicBinding<Boolean> readMapsFromResourcesBinding;
private Subscription readMapsFromResourcesBindingSubscription;
@Inject
public AppSetupWithP2P(EncryptionService encryptionService,
@ -57,7 +62,6 @@ public class AppSetupWithP2P extends AppSetup {
public void initPersistedDataHosts() {
ArrayList<PersistedDataHost> persistedDataHosts = new ArrayList<>();
persistedDataHosts.add(tradeStatisticsManager);
persistedDataHosts.add(accountAgeWitnessService);
persistedDataHosts.add(p2PService);
// we apply at startup the reading of persisted data but don't want to get it triggered in the constructor
@ -73,14 +77,19 @@ public class AppSetupWithP2P extends AppSetup {
@Override
protected void initBasicServices() {
BooleanProperty result = SetupUtils.loadEntryMap(p2PService);
result.addListener((observable, oldValue, newValue) -> {
readMapsFromResourcesBinding = EasyBind.combine(SetupUtils.readPersistableNetworkPayloadMapFromResources(p2PService),
SetupUtils.readEntryMapFromResources(p2PService),
(result1, result2) -> {
return result1 && result2;
});
readMapsFromResourcesBindingSubscription = readMapsFromResourcesBinding.subscribe((observable, oldValue, newValue) -> {
if (newValue)
startInitP2PNetwork();
});
}
private void startInitP2PNetwork() {
readMapsFromResourcesBindingSubscription.unsubscribe();
p2pNetWorkReady = initP2PNetwork();
p2pNetWorkReady.addListener((observable, oldValue, newValue) -> {
if (newValue)
@ -165,7 +174,7 @@ public class AppSetupWithP2P extends AppSetup {
p2PService.onAllServicesInitialized();
tradeStatisticsManager.onAllServicesInitialized();
accountAgeWitnessService.onAllServicesInitialized();
}
}

27
core/src/main/java/io/bisq/core/app/SetupUtils.java
View File

@ -79,12 +79,12 @@ public class SetupUtils {
checkCryptoThread.start();
}
public static BooleanProperty loadEntryMap(P2PService p2PService) {
public static BooleanProperty readEntryMapFromResources(P2PService p2PService) {
BooleanProperty result = new SimpleBooleanProperty();
Thread loadEntryMapThread = new Thread() {
Thread thread = new Thread() {
@Override
public void run() {
Thread.currentThread().setName("loadEntryMapThread");
Thread.currentThread().setName("readEntryMapFromResources");
// Used to load different EntryMap files per base currency (EntryMap_BTC_MAINNET, EntryMap_LTC,...)
final BaseCurrencyNetwork baseCurrencyNetwork = BisqEnvironment.getBaseCurrencyNetwork();
final String storageFileName = "EntryMap_"
@ -94,7 +94,26 @@ public class SetupUtils {
UserThread.execute(() -> result.set(true));
}
};
loadEntryMapThread.start();
thread.start();
return result;
}
public static BooleanProperty readPersistableNetworkPayloadMapFromResources(P2PService p2PService) {
BooleanProperty result = new SimpleBooleanProperty();
Thread thread = new Thread() {
@Override
public void run() {
Thread.currentThread().setName("readPersistableNetworkPayloadMapFromResources");
// Used to load different EntryMap files per base currency (EntryMap_BTC_MAINNET, EntryMap_LTC,...)
final BaseCurrencyNetwork baseCurrencyNetwork = BisqEnvironment.getBaseCurrencyNetwork();
final String storageFileName = "PersistableNetworkPayloadMap_"
+ baseCurrencyNetwork.getCurrencyCode() + "_"
+ baseCurrencyNetwork.getNetwork();
p2PService.readPersistableNetworkPayloadMapFromResources(storageFileName);
UserThread.execute(() -> result.set(true));
}
};
thread.start();
return result;
}
}

8
core/src/main/java/io/bisq/core/dao/compensation/CompensationRequestPayload.java
View File

@ -20,12 +20,13 @@ package io.bisq.core.dao.compensation;
import io.bisq.common.app.Version;
import io.bisq.common.crypto.Sig;
import io.bisq.common.proto.persistable.PersistableEnvelope;
import io.bisq.common.proto.persistable.PersistablePayload;
import io.bisq.common.util.JsonExclude;
import io.bisq.common.util.Utilities;
import io.bisq.generated.protobuffer.PB;
import io.bisq.network.p2p.NodeAddress;
import io.bisq.network.p2p.storage.payload.LazyProcessedStoragePayload;
import io.bisq.network.p2p.storage.payload.PersistableNetworkPayload;
import io.bisq.network.p2p.storage.payload.LazyProcessedPayload;
import io.bisq.network.p2p.storage.payload.ProtectedStoragePayload;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.bitcoinj.core.Coin;
@ -41,8 +42,7 @@ import java.util.concurrent.TimeUnit;
@Slf4j
@Data
// TODO There will be another object for PersistableEnvelope
public final class CompensationRequestPayload implements LazyProcessedStoragePayload, PersistableNetworkPayload, PersistableEnvelope {
public final class CompensationRequestPayload implements LazyProcessedPayload, ProtectedStoragePayload, PersistablePayload, PersistableEnvelope {
private final String uid;
private final String name;
private final String title;

87
core/src/main/java/io/bisq/core/payment/AccountAgeWitness.java
View File

@ -18,57 +18,40 @@
package io.bisq.core.payment;
import com.google.protobuf.ByteString;
import io.bisq.common.crypto.Sig;
import io.bisq.common.proto.persistable.PersistableEnvelope;
import io.bisq.common.util.Utilities;
import io.bisq.generated.protobuffer.PB;
import io.bisq.network.p2p.storage.payload.LazyProcessedStoragePayload;
import io.bisq.network.p2p.storage.P2PDataStorage;
import io.bisq.network.p2p.storage.payload.LazyProcessedPayload;
import io.bisq.network.p2p.storage.payload.PersistableNetworkPayload;
import lombok.EqualsAndHashCode;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import javax.annotation.Nullable;
import java.security.PublicKey;
import java.util.Date;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
// Object has 529 raw bytes (535 bytes is size of PB.AccountAgeWitness -> extraDataMap is null)
// Object has about 94 raw bytes (about 101 bytes is size of PB object)
// With 100 000 entries we get 53.5 MB of data. Old entries will be shipped with the MapEntry resource file,
// so only the newly added objects since the last release will not be loaded over the P2P network.
// TODO Get rid of sigPubKey and replace by hash of sigPubKey. That will reduce the data size to 118 bytes.
// Using EC signatures would produce longer signatures (71 bytes)
@Slf4j
@EqualsAndHashCode(exclude = {"signaturePubKey"})
@Value
public class AccountAgeWitness implements LazyProcessedStoragePayload, PersistableNetworkPayload {
public class AccountAgeWitness implements LazyProcessedPayload, PersistableNetworkPayload, PersistableEnvelope {
private final byte[] hash; // Ripemd160(Sha256(data)) hash 20 bytes
private final byte[] sigPubKey; // about 443 bytes -> 20
private final byte[] signature; // 46 bytes ->
private final byte[] sigPubKeyHash; // Ripemd160(Sha256(sigPubKey)) hash 20 bytes
private final byte[] signature; // about 46 bytes
private final long date; // 8 byte
// Only used as cache for getOwnerPubKey
transient private final PublicKey signaturePubKey;
// Should be only used in emergency case if we need to add data but do not want to break backward compatibility
// at the P2P network storage checks. The hash of the object will be used to verify if the data is valid. Any new
// field in a class would break that hash and therefore break the storage mechanism.
@Nullable
private Map<String, String> extraDataMap;
public AccountAgeWitness(byte[] hash,
byte[] sigPubKey,
byte[] sigPubKeyHash,
byte[] signature) {
this(hash,
sigPubKey,
sigPubKeyHash,
signature,
new Date().getTime()/* - TimeUnit.DAYS.toMillis(0)*/, //for testing
null);
new Date().getTime());
}
///////////////////////////////////////////////////////////////////////////////////////////
@ -76,30 +59,23 @@ public class AccountAgeWitness implements LazyProcessedStoragePayload, Persistab
///////////////////////////////////////////////////////////////////////////////////////////
private AccountAgeWitness(byte[] hash,
byte[] sigPubKey,
byte[] sigPubKeyHash,
byte[] signature,
long date,
@Nullable Map<String, String> extraDataMap) {
long date) {
this.hash = hash;
this.sigPubKey = sigPubKey;
this.sigPubKeyHash = sigPubKeyHash;
this.signature = signature;
this.date = date;
this.extraDataMap = extraDataMap;
log.debug("Date=" + new Date(date));
signaturePubKey = Sig.getPublicKeyFromBytes(sigPubKey);
this.date = date/* - TimeUnit.DAYS.toMillis(90)*/;
}
@Override
public PB.StoragePayload toProtoMessage() {
public PB.PersistableNetworkPayload toProtoMessage() {
final PB.AccountAgeWitness.Builder builder = PB.AccountAgeWitness.newBuilder()
.setHash(ByteString.copyFrom(hash))
.setSigPubKey(ByteString.copyFrom(sigPubKey))
.setSigPubKeyHash(ByteString.copyFrom(sigPubKeyHash))
.setSignature(ByteString.copyFrom(signature))
.setDate(date);
Optional.ofNullable(extraDataMap).ifPresent(builder::putAllExtraData);
return PB.StoragePayload.newBuilder().setAccountAgeWitness(builder).build();
return PB.PersistableNetworkPayload.newBuilder().setAccountAgeWitness(builder).build();
}
public PB.AccountAgeWitness toProtoAccountAgeWitness() {
@ -109,10 +85,9 @@ public class AccountAgeWitness implements LazyProcessedStoragePayload, Persistab
public static AccountAgeWitness fromProto(PB.AccountAgeWitness proto) {
return new AccountAgeWitness(
proto.getHash().toByteArray(),
proto.getSigPubKey().toByteArray(),
proto.getSigPubKeyHash().toByteArray(),
proto.getSignature().toByteArray(),
proto.getDate(),
CollectionUtils.isEmpty(proto.getExtraDataMap()) ? null : proto.getExtraDataMap());
proto.getDate());
}
@ -120,16 +95,6 @@ public class AccountAgeWitness implements LazyProcessedStoragePayload, Persistab
// Getters
///////////////////////////////////////////////////////////////////////////////////////////
//TODO needed?
@Override
public long getTTL() {
return TimeUnit.SECONDS.toMillis(1);
}
@Override
public PublicKey getOwnerPubKey() {
return signaturePubKey;
}
//TODO impl. here or in caller?
// We allow max 1 day time difference
@ -137,7 +102,17 @@ public class AccountAgeWitness implements LazyProcessedStoragePayload, Persistab
return new Date().getTime() - date < TimeUnit.DAYS.toMillis(1);
}
public String getHashAsHex() {
return Utilities.encodeToHex(hash);
public P2PDataStorage.ByteArray getHashAsByteArray() {
return new P2PDataStorage.ByteArray(hash);
}
@Override
public String toString() {
return "AccountAgeWitness{" +
"\n hash=" + Utilities.bytesAsHexString(hash) +
",\n sigPubKeyHash=" + Utilities.bytesAsHexString(sigPubKeyHash) +
",\n signature=" + Utilities.bytesAsHexString(signature) +
",\n date=" + new Date(date) +
"\n}";
}
}

56
core/src/main/java/io/bisq/core/payment/AccountAgeWitnessMap.java
View File

@ -1,56 +0,0 @@
/*
* This file is part of Bisq.
*
* Bisq is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* Bisq is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Bisq. If not, see <http://www.gnu.org/licenses/>.
*/
package io.bisq.core.payment;
import com.google.protobuf.Message;
import io.bisq.common.proto.persistable.PersistableEnvelope;
import io.bisq.common.proto.persistable.PersistableHashMap;
import io.bisq.generated.protobuffer.PB;
import java.util.Map;
import java.util.stream.Collectors;
// Key in PB map cannot be byte array, so we use a hex string of the bytes array
public class AccountAgeWitnessMap extends PersistableHashMap<String, AccountAgeWitness> {
public AccountAgeWitnessMap(Map<String, AccountAgeWitness> map) {
super(map);
}
@Override
public Message toProtoMessage() {
Map<String, PB.AccountAgeWitness> protoMap = getMap().entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> {
return e.getValue().toProtoAccountAgeWitness();
}));
final PB.AccountAgeWitnessMap.Builder builder = PB.AccountAgeWitnessMap.newBuilder();
builder.putAllAccountAgeWitnessMap(protoMap);
return PB.PersistableEnvelope.newBuilder()
.setAccountAgeWitnessMap(builder).build();
}
public static PersistableEnvelope fromProto(PB.AccountAgeWitnessMap proto) {
Map<String, AccountAgeWitness> map = proto.getAccountAgeWitnessMapMap().entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> {
return AccountAgeWitness.fromProto(e.getValue());
}));
return new AccountAgeWitnessMap(map);
}
}

105
core/src/main/java/io/bisq/core/payment/AccountAgeWitnessService.java
View File

@ -22,16 +22,12 @@ import io.bisq.common.crypto.Hash;
import io.bisq.common.crypto.KeyRing;
import io.bisq.common.crypto.Sig;
import io.bisq.common.locale.CurrencyUtil;
import io.bisq.common.proto.persistable.PersistedDataHost;
import io.bisq.common.storage.Storage;
import io.bisq.common.util.MathUtils;
import io.bisq.common.util.Utilities;
import io.bisq.core.offer.Offer;
import io.bisq.core.payment.payload.PaymentAccountPayload;
import io.bisq.network.p2p.P2PService;
import io.bisq.network.p2p.storage.HashMapChangedListener;
import io.bisq.network.p2p.storage.payload.ProtectedStorageEntry;
import io.bisq.network.p2p.storage.payload.ProtectedStoragePayload;
import io.bisq.network.p2p.storage.P2PDataStorage;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
@ -42,7 +38,7 @@ import java.util.*;
import java.util.concurrent.TimeUnit;
@Slf4j
public class AccountAgeWitnessService implements PersistedDataHost {
public class AccountAgeWitnessService {
public enum AccountAge {
LESS_ONE_MONTH,
@ -51,73 +47,48 @@ public class AccountAgeWitnessService implements PersistedDataHost {
}
private final KeyRing keyRing;
private final Storage<AccountAgeWitnessMap> storage;
private final P2PService p2PService;
private final Map<String, AccountAgeWitness> accountAgeWitnessMap = new HashMap<>();
private final Map<P2PDataStorage.ByteArray, AccountAgeWitness> accountAgeWitnessMap = new HashMap<>();
@Inject
public AccountAgeWitnessService(KeyRing keyRing, Storage<AccountAgeWitnessMap> storage, P2PService p2PService) {
public AccountAgeWitnessService(KeyRing keyRing, P2PService p2PService) {
this.keyRing = keyRing;
this.storage = storage;
this.p2PService = p2PService;
}
@Override
public void readPersisted() {
//TODO remove?
AccountAgeWitnessMap persisted = storage.initAndGetPersistedWithFileName("AccountAgeWitnessMap");
//if (persisted != null)
// accountAgeWitnessMap = persisted.getMap();
// else
}
public void onAllServicesInitialized() {
p2PService.addHashSetChangedListener(new HashMapChangedListener() {
@Override
public void onAdded(ProtectedStorageEntry data) {
final ProtectedStoragePayload protectedStoragePayload = data.getProtectedStoragePayload();
if (protectedStoragePayload instanceof AccountAgeWitness) {
add((AccountAgeWitness) protectedStoragePayload, true);
}
}
@Override
public void onRemoved(ProtectedStorageEntry data) {
// We don't remove items
}
p2PService.getP2PDataStorage().addPersistableNetworkPayloadMapListener(payload -> {
if (payload instanceof AccountAgeWitness)
addToMap((AccountAgeWitness) payload);
});
// At startup the P2PDataStorage initializes earlier, otherwise we ge the listener called.
final List<ProtectedStorageEntry> list = new ArrayList<>(p2PService.getDataMap().values());
list.forEach(e -> {
final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload();
if (protectedStoragePayload instanceof AccountAgeWitness)
add((AccountAgeWitness) protectedStoragePayload, false);
p2PService.getP2PDataStorage().getPersistableNetworkPayloadCollection().getMap().entrySet().forEach(e -> {
if (e.getValue() instanceof AccountAgeWitness)
addToMap((AccountAgeWitness) e.getValue());
});
}
private void add(AccountAgeWitness accountAgeWitness, boolean storeLocally) {
log.debug("add (accountAgeWitnessMap.put) hash=" + accountAgeWitness.getHashAsHex());
accountAgeWitnessMap.put(accountAgeWitness.getHashAsHex(), accountAgeWitness);
//TODO do we need to store it? its in EntryMap anyway...
// if (storeLocally)
// storage.queueUpForSave(new AccountAgeWitnessMap(accountAgeWitnessMap), 2000);
private void addToMap(AccountAgeWitness accountAgeWitness) {
log.debug("addToMap hash=" + Utilities.bytesAsHexString(accountAgeWitness.getHash()));
if (!accountAgeWitnessMap.containsKey(accountAgeWitness.getHashAsByteArray()))
accountAgeWitnessMap.put(accountAgeWitness.getHashAsByteArray(), accountAgeWitness);
}
public void publishAccountAgeWitness(PaymentAccountPayload paymentAccountPayload) {
try {
AccountAgeWitness accountAgeWitness = getAccountAgeWitness(paymentAccountPayload);
if (!accountAgeWitnessMap.containsKey(accountAgeWitness.getHashAsHex()))
p2PService.addData(accountAgeWitness, true);
if (!accountAgeWitnessMap.containsKey(accountAgeWitness.getHashAsByteArray()))
p2PService.addPersistableNetworkPayload(accountAgeWitness);
} catch (CryptoException e) {
e.printStackTrace();
log.error(e.toString());
}
}
public Optional<AccountAgeWitness> getWitnessByHash(String hash) {
return accountAgeWitnessMap.containsKey(hash) ? Optional.of(accountAgeWitnessMap.get(hash)) : Optional.<AccountAgeWitness>empty();
public Optional<AccountAgeWitness> getWitnessByHash(String hashAsHex) {
P2PDataStorage.ByteArray hashAsByteArray = new P2PDataStorage.ByteArray(Utilities.decodeFromHex(hashAsHex));
return accountAgeWitnessMap.containsKey(hashAsByteArray) ? Optional.of(accountAgeWitnessMap.get(hashAsByteArray)) : Optional.<AccountAgeWitness>empty();
}
public Optional<AccountAgeWitness> getWitnessByPaymentAccountPayload(PaymentAccountPayload paymentAccountPayload) {
@ -154,9 +125,9 @@ public class AccountAgeWitnessService implements PersistedDataHost {
private AccountAgeWitness getAccountAgeWitness(PaymentAccountPayload paymentAccountPayload) throws CryptoException {
byte[] hash = getWitnessHash(paymentAccountPayload);
byte[] signature = Sig.sign(keyRing.getSignatureKeyPair().getPrivate(), hash);
byte[] sigPubKey = keyRing.getPubKeyRing().getSignaturePubKeyBytes();
byte[] sigPubKeyHash = Hash.getSha256Ripemd160hash(keyRing.getPubKeyRing().getSignaturePubKeyBytes());
return new AccountAgeWitness(hash,
sigPubKey,
sigPubKeyHash,
signature);
}
@ -169,11 +140,16 @@ public class AccountAgeWitnessService implements PersistedDataHost {
}
private byte[] getWitnessHash(PaymentAccountPayload paymentAccountPayload, byte[] salt) {
salt = new byte[]{};
byte[] ageWitnessInputData = paymentAccountPayload.getAgeWitnessInputData();
final byte[] combined = ArrayUtils.addAll(ageWitnessInputData, salt);
final byte[] hash = Hash.getSha256Ripemd160hash(combined);
final byte[] hash = Hash.getSha256Ripemd160hash(combined);
log.debug("getWitnessHash paymentAccountPayload={}, salt={}, ageWitnessInputData={}, combined={}, hash={}",
paymentAccountPayload.getPaymentDetails(), Utilities.encodeToHex(salt), Utilities.encodeToHex(ageWitnessInputData), Utilities.encodeToHex(combined), Utilities.encodeToHex(hash));
paymentAccountPayload.getPaymentDetails(),
Utilities.encodeToHex(salt),
Utilities.encodeToHex(ageWitnessInputData),
Utilities.encodeToHex(combined),
Utilities.encodeToHex(hash));
return hash;
}
@ -191,16 +167,16 @@ public class AccountAgeWitnessService implements PersistedDataHost {
// We want to fade in the limit over 2 months to avoid that all users get limited to 25% of the limit when
// we deploy that feature.
final Date now = new Date();
final Date dez = new GregorianCalendar(2017, GregorianCalendar.DECEMBER, 1).getTime();
/* final Date dez = new GregorianCalendar(2017, GregorianCalendar.DECEMBER, 1).getTime();
final Date jan = new GregorianCalendar(2017, GregorianCalendar.JANUARY, 1).getTime();
final Date feb = new GregorianCalendar(2017, GregorianCalendar.FEBRUARY, 1).getTime();
*/
// testing
/*
final Date dez = new GregorianCalendar(2016, GregorianCalendar.DECEMBER, 1).getTime();
final Date dez = new GregorianCalendar(2016, GregorianCalendar.DECEMBER, 1).getTime();
final Date jan = new GregorianCalendar(2016, GregorianCalendar.JANUARY, 1).getTime();
final Date feb = new GregorianCalendar(2016, GregorianCalendar.FEBRUARY, 1).getTime();
*/
switch (accountAgeCategory) {
case TWO_MONTHS_OR_MORE:
factor = 1;
@ -250,7 +226,7 @@ public class AccountAgeWitnessService implements PersistedDataHost {
// Check if peer's pubkey is matching the one from the witness data
if (!verifySigPubKey(witness.getSigPubKey(), peersPublicKey))
if (!verifySigPubKeyHash(witness.getSigPubKeyHash(), peersPublicKey))
return false;
final byte[] combined = ArrayUtils.addAll(peersAgeWitnessInputData, peersSalt);
@ -279,12 +255,15 @@ public class AccountAgeWitnessService implements PersistedDataHost {
return result;
}
boolean verifySigPubKey(byte[] sigPubKey,
PublicKey peersPublicKey) {
final boolean result = Arrays.equals(Sig.getPublicKeyBytes(peersPublicKey), sigPubKey);
boolean verifySigPubKeyHash(byte[] sigPubKeyHash,
PublicKey peersPublicKey) {
final byte[] peersPublicKeyHash = Hash.getSha256Ripemd160hash(Sig.getPublicKeyBytes(peersPublicKey));
final boolean result = Arrays.equals(peersPublicKeyHash, sigPubKeyHash);
if (!result)
log.warn("sigPubKey is not matching peers peersPublicKey. " +
"sigPubKey={}, peersPublicKey={}", Utilities.bytesAsHexString(sigPubKey), peersPublicKey);
log.warn("sigPubKeyHash is not matching peers peersPublicKey. " +
"sigPubKeyHash={}, peersPublicKeyHash={}",
Utilities.bytesAsHexString(sigPubKeyHash),
peersPublicKeyHash);
return result;
}

8
core/src/main/java/io/bisq/core/payment/PaymentAccountUtil.java
View File

@ -1,6 +1,7 @@
package io.bisq.core.payment;
import io.bisq.common.locale.TradeCurrency;
import io.bisq.common.util.Utilities;
import io.bisq.core.offer.Offer;
import io.bisq.core.payment.payload.PaymentMethod;
import javafx.collections.FXCollections;
@ -109,11 +110,11 @@ public class PaymentAccountUtil {
.filter(paymentAccount -> isPaymentAccountValidForOffer(offer, paymentAccount))
.sorted((o1, o2) -> {
final Optional<AccountAgeWitness> witness1 = accountAgeWitnessService.getWitnessByPaymentAccountPayload(o1.getPaymentAccountPayload());
log.debug("witness1 isPresent={}, HashAsHex={}, date={}", witness1.isPresent(), witness1.get().getHashAsHex(), new Date(witness1.get().getDate()));
log.debug("witness1 isPresent={}, HashAsHex={}, date={}", witness1.isPresent(), Utilities.bytesAsHexString(witness1.get().getHash()), new Date(witness1.get().getDate()));
long age1 = witness1.isPresent() ? accountAgeWitnessService.getAccountAge(witness1.get()) : 0;
final Optional<AccountAgeWitness> witness2 = accountAgeWitnessService.getWitnessByPaymentAccountPayload(o2.getPaymentAccountPayload());
log.debug("witness2 isPresent={}, HashAsHex={}, date={}", witness2.isPresent(), witness2.get().getHashAsHex(), new Date(witness2.get().getDate()));
log.debug("witness2 isPresent={}, HashAsHex={}, date={}", witness2.isPresent(), Utilities.bytesAsHexString(witness2.get().getHash()), new Date(witness2.get().getDate()));
long age2 = witness2.isPresent() ? accountAgeWitnessService.getAccountAge(witness2.get()) : 0;
log.debug("AccountName 1 " + o1.getAccountName());
log.debug("AccountName 2 " + o2.getAccountName());
@ -125,7 +126,8 @@ public class PaymentAccountUtil {
}).collect(Collectors.toList());
list.stream().forEach(e -> log.error("getMostMaturePaymentAccountForOffer AccountName={}, witnessHashAsHex={}", e.getAccountName(), accountAgeWitnessService.getWitnessHashAsHex(e.getPaymentAccountPayload())));
final Optional<PaymentAccount> first = list.stream().findFirst();
log.debug("first={}", first.get().getAccountName());
if (first.isPresent())
log.debug("first={}", first.get().getAccountName());
return first;
}
}

17
core/src/main/java/io/bisq/core/proto/CoreProtoResolver.java
View File

@ -2,6 +2,8 @@ package io.bisq.core.proto;
import io.bisq.common.proto.ProtoResolver;
import io.bisq.common.proto.ProtobufferException;
import io.bisq.common.proto.persistable.PersistableEnvelope;
import io.bisq.core.payment.AccountAgeWitness;
import io.bisq.core.payment.payload.*;
import io.bisq.generated.protobuffer.PB;
import lombok.extern.slf4j.Slf4j;
@ -64,4 +66,19 @@ public class CoreProtoResolver implements ProtoResolver {
throw new ProtobufferException("PB.PaymentAccountPayload is null");
}
}
@Override
public PersistableEnvelope fromProto(PB.PersistableNetworkPayload proto) {
if (proto != null) {
switch (proto.getMessageCase()) {
case ACCOUNT_AGE_WITNESS:
return AccountAgeWitness.fromProto(proto.getAccountAgeWitness());
default:
throw new ProtobufferException("Unknown proto message case (PB.PersistableNetworkPayload). messageCase=" + proto.getMessageCase());
}
} else {
log.error("PB.PersistableNetworkPayload is null");
throw new ProtobufferException("PB.PersistableNetworkPayload is null");
}
}
}

9
core/src/main/java/io/bisq/core/proto/network/CoreNetworkProtoResolver.java
View File

@ -31,10 +31,7 @@ import io.bisq.network.p2p.peers.keepalive.messages.Ping;
import io.bisq.network.p2p.peers.keepalive.messages.Pong;
import io.bisq.network.p2p.peers.peerexchange.messages.GetPeersRequest;
import io.bisq.network.p2p.peers.peerexchange.messages.GetPeersResponse;
import io.bisq.network.p2p.storage.messages.AddDataMessage;
import io.bisq.network.p2p.storage.messages.RefreshOfferMessage;
import io.bisq.network.p2p.storage.messages.RemoveDataMessage;
import io.bisq.network.p2p.storage.messages.RemoveMailboxDataMessage;
import io.bisq.network.p2p.storage.messages.*;
import io.bisq.network.p2p.storage.payload.MailboxStoragePayload;
import io.bisq.network.p2p.storage.payload.ProtectedMailboxStorageEntry;
import io.bisq.network.p2p.storage.payload.ProtectedStorageEntry;
@ -120,7 +117,9 @@ public class CoreNetworkProtoResolver extends CoreProtoResolver implements Netwo
return GetBsqBlocksResponse.fromProto(proto.getGetBsqBlocksResponse(), messageVersion);
case NEW_BSQ_BLOCK_BROADCAST_MESSAGE:
return NewBsqBlockBroadcastMessage.fromProto(proto.getNewBsqBlockBroadcastMessage(), messageVersion);
case ADD_PERSISTABLE_NETWORK_PAYLOAD_MESSAGE:
return AddPersistableNetworkPayloadMessage.fromProto(proto.getAddPersistableNetworkPayloadMessage(), this, messageVersion);
default:
throw new ProtobufferException("Unknown proto message case (PB.NetworkEnvelope). messageCase=" + proto.getMessageCase());
}

6
core/src/main/java/io/bisq/core/proto/persistable/CorePersistenceProtoResolver.java
View File

@ -13,7 +13,6 @@ import io.bisq.core.btc.wallet.BtcWalletService;
import io.bisq.core.dao.blockchain.parse.BsqChainState;
import io.bisq.core.dao.compensation.CompensationRequestPayload;
import io.bisq.core.dao.vote.VoteItemsList;
import io.bisq.core.payment.AccountAgeWitnessMap;
import io.bisq.core.payment.PaymentAccountList;
import io.bisq.core.proto.CoreProtoResolver;
import io.bisq.core.trade.TradableList;
@ -23,6 +22,7 @@ import io.bisq.core.user.UserPayload;
import io.bisq.generated.protobuffer.PB;
import io.bisq.network.p2p.peers.peerexchange.PeerList;
import io.bisq.network.p2p.storage.PersistableEntryMap;
import io.bisq.network.p2p.storage.PersistableNetworkPayloadCollection;
import io.bisq.network.p2p.storage.SequenceNumberMap;
import lombok.extern.slf4j.Slf4j;
@ -66,8 +66,6 @@ public class CorePersistenceProtoResolver extends CoreProtoResolver implements P
btcWalletService.get());
case TRADE_STATISTICS_LIST:
return TradeStatisticsList.fromProto(proto.getTradeStatisticsList());
case ACCOUNT_AGE_WITNESS_MAP:
return AccountAgeWitnessMap.fromProto(proto.getAccountAgeWitnessMap());
case DISPUTE_LIST:
return DisputeList.fromProto(proto.getDisputeList(),
this,
@ -86,6 +84,8 @@ public class CorePersistenceProtoResolver extends CoreProtoResolver implements P
return VoteItemsList.fromProto(proto.getVoteItemsList());
case BSQ_CHAIN_STATE:
return BsqChainState.fromProto(proto.getBsqChainState());
case PERSISTABLE_NETWORK_PAYLOAD_LIST:
return PersistableNetworkPayloadCollection.fromProto(proto.getPersistableNetworkPayloadList(), this);
default:
throw new ProtobufferException("Unknown proto message case(PB.PersistableEnvelope). messageCase=" + proto.getMessageCase());
}

7
core/src/main/java/io/bisq/core/trade/statistics/TradeStatistics.java
View File

@ -7,11 +7,12 @@ import io.bisq.common.monetary.Altcoin;
import io.bisq.common.monetary.AltcoinExchangeRate;
import io.bisq.common.monetary.Price;
import io.bisq.common.monetary.Volume;
import io.bisq.common.proto.persistable.PersistablePayload;
import io.bisq.common.util.JsonExclude;
import io.bisq.core.offer.OfferPayload;
import io.bisq.generated.protobuffer.PB;
import io.bisq.network.p2p.storage.payload.LazyProcessedStoragePayload;
import io.bisq.network.p2p.storage.payload.PersistableNetworkPayload;
import io.bisq.network.p2p.storage.payload.LazyProcessedPayload;
import io.bisq.network.p2p.storage.payload.ProtectedStoragePayload;
import lombok.EqualsAndHashCode;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
@ -30,7 +31,7 @@ import java.util.concurrent.TimeUnit;
@Slf4j
@EqualsAndHashCode(exclude = {"signaturePubKeyBytes", "signaturePubKey"})
@Value
public final class TradeStatistics implements LazyProcessedStoragePayload, PersistableNetworkPayload {
public final class TradeStatistics implements LazyProcessedPayload, ProtectedStoragePayload, PersistablePayload {
private final OfferPayload.Direction direction;
private final String baseCurrency;
private final String counterCurrency;

10
core/src/test/java/io/bisq/core/payment/AccountAgeWitnessServiceTest.java
View File

@ -48,7 +48,7 @@ public class AccountAgeWitnessServiceTest {
@Before
public void setup() throws CertificateException, NoSuchAlgorithmException, KeyStoreException, IOException, CryptoException {
service = new AccountAgeWitnessService(null, null, null);
service = new AccountAgeWitnessService(null, null);
keypair = Sig.generateKeyPair();
publicKey = keypair.getPublic();
}
@ -74,10 +74,10 @@ public class AccountAgeWitnessServiceTest {
@Test
public void testVerifySigPubKey() {
byte[] sigPubKey = Sig.getPublicKeyBytes(publicKey);
assertFalse(service.verifySigPubKey(new byte[0], publicKey));
assertFalse(service.verifySigPubKey(new byte[1], publicKey));
assertTrue(service.verifySigPubKey(sigPubKey, publicKey));
byte[] sigPubKeHash = Hash.getSha256Ripemd160hash(Sig.getPublicKeyBytes(publicKey));
assertFalse(service.verifySigPubKeyHash(new byte[0], publicKey));
assertFalse(service.verifySigPubKeyHash(new byte[1], publicKey));
assertTrue(service.verifySigPubKeyHash(sigPubKeHash, publicKey));
}
@Test

2
gui/src/main/java/io/bisq/gui/app/BisqApp.java
View File

@ -47,7 +47,6 @@ import io.bisq.core.dao.compensation.CompensationRequestManager;
import io.bisq.core.dao.vote.VotingManager;
import io.bisq.core.filter.FilterManager;
import io.bisq.core.offer.OpenOfferManager;
import io.bisq.core.payment.AccountAgeWitnessService;
import io.bisq.core.trade.TradeManager;
import io.bisq.core.trade.closed.ClosedTradableManager;
import io.bisq.core.trade.failed.FailedTradesManager;
@ -190,7 +189,6 @@ public class BisqApp extends Application {
persistedDataHosts.add(injector.getInstance(Navigation.class));
persistedDataHosts.add(injector.getInstance(AddressEntryList.class));
persistedDataHosts.add(injector.getInstance(TradeStatisticsManager.class));
persistedDataHosts.add(injector.getInstance(AccountAgeWitnessService.class));
persistedDataHosts.add(injector.getInstance(OpenOfferManager.class));
persistedDataHosts.add(injector.getInstance(TradeManager.class));
persistedDataHosts.add(injector.getInstance(ClosedTradableManager.class));

20
gui/src/main/java/io/bisq/gui/main/MainViewModel.java
View File

@ -166,6 +166,7 @@ public class MainViewModel implements ViewModel {
final StringProperty p2PNetworkInfo = new SimpleStringProperty();
@SuppressWarnings("FieldCanBeLocal")
private MonadicBinding<String> p2PNetworkInfoBinding;
private MonadicBinding<Boolean> readMapsFromResourcesBinding;
final BooleanProperty splashP2PNetworkAnimationVisible = new SimpleBooleanProperty(true);
final StringProperty p2pNetworkWarnMsg = new SimpleStringProperty();
final StringProperty p2PNetworkIconId = new SimpleStringProperty();
@ -196,6 +197,7 @@ public class MainViewModel implements ViewModel {
private BooleanProperty p2pNetWorkReady;
private final BooleanProperty walletInitialized = new SimpleBooleanProperty();
private boolean allBasicServicesInitialized;
private Subscription readMapsFromResourcesBindingSubscription;
///////////////////////////////////////////////////////////////////////////////////////////
@ -276,11 +278,15 @@ public class MainViewModel implements ViewModel {
}
}
private void startLoadEntryMap() {
log.info("startLoadEntryMap");
private void readMapsFromResources() {
log.info("readMapsFromResources");
BooleanProperty result = SetupUtils.loadEntryMap(p2PService);
result.addListener((observable, oldValue, newValue) -> {
readMapsFromResourcesBinding = EasyBind.combine(SetupUtils.readPersistableNetworkPayloadMapFromResources(p2PService),
SetupUtils.readEntryMapFromResources(p2PService),
(result1, result2) -> {
return result1 && result2;
});
readMapsFromResourcesBindingSubscription = readMapsFromResourcesBinding.subscribe((observable, oldValue, newValue) -> {
if (newValue)
startBasicServices();
});
@ -292,6 +298,8 @@ public class MainViewModel implements ViewModel {
private void startBasicServices() {
log.info("startBasicServices");
readMapsFromResourcesBindingSubscription.unsubscribe();
ChangeListener<Boolean> walletInitializedListener = (observable, oldValue, newValue) -> {
if (newValue && !p2pNetWorkReady.get())
showStartupTimeoutPopup();
@ -702,11 +710,11 @@ public class MainViewModel implements ViewModel {
log.info("Localhost peer detected.");
UserThread.execute(() -> {
bisqEnvironment.setBitcoinLocalhostNodeRunning(true);
startLoadEntryMap();
readMapsFromResources();
});
} catch (Throwable e) {
log.info("Localhost peer not detected.");
UserThread.execute(MainViewModel.this::startLoadEntryMap);
UserThread.execute(MainViewModel.this::readMapsFromResources);
} finally {
if (socket != null) {
try {

10
network/src/main/java/io/bisq/network/crypto/EncryptionService.java
View File

@ -20,7 +20,6 @@ package io.bisq.network.crypto;
import com.google.protobuf.InvalidProtocolBufferException;
import io.bisq.common.crypto.*;
import io.bisq.common.proto.network.NetworkEnvelope;
import io.bisq.common.proto.network.NetworkPayload;
import io.bisq.common.proto.network.NetworkProtoResolver;
import io.bisq.generated.protobuffer.PB;
import io.bisq.network.p2p.DecryptedMessageWithPubKey;
@ -111,14 +110,5 @@ public class EncryptionService {
// Pack all together
return new SealedAndSigned(encryptedSecretKey, encryptedPayloadWithHmac, signature, signatureKeyPair.getPublic());
}
/**
* @param data
* @return Hash of data
*/
public static byte[] getHash(NetworkPayload data) {
return Hash.getSha256Hash(data.toProtoMessage().toByteArray());
}
}

17
network/src/main/java/io/bisq/network/p2p/P2PService.java
View File

@ -37,10 +37,7 @@ import io.bisq.network.p2p.storage.P2PDataStorage;
import io.bisq.network.p2p.storage.messages.AddDataMessage;
import io.bisq.network.p2p.storage.messages.BroadcastMessage;
import io.bisq.network.p2p.storage.messages.RefreshOfferMessage;
import io.bisq.network.p2p.storage.payload.MailboxStoragePayload;
import io.bisq.network.p2p.storage.payload.ProtectedMailboxStorageEntry;
import io.bisq.network.p2p.storage.payload.ProtectedStorageEntry;
import io.bisq.network.p2p.storage.payload.ProtectedStoragePayload;
import io.bisq.network.p2p.storage.payload.*;
import javafx.beans.property.*;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
@ -253,6 +250,10 @@ public class P2PService implements SetupListener, MessageListener, ConnectionLis
p2PDataStorage.readEntryMapFromResources(resourceFileName);
}
public void readPersistableNetworkPayloadMapFromResources(String resourceFileName) {
p2PDataStorage.readPersistableNetworkPayloadMapFromResources(resourceFileName);
}
public void onAllServicesInitialized() {
Log.traceCall();
if (networkNode.getNodeAddress() != null) {
@ -323,11 +324,11 @@ public class P2PService implements SetupListener, MessageListener, ConnectionLis
/**
* Startup sequence:
* <p/>
* <p>
* Variant 1 (normal expected mode):
* onTorNodeReady -> requestDataManager.firstDataRequestFromAnySeedNode()
* RequestDataManager.Listener.onDataReceived && onHiddenServicePublished -> onNetworkReady()
* <p/>
* <p>
* Variant 2 (no seed node available):
* onTorNodeReady -> requestDataManager.firstDataRequestFromAnySeedNode
* retry after 20-30 sec until we get at least one seed node connected
@ -787,6 +788,10 @@ public class P2PService implements SetupListener, MessageListener, ConnectionLis
// Data storage
///////////////////////////////////////////////////////////////////////////////////////////
public boolean addPersistableNetworkPayload(PersistableNetworkPayload payload) {
return p2PDataStorage.addPersistableNetworkPayload(payload, networkNode.getNodeAddress(), true, true);
}
public boolean addData(ProtectedStoragePayload protectedStoragePayload, boolean isDataOwner) {
Log.traceCall();
checkArgument(optionalKeyRing.isPresent(), "keyRing not set. Seems that is called on a seed node which must not happen.");

29
network/src/main/java/io/bisq/network/p2p/peers/getdata/GetDataRequestHandler.java
View File

@ -15,6 +15,7 @@ import io.bisq.network.p2p.peers.getdata.messages.GetDataResponse;
import io.bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest;
import io.bisq.network.p2p.storage.P2PDataStorage;
import io.bisq.network.p2p.storage.payload.CapabilityRequiringPayload;
import io.bisq.network.p2p.storage.payload.PersistableNetworkPayload;
import io.bisq.network.p2p.storage.payload.ProtectedStorageEntry;
import io.bisq.network.p2p.storage.payload.ProtectedStoragePayload;
import org.jetbrains.annotations.NotNull;
@ -23,6 +24,7 @@ import org.slf4j.LoggerFactory;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@ -73,16 +75,16 @@ public class GetDataRequestHandler {
public void handle(GetDataRequest getDataRequest, final Connection connection) {
Log.traceCall(getDataRequest + "\n\tconnection=" + connection);
final HashSet<ProtectedStorageEntry> filteredDataSet = new HashSet<>();
final Set<ProtectedStorageEntry> filteredDataSet = new HashSet<>();
final Set<Integer> lookupSet = new HashSet<>();
Set<P2PDataStorage.ByteArray> excludedItems = getDataRequest.getExcludedKeys() != null ?
getDataRequest.getExcludedKeys().stream()
.map(P2PDataStorage.ByteArray::new)
.collect(Collectors.toSet())
: new HashSet<>();
for (ProtectedStorageEntry protectedStorageEntry : dataStorage.getFilteredValues(excludedItems)) {
Set<P2PDataStorage.ByteArray> excludedKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedKeys());
Set<ProtectedStorageEntry> filteredSet = dataStorage.getMap().entrySet().stream()
.filter(e -> !excludedKeysAsByteArray.contains(e.getKey()))
.map(Map.Entry::getValue)
.collect(Collectors.toSet());
for (ProtectedStorageEntry protectedStorageEntry : filteredSet) {
final ProtectedStoragePayload protectedStoragePayload = protectedStorageEntry.getProtectedStoragePayload();
boolean doAdd = false;
if (protectedStoragePayload instanceof CapabilityRequiringPayload) {
@ -120,7 +122,16 @@ public class GetDataRequestHandler {
filteredDataSet.add(protectedStorageEntry);
}
}
GetDataResponse getDataResponse = new GetDataResponse(filteredDataSet, getDataRequest.getNonce(),
Set<P2PDataStorage.ByteArray> excludedPnpKeysAsByteArray = P2PDataStorage.ByteArray.convertBytesSetToByteArraySet(getDataRequest.getExcludedPnpKeys());
Set<PersistableNetworkPayload> filteredPnpSet = dataStorage.getPersistableNetworkPayloadCollection().getMap().entrySet().stream()
.filter(e -> !excludedPnpKeysAsByteArray.contains(e.getKey()))
.map(Map.Entry::getValue)
.collect(Collectors.toSet());
GetDataResponse getDataResponse = new GetDataResponse(filteredDataSet,
filteredPnpSet,
getDataRequest.getNonce(),
getDataRequest instanceof GetUpdatedDataRequest);
if (timeoutTimer == null) {

62
network/src/main/java/io/bisq/network/p2p/peers/getdata/RequestDataHandler.java
View File

@ -7,6 +7,8 @@ import io.bisq.common.Timer;
import io.bisq.common.UserThread;
import io.bisq.common.app.Log;
import io.bisq.common.proto.network.NetworkEnvelope;
import io.bisq.common.proto.network.NetworkPayload;
import io.bisq.common.proto.persistable.PersistablePayload;
import io.bisq.network.p2p.NodeAddress;
import io.bisq.network.p2p.network.CloseConnectionReason;
import io.bisq.network.p2p.network.Connection;
@ -18,7 +20,7 @@ import io.bisq.network.p2p.peers.getdata.messages.GetDataResponse;
import io.bisq.network.p2p.peers.getdata.messages.GetUpdatedDataRequest;
import io.bisq.network.p2p.peers.getdata.messages.PreliminaryGetDataRequest;
import io.bisq.network.p2p.storage.P2PDataStorage;
import io.bisq.network.p2p.storage.payload.LazyProcessedStoragePayload;
import io.bisq.network.p2p.storage.payload.LazyProcessedPayload;
import io.bisq.network.p2p.storage.payload.PersistableNetworkPayload;
import io.bisq.network.p2p.storage.payload.ProtectedStorageEntry;
import io.bisq.network.p2p.storage.payload.ProtectedStoragePayload;
@ -98,14 +100,18 @@ public class RequestDataHandler implements MessageListener {
// an object gets removed in between PreliminaryGetDataRequest and the GetUpdatedDataRequest and we would
// miss that event if we do not load the full set or use some delta handling.
Set<byte[]> excludedKeys = dataStorage.getMap().entrySet().stream()
.filter(e -> e.getValue().getProtectedStoragePayload() instanceof PersistableNetworkPayload)
.filter(e -> e.getValue().getProtectedStoragePayload() instanceof PersistablePayload)
.map(e -> e.getKey().bytes)
.collect(Collectors.toSet());
Set<byte[]> excludedPnpKeys = dataStorage.getPersistableNetworkPayloadCollection().getMap().entrySet().stream()
.map(e -> e.getKey().bytes)
.collect(Collectors.toSet());
if (isPreliminaryDataRequest)
getDataRequest = new PreliminaryGetDataRequest(nonce, excludedKeys);
getDataRequest = new PreliminaryGetDataRequest(nonce, excludedKeys, excludedPnpKeys);
else
getDataRequest = new GetUpdatedDataRequest(networkNode.getNodeAddress(), nonce, excludedKeys);
getDataRequest = new GetUpdatedDataRequest(networkNode.getNodeAddress(), nonce, excludedKeys, excludedPnpKeys);
if (timeoutTimer == null) {
timeoutTimer = UserThread.runAfter(() -> { // setup before sending to avoid race conditions
@ -169,8 +175,8 @@ public class RequestDataHandler implements MessageListener {
Log.traceCall(networkEnvelop.toString() + "\n\tconnection=" + connection);
if (!stopped) {
GetDataResponse getDataResponse = (GetDataResponse) networkEnvelop;
Map<String, Set<ProtectedStoragePayload>> payloadByClassName = new HashMap<>();
final HashSet<ProtectedStorageEntry> dataSet = getDataResponse.getDataSet();
Map<String, Set<NetworkPayload>> payloadByClassName = new HashMap<>();
final Set<ProtectedStorageEntry> dataSet = getDataResponse.getDataSet();
dataSet.stream().forEach(e -> {
final ProtectedStoragePayload protectedStoragePayload = e.getProtectedStoragePayload();
if (protectedStoragePayload == null) {
@ -185,10 +191,26 @@ public class RequestDataHandler implements MessageListener {
payloadByClassName.get(className).add(protectedStoragePayload);
});
Set<PersistableNetworkPayload> persistableNetworkPayloadSet = getDataResponse.getPersistableNetworkPayloadSet();
if (persistableNetworkPayloadSet != null) {
persistableNetworkPayloadSet.stream().forEach(persistableNetworkPayload -> {
// For logging different data types
String className = persistableNetworkPayload.getClass().getSimpleName();
if (!payloadByClassName.containsKey(className))
payloadByClassName.put(className, new HashSet<>());
payloadByClassName.get(className).add(persistableNetworkPayload);
});
}
// Log different data types
StringBuilder sb = new StringBuilder();
sb.append("\n#################################################################");
sb.append("\nReceived ").append(dataSet.size()).append(" instances of storage payload\n");
final int items = dataSet.size() +
(persistableNetworkPayloadSet != null ? persistableNetworkPayloadSet.size() : 0);
sb.append("\nReceived ").append(items).append(" instances\n");
payloadByClassName.entrySet().stream().forEach(e -> sb.append(e.getKey())
.append(": ")
.append(e.getValue().size())
@ -204,16 +226,27 @@ public class RequestDataHandler implements MessageListener {
final NodeAddress sender = connection.getPeersNodeAddressOptional().get();
List<ProtectedStorageEntry> processDelayedItems = new ArrayList<>();
List<NetworkPayload> processDelayedItems = new ArrayList<>();
dataSet.stream().forEach(e -> {
if (e.getProtectedStoragePayload() instanceof LazyProcessedStoragePayload)
if (e.getProtectedStoragePayload() instanceof LazyProcessedPayload) {
processDelayedItems.add(e);
else {
} else {
// We dont broadcast here (last param) as we are only connected to the seed node and would be pointless
dataStorage.add(e, sender, null, false, false);
}
});
if (persistableNetworkPayloadSet != null) {
persistableNetworkPayloadSet.stream().forEach(e -> {
if (e instanceof LazyProcessedPayload) {
processDelayedItems.add(e);
} else {
// We dont broadcast here (last param) as we are only connected to the seed node and would be pointless
dataStorage.addPersistableNetworkPayload(e, sender, false, false);
}
});
}
// We process the LazyProcessedStoragePayload items (TradeStatistics) in batches with a delay in between.
// We want avoid that the UI get stuck when processing many entries.
// The dataStorage.add call is a bit expensive as sig checks is done there.
@ -234,8 +267,13 @@ public class RequestDataHandler implements MessageListener {
for (int i = 0; i < chunks && startIndex < size; i++, startIndex += chunkSize) {
long delay = (i + 1) * 200;
int endIndex = Math.min(size, startIndex + chunkSize);
List<ProtectedStorageEntry> subList = processDelayedItems.subList(startIndex, endIndex);
UserThread.runAfter(() -> subList.stream().forEach(protectedStorageEntry -> dataStorage.add(protectedStorageEntry, sender, null, false, false)), delay, TimeUnit.MILLISECONDS);
List<NetworkPayload> subList = processDelayedItems.subList(startIndex, endIndex);
UserThread.runAfter(() -> subList.stream().forEach(item -> {
if (item instanceof ProtectedStorageEntry)
dataStorage.add((ProtectedStorageEntry) item, sender, null, false, false);
else if (item instanceof PersistableNetworkPayload)
dataStorage.addPersistableNetworkPayload((PersistableNetworkPayload) item, sender, false, false);
}), delay, TimeUnit.MILLISECONDS);
}
cleanup();

12
network/src/main/java/io/bisq/network/p2p/peers/getdata/messages/GetDataRequest.java
View File

@ -6,6 +6,7 @@ import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.ToString;
import javax.annotation.Nullable;
import java.util.Set;
@EqualsAndHashCode(callSuper = true)
@ -13,11 +14,20 @@ import java.util.Set;
@ToString
public abstract class GetDataRequest extends NetworkEnvelope implements ExtendedDataSizePermission {
protected final int nonce;
// Keys for ProtectedStorageEntry items to be excluded from the request because the peer has them already
protected final Set<byte[]> excludedKeys;
// Keys for PersistableNetworkPayload items to be excluded from the request because the peer has them already
// We added that in v 0.6 and we would get a null object from older peers, so keep it annotated with @Nullable
@Nullable
protected final Set<byte[]> excludedPnpKeys;
public GetDataRequest(int messageVersion, int nonce, Set<byte[]> excludedKeys) {
public GetDataRequest(int messageVersion,
int nonce,
Set<byte[]> excludedKeys,
@Nullable Set<byte[]> excludedPnpKeys) {
super(messageVersion);
this.nonce = nonce;
this.excludedKeys = excludedKeys;
this.excludedPnpKeys = excludedPnpKeys;
}
}

78
network/src/main/java/io/bisq/network/p2p/peers/getdata/messages/GetDataResponse.java
View File

@ -7,66 +7,106 @@ import io.bisq.common.proto.network.NetworkProtoResolver;
import io.bisq.generated.protobuffer.PB;
import io.bisq.network.p2p.ExtendedDataSizePermission;
import io.bisq.network.p2p.SupportedCapabilitiesMessage;
import io.bisq.network.p2p.storage.payload.PersistableNetworkPayload;
import io.bisq.network.p2p.storage.payload.ProtectedMailboxStorageEntry;
import io.bisq.network.p2p.storage.payload.ProtectedStorageEntry;
import lombok.EqualsAndHashCode;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
@EqualsAndHashCode(callSuper = true)
@Value
public final class GetDataResponse extends NetworkEnvelope implements SupportedCapabilitiesMessage, ExtendedDataSizePermission {
private final HashSet<ProtectedStorageEntry> dataSet;
// Set of ProtectedStorageEntry objects
private final Set<ProtectedStorageEntry> dataSet;
// Set of PersistableNetworkPayload objects
// We added that in v 0.6 and we would get a null object from older peers, so keep it annotated with @Nullable
@Nullable
private final Set<PersistableNetworkPayload> persistableNetworkPayloadSet;
private final int requestNonce;
private final boolean isGetUpdatedDataResponse;
private final ArrayList<Integer> supportedCapabilities = Capabilities.getCapabilities();
public GetDataResponse(HashSet<ProtectedStorageEntry> dataSet, int requestNonce, boolean isGetUpdatedDataResponse) {
this(dataSet, requestNonce, isGetUpdatedDataResponse, Version.getP2PMessageVersion());
public GetDataResponse(Set<ProtectedStorageEntry> dataSet,
@Nullable Set<PersistableNetworkPayload> persistableNetworkPayloadSet,
int requestNonce,
boolean isGetUpdatedDataResponse) {
this(dataSet,
persistableNetworkPayloadSet,
requestNonce,
isGetUpdatedDataResponse,
Version.getP2PMessageVersion());
}
///////////////////////////////////////////////////////////////////////////////////////////
// PROTO BUFFER
///////////////////////////////////////////////////////////////////////////////////////////
private GetDataResponse(HashSet<ProtectedStorageEntry> dataSet, int requestNonce, boolean isGetUpdatedDataResponse, int messageVersion) {
private GetDataResponse(Set<ProtectedStorageEntry> dataSet,
@Nullable Set<PersistableNetworkPayload> persistableNetworkPayloadSet,
int requestNonce,
boolean isGetUpdatedDataResponse,
int messageVersion) {
super(messageVersion);
this.dataSet = dataSet;
this.persistableNetworkPayloadSet = persistableNetworkPayloadSet;
this.requestNonce = requestNonce;
this.isGetUpdatedDataResponse = isGetUpdatedDataResponse;
}
@Override
public PB.NetworkEnvelope toProtoNetworkEnvelope() {
final PB.GetDataResponse.Builder builder = PB.GetDataResponse.newBuilder()
.addAllDataSet(dataSet.stream()
.map(protectedStorageEntry -> protectedStorageEntry instanceof ProtectedMailboxStorageEntry ?
PB.StorageEntryWrapper.newBuilder()
.setProtectedMailboxStorageEntry((PB.ProtectedMailboxStorageEntry) protectedStorageEntry.toProtoMessage())
.build()
:
PB.StorageEntryWrapper.newBuilder()
.setProtectedStorageEntry((PB.ProtectedStorageEntry) protectedStorageEntry.toProtoMessage())
.build())
.collect(Collectors.toList()))
.setRequestNonce(requestNonce)
.setIsGetUpdatedDataResponse(isGetUpdatedDataResponse)
.addAllSupportedCapabilities(supportedCapabilities);
Optional.ofNullable(persistableNetworkPayloadSet).ifPresent(set -> builder.addAllPersistableNetworkPayloadItems(set.stream()
.map(PersistableNetworkPayload::toProtoMessage)
.collect(Collectors.toList())));
return getNetworkEnvelopeBuilder()
.setGetDataResponse(PB.GetDataResponse.newBuilder()
.addAllDataSet(dataSet.stream()
.map(protectedStorageEntry -> protectedStorageEntry instanceof ProtectedMailboxStorageEntry ?
PB.StorageEntryWrapper.newBuilder()
.setProtectedMailboxStorageEntry((PB.ProtectedMailboxStorageEntry) protectedStorageEntry.toProtoMessage())
.build()
:
PB.StorageEntryWrapper.newBuilder()
.setProtectedStorageEntry((PB.ProtectedStorageEntry) protectedStorageEntry.toProtoMessage())
.build())
.collect(Collectors.toList()))
.setRequestNonce(requestNonce)
.setIsGetUpdatedDataResponse(isGetUpdatedDataResponse)
.addAllSupportedCapabilities(supportedCapabilities))
.setGetDataResponse(builder)
.build();
}
public static GetDataResponse fromProto(PB.GetDataResponse proto, NetworkProtoResolver resolver, int messageVersion) {
HashSet<ProtectedStorageEntry> dataSet = new HashSet<>(
Set<ProtectedStorageEntry> dataSet = new HashSet<>(
proto.getDataSetList().stream()
.map(entry -> (ProtectedStorageEntry) resolver.fromProto(entry))
.collect(Collectors.toSet()));
Set<PersistableNetworkPayload> persistableNetworkPayloadSet = proto.getPersistableNetworkPayloadItemsList().isEmpty() ?
null :
new HashSet<>(
proto.getPersistableNetworkPayloadItemsList().stream()
.map(e -> (PersistableNetworkPayload) resolver.fromProto(e))
.collect(Collectors.toSet()));
//PersistableNetworkPayload
return new GetDataResponse(dataSet,
persistableNetworkPayloadSet,
proto.getRequestNonce(),
proto.getIsGetUpdatedDataResponse(),
messageVersion);

37
network/src/main/java/io/bisq/network/p2p/peers/getdata/messages/GetUpdatedDataRequest.java
View File

@ -9,6 +9,8 @@ import io.bisq.network.p2p.SendersNodeAddressMessage;
import lombok.EqualsAndHashCode;
import lombok.Value;
import javax.annotation.Nullable;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -21,11 +23,13 @@ public final class GetUpdatedDataRequest extends GetDataRequest implements Sende
public GetUpdatedDataRequest(NodeAddress senderNodeAddress,
int nonce,
Set<byte[]> excludedKeys) {
Set<byte[]> excludedKeys,
@Nullable Set<byte[]> excludedPnpKeys) {
this(senderNodeAddress,
nonce,
excludedKeys,
Version.getP2PMessageVersion());
Version.getP2PMessageVersion(),
excludedPnpKeys);
}
@ -36,23 +40,31 @@ public final class GetUpdatedDataRequest extends GetDataRequest implements Sende
private GetUpdatedDataRequest(NodeAddress senderNodeAddress,
int nonce,
Set<byte[]> excludedKeys,
int messageVersion) {
int messageVersion,
@Nullable Set<byte[]> excludedPnpKeys) {
super(messageVersion,
nonce,
excludedKeys);
excludedKeys,
excludedPnpKeys);
checkNotNull(senderNodeAddress, "senderNodeAddress must not be null at GetUpdatedDataRequest");
this.senderNodeAddress = senderNodeAddress;
}
@Override
public PB.NetworkEnvelope toProtoNetworkEnvelope() {
final PB.GetUpdatedDataRequest.Builder builder = PB.GetUpdatedDataRequest.newBuilder()
.setSenderNodeAddress(senderNodeAddress.toProtoMessage())
.setNonce(nonce)
.addAllExcludedKeys(excludedKeys.stream()
.map(ByteString::copyFrom)
.collect(Collectors.toList()));
Optional.ofNullable(excludedPnpKeys).ifPresent(excludedPnpKeys -> builder.addAllExcludedPnpKeys(excludedPnpKeys.stream()
.map(ByteString::copyFrom)
.collect(Collectors.toList())));
return getNetworkEnvelopeBuilder()
.setGetUpdatedDataRequest(PB.GetUpdatedDataRequest.newBuilder()
.setSenderNodeAddress(senderNodeAddress.toProtoMessage())
.setNonce(nonce)
.addAllExcludedKeys(excludedKeys.stream()
.map(ByteString::copyFrom)
.collect(Collectors.toList())))
.setGetUpdatedDataRequest(builder)
.build();
}
@ -60,6 +72,9 @@ public final class GetUpdatedDataRequest extends GetDataRequest implements Sende
return new GetUpdatedDataRequest(NodeAddress.fromProto(proto.getSenderNodeAddress()),
proto.getNonce(),
ProtoUtil.byteSetFromProtoByteStringList(proto.getExcludedKeysList()),
messageVersion);
messageVersion,
proto.getExcludedPnpKeysList().isEmpty() ?
null :
ProtoUtil.byteSetFromProtoByteStringList(proto.getExcludedPnpKeysList()));
}
}

40
network/src/main/java/io/bisq/network/p2p/peers/getdata/messages/PreliminaryGetDataRequest.java
View File

@ -9,19 +9,25 @@ import io.bisq.network.p2p.AnonymousMessage;
import io.bisq.network.p2p.SupportedCapabilitiesMessage;
import lombok.EqualsAndHashCode;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@Slf4j
@EqualsAndHashCode(callSuper = true)
@Value
public final class PreliminaryGetDataRequest extends GetDataRequest implements AnonymousMessage, SupportedCapabilitiesMessage {
// ordinals of enum
private final ArrayList<Integer> supportedCapabilities = Capabilities.getCapabilities();
public PreliminaryGetDataRequest(int nonce, Set<byte[]> excludedKeys) {
this(nonce, excludedKeys, Version.getP2PMessageVersion());
public PreliminaryGetDataRequest(int nonce,
Set<byte[]> excludedKeys,
@Nullable Set<byte[]> excludedPnpKeys) {
this(nonce, excludedKeys, Version.getP2PMessageVersion(), excludedPnpKeys);
}
@ -29,25 +35,37 @@ public final class PreliminaryGetDataRequest extends GetDataRequest implements A
// PROTO BUFFER
///////////////////////////////////////////////////////////////////////////////////////////
private PreliminaryGetDataRequest(int nonce, Set<byte[]> excludedKeys, int messageVersion) {
super(messageVersion, nonce, excludedKeys);
private PreliminaryGetDataRequest(int nonce,
Set<byte[]> excludedKeys,
int messageVersion,
@Nullable Set<byte[]> excludedPnpKeys) {
super(messageVersion, nonce, excludedKeys, excludedPnpKeys);
}
@Override
public PB.NetworkEnvelope toProtoNetworkEnvelope() {
final PB.PreliminaryGetDataRequest.Builder builder = PB.PreliminaryGetDataRequest.newBuilder()
.setNonce(nonce)
.addAllExcludedKeys(excludedKeys.stream()
.map(ByteString::copyFrom)
.collect(Collectors.toList()))
.addAllSupportedCapabilities(supportedCapabilities);
Optional.ofNullable(excludedPnpKeys).ifPresent(excludedPnpKeys -> builder.addAllExcludedPnpKeys(excludedPnpKeys.stream()
.map(ByteString::copyFrom)
.collect(Collectors.toList())));
return getNetworkEnvelopeBuilder()
.setPreliminaryGetDataRequest(PB.PreliminaryGetDataRequest.newBuilder()
.setNonce(nonce)
.addAllExcludedKeys(excludedKeys.stream()
.map(ByteString::copyFrom)
.collect(Collectors.toList()))
.addAllSupportedCapabilities(supportedCapabilities))
.setPreliminaryGetDataRequest(builder)
.build();
}
public static PreliminaryGetDataRequest fromProto(PB.PreliminaryGetDataRequest proto, int messageVersion) {
return new PreliminaryGetDataRequest(proto.getNonce(),
ProtoUtil.byteSetFromProtoByteStringList(proto.getExcludedKeysList()),
messageVersion);
messageVersion,
proto.getExcludedPnpKeysList().isEmpty() ?
null :
ProtoUtil.byteSetFromProtoByteStringList(proto.getExcludedPnpKeysList()));
}
}

121
network/src/main/java/io/bisq/network/p2p/storage/P2PDataStorage.java
View File

@ -6,6 +6,7 @@ import io.bisq.common.Timer;
import io.bisq.common.UserThread;
import io.bisq.common.app.Log;
import io.bisq.common.crypto.CryptoException;
import io.bisq.common.crypto.Hash;
import io.bisq.common.crypto.Sig;
import io.bisq.common.proto.network.NetworkEnvelope;
import io.bisq.common.proto.network.NetworkPayload;
@ -18,7 +19,6 @@ import io.bisq.common.storage.Storage;
import io.bisq.common.util.Tuple2;
import io.bisq.common.util.Utilities;
import io.bisq.generated.protobuffer.PB;
import io.bisq.network.crypto.EncryptionService;
import io.bisq.network.p2p.NodeAddress;
import io.bisq.network.p2p.network.*;
import io.bisq.network.p2p.peers.BroadcastHandler;
@ -26,6 +26,7 @@ import io.bisq.network.p2p.peers.Broadcaster;
import io.bisq.network.p2p.storage.messages.*;
import io.bisq.network.p2p.storage.payload.*;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.ToString;
import org.apache.commons.lang3.StringUtils;
import org.bouncycastle.util.encoders.Hex;
@ -38,7 +39,6 @@ import java.nio.file.Paths;
import java.security.KeyPair;
import java.security.PublicKey;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.TimeUnit;
@ -58,14 +58,22 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
private final Broadcaster broadcaster;
private final File storageDir;
@Getter
private final Map<ByteArray, ProtectedStorageEntry> map = new ConcurrentHashMap<>();
private final CopyOnWriteArraySet<HashMapChangedListener> hashMapChangedListeners = new CopyOnWriteArraySet<>();
private Timer removeExpiredEntriesTimer;
private final SequenceNumberMap sequenceNumberMap = new SequenceNumberMap();
private final Storage<SequenceNumberMap> sequenceNumberMapStorage;
private final SequenceNumberMap sequenceNumberMap = new SequenceNumberMap();
private final Storage<PersistableEntryMap> persistedEntryMapStorage;
private PersistableEntryMap persistableEntryMap;
@Getter
private PersistableNetworkPayloadCollection persistableNetworkPayloadCollection;
private final Storage<PersistableNetworkPayloadCollection> persistableNetworkPayloadMapStorage;
private final CopyOnWriteArraySet<PersistableNetworkPayloadMapListener> persistableNetworkPayloadMapListeners = new CopyOnWriteArraySet<>();
///////////////////////////////////////////////////////////////////////////////////////////
// Constructor
@ -85,6 +93,9 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
persistedEntryMapStorage = new Storage<>(storageDir, persistenceProtoResolver);
sequenceNumberMapStorage.setNumMaxBackupFiles(5);
persistedEntryMapStorage.setNumMaxBackupFiles(1);
persistableNetworkPayloadMapStorage = new Storage<>(storageDir, persistenceProtoResolver);
persistableNetworkPayloadMapStorage.setNumMaxBackupFiles(1);
}
@Override
@ -94,7 +105,7 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
sequenceNumberMap.setMap(getPurgedSequenceNumberMap(persistedSequenceNumberMap.getMap()));
// PersistedEntryMap cannot be set here as we dont know yet the selected base currency
// We get it called in readPersistedEntryMap once ready
// We get it called in readEntryMapFromResources once ready
}
// This method is called at startup in a non-user thread.
@ -136,6 +147,43 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
}
}
// This method is called at startup in a non-user thread.
// We should not have any threading issues here as the p2p network is just initializing
public synchronized void readPersistableNetworkPayloadMapFromResources(String resourceFileName) {
final String storageFileName = "PersistableNetworkPayloadMap";
File dbDir = new File(storageDir.getAbsolutePath());
if (!dbDir.exists() && !dbDir.mkdir())
log.warn("make dir failed.\ndbDir=" + dbDir.getAbsolutePath());
final File destinationFile = new File(Paths.get(storageDir.getAbsolutePath(), storageFileName).toString());
if (!destinationFile.exists()) {
try {
log.info("We copy resource to file: resourceFileName={}, destinationFile={}", resourceFileName, destinationFile);
FileUtil.resourceToFile(resourceFileName, destinationFile);
} catch (ResourceNotFoundException e) {
log.info("Could not find resourceFile " + resourceFileName + ". That is expected if none is provided yet.");
} catch (Throwable e) {
log.error("Could not copy resourceFile " + resourceFileName + " to " +
destinationFile.getAbsolutePath() + ".\n" + e.getMessage());
e.printStackTrace();
}
} else {
log.debug(storageFileName + " file exists already.");
}
persistableNetworkPayloadCollection = persistableNetworkPayloadMapStorage.initAndGetPersistedWithFileName(storageFileName);
if (persistableNetworkPayloadCollection != null) {
log.info("persistableNetworkPayloadMap size=" + persistableNetworkPayloadCollection.getMap().size());
// In case another object is already listening...
if (!persistableNetworkPayloadMapListeners.isEmpty())
persistableNetworkPayloadCollection.getMap().values().stream()
.forEach(payload -> persistableNetworkPayloadMapListeners.stream().forEach(e -> e.onAdded(payload)));
} else {
persistableNetworkPayloadCollection = new PersistableNetworkPayloadCollection();
}
}
///////////////////////////////////////////////////////////////////////////////////////////
// API
@ -195,6 +243,9 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
removeMailboxData(((RemoveMailboxDataMessage) networkEnvelop).getProtectedMailboxStorageEntry(), peersNodeAddress, false);
} else if (networkEnvelop instanceof RefreshOfferMessage) {
refreshTTL((RefreshOfferMessage) networkEnvelop, peersNodeAddress, false);
} else if (networkEnvelop instanceof AddPersistableNetworkPayloadMessage) {
addPersistableNetworkPayload(((AddPersistableNetworkPayloadMessage) networkEnvelop).getPersistableNetworkPayload(),
peersNodeAddress, false, true);
}
});
}
@ -266,6 +317,26 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
// API
///////////////////////////////////////////////////////////////////////////////////////////
public boolean addPersistableNetworkPayload(PersistableNetworkPayload payload,
@Nullable NodeAddress sender,
boolean isDataOwner,
boolean allowBroadcast) {
log.debug("addPersistableNetworkPayload payload={}", payload);
final ByteArray hashAsByteArray = new ByteArray(payload.getHash());
boolean containsKey = persistableNetworkPayloadCollection.getMap().containsKey(hashAsByteArray);
if (!containsKey) {
persistableNetworkPayloadCollection.getMap().put(hashAsByteArray, payload);
persistableNetworkPayloadMapStorage.queueUpForSave(persistableNetworkPayloadCollection, 2000);
persistableNetworkPayloadMapListeners.stream().forEach(e -> e.onAdded(payload));
if (allowBroadcast)
broadcaster.broadcast(new AddPersistableNetworkPayloadMessage(payload), sender, null, isDataOwner);
return true;
} else {
log.trace("We have that payload already in our map");
return false;
}
}
public boolean add(ProtectedStorageEntry protectedStorageEntry, @Nullable NodeAddress sender,
@Nullable BroadcastHandler.Listener listener, boolean isDataOwner) {
Log.traceCall("with allowBroadcast=true");
@ -420,11 +491,6 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
return result;
}
public Map<ByteArray, ProtectedStorageEntry> getMap() {
return map;
}
public ProtectedStorageEntry getProtectedStorageEntry(ProtectedStoragePayload protectedStoragePayload, KeyPair ownerStoragePubKey)
throws CryptoException {
ByteArray hashOfData = getHashAsByteArray(protectedStoragePayload);
@ -434,7 +500,7 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
else
sequenceNumber = 1;
byte[] hashOfDataAndSeqNr = EncryptionService.getHash(new DataAndSeqNrPair(protectedStoragePayload, sequenceNumber));
byte[] hashOfDataAndSeqNr = P2PDataStorage.getHash(new DataAndSeqNrPair(protectedStoragePayload, sequenceNumber));
byte[] signature = Sig.sign(ownerStoragePubKey.getPrivate(), hashOfDataAndSeqNr);
return new ProtectedStorageEntry(protectedStoragePayload, ownerStoragePubKey.getPublic(), sequenceNumber, signature);
}
@ -448,7 +514,7 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
else
sequenceNumber = 1;
byte[] hashOfDataAndSeqNr = EncryptionService.getHash(new DataAndSeqNrPair(protectedStoragePayload, sequenceNumber));
byte[] hashOfDataAndSeqNr = P2PDataStorage.getHash(new DataAndSeqNrPair(protectedStoragePayload, sequenceNumber));
byte[] signature = Sig.sign(ownerStoragePubKey.getPrivate(), hashOfDataAndSeqNr);
return new RefreshOfferMessage(hashOfDataAndSeqNr, signature, hashOfPayload.bytes, sequenceNumber);
}
@ -463,7 +529,7 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
else
sequenceNumber = 1;
byte[] hashOfDataAndSeqNr = EncryptionService.getHash(new DataAndSeqNrPair(expirableMailboxStoragePayload, sequenceNumber));
byte[] hashOfDataAndSeqNr = P2PDataStorage.getHash(new DataAndSeqNrPair(expirableMailboxStoragePayload, sequenceNumber));
byte[] signature = Sig.sign(storageSignaturePubKey.getPrivate(), hashOfDataAndSeqNr);
return new ProtectedMailboxStorageEntry(expirableMailboxStoragePayload,
storageSignaturePubKey.getPublic(), sequenceNumber, signature, receiversPublicKey);
@ -477,11 +543,12 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
hashMapChangedListeners.remove(hashMapChangedListener);
}
public Set<ProtectedStorageEntry> getFilteredValues(Set<ByteArray> excludedKeys) {
return map.entrySet().stream()
.filter(e -> !excludedKeys.contains(e.getKey()))
.map(Entry::getValue)
.collect(Collectors.toSet());
public void addPersistableNetworkPayloadMapListener(PersistableNetworkPayloadMapListener listener) {
persistableNetworkPayloadMapListeners.add(listener);
}
public void removePersistableNetworkPayloadMapListener(PersistableNetworkPayloadMapListener listener) {
persistableNetworkPayloadMapListeners.remove(listener);
}
@ -559,7 +626,7 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
}
private boolean checkSignature(ProtectedStorageEntry protectedStorageEntry) {
byte[] hashOfDataAndSeqNr = EncryptionService.getHash(new DataAndSeqNrPair(protectedStorageEntry.getProtectedStoragePayload(), protectedStorageEntry.getSequenceNumber()));
byte[] hashOfDataAndSeqNr = P2PDataStorage.getHash(new DataAndSeqNrPair(protectedStorageEntry.getProtectedStoragePayload(), protectedStorageEntry.getSequenceNumber()));
return checkSignature(protectedStorageEntry.getOwnerPubKey(), hashOfDataAndSeqNr, protectedStorageEntry.getSignature());
}
@ -630,7 +697,7 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
}
private ByteArray getHashAsByteArray(ExpirablePayload data) {
return new ByteArray(EncryptionService.getHash(data));
return new ByteArray(P2PDataStorage.getHash(data));
}
// Get a new map with entries older than PURGE_AGE_DAYS purged from the given map.
@ -679,6 +746,14 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
}
}
/**
* @param data
* @return Hash of data
*/
public static byte[] getHash(NetworkPayload data) {
return Hash.getSha256Hash(data.toProtoMessage().toByteArray());
}
///////////////////////////////////////////////////////////////////////////////////////////
// Static class
@ -751,6 +826,14 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers
public String getHex() {
return Utilities.encodeToHex(bytes);
}
public static Set<P2PDataStorage.ByteArray> convertBytesSetToByteArraySet(Set<byte[]> set) {
return set != null ?
set.stream()
.map(P2PDataStorage.ByteArray::new)
.collect(Collectors.toSet())
: new HashSet<>();
}
}
/**

75
network/src/main/java/io/bisq/network/p2p/storage/PersistableNetworkPayloadCollection.java Normal file
View File

@ -0,0 +1,75 @@
/*
* This file is part of Bisq.
*
* Bisq is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* Bisq is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Bisq. If not, see <http://www.gnu.org/licenses/>.
*/
package io.bisq.network.p2p.storage;
import com.google.protobuf.Message;
import io.bisq.common.proto.persistable.PersistableEnvelope;
import io.bisq.common.proto.persistable.PersistenceProtoResolver;
import io.bisq.generated.protobuffer.PB;
import io.bisq.network.p2p.storage.payload.PersistableNetworkPayload;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
// That class wraps a map but is represented in PB as a list to reduce data size (no key).
// PB also does not support a byte array as key and would require some quirks to support such a map (using hex string
// would render our 20 byte keys to 40 bytes as HEX encoded).
@Slf4j
public class PersistableNetworkPayloadCollection implements PersistableEnvelope {
@Getter
private Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new ConcurrentHashMap<>();
public PersistableNetworkPayloadCollection() {
}
///////////////////////////////////////////////////////////////////////////////////////////
// PROTO BUFFER
///////////////////////////////////////////////////////////////////////////////////////////
private PersistableNetworkPayloadCollection(Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map) {
this.map.putAll(map);
}
public Message toProtoMessage() {
// Protobuffer maps don't support bytes as key so we use a hex string
Set<PB.PersistableNetworkPayload> values = map.values().stream()
.map(PersistableNetworkPayload::toProtoMessage)
.collect(Collectors.toSet());
return PB.PersistableEnvelope.newBuilder()
.setPersistableNetworkPayloadList(PB.PersistableNetworkPayloadList.newBuilder()
.addAllItems(values))
.build();
}
public static PersistableEnvelope fromProto(PB.PersistableNetworkPayloadList proto,
PersistenceProtoResolver resolver) {
Map<P2PDataStorage.ByteArray, PersistableNetworkPayload> map = new HashMap<>();
proto.getItemsList().stream()
.forEach(e -> {
PersistableNetworkPayload payload = PersistableNetworkPayload.fromProto(e, resolver);
map.put(new P2PDataStorage.ByteArray(payload.getHash()), payload);
});
return new PersistableNetworkPayloadCollection(map);
}
}

7
network/src/main/java/io/bisq/network/p2p/storage/PersistableNetworkPayloadMapListener.java Normal file
View File

@ -0,0 +1,7 @@
package io.bisq.network.p2p.storage;
import io.bisq.network.p2p.storage.payload.PersistableNetworkPayload;
public interface PersistableNetworkPayloadMapListener {
void onAdded(PersistableNetworkPayload payload);
}

43
network/src/main/java/io/bisq/network/p2p/storage/messages/AddPersistableNetworkPayloadMessage.java Normal file
View File

@ -0,0 +1,43 @@
package io.bisq.network.p2p.storage.messages;
import io.bisq.common.app.Version;
import io.bisq.common.proto.network.NetworkProtoResolver;
import io.bisq.generated.protobuffer.PB;
import io.bisq.network.p2p.storage.payload.PersistableNetworkPayload;
import lombok.EqualsAndHashCode;
import lombok.Value;
@EqualsAndHashCode(callSuper = true)
@Value
public final class AddPersistableNetworkPayloadMessage extends BroadcastMessage {
private final PersistableNetworkPayload persistableNetworkPayload;
public AddPersistableNetworkPayloadMessage(PersistableNetworkPayload persistableNetworkPayload) {
this(persistableNetworkPayload, Version.getP2PMessageVersion());
}
///////////////////////////////////////////////////////////////////////////////////////////
// PROTO BUFFER
///////////////////////////////////////////////////////////////////////////////////////////
private AddPersistableNetworkPayloadMessage(PersistableNetworkPayload persistableNetworkPayload, int messageVersion) {
super(messageVersion);
this.persistableNetworkPayload = persistableNetworkPayload;
}
@Override
public PB.NetworkEnvelope toProtoNetworkEnvelope() {
return getNetworkEnvelopeBuilder()
.setAddPersistableNetworkPayloadMessage(PB.AddPersistableNetworkPayloadMessage.newBuilder()
.setPayload(persistableNetworkPayload.toProtoMessage()))
.build();
}
public static AddPersistableNetworkPayloadMessage fromProto(PB.AddPersistableNetworkPayloadMessage proto,
NetworkProtoResolver resolver,
int messageVersion) {
return new AddPersistableNetworkPayloadMessage((PersistableNetworkPayload) resolver.fromProto(proto.getPayload()),
messageVersion);
}
}

6
network/src/main/java/io/bisq/network/p2p/storage/payload/LazyProcessedStoragePayload.java → network/src/main/java/io/bisq/network/p2p/storage/payload/LazyProcessedPayload.java
View File

@ -1,8 +1,10 @@
package io.bisq.network.p2p.storage.payload;
import io.bisq.common.Payload;
/**
* Marker interface for payload which gets delayed processed at startup so we don't hit performance too much.
* Used for TradeStatistics.
* Used for TradeStatistics and AccountAgeWitness.
*/
public interface LazyProcessedStoragePayload extends ProtectedStoragePayload {
public interface LazyProcessedPayload extends Payload {
}

17
network/src/main/java/io/bisq/network/p2p/storage/payload/PersistableNetworkPayload.java
View File

@ -1,10 +1,25 @@
package io.bisq.network.p2p.storage.payload;
import io.bisq.common.proto.ProtoResolver;
import io.bisq.common.proto.network.NetworkPayload;
import io.bisq.common.proto.persistable.PersistablePayload;
import io.bisq.generated.protobuffer.PB;
/**
* Marker interface for NetworkPayload which gets persisted in EntryMap.
* Marker interface for NetworkPayload which gets persisted in PersistableNetworkPayloadMap.
* We store it as a list in PB to keep storage size small (map would use hash as key which is in data object anyway).
* Not using a map also give more tolerance with data structure changes.
* This data structure does not use a verification of the owners signature. ProtectedStoragePayload is used if that is required.
* Currently we use it only for the AccountAgeWitness data.
*/
public interface PersistableNetworkPayload extends NetworkPayload, PersistablePayload {
static PersistableNetworkPayload fromProto(PB.PersistableNetworkPayload payload, ProtoResolver resolver) {
return (PersistableNetworkPayload) resolver.fromProto(payload);
}
PB.PersistableNetworkPayload toProtoMessage();
// Hash which will be used as key in the in-memory hashMap
byte [] getHash();
}

4
network/src/main/resources/PersistableNetworkPayloadMap_BTC_REGTEST Normal file
View File

@ -0,0 +1,4 @@
izg
e
c
L9┴·ц▄!╠c {⌡шR>√з╡<оОвИд─Ь5▌─╥⌠.0,n╩ЖaМжнО┌и╓╛А≈ЪH╞W╓x}]g╒цSЛЙI÷╘5PЩ ▒ю ╣Р+

10
network/src/test/java/io/bisq/network/p2p/TestUtils.java
View File

@ -148,6 +148,11 @@ public class TestUtils {
return null;
}
@Override
public PersistableEnvelope fromProto(PB.PersistableNetworkPayload persistable) {
return null;
}
@Override
public NetworkEnvelope fromProto(PB.NetworkEnvelope envelope) {
return null;
@ -176,6 +181,11 @@ public class TestUtils {
public PersistableEnvelope fromProto(PB.PersistableEnvelope persistable) {
return null;
}
@Override
public PersistableEnvelope fromProto(PB.PersistableNetworkPayload persistable) {
return null;
}
};
}
}

2
network/src/test/java/io/bisq/network/p2p/storage/ProtectedDataStorageTest.java
View File

@ -109,7 +109,7 @@ public class ProtectedDataStorageTest {
Assert.assertEquals(1, dataStorage1.getMap().size());
int newSequenceNumber = data.getSequenceNumber() + 1;
byte[] hashOfDataAndSeqNr = EncryptionService.getHash(new P2PDataStorage.DataAndSeqNrPair(data.getProtectedStoragePayload(), newSequenceNumber));
byte[] hashOfDataAndSeqNr = P2PDataStorage.getHash(new P2PDataStorage.DataAndSeqNrPair(data.getProtectedStoragePayload(), newSequenceNumber));
byte[] signature = Sig.sign(storageSignatureKeyPair1.getPrivate(), hashOfDataAndSeqNr);
ProtectedStorageEntry dataToRemove = new ProtectedStorageEntry(data.getProtectedStoragePayload(), data.getOwnerPubKey(), newSequenceNumber, signature);
Assert.assertTrue(dataStorage1.remove(dataToRemove, null, true));