Add TradeStatistics3 and related classes

Add TRADE_STATISTICS_3 Capability
Add TradeStatistics3 to proto resolvers
Make message TradeStatistics2 deprecated

common/src/main/java/bisq/common/app/Capability.java

@@ -41,5 +41,6 @@ public enum Capability {
     MEDIATION,                          // Supports mediation feature
     REFUND_AGENT,                       // Supports refund agents
     TRADE_STATISTICS_HASH_UPDATE,       // We changed the hash method in 1.2.0 and that requires update to 1.2.2 for handling it correctly, otherwise the seed nodes have to process too much data.
-    NO_ADDRESS_PRE_FIX                  // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix.
+    NO_ADDRESS_PRE_FIX,                 // At 1.4.0 we removed the prefix filter for mailbox messages. If a peer has that capability we do not sent the prefix.
+    TRADE_STATISTICS_3                  // We used a new reduced trade statistics model from v1.4.0 on
 }

core/src/main/java/bisq/core/proto/CoreProtoResolver.java

@@ -54,6 +54,7 @@ import bisq.core.payment.payload.VenmoAccountPayload;
 import bisq.core.payment.payload.WeChatPayAccountPayload;
 import bisq.core.payment.payload.WesternUnionAccountPayload;
 import bisq.core.trade.statistics.TradeStatistics2;
+import bisq.core.trade.statistics.TradeStatistics3;
 
 import bisq.common.proto.ProtoResolver;
 import bisq.common.proto.ProtobufferRuntimeException;
@@ -178,6 +179,8 @@ public class CoreProtoResolver implements ProtoResolver {
                     return BlindVotePayload.fromProto(proto.getBlindVotePayload());
                 case SIGNED_WITNESS:
                     return SignedWitness.fromProto(proto.getSignedWitness());
+                case TRADE_STATISTICS3:
+                    return TradeStatistics3.fromProto(proto.getTradeStatistics3());
                 default:
                     throw new ProtobufferRuntimeException("Unknown proto message case (PB.PersistableNetworkPayload). messageCase=" + proto.getMessageCase());
             }

core/src/main/java/bisq/core/proto/persistable/CorePersistenceProtoResolver.java

@@ -39,6 +39,7 @@ import bisq.core.support.dispute.mediation.MediationDisputeList;
 import bisq.core.support.dispute.refund.RefundDisputeList;
 import bisq.core.trade.TradableList;
 import bisq.core.trade.statistics.TradeStatistics2Store;
+import bisq.core.trade.statistics.TradeStatistics3Store;
 import bisq.core.user.PreferencesPayload;
 import bisq.core.user.UserPayload;
 
@@ -126,6 +127,8 @@ public class CorePersistenceProtoResolver extends CoreProtoResolver implements P
                     return UnconfirmedBsqChangeOutputList.fromProto(proto.getUnconfirmedBsqChangeOutputList());
                 case SIGNED_WITNESS_STORE:
                     return SignedWitnessStore.fromProto(proto.getSignedWitnessStore());
+                case TRADE_STATISTICS3_STORE:
+                    return TradeStatistics3Store.fromProto(proto.getTradeStatistics3Store());
 
                 default:
                     throw new ProtobufferRuntimeException("Unknown proto message case(PB.PersistableEnvelope). " +

core/src/main/java/bisq/core/setup/CoreNetworkCapabilities.java

@@ -40,7 +40,8 @@ public class CoreNetworkCapabilities {
                 Capability.SIGNED_ACCOUNT_AGE_WITNESS,
                 Capability.REFUND_AGENT,
                 Capability.TRADE_STATISTICS_HASH_UPDATE,
-                Capability.NO_ADDRESS_PRE_FIX
+                Capability.NO_ADDRESS_PRE_FIX,
+                Capability.TRADE_STATISTICS_3
         );
 
         if (config.daoActivated) {

core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java

@@ -0,0 +1,320 @@
+/*
+ * This file is part of Bisq.
+ *
+ * Bisq is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Bisq is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package bisq.core.trade.statistics;
+
+import bisq.core.monetary.Altcoin;
+import bisq.core.monetary.AltcoinExchangeRate;
+import bisq.core.monetary.Price;
+import bisq.core.monetary.Volume;
+import bisq.core.offer.OfferUtil;
+
+import bisq.network.p2p.storage.payload.CapabilityRequiringPayload;
+import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
+import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload;
+
+import bisq.common.app.Capabilities;
+import bisq.common.app.Capability;
+import bisq.common.crypto.Hash;
+import bisq.common.proto.ProtoUtil;
+import bisq.common.util.CollectionUtils;
+import bisq.common.util.ExtraDataMapValidator;
+import bisq.common.util.JsonExclude;
+import bisq.common.util.Utilities;
+
+import com.google.protobuf.ByteString;
+
+import org.bitcoinj.core.Coin;
+import org.bitcoinj.utils.ExchangeRate;
+import org.bitcoinj.utils.Fiat;
+
+import com.google.common.base.Charsets;
+
+import java.util.Date;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Value;
+import lombok.extern.slf4j.Slf4j;
+
+import javax.annotation.Nullable;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * This new trade statistics class uses only the bare minimum of data.
+ * Data size is about 50 bytes in average
+ */
+@Slf4j
+@Value
+public final class TradeStatistics3 implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload,
+        CapabilityRequiringPayload {
+
+    // This enum must not change the order as we use the ordinal for storage to reduce data size.
+    // The payment method string can be quite long and would consume 15% more space.
+    // When we get a new payment method we can add it to the enum at the end. Old users would add it as string if not
+    // recognized.
+    private enum PaymentMethodMapper {
+        OK_PAY,
+        CASH_APP,
+        VENMO,
+        AUSTRALIA_PAYID, // seems there is a dev trade
+        UPHOLD,
+        MONEY_BEAM,
+        POPMONEY,
+        REVOLUT,
+        PERFECT_MONEY,
+        SEPA,
+        SEPA_INSTANT,
+        FASTER_PAYMENTS,
+        NATIONAL_BANK,
+        JAPAN_BANK,
+        SAME_BANK,
+        SPECIFIC_BANKS,
+        SWISH,
+        ALI_PAY,
+        WECHAT_PAY,
+        CLEAR_X_CHANGE,
+        CHASE_QUICK_PAY,
+        INTERAC_E_TRANSFER,
+        US_POSTAL_MONEY_ORDER,
+        CASH_DEPOSIT,
+        MONEY_GRAM,
+        WESTERN_UNION,
+        HAL_CASH,
+        F2F,
+        BLOCK_CHAINS,
+        PROMPT_PAY,
+        ADVANCED_CASH,
+        BLOCK_CHAINS_INSTANT
+    }
+
+    private final String currency;
+    private final long price;
+    private final long amount;
+    private final String paymentMethod;
+    // As only seller is publishing it is the sellers trade date
+    private final long date;
+
+    // Old converted trade stat objects might not have it set
+    @Nullable
+    @JsonExclude
+    private final String mediator;  // todo entries from old data could be pruned
+    @Nullable
+    @JsonExclude
+    private final String refundAgent;
+
+    // todo should we add referrerId as well? get added to extra map atm but not used so far
+
+    // Hash get set in constructor from json of all the other data fields (with hash = null).
+    @JsonExclude
+    private final byte[] hash;
+    // Should be only used in emergency case if we need to add data but do not want to break backward compatibility
+    // at the P2P network storage checks. The hash of the object will be used to verify if the data is valid. Any new
+    // field in a class would break that hash and therefore break the storage mechanism.
+    @Nullable
+    @JsonExclude
+    private Map<String, String> extraDataMap;
+
+    public TradeStatistics3(String currency,
+                            long price,
+                            long amount,
+                            String paymentMethod,
+                            long date,
+                            String mediator,
+                            String refundAgent,
+                            @Nullable Map<String, String> extraDataMap) {
+        this(currency,
+                price,
+                amount,
+                paymentMethod,
+                date,
+                mediator,
+                refundAgent,
+                extraDataMap,
+                null);
+    }
+
+    // Used from conversion method where we use the hash of the TradeStatistics2 objects to avoid duplicate entries
+    public TradeStatistics3(String currency,
+                            long price,
+                            long amount,
+                            String paymentMethod,
+                            long date,
+                            String mediator,
+                            String refundAgent,
+                            @Nullable byte[] hash) {
+        this(currency,
+                price,
+                amount,
+                paymentMethod,
+                date,
+                mediator,
+                refundAgent,
+                null,
+                hash);
+    }
+
+    ///////////////////////////////////////////////////////////////////////////////////////////
+    // PROTO BUFFER
+    ///////////////////////////////////////////////////////////////////////////////////////////
+
+    private TradeStatistics3(String currency,
+                             long price,
+                             long amount,
+                             String paymentMethod,
+                             long date,
+                             @Nullable String mediator,
+                             @Nullable String refundAgent,
+                             @Nullable Map<String, String> extraDataMap,
+                             @Nullable byte[] hash) {
+        this.currency = currency;
+        this.price = price;
+        this.amount = amount;
+        String tempPaymentMethod;
+        try {
+            tempPaymentMethod = String.valueOf(PaymentMethodMapper.valueOf(paymentMethod).ordinal());
+        } catch (Throwable t) {
+            tempPaymentMethod = paymentMethod;
+        }
+        this.paymentMethod = tempPaymentMethod;
+        this.date = date;
+        this.mediator = mediator;
+        this.refundAgent = refundAgent;
+        this.extraDataMap = ExtraDataMapValidator.getValidatedExtraDataMap(extraDataMap);
+
+        this.hash = hash == null ? createHash() : hash;
+    }
+
+    public byte[] createHash() {
+        // We create hash from all fields excluding hash itself. We use json as simple data serialisation.
+        // TradeDate is different for both peers so we ignore it for hash. ExtraDataMap is ignored as well as at
+        // software updates we might have different entries which would cause a different hash.
+        return Hash.getSha256Ripemd160hash(Utilities.objectToJson(this).getBytes(Charsets.UTF_8));
+    }
+
+    private protobuf.TradeStatistics3.Builder getBuilder() {
+        protobuf.TradeStatistics3.Builder builder = protobuf.TradeStatistics3.newBuilder()
+                .setCurrency(currency)
+                .setPrice(price)
+                .setAmount(amount)
+                .setPaymentMethod(paymentMethod)
+                .setDate(date)
+                .setHash(ByteString.copyFrom(hash));
+        Optional.ofNullable(mediator).ifPresent(builder::setMediator);
+        Optional.ofNullable(refundAgent).ifPresent(builder::setRefundAgent);
+        Optional.ofNullable(extraDataMap).ifPresent(builder::putAllExtraData);
+        return builder;
+    }
+
+    public protobuf.TradeStatistics3 toProtoTradeStatistics3() {
+        return getBuilder().build();
+    }
+
+    @Override
+    public protobuf.PersistableNetworkPayload toProtoMessage() {
+        return protobuf.PersistableNetworkPayload.newBuilder().setTradeStatistics3(getBuilder()).build();
+    }
+
+    public static TradeStatistics3 fromProto(protobuf.TradeStatistics3 proto) {
+        return new TradeStatistics3(
+                proto.getCurrency(),
+                proto.getPrice(),
+                proto.getAmount(),
+                proto.getPaymentMethod(),
+                proto.getDate(),
+                ProtoUtil.stringOrNullFromProto(proto.getMediator()),
+                ProtoUtil.stringOrNullFromProto(proto.getRefundAgent()),
+                CollectionUtils.isEmpty(proto.getExtraDataMap()) ? null : proto.getExtraDataMap(),
+                proto.getHash().toByteArray());
+    }
+
+
+    ///////////////////////////////////////////////////////////////////////////////////////////
+    // API
+    ///////////////////////////////////////////////////////////////////////////////////////////
+
+    @Override
+    public byte[] getHash() {
+        return hash;
+    }
+
+    @Override
+    public boolean verifyHashSize() {
+        checkNotNull(hash, "hash must not be null");
+        return hash.length == 20;
+    }
+
+    @Override
+    public Capabilities getRequiredCapabilities() {
+        return new Capabilities(Capability.TRADE_STATISTICS_3);
+    }
+
+    public String getPaymentMethod() {
+        try {
+            return PaymentMethodMapper.values()[Integer.parseInt(paymentMethod)].name();
+        } catch (Throwable ignore) {
+            return paymentMethod;
+        }
+    }
+
+    public Date getTradeDate() {
+        return new Date(date);
+    }
+
+    public Price getTradePrice() {
+        return Price.valueOf(currency, price);
+    }
+
+    public Coin getTradeAmount() {
+        return Coin.valueOf(amount);
+    }
+
+    public Volume getTradeVolume() {
+        if (getTradePrice().getMonetary() instanceof Altcoin) {
+            return new Volume(new AltcoinExchangeRate((Altcoin) getTradePrice().getMonetary()).coinToAltcoin(getTradeAmount()));
+        } else {
+            Volume volume = new Volume(new ExchangeRate((Fiat) getTradePrice().getMonetary()).coinToFiat(getTradeAmount()));
+            return OfferUtil.getRoundedFiatVolume(volume);
+        }
+    }
+
+    public boolean isValid() {
+        return amount > 0 &&
+                price > 0 &&
+                date > 0 &&
+                paymentMethod != null &&
+                !paymentMethod.isEmpty() &&
+                currency != null &&
+                !currency.isEmpty();
+    }
+
+    @Override
+    public String toString() {
+        return "TradeStatistics3{" +
+                "\n     currency='" + currency + '\'' +
+                ",\n     price=" + price +
+                ",\n     amount=" + amount +
+                ",\n     paymentMethod='" + paymentMethod + '\'' +
+                ",\n     date=" + date +
+                ",\n     mediator='" + mediator + '\'' +
+                ",\n     refundAgent='" + refundAgent + '\'' +
+                ",\n     hash=" + Utilities.bytesAsHexString(hash) +
+                ",\n     extraDataMap=" + extraDataMap +
+                "\n}";
+    }
+}

core/src/main/java/bisq/core/trade/statistics/TradeStatistics3StorageService.java

@@ -0,0 +1,84 @@
+/*
+ * This file is part of Bisq.
+ *
+ * bisq is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * bisq is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with bisq. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package bisq.core.trade.statistics;
+
+import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
+import bisq.network.p2p.storage.persistence.HistoricalDataStoreService;
+
+import bisq.common.config.Config;
+import bisq.common.persistence.PersistenceManager;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import java.io.File;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Singleton
+@Slf4j
+public class TradeStatistics3StorageService extends HistoricalDataStoreService<TradeStatistics3Store> {
+    private static final String FILE_NAME = "TradeStatistics3Store";
+
+
+    ///////////////////////////////////////////////////////////////////////////////////////////
+    // Constructor
+    ///////////////////////////////////////////////////////////////////////////////////////////
+
+    @Inject
+    public TradeStatistics3StorageService(@Named(Config.STORAGE_DIR) File storageDir,
+                                          PersistenceManager<TradeStatistics3Store> persistenceManager) {
+        super(storageDir, persistenceManager);
+    }
+
+
+    ///////////////////////////////////////////////////////////////////////////////////////////
+    // API
+    ///////////////////////////////////////////////////////////////////////////////////////////
+
+    @Override
+    public String getFileName() {
+        return FILE_NAME;
+    }
+
+    @Override
+    protected void initializePersistenceManager() {
+        persistenceManager.initialize(store, PersistenceManager.Source.NETWORK);
+    }
+
+    @Override
+    public boolean canHandle(PersistableNetworkPayload payload) {
+        return payload instanceof TradeStatistics3;
+    }
+
+
+    ///////////////////////////////////////////////////////////////////////////////////////////
+    // Protected
+    ///////////////////////////////////////////////////////////////////////////////////////////
+
+    @Override
+    protected TradeStatistics3Store createStore() {
+        return new TradeStatistics3Store();
+    }
+
+    public void persistNow() {
+        persistenceManager.persistNow(() -> {
+        });
+    }
+}

core/src/main/java/bisq/core/trade/statistics/TradeStatistics3Store.java

@@ -0,0 +1,73 @@
+/*
+ * This file is part of Bisq.
+ *
+ * Bisq is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Bisq is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package bisq.core.trade.statistics;
+
+import bisq.network.p2p.storage.P2PDataStorage;
+import bisq.network.p2p.storage.persistence.PersistableNetworkPayloadStore;
+
+import com.google.protobuf.Message;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * We store only the payload in the PB file to save disc space. The hash of the payload can be created anyway and
+ * is only used as key in the map. So we have a hybrid data structure which is represented as list in the protobuffer
+ * definition and provide a hashMap for the domain access.
+ */
+@Slf4j
+public class TradeStatistics3Store extends PersistableNetworkPayloadStore<TradeStatistics3> {
+
+    TradeStatistics3Store() {
+    }
+
+
+    ///////////////////////////////////////////////////////////////////////////////////////////
+    // PROTO BUFFER
+    ///////////////////////////////////////////////////////////////////////////////////////////
+
+    private TradeStatistics3Store(List<TradeStatistics3> list) {
+        list.forEach(item -> map.put(new P2PDataStorage.ByteArray(item.getHash()), item));
+    }
+
+    public Message toProtoMessage() {
+        return protobuf.PersistableEnvelope.newBuilder()
+                .setTradeStatistics3Store(getBuilder())
+                .build();
+    }
+
+    private protobuf.TradeStatistics3Store.Builder getBuilder() {
+        List<protobuf.TradeStatistics3> protoList = map.values().stream()
+                .map(payload -> (TradeStatistics3) payload)
+                .map(TradeStatistics3::toProtoTradeStatistics3)
+                .collect(Collectors.toList());
+        return protobuf.TradeStatistics3Store.newBuilder().addAllItems(protoList);
+    }
+
+    public static TradeStatistics3Store fromProto(protobuf.TradeStatistics3Store proto) {
+        List<TradeStatistics3> list = proto.getItemsList().stream()
+                .map(TradeStatistics3::fromProto).collect(Collectors.toList());
+        return new TradeStatistics3Store(list);
+    }
+
+    public boolean containsKey(P2PDataStorage.ByteArray hash) {
+        return map.containsKey(hash);
+    }
+}

core/src/main/java/bisq/core/trade/statistics/TradeStatisticsConverter.java

@@ -0,0 +1,147 @@
+/*
+ * This file is part of Bisq.
+ *
+ * Bisq is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Bisq is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package bisq.core.trade.statistics;
+
+import bisq.network.p2p.BootstrapListener;
+import bisq.network.p2p.P2PService;
+import bisq.network.p2p.storage.P2PDataStorage;
+import bisq.network.p2p.storage.payload.PersistableNetworkPayload;
+import bisq.network.p2p.storage.persistence.AppendOnlyDataStoreService;
+
+import bisq.common.config.Config;
+import bisq.common.file.FileUtil;
+
+import com.google.inject.Inject;
+
+import javax.inject.Named;
+
+import java.io.File;
+import java.io.IOException;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class TradeStatisticsConverter {
+
+    @Inject
+    public TradeStatisticsConverter(P2PService p2PService,
+                                    P2PDataStorage p2PDataStorage,
+                                    TradeStatistics2StorageService tradeStatistics2StorageService,
+                                    TradeStatistics3StorageService tradeStatistics3StorageService,
+                                    AppendOnlyDataStoreService appendOnlyDataStoreService,
+                                    @Named(Config.STORAGE_DIR) File storageDir) {
+        File tradeStatistics2Store = new File(storageDir, "TradeStatistics2Store");
+        appendOnlyDataStoreService.addService(tradeStatistics2StorageService);
+
+        p2PService.addP2PServiceListener(new BootstrapListener() {
+            @Override
+            public void onTorNodeReady() {
+                if (!tradeStatistics2Store.exists()) {
+                    return;
+                }
+
+                // We convert early once tor is initialized but still not ready to receive data
+                var mapOfLiveData = tradeStatistics3StorageService.getMapOfLiveData();
+                convertToTradeStatistics3(tradeStatistics2StorageService.getMapOfAllData().values())
+                        .forEach(e -> mapOfLiveData.put(new P2PDataStorage.ByteArray(e.getHash()), e));
+                tradeStatistics3StorageService.persistNow();
+                try {
+                    log.info("We delete now the old trade statistics file as it was converted to the new format.");
+                    FileUtil.deleteFileIfExists(tradeStatistics2Store);
+                } catch (IOException e) {
+                    e.printStackTrace();
+                    log.error(e.toString());
+                }
+            }
+
+            @Override
+            public void onUpdatedDataReceived() {
+            }
+        });
+
+        // We listen to old TradeStatistics2 objects, convert and store them and rebroadcast.
+        p2PDataStorage.addAppendOnlyDataStoreListener(payload -> {
+            if (payload instanceof TradeStatistics2) {
+                TradeStatistics3 tradeStatistics3 = convertToTradeStatistics3((TradeStatistics2) payload, true);
+                // We add it to the p2PDataStorage, which handles to get the data stored in the maps and maybe
+                // re-broadcast as tradeStatistics3 object if not already received.
+                p2PDataStorage.addPersistableNetworkPayload(tradeStatistics3, null, true);
+            }
+        });
+    }
+
+    private static Set<PersistableNetworkPayload> convertToTradeStatistics3(Collection<PersistableNetworkPayload> persistableNetworkPayloads) {
+        Set<PersistableNetworkPayload> result = new HashSet<>();
+        long ts = System.currentTimeMillis();
+
+        // We might have duplicate entries from both traders as the trade date was different from old clients.
+        // This should not be the case with converting old persisted data as we did filter those out but it is the case
+        // when we receive old trade stat objects from the network of 2 not updated traders.
+        // The hash was ignoring the trade date so we use that to get a unique list
+        Map<P2PDataStorage.ByteArray, TradeStatistics2> mapWithoutDuplicates = new HashMap<>();
+        persistableNetworkPayloads.stream()
+                .filter(e -> e instanceof TradeStatistics2)
+                .map(e -> (TradeStatistics2) e)
+                .filter(TradeStatistics2::isValid)
+                .forEach(e -> mapWithoutDuplicates.putIfAbsent(new P2PDataStorage.ByteArray(e.getHash()), e));
+
+        log.info("We convert the existing {} trade statistics objects to the new format. " +
+                "This might take a bit but is only done once.", mapWithoutDuplicates.size());
+
+        mapWithoutDuplicates.values().stream()
+                .map(e -> convertToTradeStatistics3(e, false))
+                .filter(TradeStatistics3::isValid)
+                .forEach(result::add);
+
+        log.info("Conversion to {} new trade statistic objects has been completed after {} ms",
+                result.size(), System.currentTimeMillis() - ts);
+
+        return result;
+    }
+
+    private static TradeStatistics3 convertToTradeStatistics3(TradeStatistics2 tradeStatistics2, boolean fromNetwork) {
+        Map<String, String> extraDataMap = tradeStatistics2.getExtraDataMap();
+        String mediator = extraDataMap != null ? extraDataMap.get(TradeStatistics2.MEDIATOR_ADDRESS) : null;
+        String refundAgent = extraDataMap != null ? extraDataMap.get(TradeStatistics2.REFUND_AGENT_ADDRESS) : null;
+        long time = tradeStatistics2.getTradeDate().getTime();
+        byte[] hash = null;
+        if (fromNetwork) {
+            // We need to avoid that we duplicate tradeStatistics2 objects in case both traders have not udpated yet.
+            // Before v1.4.0 both traders published the trade statistics. If one trader has updated he will check
+            // the capabilities of the peer and if the peer has not updated he will leave publishing to the peer, so we
+            // do not have the problem of duplicated objects.
+            // To ensure we add only one object we will use the hash of the tradeStatistics2 object which is the same
+            // for both traders as it excluded the trade date which is different for both.
+            hash = tradeStatistics2.getHash();
+        }
+        return new TradeStatistics3(tradeStatistics2.getCurrencyCode(),
+                tradeStatistics2.getTradePrice().getValue(),
+                tradeStatistics2.getTradeAmount().getValue(),
+                tradeStatistics2.getOfferPaymentMethod(),
+                time,
+                mediator,
+                refundAgent,
+                hash);
+    }
+}

monitor/src/main/java/bisq/monitor/Monitor.java

@@ -90,7 +90,8 @@ public class Monitor {
                 Capability.DAO_STATE,
                 Capability.BUNDLE_OF_ENVELOPES,
                 Capability.REFUND_AGENT,
-                Capability.MEDIATION);
+                Capability.MEDIATION,
+                Capability.TRADE_STATISTICS_3);
 
         // assemble Metrics
         // - create reporters

proto/src/main/proto/pb.proto

@@ -524,10 +524,11 @@ message StoragePayload {
 message PersistableNetworkPayload {
     oneof message {
         AccountAgeWitness account_age_witness = 1;
-        TradeStatistics2 trade_statistics2 = 2;
+        TradeStatistics2 trade_statistics2 = 2 [deprecated = true];
         ProposalPayload proposal_payload = 3;
         BlindVotePayload blind_vote_payload = 4;
         SignedWitness signed_witness = 5;
+        TradeStatistics3 trade_statistics3 = 6;
     }
 }
 
@@ -672,22 +673,34 @@ message TradeStatistics {
 }
 
 message TradeStatistics2 {
-    string base_currency = 1;
-    string counter_currency = 2;
-    OfferPayload.Direction direction = 3;
-    int64 trade_price = 4;
-    int64 trade_amount = 5;
-    int64 trade_date = 6;
-    string payment_method_id = 7;
-    int64 offer_date = 8;
-    bool offer_use_market_based_price = 9;
-    double offer_market_price_margin = 10;
-    int64 offer_amount = 11;
-    int64 offer_min_amount = 12;
-    string offer_id = 13;
-    string deposit_tx_id = 14;
-    bytes hash = 15;
-    map<string, string> extra_data = 16;
+    string base_currency = 1 [deprecated = true];
+    string counter_currency = 2 [deprecated = true];
+    OfferPayload.Direction direction = 3 [deprecated = true];
+    int64 trade_price = 4 [deprecated = true];
+    int64 trade_amount = 5 [deprecated = true];
+    int64 trade_date = 6 [deprecated = true];
+    string payment_method_id = 7 [deprecated = true];
+    int64 offer_date = 8 [deprecated = true];
+    bool offer_use_market_based_price = 9 [deprecated = true];
+    double offer_market_price_margin = 10 [deprecated = true];
+    int64 offer_amount = 11 [deprecated = true];
+    int64 offer_min_amount = 12 [deprecated = true];
+    string offer_id = 13 [deprecated = true];
+    string deposit_tx_id = 14 [deprecated = true];
+    bytes hash = 15 [deprecated = true];
+    map<string, string> extra_data = 16 [deprecated = true];
+}
+
+message TradeStatistics3 {
+    string currency = 1;
+    int64 price = 2;
+    int64 amount = 3;
+    string payment_method = 4;
+    int64 date = 5;
+    string mediator = 6;
+    string refund_agent = 7;
+    bytes hash = 8;
+    map<string, string> extra_data = 9;
 }
 
 message MailboxStoragePayload {
@@ -1152,7 +1165,7 @@ message PersistableEnvelope {
         // BsqState bsq_state = 12; // not used but as other non-dao data have a higher index number we leave it to make clear that we cannot change following indexes
 
         AccountAgeWitnessStore account_age_witness_store = 13;
-        TradeStatistics2Store trade_statistics2_store = 14;
+        TradeStatistics2Store trade_statistics2_store = 14 [deprecated = true];
 
         // PersistableNetworkPayloadList persistable_network_payload_list = 15; // long deprecated & migration away from it is already done
 
@@ -1171,6 +1184,7 @@ message PersistableEnvelope {
         SignedWitnessStore signed_witness_store = 28;
         MediationDisputeList mediation_dispute_list = 29;
         RefundDisputeList refund_dispute_list = 30;
+        TradeStatistics3Store trade_statistics3_store = 31;
     }
 }
 
@@ -1211,8 +1225,13 @@ message SignedWitnessStore {
 }
 
 // We use a list not a hash map to save disc space. The hash can be calculated from the payload anyway
+// Deprecated
 message TradeStatistics2Store {
-    repeated TradeStatistics2 items = 1;
+    repeated TradeStatistics2 items = 1 [deprecated = true];
+}
+
+message TradeStatistics3Store {
+    repeated TradeStatistics3 items = 1;
 }
 
 message PeerList {