diff --git a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java index e585eff1fa..08963dab47 100644 --- a/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java +++ b/core/src/main/java/bisq/core/trade/statistics/TradeStatistics3.java @@ -24,6 +24,7 @@ import bisq.core.monetary.Volume; import bisq.core.offer.OfferUtil; import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; +import bisq.network.p2p.storage.payload.DateSortedTruncatablePayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; import bisq.network.p2p.storage.payload.ProcessOncePersistableNetworkPayload; @@ -64,7 +65,7 @@ import static com.google.common.base.Preconditions.checkNotNull; @Slf4j @Getter public final class TradeStatistics3 implements ProcessOncePersistableNetworkPayload, PersistableNetworkPayload, - CapabilityRequiringPayload { + CapabilityRequiringPayload, DateSortedTruncatablePayload { // This enum must not change the order as we use the ordinal for storage to reduce data size. // The payment method string can be quite long and would consume 15% more space. @@ -267,6 +268,16 @@ public final class TradeStatistics3 implements ProcessOncePersistableNetworkPayl return new Capabilities(Capability.TRADE_STATISTICS_3); } + @Override + public Date getDate() { + return getTradeDate(); + } + + @Override + public int maxItems() { + return 3000; + } + public void pruneOptionalData() { mediator = null; refundAgent = null; diff --git a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java index 7b9e3714ce..9d6e0a705e 100644 --- a/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java +++ b/p2p/src/main/java/bisq/network/p2p/storage/P2PDataStorage.java @@ -37,6 +37,7 @@ import bisq.network.p2p.storage.messages.RefreshOfferMessage; import bisq.network.p2p.storage.messages.RemoveDataMessage; import bisq.network.p2p.storage.messages.RemoveMailboxDataMessage; import bisq.network.p2p.storage.payload.CapabilityRequiringPayload; +import bisq.network.p2p.storage.payload.DateSortedTruncatablePayload; import bisq.network.p2p.storage.payload.DateTolerantPayload; import bisq.network.p2p.storage.payload.MailboxStoragePayload; import bisq.network.p2p.storage.payload.PersistableNetworkPayload; @@ -96,7 +97,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; @@ -317,30 +317,58 @@ public class P2PDataStorage implements MessageListener, ConnectionListener, Pers * by a given set of keys and peer capabilities. */ static private Set filterKnownHashes( - Map mapToFilter, - Function objToPayloadFunction, + Map toFilter, + Function objToPayload, Set knownHashes, Capabilities peerCapabilities, int maxEntries, - AtomicBoolean wasTruncated) { + AtomicBoolean outTruncated) { - AtomicInteger limit = new AtomicInteger(maxEntries); + log.info("Num knownHashes {}", knownHashes.size()); - Set filteredResults = mapToFilter.entrySet().stream() - .filter(e -> !knownHashes.contains(e.getKey())) - .filter(e -> limit.decrementAndGet() >= 0) + Set> entries = toFilter.entrySet(); + List dateSortedTruncatablePayloads = entries.stream() + .filter(entry -> entry.getValue() instanceof DateSortedTruncatablePayload) + .filter(entry -> !knownHashes.contains(entry.getKey())) .map(Map.Entry::getValue) - .filter(networkPayload -> shouldTransmitPayloadToPeer(peerCapabilities, - objToPayloadFunction.apply(networkPayload))) - .collect(Collectors.toSet()); - - if (limit.get() < 0) { - wasTruncated.set(true); + .filter(payload -> shouldTransmitPayloadToPeer(peerCapabilities, objToPayload.apply(payload))) + .sorted(Comparator.comparing(payload -> ((DateSortedTruncatablePayload) payload).getDate())) + .collect(Collectors.toList()); + log.info("Num filtered dateSortedTruncatablePayloads {}", dateSortedTruncatablePayloads.size()); + if (!dateSortedTruncatablePayloads.isEmpty()) { + int maxItems = ((DateSortedTruncatablePayload) dateSortedTruncatablePayloads.get(0)).maxItems(); + if (dateSortedTruncatablePayloads.size() > maxItems) { + int fromIndex = dateSortedTruncatablePayloads.size() - maxItems; + int toIndex = dateSortedTruncatablePayloads.size(); + dateSortedTruncatablePayloads = dateSortedTruncatablePayloads.subList(fromIndex, toIndex); + log.info("Num truncated dateSortedTruncatablePayloads {}", dateSortedTruncatablePayloads.size()); + } } - return filteredResults; + List filteredResults = entries.stream() + .filter(entry -> !(entry.getValue() instanceof DateSortedTruncatablePayload)) + .filter(entry -> !knownHashes.contains(entry.getKey())) + .map(Map.Entry::getValue) + .filter(payload -> shouldTransmitPayloadToPeer(peerCapabilities, objToPayload.apply(payload))) + .collect(Collectors.toList()); + log.info("Num filtered non-dateSortedTruncatablePayloads {}", filteredResults.size()); + + // The non-dateSortedTruncatablePayloads have higher prio, so we added dateSortedTruncatablePayloads + // after those so in case we need to truncate we first truncate the dateSortedTruncatablePayloads. + filteredResults.addAll(dateSortedTruncatablePayloads); + + if (filteredResults.size() > maxEntries) { + filteredResults = filteredResults.subList(0, maxEntries); + outTruncated.set(true); + log.info("Num truncated filteredResults {}", filteredResults.size()); + } else { + log.info("Num filteredResults {}", filteredResults.size()); + } + + return new HashSet<>(filteredResults); } + private Set getKeysAsByteSet(Map map) { return map.keySet().stream() .map(e -> e.bytes) diff --git a/p2p/src/main/java/bisq/network/p2p/storage/payload/DateSortedTruncatablePayload.java b/p2p/src/main/java/bisq/network/p2p/storage/payload/DateSortedTruncatablePayload.java new file mode 100644 index 0000000000..8e7d830a39 --- /dev/null +++ b/p2p/src/main/java/bisq/network/p2p/storage/payload/DateSortedTruncatablePayload.java @@ -0,0 +1,31 @@ +/* + * This file is part of Bisq. + * + * Bisq is free software: you can redistribute it and/or modify it + * under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Bisq is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public + * License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Bisq. If not, see . + */ + +package bisq.network.p2p.storage.payload; + +import java.util.Date; + +/** + * Marker interface for PersistableNetworkPayloads which get truncated at initial data response in case we exceed + * the max items defined for that type of object. The truncation happens on a sorted list where we use the date for + * sorting so in case of truncation we prefer to receive the most recent data. + */ +public interface DateSortedTruncatablePayload extends PersistableNetworkPayload { + Date getDate(); + + int maxItems(); +}