Merge pull request #4745 from chimp1984/fix-bug-with-reading-historical-data

Fix bug with reading historical data

common/src/main/java/bisq/common/setup/CommonSetup.java

@@ -102,11 +102,13 @@ public class CommonSetup {
 
     protected static void setupSigIntHandlers(GracefulShutDownHandler gracefulShutDownHandler) {
         Signal.handle(new Signal("INT"), signal -> {
+            log.info("Received {}", signal);
             UserThread.execute(() -> gracefulShutDownHandler.gracefulShutDown(() -> {
             }));
         });
 
         Signal.handle(new Signal("TERM"), signal -> {
+            log.info("Received {}", signal);
             UserThread.execute(() -> gracefulShutDownHandler.gracefulShutDown(() -> {
             }));
         });

core/src/main/java/bisq/core/app/BisqExecutable.java

@@ -168,8 +168,8 @@ public abstract class BisqExecutable implements GracefulShutDownHandler, BisqSet
         }
 
         AtomicInteger remaining = new AtomicInteger(hosts.size());
-        hosts.forEach(e -> {
+        hosts.forEach(host -> {
-            e.readPersisted(() -> {
+            host.readPersisted(() -> {
                 if (remaining.decrementAndGet() == 0) {
                     UserThread.execute(completeHandler);
                 }

core/src/main/java/bisq/core/app/misc/AppSetupWithP2P.java

@@ -19,7 +19,6 @@ package bisq.core.app.misc;
 
 import bisq.core.account.sign.SignedWitnessService;
 import bisq.core.account.witness.AccountAgeWitnessService;
-import bisq.core.app.TorSetup;
 import bisq.core.filter.FilterManager;
 import bisq.core.trade.statistics.TradeStatisticsManager;
 
@@ -28,6 +27,8 @@ import bisq.network.p2p.P2PServiceListener;
 import bisq.network.p2p.network.CloseConnectionReason;
 import bisq.network.p2p.network.Connection;
 import bisq.network.p2p.network.ConnectionListener;
+import bisq.network.p2p.peers.PeerManager;
+import bisq.network.p2p.storage.P2PDataStorage;
 
 import bisq.common.config.Config;
 import bisq.common.proto.persistable.PersistedDataHost;
@@ -47,32 +48,36 @@ public class AppSetupWithP2P extends AppSetup {
     protected final AccountAgeWitnessService accountAgeWitnessService;
     private final SignedWitnessService signedWitnessService;
     protected final FilterManager filterManager;
-    private final TorSetup torSetup;
+    private final P2PDataStorage p2PDataStorage;
-    protected BooleanProperty p2pNetWorkReady;
+    private final PeerManager peerManager;
     protected final TradeStatisticsManager tradeStatisticsManager;
     protected ArrayList<PersistedDataHost> persistedDataHosts;
+    protected BooleanProperty p2pNetWorkReady;
 
     @Inject
     public AppSetupWithP2P(P2PService p2PService,
+                           P2PDataStorage p2PDataStorage,
+                           PeerManager peerManager,
                            TradeStatisticsManager tradeStatisticsManager,
                            AccountAgeWitnessService accountAgeWitnessService,
                            SignedWitnessService signedWitnessService,
                            FilterManager filterManager,
-                           TorSetup torSetup,
                            Config config) {
         super(config);
         this.p2PService = p2PService;
+        this.p2PDataStorage = p2PDataStorage;
+        this.peerManager = peerManager;
         this.tradeStatisticsManager = tradeStatisticsManager;
         this.accountAgeWitnessService = accountAgeWitnessService;
         this.signedWitnessService = signedWitnessService;
         this.filterManager = filterManager;
-        this.torSetup = torSetup;
         this.persistedDataHosts = new ArrayList<>();
     }
 
     @Override
     public void initPersistedDataHosts() {
-        persistedDataHosts.add(p2PService);
+        persistedDataHosts.add(p2PDataStorage);
+        persistedDataHosts.add(peerManager);
 
         // we apply at startup the reading of persisted data but don't want to get it triggered in the constructor
         persistedDataHosts.forEach(e -> {
@@ -88,7 +93,7 @@ public class AppSetupWithP2P extends AppSetup {
     @Override
     protected void initBasicServices() {
         String postFix = "_" + config.baseCurrencyNetwork.name();
-        p2PService.getP2PDataStorage().readFromResources(postFix, this::startInitP2PNetwork);
+        p2PDataStorage.readFromResources(postFix, this::startInitP2PNetwork);
     }
 
     private void startInitP2PNetwork() {

core/src/main/java/bisq/core/app/misc/AppSetupWithP2PAndDAO.java

@@ -19,7 +19,6 @@ package bisq.core.app.misc;
 
 import bisq.core.account.sign.SignedWitnessService;
 import bisq.core.account.witness.AccountAgeWitnessService;
-import bisq.core.app.TorSetup;
 import bisq.core.dao.DaoSetup;
 import bisq.core.dao.governance.ballot.BallotListService;
 import bisq.core.dao.governance.blindvote.MyBlindVoteListService;
@@ -31,6 +30,8 @@ import bisq.core.filter.FilterManager;
 import bisq.core.trade.statistics.TradeStatisticsManager;
 
 import bisq.network.p2p.P2PService;
+import bisq.network.p2p.peers.PeerManager;
+import bisq.network.p2p.storage.P2PDataStorage;
 
 import bisq.common.config.Config;
 
@@ -44,6 +45,8 @@ public class AppSetupWithP2PAndDAO extends AppSetupWithP2P {
 
     @Inject
     public AppSetupWithP2PAndDAO(P2PService p2PService,
+                                 P2PDataStorage p2PDataStorage,
+                                 PeerManager peerManager,
                                  TradeStatisticsManager tradeStatisticsManager,
                                  AccountAgeWitnessService accountAgeWitnessService,
                                  SignedWitnessService signedWitnessService,
@@ -55,14 +58,14 @@ public class AppSetupWithP2PAndDAO extends AppSetupWithP2P {
                                  MyProposalListService myProposalListService,
                                  MyReputationListService myReputationListService,
                                  MyProofOfBurnListService myProofOfBurnListService,
-                                 TorSetup torSetup,
                                  Config config) {
         super(p2PService,
+                p2PDataStorage,
+                peerManager,
                 tradeStatisticsManager,
                 accountAgeWitnessService,
                 signedWitnessService,
                 filterManager,
-                torSetup,
                 config);
 
         this.daoSetup = daoSetup;

core/src/main/java/bisq/core/setup/CorePersistedDataHost.java

@@ -35,7 +35,8 @@ import bisq.core.trade.failed.FailedTradesManager;
 import bisq.core.user.Preferences;
 import bisq.core.user.User;
 
-import bisq.network.p2p.P2PService;
+import bisq.network.p2p.peers.PeerManager;
+import bisq.network.p2p.storage.P2PDataStorage;
 
 import bisq.common.config.Config;
 import bisq.common.proto.persistable.PersistedDataHost;
@@ -63,7 +64,8 @@ public class CorePersistedDataHost {
         persistedDataHosts.add(injector.getInstance(ArbitrationDisputeListService.class));
         persistedDataHosts.add(injector.getInstance(MediationDisputeListService.class));
         persistedDataHosts.add(injector.getInstance(RefundDisputeListService.class));
-        persistedDataHosts.add(injector.getInstance(P2PService.class));
+        persistedDataHosts.add(injector.getInstance(P2PDataStorage.class));
+        persistedDataHosts.add(injector.getInstance(PeerManager.class));
 
         if (injector.getInstance(Config.class).daoActivated) {
             persistedDataHosts.add(injector.getInstance(BallotListService.class));

desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java

@@ -114,7 +114,6 @@ public class BsqDashboardView extends ActivatableView<GridPane, Void> implements
 
     private Coin availableAmount;
     private int gridRow = 0;
-    double percentToTrim = 5;
     double howManyStdDevsConstituteOutlier = 10;
 
 

p2p/src/main/java/bisq/network/p2p/P2PService.java

@@ -52,7 +52,6 @@ import bisq.common.crypto.KeyRing;
 import bisq.common.crypto.PubKeyRing;
 import bisq.common.proto.ProtobufferException;
 import bisq.common.proto.network.NetworkEnvelope;
-import bisq.common.proto.persistable.PersistedDataHost;
 import bisq.common.util.Utilities;
 
 import com.google.inject.Inject;
@@ -105,7 +104,7 @@ import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 public class P2PService implements SetupListener, MessageListener, ConnectionListener, RequestDataManager.Listener,
-        HashMapChangedListener, PersistedDataHost {
+        HashMapChangedListener {
     private static final Logger log = LoggerFactory.getLogger(P2PService.class);
 
     private final SeedNodeRepository seedNodeRepository;
@@ -184,12 +183,6 @@ public class P2PService implements SetupListener, MessageListener, ConnectionLis
         });
     }
 
-    @Override
-    public void readPersisted(Runnable completeHandler) {
-        p2PDataStorage.readPersisted(completeHandler);
-        peerManager.readPersisted(completeHandler);
-    }
-
 
     ///////////////////////////////////////////////////////////////////////////////////////////
     // API

p2p/src/main/java/bisq/network/p2p/storage/persistence/HistoricalDataStoreService.java

@@ -29,6 +29,7 @@ import java.io.File;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import lombok.extern.slf4j.Slf4j;
 
@@ -154,14 +155,18 @@ public abstract class HistoricalDataStoreService<T extends PersistableNetworkPay
             ImmutableMap.Builder<P2PDataStorage.ByteArray, PersistableNetworkPayload> allHistoricalPayloadsBuilder = ImmutableMap.builder();
             ImmutableMap.Builder<String, PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> storesByVersionBuilder = ImmutableMap.builder();
 
+            AtomicInteger numFiles = new AtomicInteger(Version.HISTORICAL_RESOURCE_FILE_VERSION_TAGS.size());
             Version.HISTORICAL_RESOURCE_FILE_VERSION_TAGS.forEach(version -> readHistoricalStoreFromResources(version,
                     postFix,
                     allHistoricalPayloadsBuilder,
-                    storesByVersionBuilder));
+                    storesByVersionBuilder,
-
+                    () -> {
-            allHistoricalPayloads = allHistoricalPayloadsBuilder.build();
+                        if (numFiles.decrementAndGet() == 0) {
-            storesByVersion = storesByVersionBuilder.build();
+                            allHistoricalPayloads = allHistoricalPayloadsBuilder.build();
-            completeHandler.run();
+                            storesByVersion = storesByVersionBuilder.build();
+                            completeHandler.run();
+                        }
+                    }));
         });
     }
 
@@ -173,21 +178,24 @@ public abstract class HistoricalDataStoreService<T extends PersistableNetworkPay
     private void readHistoricalStoreFromResources(String version,
                                                   String postFix,
                                                   ImmutableMap.Builder<P2PDataStorage.ByteArray, PersistableNetworkPayload> allHistoricalDataBuilder,
-                                                  ImmutableMap.Builder<String, PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> storesByVersionBuilder) {
+                                                  ImmutableMap.Builder<String, PersistableNetworkPayloadStore<? extends PersistableNetworkPayload>> storesByVersionBuilder,
+                                                  Runnable completeHandler) {
         String fileName = getFileName() + "_" + version;
         boolean wasCreatedFromResources = makeFileFromResourceFile(fileName, postFix);
 
         // If resource file does not exist we do not create a new store as it would never get filled.
-        persistenceManager.readPersisted(getFileName(), persisted -> {
+        persistenceManager.readPersisted(fileName, persisted -> {
                     storesByVersionBuilder.put(version, persisted);
                     allHistoricalDataBuilder.putAll(persisted.getMap());
-
+                    log.info("We have read from {} {} historical items.", fileName, persisted.getMap().size());
                     if (wasCreatedFromResources) {
                         pruneStore(persisted, version);
                     }
+                    completeHandler.run();
                 },
                 () -> {
                     log.warn("Resource file with file name {} does not exits.", fileName);
+                    completeHandler.run();
                 });
     }