Remove outliers when calculating BSQ rate

Add percentage for upper and lower threshold for outlier detection.

core/src/main/java/bisq/core/user/Preferences.java

@@ -416,6 +416,11 @@ public final class Preferences implements PersistedDataHost, BridgeAddressProvid
         requestPersistence();
     }
 
+    public void setBsqAverageTrimThreshold(double bsqAverageTrimThreshold) {
+        prefPayload.setBsqAverageTrimThreshold(bsqAverageTrimThreshold);
+        requestPersistence();
+    }
+
     public Optional<AutoConfirmSettings> findAutoConfirmSettings(String currencyCode) {
         return prefPayload.getAutoConfirmSettingsList().stream()
                 .filter(e -> e.getCurrencyCode().equals(currencyCode))
@@ -1034,6 +1039,8 @@ public final class Preferences implements PersistedDataHost, BridgeAddressProvid
 
         void setTacAcceptedV120(boolean tacAccepted);
 
+        void setBsqAverageTrimThreshold(double bsqAverageTrimThreshold);
+
         void setAutoConfirmSettings(AutoConfirmSettings autoConfirmSettings);
     }
 }

core/src/main/java/bisq/core/user/PreferencesPayload.java

@@ -124,6 +124,7 @@ public final class PreferencesPayload implements PersistableEnvelope {
     private double buyerSecurityDepositAsPercentForCrypto = getDefaultBuyerSecurityDepositAsPercent();
     private int blockNotifyPort;
     private boolean tacAcceptedV120;
+    private double bsqAverageTrimThreshold = 0.05;
 
     // Added at 1.3.8
     private List<AutoConfirmSettings> autoConfirmSettingsList = new ArrayList<>();
@@ -188,9 +189,10 @@ public final class PreferencesPayload implements PersistableEnvelope {
                 .setBuyerSecurityDepositAsPercentForCrypto(buyerSecurityDepositAsPercentForCrypto)
                 .setBlockNotifyPort(blockNotifyPort)
                 .setTacAcceptedV120(tacAcceptedV120)
+                .setBsqAverageTrimThreshold(bsqAverageTrimThreshold)
                 .addAllAutoConfirmSettings(autoConfirmSettingsList.stream()
-                    .map(autoConfirmSettings -> ((protobuf.AutoConfirmSettings) autoConfirmSettings.toProtoMessage()))
+                        .map(autoConfirmSettings -> ((protobuf.AutoConfirmSettings) autoConfirmSettings.toProtoMessage()))
-                    .collect(Collectors.toList()));
+                        .collect(Collectors.toList()));
 
         Optional.ofNullable(backupDirectory).ifPresent(builder::setBackupDirectory);
         Optional.ofNullable(preferredTradeCurrency).ifPresent(e -> builder.setPreferredTradeCurrency((protobuf.TradeCurrency) e.toProtoMessage()));
@@ -280,6 +282,7 @@ public final class PreferencesPayload implements PersistableEnvelope {
                 proto.getBuyerSecurityDepositAsPercentForCrypto(),
                 proto.getBlockNotifyPort(),
                 proto.getTacAcceptedV120(),
+                proto.getBsqAverageTrimThreshold(),
                 proto.getAutoConfirmSettingsList().isEmpty() ? new ArrayList<>() :
                         new ArrayList<>(proto.getAutoConfirmSettingsList().stream()
                                 .map(AutoConfirmSettings::fromProto)

core/src/main/resources/i18n/displayStrings.properties

@@ -1184,6 +1184,7 @@ setting.preferences.general=General preferences
 setting.preferences.explorer=Bitcoin Explorer
 setting.preferences.explorer.bsq=Bisq Explorer
 setting.preferences.deviation=Max. deviation from market price
+setting.preferences.bsqAverageTrimThreshold=Outlier threshold for BSQ rate
 setting.preferences.avoidStandbyMode=Avoid standby mode
 setting.preferences.autoConfirmXMR=XMR auto-confirm
 setting.preferences.autoConfirmEnabled=Enabled

desktop/src/main/java/bisq/desktop/main/dao/economy/dashboard/BsqDashboardView.java

@@ -20,6 +20,7 @@ package bisq.desktop.main.dao.economy.dashboard;
 import bisq.desktop.common.view.ActivatableView;
 import bisq.desktop.common.view.FxmlView;
 import bisq.desktop.components.TextFieldWithIcon;
+import bisq.desktop.util.AxisInlierUtils;
 
 import bisq.core.dao.DaoFacade;
 import bisq.core.dao.state.DaoStateListener;
@@ -112,8 +113,10 @@ public class BsqDashboardView extends ActivatableView<GridPane, Void> implements
     private Label marketPriceLabel;
 
     private Coin availableAmount;
-
     private int gridRow = 0;
+    double percentToTrim = 5;
+    double howManyStdDevsConstituteOutlier = 10;
+
 
     ///////////////////////////////////////////////////////////////////////////////////////////
     // Constructor, lifecycle
@@ -121,10 +124,10 @@ public class BsqDashboardView extends ActivatableView<GridPane, Void> implements
 
     @Inject
     public BsqDashboardView(DaoFacade daoFacade,
-                             TradeStatisticsManager tradeStatisticsManager,
+                            TradeStatisticsManager tradeStatisticsManager,
-                             PriceFeedService priceFeedService,
+                            PriceFeedService priceFeedService,
-                             Preferences preferences,
+                            Preferences preferences,
-                             BsqFormatter bsqFormatter) {
+                            BsqFormatter bsqFormatter) {
         this.daoFacade = daoFacade;
         this.tradeStatisticsManager = tradeStatisticsManager;
         this.priceFeedService = priceFeedService;
@@ -134,7 +137,6 @@ public class BsqDashboardView extends ActivatableView<GridPane, Void> implements
 
     @Override
     public void initialize() {
-
         ADJUSTERS.put(DAY, TemporalAdjusters.ofDateAdjuster(d -> d));
 
         createKPIs();
@@ -368,15 +370,24 @@ public class BsqDashboardView extends ActivatableView<GridPane, Void> implements
     }
 
     private long updateAveragePriceField(TextField textField, int days, boolean isUSDField) {
+        double percentToTrim = Math.max(0, Math.min(49, preferences.getBsqAverageTrimThreshold() * 100));
         Date pastXDays = getPastDate(days);
-        List<TradeStatistics3> bsqTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream()
+        List<TradeStatistics3> bsqAllTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream()
                 .filter(e -> e.getCurrency().equals("BSQ"))
                 .filter(e -> e.getDate().after(pastXDays))
                 .collect(Collectors.toList());
-        List<TradeStatistics3> usdTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream()
+        List<TradeStatistics3> bsqTradePastXDays = percentToTrim > 0 ?
+                removeOutliers(bsqAllTradePastXDays, percentToTrim) :
+                bsqAllTradePastXDays;
+
+        List<TradeStatistics3> usdAllTradePastXDays = tradeStatisticsManager.getObservableTradeStatisticsSet().stream()
                 .filter(e -> e.getCurrency().equals("USD"))
                 .filter(e -> e.getDate().after(pastXDays))
                 .collect(Collectors.toList());
+        List<TradeStatistics3> usdTradePastXDays = percentToTrim > 0 ?
+                removeOutliers(usdAllTradePastXDays, percentToTrim) :
+                usdAllTradePastXDays;
+
         long average = isUSDField ? getUSDAverage(bsqTradePastXDays, usdTradePastXDays) :
                 getBTCAverage(bsqTradePastXDays);
         Price avgPrice = isUSDField ? Price.valueOf("USD", average) :
@@ -390,11 +401,26 @@ public class BsqDashboardView extends ActivatableView<GridPane, Void> implements
         return average;
     }
 
-    private long getBTCAverage(List<TradeStatistics3> bsqList) {
+    private List<TradeStatistics3> removeOutliers(List<TradeStatistics3> list, double percentToTrim) {
+        List<Double> yValues = list.stream()
+                .filter(TradeStatistics3::isValid)
+                .map(e -> (double) e.getPrice())
+                .collect(Collectors.toList());
+
+        Tuple2<Double, Double> tuple = AxisInlierUtils.findInlierRange(yValues, percentToTrim, howManyStdDevsConstituteOutlier);
+        double lowerBound = tuple.first;
+        double upperBound = tuple.second;
+        return list.stream()
+                .filter(e -> e.getPrice() > lowerBound)
+                .filter(e -> e.getPrice() < upperBound)
+                .collect(Collectors.toList());
+    }
+
+    private long getBTCAverage(List<TradeStatistics3> list) {
         long accumulatedVolume = 0;
         long accumulatedAmount = 0;
 
-        for (TradeStatistics3 item : bsqList) {
+        for (TradeStatistics3 item : list) {
             accumulatedVolume += item.getTradeVolume().getValue();
             accumulatedAmount += item.getTradeAmount().getValue(); // Amount of BTC traded
         }

desktop/src/main/java/bisq/desktop/main/settings/preferences/PreferencesView.java

@@ -148,11 +148,11 @@ public class PreferencesView extends ActivatableViewAndModel<GridPane, Preferenc
     private ObservableList<CryptoCurrency> cryptoCurrencies;
     private ObservableList<CryptoCurrency> allCryptoCurrencies;
     private ObservableList<TradeCurrency> tradeCurrencies;
-    private InputTextField deviationInputTextField;
+    private InputTextField deviationInputTextField, bsqAverageTrimThresholdTextField;
-    private ChangeListener<String> deviationListener, ignoreTradersListListener, ignoreDustThresholdListener,
+    private ChangeListener<String> deviationListener, bsqAverageTrimThresholdListener, ignoreTradersListListener, ignoreDustThresholdListener,
             rpcUserListener, rpcPwListener, blockNotifyPortListener,
             autoConfTradeLimitListener, autoConfServiceAddressListener;
-    private ChangeListener<Boolean> deviationFocusedListener;
+    private ChangeListener<Boolean> deviationFocusedListener, bsqAverageTrimThresholdFocusedListener;
     private ChangeListener<Boolean> useCustomFeeCheckboxListener;
     private ChangeListener<Number> transactionFeeChangeListener;
     private final boolean daoOptionsSet;
@@ -318,7 +318,6 @@ public class PreferencesView extends ActivatableViewAndModel<GridPane, Preferenc
         // deviation
         deviationInputTextField = addInputTextField(root, ++gridRow,
                 Res.get("setting.preferences.deviation"));
-
         deviationListener = (observable, oldValue, newValue) -> {
             try {
                 double value = ParsingUtils.parsePercentStringToDouble(newValue);
@@ -327,16 +326,16 @@ public class PreferencesView extends ActivatableViewAndModel<GridPane, Preferenc
                     preferences.setMaxPriceDistanceInPercent(value);
                 } else {
                     new Popup().warning(Res.get("setting.preferences.deviationToLarge", maxDeviation * 100)).show();
-                    UserThread.runAfter(() -> deviationInputTextField.setText(FormattingUtils.formatPercentagePrice(preferences.getMaxPriceDistanceInPercent())), 100, TimeUnit.MILLISECONDS);
+                    UserThread.runAfter(() -> deviationInputTextField.setText(FormattingUtils.formatToPercentWithSymbol(preferences.getMaxPriceDistanceInPercent())), 100, TimeUnit.MILLISECONDS);
                 }
             } catch (NumberFormatException t) {
                 log.error("Exception at parseDouble deviation: " + t.toString());
-                UserThread.runAfter(() -> deviationInputTextField.setText(FormattingUtils.formatPercentagePrice(preferences.getMaxPriceDistanceInPercent())), 100, TimeUnit.MILLISECONDS);
+                UserThread.runAfter(() -> deviationInputTextField.setText(FormattingUtils.formatToPercentWithSymbol(preferences.getMaxPriceDistanceInPercent())), 100, TimeUnit.MILLISECONDS);
             }
         };
         deviationFocusedListener = (observable1, oldValue1, newValue1) -> {
             if (oldValue1 && !newValue1)
-                UserThread.runAfter(() -> deviationInputTextField.setText(FormattingUtils.formatPercentagePrice(preferences.getMaxPriceDistanceInPercent())), 100, TimeUnit.MILLISECONDS);
+                UserThread.runAfter(() -> deviationInputTextField.setText(FormattingUtils.formatToPercentWithSymbol(preferences.getMaxPriceDistanceInPercent())), 100, TimeUnit.MILLISECONDS);
         };
 
         // ignoreTraders
@@ -617,7 +616,7 @@ public class PreferencesView extends ActivatableViewAndModel<GridPane, Preferenc
     }
 
     private void initializeDaoOptions() {
-        daoOptionsTitledGroupBg = addTitledGroupBg(root, ++gridRow, 3, Res.get("setting.preferences.daoOptions"), Layout.GROUP_DISTANCE);
+        daoOptionsTitledGroupBg = addTitledGroupBg(root, ++gridRow, 4, Res.get("setting.preferences.daoOptions"), Layout.GROUP_DISTANCE);
         resyncDaoFromResourcesButton = addButton(root, gridRow, Res.get("setting.preferences.dao.resyncFromResources.label"), Layout.TWICE_FIRST_ROW_AND_GROUP_DISTANCE);
         resyncDaoFromResourcesButton.setMaxWidth(Double.MAX_VALUE);
         GridPane.setHgrow(resyncDaoFromResourcesButton, Priority.ALWAYS);
@@ -626,6 +625,36 @@ public class PreferencesView extends ActivatableViewAndModel<GridPane, Preferenc
         resyncDaoFromGenesisButton.setMaxWidth(Double.MAX_VALUE);
         GridPane.setHgrow(resyncDaoFromGenesisButton, Priority.ALWAYS);
 
+        bsqAverageTrimThresholdTextField = addInputTextField(root, ++gridRow,
+                Res.get("setting.preferences.bsqAverageTrimThreshold"));
+        bsqAverageTrimThresholdTextField.setText(FormattingUtils.formatToPercentWithSymbol(preferences.getBsqAverageTrimThreshold()));
+
+        bsqAverageTrimThresholdListener = (observable, oldValue, newValue) -> {
+            try {
+                double value = ParsingUtils.parsePercentStringToDouble(newValue);
+                double maxValue = 0.49;
+                checkArgument(value >= 0, "Input must be positive");
+                if (value <= maxValue) {
+                    preferences.setBsqAverageTrimThreshold(value);
+                } else {
+                    new Popup().warning(Res.get("setting.preferences.deviationToLarge",
+                            maxValue * 100)).show();
+                    UserThread.runAfter(() -> bsqAverageTrimThresholdTextField.setText(FormattingUtils.formatToPercentWithSymbol(
+                            preferences.getBsqAverageTrimThreshold())), 100, TimeUnit.MILLISECONDS);
+                }
+            } catch (NumberFormatException t) {
+                log.error("Exception: " + t.toString());
+                UserThread.runAfter(() -> bsqAverageTrimThresholdTextField.setText(FormattingUtils.formatToPercentWithSymbol(
+                        preferences.getBsqAverageTrimThreshold())), 100, TimeUnit.MILLISECONDS);
+            }
+        };
+        bsqAverageTrimThresholdFocusedListener = (observable1, oldValue1, newValue1) -> {
+            if (oldValue1 && !newValue1)
+                UserThread.runAfter(() -> bsqAverageTrimThresholdTextField.setText(FormattingUtils.formatToPercentWithSymbol(
+                        preferences.getBsqAverageTrimThreshold())), 100, TimeUnit.MILLISECONDS);
+        };
+
+
         isDaoFullNodeToggleButton = addSlideToggleButton(root, ++gridRow, Res.get("setting.preferences.dao.isDaoFullNode"));
         rpcUserTextField = addInputTextField(root, ++gridRow, Res.get("setting.preferences.dao.rpcUser"));
         rpcUserTextField.setVisible(false);
@@ -861,7 +890,7 @@ public class PreferencesView extends ActivatableViewAndModel<GridPane, Preferenc
         });
         bsqBlockChainExplorerComboBox.setOnAction(e -> preferences.setBsqBlockChainExplorer(bsqBlockChainExplorerComboBox.getSelectionModel().getSelectedItem()));
 
-        deviationInputTextField.setText(FormattingUtils.formatPercentagePrice(preferences.getMaxPriceDistanceInPercent()));
+        deviationInputTextField.setText(FormattingUtils.formatToPercentWithSymbol(preferences.getMaxPriceDistanceInPercent()));
         deviationInputTextField.textProperty().addListener(deviationListener);
         deviationInputTextField.focusedProperty().addListener(deviationFocusedListener);
 
@@ -950,6 +979,10 @@ public class PreferencesView extends ActivatableViewAndModel<GridPane, Preferenc
     private void activateDaoPreferences() {
         boolean daoFullNode = preferences.isDaoFullNode();
         isDaoFullNodeToggleButton.setSelected(daoFullNode);
+
+        bsqAverageTrimThresholdTextField.textProperty().addListener(bsqAverageTrimThresholdListener);
+        bsqAverageTrimThresholdTextField.focusedProperty().addListener(bsqAverageTrimThresholdFocusedListener);
+
         String rpcUser = preferences.getRpcUser();
         String rpcPw = preferences.getRpcPw();
         int blockNotifyPort = preferences.getBlockNotifyPort();
@@ -1091,6 +1124,8 @@ public class PreferencesView extends ActivatableViewAndModel<GridPane, Preferenc
     private void deactivateDaoPreferences() {
         resyncDaoFromResourcesButton.setOnAction(null);
         resyncDaoFromGenesisButton.setOnAction(null);
+        bsqAverageTrimThresholdTextField.textProperty().removeListener(bsqAverageTrimThresholdListener);
+        bsqAverageTrimThresholdTextField.focusedProperty().removeListener(bsqAverageTrimThresholdFocusedListener);
         isDaoFullNodeToggleButton.setOnAction(null);
         rpcUserTextField.textProperty().removeListener(rpcUserListener);
         rpcPwTextField.textProperty().removeListener(rpcPwListener);

desktop/src/main/java/bisq/desktop/util/AxisInlierUtils.java

@@ -91,7 +91,7 @@ public class AxisInlierUtils {
     /* Finds the minimum and maximum inlier values. The returned values may be NaN.
      * See `computeInlierThreshold` for the definition of inlier.
      */
-    private static Tuple2<Double, Double> findInlierRange(
+    public static Tuple2<Double, Double> findInlierRange(
             List<Double> yValues,
             double percentToTrim,
             double howManyStdDevsConstituteOutlier

proto/src/main/proto/pb.proto

@@ -1576,6 +1576,7 @@ message PreferencesPayload {
     int32 css_theme = 54;
     bool tac_accepted_v120 = 55;
     repeated AutoConfirmSettings auto_confirm_settings = 56;
+    double bsq_average_trim_threshold = 57;
 }
 
 message AutoConfirmSettings {