Merge branch 'dao-permute-blindvotelist-if-not-majority-hash' into dao-fix-vote-result-with-permutated-blindvotelist

# Conflicts:
#	core/src/main/java/bisq/core/dao/governance/votereveal/VoteRevealService.java

common/src/main/java/bisq/common/util/PermutationUtil.java

@@ -0,0 +1,107 @@
+/*
+ * This file is part of Bisq.
+ *
+ * Bisq is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Bisq is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package bisq.common.util;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class PermutationUtil {
+
+    /**
+     * @param list                  Original list
+     * @param indicesToRemove       List of indices to remove
+     * @param <T>                   Type of List items
+     * @return Partial list where items at indices of indicesToRemove have been removed
+     */
+    public static <T> List<T> getPartialList(List<T> list, List<Integer> indicesToRemove) {
+        List<T> altered = new ArrayList<>(list);
+
+        // Eliminate duplicates
+        indicesToRemove = new ArrayList<>(new HashSet<>(indicesToRemove));
+
+        // Sort
+        Collections.sort(indicesToRemove);
+
+        // Reverse list.
+        // We need to remove from highest index downwards to not change order of remaining indices
+        Collections.reverse(indicesToRemove);
+
+        indicesToRemove.forEach(index -> {
+            if (altered.size() > index && index >= 0)
+                altered.remove((int) index);
+        });
+        return altered;
+    }
+
+    /**
+     * Returns a list of all possible permutations of a give sorted list ignoring duplicates.
+     * E.g. List [A,B,C] results in this list of permutations: [[A], [B], [A,B], [C], [A,C], [B,C], [A,B,C]]
+     * Number of variations and iterations grows with 2^n - 1 where n is the number of items in the list.
+     * With 20 items we reach about 1 million iterations and it takes about 0.5 sec.
+     * To avoid performance issues we added the maxIterations parameter to stop once the number of iterations has
+     * reached the maxIterations and return in such a case the list of permutations we have been able to create.
+     * Depending on the type of object which is stored in the list the memory usage should to be considered as well for
+     * choosing the right maxIterations value.
+     *
+     * @param list              List from which we create permutations
+     * @param maxIterations     Max. number of iterations including inner iterations
+     * @param <T>               Type of list items
+     * @return List of possible permutations of the original list
+     */
+    public static <T> List<List<T>> findAllPermutations(List<T> list, int maxIterations) {
+        List<List<T>> result = new ArrayList<>();
+        int counter = 0;
+        long ts = System.currentTimeMillis();
+        for (T item : list) {
+            counter++;
+            if (counter > maxIterations) {
+                log.warn("We reached maxIterations of our allowed iterations and return current state of the result. " +
+                        "counter={}", counter);
+                return result;
+            }
+
+            List<List<T>> subLists = new ArrayList<>();
+            for (int n = 0; n < result.size(); n++) {
+                counter++;
+                if (counter > maxIterations) {
+                    log.warn("We reached maxIterations of our allowed iterations and return current state of the result. " +
+                            "counter={}", counter);
+                    return result;
+                }
+                List<T> subList = new ArrayList<>(result.get(n));
+                subList.add(item);
+                subLists.add(subList);
+            }
+
+            // add single item
+            result.add(new ArrayList<>(Collections.singletonList(item)));
+
+            // add subLists
+            result.addAll(subLists);
+        }
+
+        log.info("findAllPermutations took {} ms for {} items and {} iterations. Heap size used: {} MB",
+                (System.currentTimeMillis() - ts), list.size(), counter, Profiler.getUsedMemoryInMB());
+        return result;
+    }
+}

common/src/test/java/bisq/common/util/PermutationTest.java

@@ -0,0 +1,432 @@
+/*
+ * This file is part of Bisq.
+ *
+ * bisq is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * bisq is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with bisq. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package bisq.common.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+public class PermutationTest {
+
+
+    @Test
+    public void testGetPartialList() {
+        String blindVote0 = "blindVote0";
+        String blindVote1 = "blindVote1";
+        String blindVote2 = "blindVote2";
+        String blindVote3 = "blindVote3";
+        String blindVote4 = "blindVote4";
+        String blindVote5 = "blindVote5";
+
+        List<String> list = new ArrayList<>(Arrays.asList(blindVote0, blindVote1, blindVote2, blindVote3, blindVote4, blindVote5));
+        List<Integer> indicesToRemove = Arrays.asList(0, 3);
+        List<String> expected = new ArrayList<>(Arrays.asList(blindVote1, blindVote2, blindVote4, blindVote5));
+        List<String> result = PermutationUtil.getPartialList(list, indicesToRemove);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // remove nothing
+        indicesToRemove = new ArrayList<>();
+        expected = new ArrayList<>(list);
+        result = PermutationUtil.getPartialList(list, indicesToRemove);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // remove first
+        indicesToRemove = indicesToRemove = Collections.singletonList(0);
+        expected = new ArrayList<>(list);
+        expected.remove(0);
+        result = PermutationUtil.getPartialList(list, indicesToRemove);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // remove last
+        indicesToRemove = indicesToRemove = Collections.singletonList(5);
+        expected = new ArrayList<>(list);
+        expected.remove(5);
+        result = PermutationUtil.getPartialList(list, indicesToRemove);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // remove all
+        indicesToRemove = indicesToRemove = Arrays.asList(0, 1, 2, 3, 4, 5);
+        expected = new ArrayList<>();
+        result = PermutationUtil.getPartialList(list, indicesToRemove);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // wrong sorting of indices
+        indicesToRemove = indicesToRemove = Arrays.asList(4, 0, 1);
+        expected = expected = new ArrayList<>(Arrays.asList(blindVote2, blindVote3, blindVote5));
+        result = PermutationUtil.getPartialList(list, indicesToRemove);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // wrong sorting of indices
+        indicesToRemove = indicesToRemove = Arrays.asList(0, 0);
+        expected = expected = new ArrayList<>(Arrays.asList(blindVote1, blindVote2, blindVote3, blindVote4, blindVote5));
+        result = PermutationUtil.getPartialList(list, indicesToRemove);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // don't remove as invalid index
+        indicesToRemove = indicesToRemove = Collections.singletonList(9);
+        expected = new ArrayList<>(list);
+        result = PermutationUtil.getPartialList(list, indicesToRemove);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // don't remove as invalid index
+        indicesToRemove = indicesToRemove = Collections.singletonList(-2);
+        expected = new ArrayList<>(list);
+        result = PermutationUtil.getPartialList(list, indicesToRemove);
+        assertTrue(expected.toString().equals(result.toString()));
+    }
+
+    @Test
+    public void testFindAllPermutations() {
+        String blindVote0 = "blindVote0";
+        String blindVote1 = "blindVote1";
+        String blindVote2 = "blindVote2";
+        String blindVote3 = "blindVote3";
+        String blindVote4 = "blindVote4";
+
+        // Up to about 1M iterations performance is acceptable (0.5 sec)
+        // findAllPermutations took 580 ms for 20 items and 1048575 iterations
+        // findAllPermutations took 10 ms for 15 items and 32767 iterations
+        // findAllPermutations took 0 ms for 10 items and 1023 iterations
+        int limit = 1048575;
+        List<String> list;
+        List<List<String>> expected;
+        List<List<String>> result;
+        List<String> subList;
+
+
+       /* list = new ArrayList<>();
+        for (int i = 0; i < 20; i++) {
+            list.add("blindVote"+i);
+        }
+        PermutationUtil.findAllPermutations(list, limit);*/
+
+
+        list = new ArrayList<>();
+        expected = new ArrayList<>();
+        result = PermutationUtil.findAllPermutations(list, limit);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        list = new ArrayList<>(Arrays.asList(blindVote0));
+        expected = new ArrayList<>();
+        expected.add(list);
+        result = PermutationUtil.findAllPermutations(list, limit);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // 2 items -> 3 variations
+        list = new ArrayList<>(Arrays.asList(blindVote0, blindVote1));
+        expected = new ArrayList<>();
+        expected.add(Arrays.asList(list.get(0)));
+
+        expected.add(Arrays.asList(list.get(1)));
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        expected.add(subList);
+
+        result = PermutationUtil.findAllPermutations(list, limit);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // 3 items -> 7 variations
+        list = new ArrayList<>(Arrays.asList(blindVote0, blindVote1, blindVote2));
+        expected = new ArrayList<>();
+        expected.add(Arrays.asList(list.get(0)));
+
+        expected.add(Arrays.asList(list.get(1)));
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        expected.add(subList);
+
+        expected.add(Arrays.asList(list.get(2)));
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(2));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        expected.add(subList);
+
+        result = PermutationUtil.findAllPermutations(list, limit);
+        assertTrue(expected.toString().equals(result.toString()));
+
+        // 4 items -> 15 variations
+        list = new ArrayList<>(Arrays.asList(blindVote0, blindVote1, blindVote2, blindVote3));
+        expected = new ArrayList<>();
+        expected.add(Arrays.asList(list.get(0)));
+
+        expected.add(Arrays.asList(list.get(1)));
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        expected.add(subList);
+
+        expected.add(Arrays.asList(list.get(2)));
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(2));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        expected.add(subList);
+
+        expected.add(Arrays.asList(list.get(3)));
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        result = PermutationUtil.findAllPermutations(list, limit);
+        assertTrue(expected.toString().equals(result.toString()));
+
+
+        // 5 items -> 31 variations
+        list = new ArrayList<>(Arrays.asList(blindVote0, blindVote1, blindVote2, blindVote3, blindVote4));
+        expected = new ArrayList<>();
+        expected.add(Arrays.asList(list.get(0)));
+
+        expected.add(Arrays.asList(list.get(1)));
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        expected.add(subList);
+
+        expected.add(Arrays.asList(list.get(2)));
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(2));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        expected.add(subList);
+
+        expected.add(Arrays.asList(list.get(3)));
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        expected.add(subList);
+
+        expected.add(Arrays.asList(list.get(4)));
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(2));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(2));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(3));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(3));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(3));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(3));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        subList = new ArrayList<>();
+        subList.add(list.get(0));
+        subList.add(list.get(1));
+        subList.add(list.get(2));
+        subList.add(list.get(3));
+        subList.add(list.get(4));
+        expected.add(subList);
+
+        result = PermutationUtil.findAllPermutations(list, limit);
+        assertTrue(expected.toString().equals(result.toString()));
+
+
+    }
+
+
+}

core/src/main/java/bisq/core/dao/governance/voteresult/VoteResultConsensus.java

@@ -68,7 +68,7 @@ public class VoteResultConsensus {
     // hex encoded hashOfProposalList for comparision
     @Nullable
     public static byte[] getMajorityHash(List<VoteResultService.HashWithStake> hashWithStakeList)
-            throws VoteResultException.ConsensusException, VoteResultException.ValidationException {
+            throws VoteResultException.ValidationException {
         try {
             checkArgument(!hashWithStakeList.isEmpty(), "hashWithStakeList must not be empty");
         } catch (Throwable t) {

core/src/main/java/bisq/core/dao/governance/voteresult/VoteResultService.java

@@ -56,6 +56,7 @@ import bisq.core.locale.CurrencyUtil;
 import bisq.network.p2p.storage.P2PDataStorage;
 
 import bisq.common.util.MathUtils;
+import bisq.common.util.PermutationUtil;
 import bisq.common.util.Utilities;
 
 import javax.inject.Inject;
@@ -417,28 +418,39 @@ public class VoteResultService implements DaoStateListener, DaoSetupService {
             // It still could be that we have additional blind votes so our hash does not match. We can try to permute
             // our list with excluding items to see if we get a matching list. If not last resort is to request the
             // missing items from the network.
-            List<BlindVote> permutatedListMatchingMajority = findPermutatedListMatchingMajority(majorityVoteListHash);
-            if (!permutatedListMatchingMajority.isEmpty()) {
-                log.info("We found a permutation of our blindVote list which matches the majority view. " +
-                        "permutatedListMatchingMajority={}", permutatedListMatchingMajority);
+            Optional<List<BlindVote>> permutatedList = findPermutatedListMatchingMajority(majorityVoteListHash);
+            if (permutatedList.isPresent()) {
                 //TODO do we need to apply/store it for later use?
+
+                return true;
             } else {
                 log.info("We did not find a permutation of our blindVote list which matches the majority view. " +
                         "We will request the blindVote data from the peers.");
                 // This is async operation. We will restart the whole verification process once we received the data.
+                // TODO implement
                 requestBlindVoteListFromNetwork(majorityVoteListHash);
             }
         }
         return matches;
     }
 
-    private List<BlindVote> findPermutatedListMatchingMajority(byte[] majorityVoteListHash) {
+    private Optional<List<BlindVote>> findPermutatedListMatchingMajority(byte[] majorityVoteListHash) {
         List<BlindVote> list = BlindVoteConsensus.getSortedBlindVoteListOfCycle(blindVoteListService);
-        while (!list.isEmpty() && !isListMatchingMajority(majorityVoteListHash, list)) {
-            // We remove first item as it will be sorted anyway...
-            list.remove(0);
+        long ts = System.currentTimeMillis();
+        List<List<BlindVote>> result = PermutationUtil.findAllPermutations(list, 1000000);
+        for (List<BlindVote> variation : result) {
+            if (isListMatchingMajority(majorityVoteListHash, variation)) {
+                log.info("We found a variation of the blind vote list which matches the majority hash. variation={}",
+                        variation);
+                log.info("findPermutatedListMatchingMajority for {} items took {} ms.",
+                        list.size(), (System.currentTimeMillis() - ts));
+                return Optional.of(variation);
+            }
         }
-        return list;
+        log.info("We did not find a variation of the blind vote list which matches the majority hash.");
+        log.info("findPermutatedListMatchingMajority for {} items took {} ms.",
+                list.size(), (System.currentTimeMillis() - ts));
+        return Optional.empty();
     }
 
     private boolean isListMatchingMajority(byte[] majorityVoteListHash, List<BlindVote> list) {

core/src/main/java/bisq/core/dao/governance/votereveal/VoteRevealService.java

@@ -147,7 +147,10 @@ public class VoteRevealService implements DaoStateListener, DaoSetupService {
 
     public byte[] getHashOfBlindVoteList() {
         List<BlindVote> blindVotes = BlindVoteConsensus.getSortedBlindVoteListOfCycle(blindVoteListService);
-        return VoteRevealConsensus.getHashOfBlindVoteList(blindVotes);
+        byte[] hashOfBlindVoteList = VoteRevealConsensus.getHashOfBlindVoteList(blindVotes);
+        log.info("blindVoteList for creating hash: " + blindVotes);
+        log.info("Sha256Ripemd160 hash of hashOfBlindVoteList " + Utilities.bytesAsHexString(hashOfBlindVoteList));
+        return hashOfBlindVoteList;
     }
 
     public void addVoteRevealTxPublishedListener(VoteRevealTxPublishedListener voteRevealTxPublishedListener) {
@@ -239,7 +242,7 @@ public class VoteRevealService implements DaoStateListener, DaoSetupService {
             // If we are not in the right phase we just add an empty hash (still need to have the hash as otherwise we
             // would not recognize the tx as vote reveal tx)
             byte[] hashOfBlindVoteList = inBlindVotePhase ? getHashOfBlindVoteList() : new byte[20];
-            log.info("Sha256Ripemd160 hash of hashOfBlindVoteList " + Utilities.bytesAsHexString(hashOfBlindVoteList));
+            log.info("revealVote: Sha256Ripemd160 hash of hashOfBlindVoteList " + Utilities.bytesAsHexString(hashOfBlindVoteList));
             byte[] opReturnData = VoteRevealConsensus.getOpReturnData(hashOfBlindVoteList, myVote.getSecretKey());
 
             // We search for my unspent stake output.