Fix error handling

Add all seeds
This commit is contained in:
chimp1984 2020-10-21 01:56:22 -05:00
parent d08b24ece0
commit 5fb48fe72f
No known key found for this signature in database
GPG key ID: 9801B4EC591F90E3
6 changed files with 124 additions and 82 deletions

View file

@ -83,7 +83,7 @@ public class GetInventoryRequester implements MessageListener, ConnectionListene
shutDown();
// We shut down our connection after work as our node is not helpful for the network.
connection.shutDown(CloseConnectionReason.CLOSE_REQUESTED_BY_PEER);
UserThread.runAfter(() -> connection.shutDown(CloseConnectionReason.CLOSE_REQUESTED_BY_PEER), 1);
}
});
}
@ -107,7 +107,9 @@ public class GetInventoryRequester implements MessageListener, ConnectionListene
Connection connection) {
connection.getPeersNodeAddressOptional().ifPresent(address -> {
if (address.equals(nodeAddress)) {
if (!closeConnectionReason.isIntended) {
errorMessageHandler.handleErrorMessage("Connected closed because of " + closeConnectionReason.name());
}
shutDown();
}
});

View file

@ -46,6 +46,8 @@ import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.Nullable;
@Slf4j
public class InventoryMonitor implements SetupListener {
@ -72,9 +74,8 @@ public class InventoryMonitor implements SetupListener {
networkNode = getNetworkNode(appDir);
getInventoryRequestManager = new GetInventoryRequestManager(networkNode);
//TODO until we use all seeds we use our custom seed node file which includes only those which have updated to our branch
// Once all seeds are updated we can remove that resource file and prefix
//String fileName = network.name().toLowerCase();
// We maintain our own list as we want to monitor also old v2 nodes which are not part of the normal seed
// node list anymore.
String networkName = network.name().toLowerCase();
String fileName = network.isMainnet() ? "inv_" + networkName : networkName;
DefaultSeedNodeRepository.readSeedNodePropertyFile(fileName)
@ -117,11 +118,28 @@ public class InventoryMonitor implements SetupListener {
new Thread(() -> {
Thread.currentThread().setName("request @ " + getShortAddress(nodeAddress, useLocalhostForP2P));
getInventoryRequestManager.request(nodeAddress,
result -> {
result -> processResponse(nodeAddress, requestInfo, result, null),
errorMessage -> processResponse(nodeAddress, requestInfo, null, errorMessage));
}).start();
});
}
private void processResponse(NodeAddress nodeAddress,
RequestInfo requestInfo,
@Nullable Map<InventoryItem, String> result,
@Nullable String errorMessage) {
if (errorMessage != null) {
log.warn("Error at connection to peer {}: {}", nodeAddress, errorMessage);
requestInfo.setErrorMessage(errorMessage);
}
if (result != null) {
log.info("nodeAddress={}, result={}", nodeAddress, result.toString());
requestInfo.setInventory(result);
long responseTime = System.currentTimeMillis();
requestInfo.setResponseTime(responseTime);
requestInfo.setInventory(result);
}
requestInfoListByNode.putIfAbsent(nodeAddress, new ArrayList<>());
List<RequestInfo> requestInfoList = requestInfoListByNode.get(nodeAddress);
@ -138,15 +156,7 @@ public class InventoryMonitor implements SetupListener {
inventoryWebServer.onNewRequestInfo(requestInfoListByNode, averageValues, requestCounter);
String json = Utilities.objectToJson(requestInfo);
jsonFileManagerByNodeAddress.get(nodeAddress).writeToDisc(json, String.valueOf(responseTime));
},
errorMessage -> {
log.warn(errorMessage);
requestInfo.setErrorMessage(errorMessage);
});
}).start();
});
jsonFileManagerByNodeAddress.get(nodeAddress).writeToDisc(json, String.valueOf(requestInfo.getRequestStartTime()));
}
private void addJsonFileManagers(List<NodeAddress> seedNodes) {

View file

@ -31,6 +31,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import static com.google.common.base.Preconditions.checkArgument;
@ -57,6 +58,7 @@ public class InventoryUtil {
checkArgument(inventoryItem.getType().equals(Integer.class));
return requestInfoSetOfOtherNodes.stream()
.map(RequestInfo::getInventory)
.filter(Objects::nonNull)
.filter(inventory -> inventory.containsKey(inventoryItem))
.mapToInt(inventory -> Integer.parseInt(inventory.get(inventoryItem)))
.average()
@ -68,6 +70,7 @@ public class InventoryUtil {
checkArgument(inventoryItem.getType().equals(Long.class));
return requestInfoSetOfOtherNodes.stream()
.map(RequestInfo::getInventory)
.filter(Objects::nonNull)
.filter(inventory -> inventory.containsKey(inventoryItem))
.mapToLong(inventory -> Long.parseLong(inventory.get(inventoryItem)))
.average()
@ -79,6 +82,7 @@ public class InventoryUtil {
checkArgument(inventoryItem.getType().equals(Double.class));
return requestInfoSetOfOtherNodes.stream()
.map(RequestInfo::getInventory)
.filter(Objects::nonNull)
.filter(inventory -> inventory.containsKey(inventoryItem))
.mapToDouble(inventory -> Double.parseDouble((inventory.get(inventoryItem))))
.average()
@ -97,6 +101,7 @@ public class InventoryUtil {
.filter(list -> !list.isEmpty())
.map(list -> list.get(list.size() - 1)) // We use last item only
.map(RequestInfo::getInventory)
.filter(Objects::nonNull)
.map(e -> e.get(inventoryItem))
.forEach(e -> {
sameItemsByValue.putIfAbsent(e, 0);
@ -109,7 +114,9 @@ public class InventoryUtil {
sameItems.sort(Comparator.comparing(o -> o.second));
Collections.reverse(sameItems);
String majority = sameItems.get(0).first;
String candidate = requestInfo.getInventory().get(inventoryItem);
Map<InventoryItem, String> inventory = requestInfo.getInventory();
if (inventory != null) {
String candidate = inventory.get(inventoryItem);
if (!majority.equals(candidate)) {
int majorityAsInt = Integer.parseInt(majority);
int candidateAsInt = Integer.parseInt(candidate);
@ -123,6 +130,7 @@ public class InventoryUtil {
}
}
}
}
return deviationSeverity;
}
@ -136,8 +144,9 @@ public class InventoryUtil {
.filter(list -> !list.isEmpty())
.map(list -> list.get(list.size() - 1)) // We use last item only
.map(RequestInfo::getInventory)
.filter(inventoryMap -> inventoryMap.get(InventoryItem.daoStateChainHeight).equals(daoStateChainHeightAsString))
.map(inventoryMap -> inventoryMap.get(inventoryItem))
.filter(Objects::nonNull)
.filter(inventory -> inventory.get(InventoryItem.daoStateChainHeight).equals(daoStateChainHeightAsString))
.map(inventory -> inventory.get(inventoryItem))
.forEach(value -> {
sameHashesPerHashListByHash.putIfAbsent(value, 0);
int prev = sameHashesPerHashListByHash.get(value);
@ -151,7 +160,9 @@ public class InventoryUtil {
// It could be that first and any following list entry has same number of hashes, but we ignore that as
// it is reason enough to alert the operators in case not all hashes are the same.
if (sameHashesPerHashList.get(0).first.equals(requestInfo.getInventory().get(inventoryItem))) {
Map<InventoryItem, String> inventory = requestInfo.getInventory();
if (inventory != null) {
if (sameHashesPerHashList.get(0).first.equals(inventory.get(inventoryItem))) {
// We are in the majority group.
// We also set a warning to make sure the operators act quickly and to check if there are
// more severe issues.
@ -160,6 +171,7 @@ public class InventoryUtil {
deviationSeverity = DeviationSeverity.ALERT;
}
}
}
return deviationSeverity;
}
}

View file

@ -92,26 +92,24 @@ public class InventoryWebServer {
.append("<th align=\"left\">Network info</th>").append("</tr>");
seedNodes.forEach(seedNode -> {
html.append("<tr valign=\"top\">");
if (map.containsKey(seedNode) && !map.get(seedNode).isEmpty()) {
List<RequestInfo> list = map.get(seedNode);
int numRequests = list.size();
RequestInfo requestInfo = list.get(numRequests - 1);
html.append("<tr valign=\"top\">")
.append("<td>").append(getSeedNodeInfo(seedNode, requestInfo, averageValues)).append("</td>")
.append("<td>").append(getRequestInfo(requestInfo, numRequests)).append("</td>")
int numResponses = list.size();
RequestInfo requestInfo = list.get(numResponses - 1);
html.append("<td>").append(getSeedNodeInfo(seedNode, requestInfo, averageValues)).append("</td>")
.append("<td>").append(getRequestInfo(requestInfo, numResponses)).append("</td>")
.append("<td>").append(getDataInfo(requestInfo, averageValues, map)).append("</td>")
.append("<td>").append(getDaoInfo(requestInfo, averageValues, map)).append("</td>")
.append("<td>").append(getNetworkInfo(requestInfo, averageValues)).append("</td>");
html.append("</tr>");
} else {
html.append("<tr valign=\"top\">")
.append("<td>").append(getSeedNodeInfo(seedNode, null, averageValues)).append("</td>")
html.append("<td>").append(getSeedNodeInfo(seedNode, null, averageValues)).append("</td>")
.append("<td>").append("n/a").append("</td>")
.append("<td>").append("n/a").append("</td>")
.append("<td>").append("n/a").append("</td>")
.append("<td>").append("n/a").append("</td>");
html.append("</tr>");
}
html.append("</tr>");
});
html.append("</table></body></html>");
@ -140,26 +138,29 @@ public class InventoryWebServer {
InventoryItem.jvmStartTime,
value -> new Date(Long.parseLong(value)).toString());
String duration = FormattingUtils.formatDurationAsWords(
String duration = jvmStartTimeString != null ?
FormattingUtils.formatDurationAsWords(
System.currentTimeMillis() - Long.parseLong(jvmStartTimeString),
true, true);
true, true) :
"n/a";
sb.append("Run duration: ").append(duration).append("<br/>");
}
return sb.toString();
}
private String getRequestInfo(RequestInfo requestInfo, int numRequests) {
private String getRequestInfo(RequestInfo requestInfo, int numResponses) {
StringBuilder sb = new StringBuilder();
DeviationSeverity deviationSeverity = numRequests == requestCounter ?
DeviationSeverity deviationSeverity = numResponses == requestCounter ?
DeviationSeverity.OK :
requestCounter - numRequests > 4 ?
requestCounter - numResponses > 4 ?
DeviationSeverity.ALERT :
DeviationSeverity.WARN;
sb.append("Number of responses: ").append(getColorTagByDeviationSeverity(deviationSeverity))
.append(numRequests).append(CLOSE_TAG);
.append(numResponses).append(CLOSE_TAG);
if (requestInfo.getResponseTime() > 0) {
long rrt = requestInfo.getResponseTime() - requestInfo.getRequestStartTime();
DeviationSeverity rrtDeviationSeverity = DeviationSeverity.OK;
if (rrt > 20_000) {
@ -170,11 +171,16 @@ public class InventoryWebServer {
String rrtString = MathUtils.roundDouble(rrt / 1000d, 3) + " sec";
sb.append("Round trip time: ").append(getColorTagByDeviationSeverity(rrtDeviationSeverity))
.append(rrtString).append(CLOSE_TAG);
} else {
sb.append("Round trip time: ").append("n/a").append(CLOSE_TAG);
}
Date requestStartTime = new Date(requestInfo.getRequestStartTime());
sb.append("Requested at: ").append(requestStartTime).append("<br/>");
Date responseTime = new Date(requestInfo.getResponseTime());
String responseTime = requestInfo.getResponseTime() > 0 ?
new Date(requestInfo.getResponseTime()).toString() :
"n/a";
sb.append("Response received at: ").append(responseTime).append("<br/>");
String errorMessage = requestInfo.getErrorMessage();
@ -359,8 +365,9 @@ public class InventoryWebServer {
String displayString = "n/a";
String deviationAsString = "";
String colorTag = getColorTagByDeviationSeverity(DeviationSeverity.OK);
if (requestInfo.getInventory().containsKey(inventoryItem)) {
valueAsString = requestInfo.getInventory().get(inventoryItem);
Map<InventoryItem, String> inventory = requestInfo.getInventory();
if (inventory != null && inventory.containsKey(inventoryItem)) {
valueAsString = inventory.get(inventoryItem);
if (averageValues != null && averageValues.containsKey(inventoryItem)) {
double average = averageValues.get(inventoryItem);
double deviation = 0;

View file

@ -24,13 +24,17 @@ import java.util.Map;
import lombok.Getter;
import lombok.Setter;
import org.jetbrains.annotations.Nullable;
@Getter
public class RequestInfo {
private final long requestStartTime;
@Setter
private long responseTime;
@Nullable
@Setter
private Map<InventoryItem, String> inventory;
@Nullable
@Setter
private String errorMessage;

View file

@ -5,8 +5,15 @@ wizseed7ab2gi3x267xahrp2pkndyrovczezzb46jk6quvguciuyqrid.onion:8000 (@wiz)
sn3emzy56u3mxzsr4geysc52feoq5qt7ja56km6gygwnszkshunn2sid.onion:8000 (@emzy)
sn4emzywye3dhjouv7jig677qepg7fnusjidw74fbwneieruhmi7fuyd.onion:8000 (@emzy)
sn5emzyvxuildv34n6jewfp2zeota4aq63fsl5yyilnvksezr3htveqd.onion:8000 (@emzy)
723ljisnynbtdohi.onion:8000 (@emzy)
s67qglwhkgkyvr74.onion:8000 (@emzy)
sn2bisqad7ncazupgbd3dcedqh5ptirgwofw63djwpdtftwhddo75oid.onion:8000 (@miker)
sn3bsq3evqkpshdmc3sbdxafkhfnk7ctop44jsxbxyys5ridsaw5abyd.onion:8000 (@miker)
sn4bsqpc7eb2ntvpsycxbzqt6fre72l4krp2fl5svphfh2eusrqtq3qd.onion:8000 (@miker)
devinv3rhon24gqf5v6ondoqgyrbzyqihzyouzv7ptltsewhfmox2zqd.onion:8000 (@devinbileck)
devinsn2teu33efff62bnvwbxmfgbfjlgqsu3ad4b4fudx3a725eqnyd.onion:8000 (@devinbileck)
devinsn3xuzxhj6pmammrxpydhwwmwp75qkksedo5dn2tlmu7jggo7id.onion:8000 (@devinbileck)
723ljisnynbtdohi.onion:8000 (@emzy)
s67qglwhkgkyvr74.onion:8000 (@emzy)
5quyxpxheyvzmb2d.onion:8000 (@miker)
rm7b56wbrcczpjvl.onion:8000 (@miker)
3f3cu2yw7u457ztq.onion:8000 (@devinbileck)
fl3mmribyxgrv63c.onion:8000 (@devinbileck)