Add memorey check and restart routine for seednode

This commit is contained in:
Manfred Karrer 2016-07-22 16:32:01 +02:00
parent 3399e8a45a
commit 3a8c09c0d5
5 changed files with 108 additions and 19 deletions

View file

@ -24,16 +24,15 @@ public class Profiler {
private static final Logger log = LoggerFactory.getLogger(Profiler.class);
public static void printSystemLoad(Logger log) {
String msg = printSystemLoadString();
log.info(msg);
log.warn(printSystemLoadString());
}
public static String printSystemLoadString() {
long used = getUsedMemory();
return "System load (nr. threads/used memory (MB)): " + Thread.activeCount() + "/" + used;
long used = getUsedMemoryInMB();
return "System load: Memory (MB)): " + used + " / Nr. of threads: " + Thread.activeCount();
}
public static long getUsedMemory() {
public static long getUsedMemoryInMB() {
Runtime runtime = Runtime.getRuntime();
long free = runtime.freeMemory() / 1024 / 1024;
long total = runtime.totalMemory() / 1024 / 1024;

View file

@ -0,0 +1,64 @@
package io.bitsquare.common.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.List;
// Borrowed from: https://dzone.com/articles/programmatically-restart-java
public class RestartUtil {
private static final Logger log = LoggerFactory.getLogger(RestartUtil.class);
/**
* Sun property pointing the main class and its arguments.
* Might not be defined on non Hotspot VM implementations.
*/
public static final String SUN_JAVA_COMMAND = "sun.java.command";
public static void restartApplication(String logPath) throws IOException {
try {
String java = System.getProperty("java.home") + "/bin/java";
List<String> vmArguments = ManagementFactory.getRuntimeMXBean().getInputArguments();
StringBuffer vmArgsOneLine = new StringBuffer();
for (String arg : vmArguments) {
// if it's the agent argument : we ignore it otherwise the
// address of the old application and the new one will be in conflict
if (!arg.contains("-agentlib")) {
vmArgsOneLine.append(arg);
vmArgsOneLine.append(" ");
}
}
// init the command to execute, add the vm args
final StringBuffer cmd = new StringBuffer(java + " " + vmArgsOneLine);
// program main and program arguments
String[] mainCommand = System.getProperty(SUN_JAVA_COMMAND).split(" ");
// program main is a jar
if (mainCommand[0].endsWith(".jar")) {
// if it's a jar, add -jar mainJar
cmd.append("-jar " + new File(mainCommand[0]).getPath());
} else {
// else it's a .class, add the classpath and mainClass
cmd.append("-cp \"" + System.getProperty("java.class.path") + "\" " + mainCommand[0]);
}
// finally add program arguments
for (int i = 1; i < mainCommand.length; i++) {
cmd.append(" ");
cmd.append(mainCommand[i]);
}
try {
final String command = "nohup " + cmd.toString() + " >/dev/null 2>" + logPath + " &";
log.warn("Executing cmd for restart:\n" + command);
Runtime.getRuntime().exec(command);
} catch (IOException e) {
e.printStackTrace();
}
} catch (Exception e) {
throw new IOException("Error while trying to restart the application", e);
}
}
}

View file

@ -78,6 +78,7 @@ public class BitsquareEnvironment extends StandardEnvironment {
private final String appName;
private final String userDataDir;
private final String appDataDir;
private final String btcNetworkDir;
private final String logLevel;
@ -119,6 +120,10 @@ public class BitsquareEnvironment extends StandardEnvironment {
}
}
public String getAppDataDir() {
return appDataDir;
}
protected BitsquareEnvironment(PropertySource commandLineProperties) {
userDataDir = commandLineProperties.containsProperty(USER_DATA_DIR_KEY) ?
(String) commandLineProperties.getProperty(USER_DATA_DIR_KEY) :

View file

@ -124,7 +124,7 @@ public class SeedNode {
});
}
private void gracefulShutDown(ResultHandler resultHandler) {
public void gracefulShutDown(ResultHandler resultHandler) {
log.debug("gracefulShutDown");
try {
if (injector != null) {

View file

@ -21,13 +21,15 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
import io.bitsquare.app.BitsquareEnvironment;
import io.bitsquare.app.BitsquareExecutable;
import io.bitsquare.common.UserThread;
import io.bitsquare.common.util.Profiler;
import io.bitsquare.common.util.RestartUtil;
import joptsimple.OptionException;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Scanner;
import java.io.IOException;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
@ -35,8 +37,10 @@ import static io.bitsquare.app.BitsquareEnvironment.*;
public class SeedNodeMain extends BitsquareExecutable {
private static final Logger log = LoggerFactory.getLogger(SeedNodeMain.class);
private static final long MAX_MEMORY_MB = 800;
private static final long CHECK_MEMORY_PERIOD_SEC = 60;
private SeedNode seedNode;
private boolean isStopped;
private volatile boolean stopped;
public static void main(String[] args) throws Exception {
final ThreadFactory threadFactory = new ThreadFactoryBuilder()
@ -79,20 +83,37 @@ public class SeedNodeMain extends BitsquareExecutable {
@Override
protected void doExecute(OptionSet options) {
SeedNode.setEnvironment(new BitsquareEnvironment(options));
final BitsquareEnvironment environment = new BitsquareEnvironment(options);
SeedNode.setEnvironment(environment);
UserThread.execute(() -> seedNode = new SeedNode());
while (!isStopped) {
try {
Scanner scanner = new Scanner(System.in);
while (scanner.hasNextLine()) {
String inputString = scanner.nextLine();
if (inputString.equals("q")) {
UserThread.execute(seedNode::shutDown);
isStopped = true;
UserThread.runPeriodically(() -> {
Profiler.printSystemLoad(log);
if (!stopped && Profiler.getUsedMemoryInMB() > MAX_MEMORY_MB) {
stopped = true;
seedNode.gracefulShutDown(() -> {
try {
final String[] tokens = environment.getAppDataDir().split("_");
String logPath = "error_" + (tokens.length > 1 ? tokens[tokens.length - 2] : "") + ".log";
RestartUtil.restartApplication(logPath);
} catch (IOException e) {
log.error(e.toString());
e.printStackTrace();
} finally {
log.warn("Shutdown complete");
System.exit(0);
}
}
} catch (Throwable ignore) {
});
}
}, CHECK_MEMORY_PERIOD_SEC);
while (true) {
try {
Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) {
e.printStackTrace();
log.error(e.getMessage());
}
}
}