Merge pull request #7983 from Roasbeef/blocking-and-mutex-profile

lnd+config: add ability to obtain blocking and mutex profiles

config.go

@@ -337,6 +337,9 @@ type Config struct {
 
 	Profile string `long:"profile" description:"Enable HTTP profiling on either a port or host:port"`
 
+	BlockingProfile int `long:"blockingprofile" description:"Used to enable a blocking profile to be served on the profiling port. This takes a value from 0 to 1, with 1 including every blocking event, and 0 including no events."`
+	MutexProfile    int `long:"mutexprofile" description:"Used to Enable a mutex profile to be served on the profiling port. This takes a value from 0 to 1, with 1 including every mutex event, and 0 including no events."`
+
 	UnsafeDisconnect   bool   `long:"unsafe-disconnect" description:"DEPRECATED: Allows the rpcserver to intentionally disconnect from peers with open channels. THIS FLAG WILL BE REMOVED IN 0.10.0"`
 	UnsafeReplay       bool   `long:"unsafe-replay" description:"Causes a link to replay the adds on its commitment txn after starting up, this enables testing of the sphinx replay logic."`
 	MaxPendingChannels int    `long:"maxpendingchannels" description:"The maximum number of incoming pending channels permitted per peer."`

docs/release-notes/release-notes-0.17.0.md

@@ -77,6 +77,11 @@ fails](https://github.com/lightningnetwork/lnd/pull/7876).
 
 # New Features
 ## Functional Enhancements
+
+* `lnd` can now optionally generate [blocking and mutex
+  profiles](https://github.com/lightningnetwork/lnd/pull/7983). These profiles
+  are useful to attempt to debug high mutex contention, or deadlock scenarios.
+
 ### Protocol Features
 * This release marks the first release that includes the new [musig2-based
   taproot channel type](https://github.com/lightningnetwork/lnd/pull/7904). As

lnd.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"net/http/pprof"
 	"os"
+	"runtime"
 	runtimePprof "runtime/pprof"
 	"strings"
 	"sync"
@@ -194,6 +195,13 @@ func Main(cfg *Config, lisCfg ListenerCfg, implCfg *ImplementationCfg,
 		pprofMux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
 		pprofMux.HandleFunc("/debug/pprof/trace", pprof.Trace)
 
+		if cfg.BlockingProfile != 0 {
+			runtime.SetBlockProfileRate(cfg.BlockingProfile)
+		}
+		if cfg.MutexProfile != 0 {
+			runtime.SetMutexProfileFraction(cfg.MutexProfile)
+		}
+
 		// Redirect all requests to the pprof handler, thus visiting
 		// `127.0.0.1:6060` will be redirected to
 		// `127.0.0.1:6060/debug/pprof`.

sample-lnd.conf

@@ -270,6 +270,18 @@
 ; 65536. The profile can be access at: http://localhost:<PORT>/debug/pprof/.
 ; profile=
 
+; Enable a blocking profile to be obtained from the profiling port. A blocking
+; profile can show where goroutines are blocking (stuck on mutexes, I/O, etc).
+; This takes a value from 0 to 1, with 0 turning off the setting, and 1 sampling
+; every blocking event (it's a rate value).
+; blockingprofile=0
+
+; Enable a mutex profile to be obtained from the profiling port. A mutex 
+; profile can show where goroutines are blocked on mutexes, and which mutexes
+; have high contention.  This takes a value from 0 to 1, with 0 turning off the
+; setting, and 1 sampling every mutex event (it's a rate value).
+; mutexprofile=0
+
 ; DEPRECATED: Allows the rpcserver to intentionally disconnect from peers with
 ; open channels. THIS FLAG WILL BE REMOVED IN 0.10.0.
 ; unsafe-disconnect=false