From 10c5d6c0bd2e0e8b922d7b7ad1e4e49eb7a49867 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 1 Feb 2022 15:48:35 -0800 Subject: [PATCH 1/2] monitoring+cfg: add new option to export gRPC perf metrics In this commit, we expose a new monitoring option to allow users to export gRPC performance metrics. These metrics can be used to see how long certain calls are taking, the total amount of time spent handling calls, broken down by service and also call. This option consumes additional memory and disk space for the Prometheus server, which is why we're opting to make it an optional flag. --- lncfg/monitoring_on.go | 6 ++++++ monitoring/monitoring_on.go | 8 ++++++++ sample-lnd.conf | 6 ++++++ 3 files changed, 20 insertions(+) diff --git a/lncfg/monitoring_on.go b/lncfg/monitoring_on.go index 872b46330..579c83638 100644 --- a/lncfg/monitoring_on.go +++ b/lncfg/monitoring_on.go @@ -13,6 +13,12 @@ type Prometheus struct { // Enable indicates whether to export lnd gRPC performance metrics to // Prometheus. Default is false. Enable bool `long:"enable" description:"enable Prometheus exporting of lnd gRPC performance metrics."` + + // PerfHistograms indicates if the additional histogram information for + // latency, and handling time of gRPC calls should be enabled. This + // generates additional data, and consume more memory for the + // Prometheus server. + PerfHistograms bool `long:"perfhistograms" description:"enable additional histogram to track gRPC call processing performance (latency, etc)"` } // DefaultPrometheus is the default configuration for the Prometheus metrics diff --git a/monitoring/monitoring_on.go b/monitoring/monitoring_on.go index 78a0f7e22..773f9cd4e 100644 --- a/monitoring/monitoring_on.go +++ b/monitoring/monitoring_on.go @@ -36,6 +36,14 @@ func ExportPrometheusMetrics(grpcServer *grpc.Server, cfg lncfg.Prometheus) erro grpc_prometheus.Register(grpcServer) + // Enable the histograms which can allow plotting latency + // distributions of inbound calls. However we guard this behind + // another flag as this can generate a lot of additional data, + // as its a high cardinality metric typically. + if cfg.PerfHistograms { + grpc_prometheus.EnableHandlingTimeHistogram() + } + http.Handle("/metrics", promhttp.Handler()) go func() { http.ListenAndServe(cfg.Listen, nil) diff --git a/sample-lnd.conf b/sample-lnd.conf index 42e08dfcf..822d47784 100644 --- a/sample-lnd.conf +++ b/sample-lnd.conf @@ -421,6 +421,12 @@ ; Specify the interface to listen on for Prometheus connections. ; prometheus.listen=0.0.0.0:8989 +; If true, then we'll export additional information that allows users to plot +; the processing latency, and total time spent across each RPC calls+service. +; This generates additional memory load for the Prometheus server, and will end +; up using more disk space over time. +; prometheus.perfhistograms=true + ; The alias your node will use, which can be up to 32 UTF-8 characters in ; length. ; alias=My Lightning ☇ From 1fd8a78537881aac641c285ed50ab4fc0b939369 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Tue, 1 Feb 2022 15:57:53 -0800 Subject: [PATCH 2/2] docs/release-notes: add entry for gRPC perf flag --- docs/release-notes/release-notes-0.15.0.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/release-notes/release-notes-0.15.0.md b/docs/release-notes/release-notes-0.15.0.md index 1b1dc2c60..57ef55e00 100644 --- a/docs/release-notes/release-notes-0.15.0.md +++ b/docs/release-notes/release-notes-0.15.0.md @@ -67,6 +67,11 @@ * Improved instructions on [how to build lnd for mobile](https://github.com/lightningnetwork/lnd/pull/6085). * [Log force-close related messages on "info" level](https://github.com/lightningnetwork/lnd/pull/6124). +## Monitoring + +A new [flag (`--prometheus.perfhistograms`) has been added to enable export of +gRPC performance metrics (latency to process `GetInfo`, etc)](https://github.com/lightningnetwork/lnd/pull/6224). + ## Code Health ### Code cleanup, refactor, typo fixes @@ -92,6 +97,7 @@ * Liviu * mateuszmp * Naveen Srinivasan +* Olaoluwa Osuntokun * randymcmillan * Rong Ou * Thebora Kompanioni