lnwallet: add configurable cache for web fee estimator

Add fee.min-update-timeout and fee.max-update-timeout config options to
allow configuration of the web fee estimator cache.
This commit is contained in:
Tom Kirkpatrick 2024-04-23 09:49:04 +02:00 committed by yyforyongyu
parent fa616ee059
commit 3837c3f12e
No known key found for this signature in database
GPG Key ID: 9BCD95C4FF296868
15 changed files with 202 additions and 55 deletions

View File

@ -77,10 +77,14 @@ type Config struct {
// ActiveNetParams details the current chain we are on.
ActiveNetParams BitcoinNetParams
// FeeURL defines the URL for fee estimation we will use. This field is
// optional.
// Deprecated: Use Fee.URL. FeeURL defines the URL for fee estimation
// we will use. This field is optional.
FeeURL string
// Fee defines settings for the web fee estimator. This field is
// optional.
Fee *lncfg.Fee
// Dialer is a function closure that will be used to establish outbound
// TCP connections to Bitcoin peers in the event of a pruned block being
// requested.
@ -243,6 +247,16 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) {
"cache: %v", err)
}
// Map the deprecated feeurl flag to fee.url.
if cfg.FeeURL != "" {
if cfg.Fee.URL != "" {
return nil, nil, errors.New("fee.url and " +
"feeurl are mutually exclusive")
}
cfg.Fee.URL = cfg.FeeURL
}
// If spv mode is active, then we'll be using a distinct set of
// chainControl interfaces that interface directly with the p2p network
// of the selected chain.
@ -682,27 +696,34 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) {
// If the fee URL isn't set, and the user is running mainnet, then
// we'll return an error to instruct them to set a proper fee
// estimator.
case cfg.FeeURL == "" && cfg.Bitcoin.MainNet &&
case cfg.Fee.URL == "" && cfg.Bitcoin.MainNet &&
cfg.Bitcoin.Node == "neutrino":
return nil, nil, fmt.Errorf("--feeurl parameter required " +
return nil, nil, fmt.Errorf("--fee.url parameter required " +
"when running neutrino on mainnet")
// Override default fee estimator if an external service is specified.
case cfg.FeeURL != "":
case cfg.Fee.URL != "":
// Do not cache fees on regtest to make it easier to execute
// manual or automated test cases.
cacheFees := !cfg.Bitcoin.RegTest
log.Infof("Using external fee estimator %v: cached=%v",
cfg.FeeURL, cacheFees)
log.Infof("Using external fee estimator %v: cached=%v: "+
"min update timeout=%v, max update timeout=%v",
cfg.Fee.URL, cacheFees, cfg.Fee.MinUpdateTimeout,
cfg.Fee.MaxUpdateTimeout)
cc.FeeEstimator = chainfee.NewWebAPIEstimator(
cc.FeeEstimator, err = chainfee.NewWebAPIEstimator(
chainfee.SparseConfFeeSource{
URL: cfg.FeeURL,
URL: cfg.Fee.URL,
},
!cacheFees,
cfg.Fee.MinUpdateTimeout,
cfg.Fee.MaxUpdateTimeout,
)
if err != nil {
return nil, nil, err
}
}
ccCleanup := func() {

View File

@ -350,7 +350,7 @@ type Config struct {
MaxPendingChannels int `long:"maxpendingchannels" description:"The maximum number of incoming pending channels permitted per peer."`
BackupFilePath string `long:"backupfilepath" description:"The target location of the channel backup file"`
FeeURL string `long:"feeurl" description:"Optional URL for external fee estimation. If no URL is specified, the method for fee estimation will depend on the chosen backend and network. Must be set for neutrino on mainnet."`
FeeURL string `long:"feeurl" description:"DEPRECATED: Use 'fee.url' option. Optional URL for external fee estimation. If no URL is specified, the method for fee estimation will depend on the chosen backend and network. Must be set for neutrino on mainnet." hidden:"true"`
Bitcoin *lncfg.Chain `group:"Bitcoin" namespace:"bitcoin"`
BtcdMode *lncfg.Btcd `group:"btcd" namespace:"btcd"`
@ -442,6 +442,8 @@ type Config struct {
DustThreshold uint64 `long:"dust-threshold" description:"Sets the dust sum threshold in satoshis for a channel after which dust HTLC's will be failed."`
Fee *lncfg.Fee `group:"fee" namespace:"fee"`
Invoices *lncfg.Invoices `group:"invoices" namespace:"invoices"`
Routing *lncfg.Routing `group:"routing" namespace:"routing"`
@ -582,6 +584,12 @@ func DefaultConfig() Config {
MinBackoff: defaultMinBackoff,
MaxBackoff: defaultMaxBackoff,
ConnectionTimeout: tor.DefaultConnTimeout,
Fee: &lncfg.Fee{
MinUpdateTimeout: lncfg.DefaultMinUpdateTimeout,
MaxUpdateTimeout: lncfg.DefaultMaxUpdateTimeout,
},
SubRPCServers: &subRPCServerConfigs{
SignRPC: &signrpc.Config{},
RouterRPC: routerrpc.DefaultConfig(),

View File

@ -550,6 +550,11 @@ func (d *DefaultWalletImpl) BuildWalletConfig(ctx context.Context,
NeutrinoCS: neutrinoCS,
ActiveNetParams: d.cfg.ActiveNetParams,
FeeURL: d.cfg.FeeURL,
Fee: &lncfg.Fee{
URL: d.cfg.Fee.URL,
MinUpdateTimeout: d.cfg.Fee.MinUpdateTimeout,
MaxUpdateTimeout: d.cfg.Fee.MaxUpdateTimeout,
},
Dialer: func(addr string) (net.Conn, error) {
return d.cfg.net.Dial(
"tcp", addr, d.cfg.ConnectionTimeout,

View File

@ -209,6 +209,17 @@
its bitcoin peers' `feefilter` values into
account](https://github.com/lightningnetwork/lnd/pull/8418).
* Web fee estimator settings have been moved into a new `fee` config group.
A new `fee.url` option has been added within this group that replaces the old
`feeurl` option, which is now deprecated. Additionally, [two new config values,
fee.min-update-timeout and fee.max-update-timeout](https://github.com/lightningnetwork/lnd/pull/8484)
are added to allow users to specify the minimum and maximum time between fee
updates from the web fee estimator. The default values are 5 minutes and 20
minutes respectively. These values are used to prevent the fee estimator from
being queried too frequently. This replaces previously hardcoded values that
were set to the same values as the new defaults. The previously deprecated
`neutrino.feeurl` option has been removed.
* [Preparatory work](https://github.com/lightningnetwork/lnd/pull/8159) for
forwarding of blinded routes was added, along with [support](https://github.com/lightningnetwork/lnd/pull/8160)
for forwarding blinded payments and [error handling](https://github.com/lightningnetwork/lnd/pull/8485).

View File

@ -202,15 +202,15 @@ type ChannelLinkConfig struct {
// receiving node is persistent.
UnsafeReplay bool
// MinFeeUpdateTimeout represents the minimum interval in which a link
// MinUpdateTimeout represents the minimum interval in which a link
// will propose to update its commitment fee rate. A random timeout will
// be selected between this and MaxFeeUpdateTimeout.
MinFeeUpdateTimeout time.Duration
// be selected between this and MaxUpdateTimeout.
MinUpdateTimeout time.Duration
// MaxFeeUpdateTimeout represents the maximum interval in which a link
// MaxUpdateTimeout represents the maximum interval in which a link
// will propose to update its commitment fee rate. A random timeout will
// be selected between this and MinFeeUpdateTimeout.
MaxFeeUpdateTimeout time.Duration
// be selected between this and MinUpdateTimeout.
MaxUpdateTimeout time.Duration
// OutgoingCltvRejectDelta defines the number of blocks before expiry of
// an htlc where we don't offer an htlc anymore. This should be at least
@ -1558,8 +1558,8 @@ func getResolutionFailure(resolution *invoices.HtlcFailResolution,
// within the link's configuration that will be used to determine when the link
// should propose an update to its commitment fee rate.
func (l *channelLink) randomFeeUpdateTimeout() time.Duration {
lower := int64(l.cfg.MinFeeUpdateTimeout)
upper := int64(l.cfg.MaxFeeUpdateTimeout)
lower := int64(l.cfg.MinUpdateTimeout)
upper := int64(l.cfg.MaxUpdateTimeout)
return time.Duration(prand.Int63n(upper-lower) + lower)
}

View File

@ -2220,11 +2220,11 @@ func newSingleLinkTestHarness(t *testing.T, chanAmt,
BatchTicker: bticker,
FwdPkgGCTicker: ticker.NewForce(15 * time.Second),
PendingCommitTicker: ticker.New(time.Minute),
// Make the BatchSize and Min/MaxFeeUpdateTimeout large enough
// Make the BatchSize and Min/MaxUpdateTimeout large enough
// to not trigger commit updates automatically during tests.
BatchSize: 10000,
MinFeeUpdateTimeout: 30 * time.Minute,
MaxFeeUpdateTimeout: 40 * time.Minute,
MinUpdateTimeout: 30 * time.Minute,
MaxUpdateTimeout: 40 * time.Minute,
MaxOutgoingCltvExpiry: DefaultMaxOutgoingCltvExpiry,
MaxFeeAllocation: DefaultMaxLinkFeeAllocation,
NotifyActiveLink: func(wire.OutPoint) {},
@ -4881,11 +4881,11 @@ func (h *persistentLinkHarness) restartLink(
BatchTicker: bticker,
FwdPkgGCTicker: ticker.New(5 * time.Second),
PendingCommitTicker: ticker.New(time.Minute),
// Make the BatchSize and Min/MaxFeeUpdateTimeout large enough
// Make the BatchSize and Min/MaxUpdateTimeout large enough
// to not trigger commit updates automatically during tests.
BatchSize: 10000,
MinFeeUpdateTimeout: 30 * time.Minute,
MaxFeeUpdateTimeout: 40 * time.Minute,
BatchSize: 10000,
MinUpdateTimeout: 30 * time.Minute,
MaxUpdateTimeout: 40 * time.Minute,
// Set any hodl flags requested for the new link.
HodlMask: hodl.MaskFromFlags(hodlFlags...),
MaxOutgoingCltvExpiry: DefaultMaxOutgoingCltvExpiry,

View File

@ -1155,8 +1155,8 @@ func (h *hopNetwork) createChannelLink(server, peer *mockServer,
BatchTicker: ticker.NewForce(testBatchTimeout),
FwdPkgGCTicker: ticker.NewForce(fwdPkgTimeout),
PendingCommitTicker: ticker.New(2 * time.Minute),
MinFeeUpdateTimeout: minFeeUpdateTimeout,
MaxFeeUpdateTimeout: maxFeeUpdateTimeout,
MinUpdateTimeout: minFeeUpdateTimeout,
MaxUpdateTimeout: maxFeeUpdateTimeout,
OnChannelFailure: func(lnwire.ChannelID, lnwire.ShortChannelID, LinkFailureError) {},
OutgoingCltvRejectDelta: 3,
MaxOutgoingCltvExpiry: DefaultMaxOutgoingCltvExpiry,

20
lncfg/fee.go Normal file
View File

@ -0,0 +1,20 @@
package lncfg
import "time"
// DefaultMinUpdateTimeout represents the minimum interval in which a
// WebAPIEstimator will request fresh fees from its API.
const DefaultMinUpdateTimeout = 5 * time.Minute
// DefaultMaxUpdateTimeout represents the maximum interval in which a
// WebAPIEstimator will request fresh fees from its API.
const DefaultMaxUpdateTimeout = 20 * time.Minute
// Fee holds the configuration options for fee estimation.
//
//nolint:lll
type Fee struct {
URL string `long:"url" description:"Optional URL for external fee estimation. If no URL is specified, the method for fee estimation will depend on the chosen backend and network. Must be set for neutrino on mainnet."`
MinUpdateTimeout time.Duration `long:"min-update-timeout" description:"The minimum interval in which fees will be updated from the specified fee URL."`
MaxUpdateTimeout time.Duration `long:"max-update-timeout" description:"The maximum interval in which fees will be updated from the specified fee URL."`
}

View File

@ -12,7 +12,6 @@ type Neutrino struct {
MaxPeers int `long:"maxpeers" description:"Max number of inbound and outbound peers"`
BanDuration time.Duration `long:"banduration" description:"How long to ban misbehaving peers. Valid time units are {s, m, h}. Minimum 1 second"`
BanThreshold uint32 `long:"banthreshold" description:"Maximum allowed ban score before disconnecting and banning misbehaving peers."`
FeeURL string `long:"feeurl" description:"DEPRECATED: Use top level 'feeurl' option. Optional URL for fee estimation. If a URL is not specified, static fees will be used for estimation." hidden:"true"`
AssertFilterHeader string `long:"assertfilterheader" description:"Optional filter header in height:hash format to assert the state of neutrino's filter header chain on startup. If the assertion does not hold, then the filter header chain will be re-synced from the genesis block."`
UserAgentName string `long:"useragentname" description:"Used to help identify ourselves to other bitcoin peers"`
UserAgentVersion string `long:"useragentversion" description:"Used to help identify ourselves to other bitcoin peers"`

View File

@ -17,7 +17,7 @@ import (
// WebFeeService defines an interface that's used to provide fee estimation
// service used in the integration tests. It must provide an URL so that a lnd
// node can be started with the flag `--feeurl` and uses the customized fee
// node can be started with the flag `--fee.url` and uses the customized fee
// estimator.
type WebFeeService interface {
// Start starts the service.

View File

@ -279,7 +279,7 @@ func (cfg *BaseNodeConfig) GenArgs() []string {
}
if cfg.FeeURL != "" {
args = append(args, "--feeurl="+cfg.FeeURL)
args = append(args, "--fee.url="+cfg.FeeURL)
}
// Put extra args in the end so the args can be overwritten.

View File

@ -28,14 +28,6 @@ const (
// less than this will result in an error.
minBlockTarget uint32 = 1
// minFeeUpdateTimeout represents the minimum interval in which a
// WebAPIEstimator will request fresh fees from its API.
minFeeUpdateTimeout = 5 * time.Minute
// maxFeeUpdateTimeout represents the maximum interval in which a
// WebAPIEstimator will request fresh fees from its API.
maxFeeUpdateTimeout = 20 * time.Minute
// WebAPIConnectionTimeout specifies the timeout value for connecting
// to the api source.
WebAPIConnectionTimeout = 5 * time.Second
@ -739,19 +731,43 @@ type WebAPIEstimator struct {
// estimates.
noCache bool
// minFeeUpdateTimeout represents the minimum interval in which the
// web estimator will request fresh fees from its API.
minFeeUpdateTimeout time.Duration
// minFeeUpdateTimeout represents the maximum interval in which the
// web estimator will request fresh fees from its API.
maxFeeUpdateTimeout time.Duration
quit chan struct{}
wg sync.WaitGroup
}
// NewWebAPIEstimator creates a new WebAPIEstimator from a given URL and a
// fallback default fee. The fees are updated whenever a new block is mined.
func NewWebAPIEstimator(api WebAPIFeeSource, noCache bool) *WebAPIEstimator {
return &WebAPIEstimator{
apiSource: api,
feeByBlockTarget: make(map[uint32]uint32),
noCache: noCache,
quit: make(chan struct{}),
func NewWebAPIEstimator(api WebAPIFeeSource, noCache bool,
minFeeUpdateTimeout time.Duration,
maxFeeUpdateTimeout time.Duration) (*WebAPIEstimator, error) {
if minFeeUpdateTimeout == 0 || maxFeeUpdateTimeout == 0 {
return nil, fmt.Errorf("minFeeUpdateTimeout and " +
"maxFeeUpdateTimeout must be greater than 0")
}
if minFeeUpdateTimeout >= maxFeeUpdateTimeout {
return nil, fmt.Errorf("minFeeUpdateTimeout target of %v "+
"cannot be greater than maxFeeUpdateTimeout of %v",
minFeeUpdateTimeout, maxFeeUpdateTimeout)
}
return &WebAPIEstimator{
apiSource: api,
feeByBlockTarget: make(map[uint32]uint32),
noCache: noCache,
quit: make(chan struct{}),
minFeeUpdateTimeout: minFeeUpdateTimeout,
maxFeeUpdateTimeout: maxFeeUpdateTimeout,
}, nil
}
// EstimateFeePerKW takes in a target for the number of blocks until an initial
@ -809,7 +825,12 @@ func (w *WebAPIEstimator) Start() error {
w.started.Do(func() {
log.Infof("Starting web API fee estimator")
w.updateFeeTicker = time.NewTicker(w.randomFeeUpdateTimeout())
feeUpdateTimeout := w.randomFeeUpdateTimeout()
log.Infof("Web API fee estimator using update timeout of %v",
feeUpdateTimeout)
w.updateFeeTicker = time.NewTicker(feeUpdateTimeout)
w.updateFeeEstimates()
w.wg.Add(1)
@ -852,9 +873,11 @@ func (w *WebAPIEstimator) RelayFeePerKW() SatPerKWeight {
// and maxFeeUpdateTimeout that will be used to determine how often the Estimator
// should retrieve fresh fees from its API.
func (w *WebAPIEstimator) randomFeeUpdateTimeout() time.Duration {
lower := int64(minFeeUpdateTimeout)
upper := int64(maxFeeUpdateTimeout)
return time.Duration(prand.Int63n(upper-lower) + lower)
lower := int64(w.minFeeUpdateTimeout)
upper := int64(w.maxFeeUpdateTimeout)
return time.Duration(
prand.Int63n(upper-lower) + lower, //nolint:gosec
).Round(time.Second)
}
// getCachedFee takes a conf target and returns the cached fee rate. When the

View File

@ -4,6 +4,7 @@ import (
"bytes"
"encoding/json"
"testing"
"time"
"github.com/btcsuite/btcd/btcutil"
"github.com/stretchr/testify/require"
@ -142,6 +143,9 @@ func TestWebAPIFeeEstimator(t *testing.T) {
// Fee rates are in sat/kb.
minFeeRate uint32 = 2000 // 500 sat/kw
maxFeeRate uint32 = 4000 // 1000 sat/kw
minFeeUpdateTimeout = 5 * time.Minute
maxFeeUpdateTimeout = 20 * time.Minute
)
testCases := []struct {
@ -199,7 +203,9 @@ func TestWebAPIFeeEstimator(t *testing.T) {
feeSource := &mockFeeSource{}
feeSource.On("GetFeeMap").Return(feeRateResp, nil)
estimator := NewWebAPIEstimator(feeSource, false)
estimator, _ := NewWebAPIEstimator(
feeSource, false, minFeeUpdateTimeout, maxFeeUpdateTimeout,
)
// Test that requesting a fee when no fees have been cached won't fail.
feeRate, err := estimator.EstimateFeePerKW(5)
@ -247,10 +253,15 @@ func TestGetCachedFee(t *testing.T) {
minFeeRate uint32 = 100
maxFeeRate uint32 = 1000
minFeeUpdateTimeout = 5 * time.Minute
maxFeeUpdateTimeout = 20 * time.Minute
)
// Create a dummy estimator without WebAPIFeeSource.
estimator := NewWebAPIEstimator(nil, false)
estimator, _ := NewWebAPIEstimator(
nil, false, minFeeUpdateTimeout, maxFeeUpdateTimeout,
)
// When the cache is empty, an error should be returned.
cachedFee, err := estimator.getCachedFee(minTarget)
@ -315,3 +326,38 @@ func TestGetCachedFee(t *testing.T) {
})
}
}
func TestRandomFeeUpdateTimeout(t *testing.T) {
t.Parallel()
var (
minFeeUpdateTimeout = 1 * time.Minute
maxFeeUpdateTimeout = 2 * time.Minute
)
estimator, _ := NewWebAPIEstimator(
nil, false, minFeeUpdateTimeout, maxFeeUpdateTimeout,
)
for i := 0; i < 1000; i++ {
timeout := estimator.randomFeeUpdateTimeout()
require.GreaterOrEqual(t, timeout, minFeeUpdateTimeout)
require.LessOrEqual(t, timeout, maxFeeUpdateTimeout)
}
}
func TestInvalidFeeUpdateTimeout(t *testing.T) {
t.Parallel()
var (
minFeeUpdateTimeout = 2 * time.Minute
maxFeeUpdateTimeout = 1 * time.Minute
)
_, err := NewWebAPIEstimator(
nil, false, minFeeUpdateTimeout, maxFeeUpdateTimeout,
)
require.Error(t, err, "NewWebAPIEstimator should return an error "+
"when minFeeUpdateTimeout > maxFeeUpdateTimeout")
}

View File

@ -1146,8 +1146,8 @@ func (p *Brontide) addLink(chanPoint *wire.OutPoint,
),
BatchSize: p.cfg.ChannelCommitBatchSize,
UnsafeReplay: p.cfg.UnsafeReplay,
MinFeeUpdateTimeout: htlcswitch.DefaultMinLinkFeeUpdateTimeout,
MaxFeeUpdateTimeout: htlcswitch.DefaultMaxLinkFeeUpdateTimeout,
MinUpdateTimeout: htlcswitch.DefaultMinLinkFeeUpdateTimeout,
MaxUpdateTimeout: htlcswitch.DefaultMaxLinkFeeUpdateTimeout,
OutgoingCltvRejectDelta: p.cfg.OutgoingCltvRejectDelta,
TowerClient: p.cfg.TowerClient,
MaxOutgoingCltvExpiry: p.cfg.MaxOutgoingCltvExpiry,

View File

@ -305,14 +305,28 @@
; The default value below is 20 MB (1024 * 1024 * 20)
; blockcachesize=20971520
; Optional URL for external fee estimation. If no URL is specified, the method
; for fee estimation will depend on the chosen backend and network. Must be set
; for neutrino on mainnet.
; DEPRECATED: Use 'fee.url' option. Optional URL for external fee estimation.
; If no URL is specified, the method for fee estimation will depend on the
; chosen backend and network. Must be set for neutrino on mainnet.
; Default:
; feeurl=
; Example:
; feeurl=https://nodes.lightning.computer/fees/v1/btc-fee-estimates.json
; Optional URL for external fee estimation. If no URL is specified, the method
; for fee estimation will depend on the chosen backend and network. Must be set
; for neutrino on mainnet.
; Default:
; fee.url=
; Example:
; fee.url=https://nodes.lightning.computer/fees/v1/btc-fee-estimates.json
; The minimum interval in which fees will be updated from the specified fee URL.
; fee.min-update-timeout=5m
; The maximum interval in which fees will be updated from the specified fee URL.
; fee.max-update-timeout=20m
; If true, then automatic network bootstrapping will not be attempted. This
; means that your node won't attempt to automatically seek out peers on the
; network.