lntemp+itest: refactor testRPCMiddlewareInterceptor

This commit is contained in:
yyforyongyu 2023-01-16 07:49:16 +08:00
parent ad77a45112
commit 84278d6a49
No known key found for this signature in database
GPG Key ID: 9BCD95C4FF296868
5 changed files with 195 additions and 131 deletions

View File

@ -611,3 +611,19 @@ func (h *HarnessRPC) ForwardingHistory(
return resp
}
type MiddlewareClient lnrpc.Lightning_RegisterRPCMiddlewareClient
// RegisterRPCMiddleware makes a RPC call to the node's RegisterRPCMiddleware
// and asserts. It also returns a cancel context which can cancel the context
// used by the client.
func (h *HarnessRPC) RegisterRPCMiddleware() (MiddlewareClient,
context.CancelFunc) {
ctxt, cancel := context.WithCancel(h.runCtx)
stream, err := h.LN.RegisterRPCMiddleware(ctxt)
h.NoError(err, "RegisterRPCMiddleware")
return stream, cancel
}

View File

@ -341,4 +341,8 @@ var allTestCasesTemp = []*lntemp.TestCase{
Name: "route fee cutoff",
TestFunc: testRouteFeeCutoff,
},
{
Name: "rpc middleware interceptor",
TestFunc: testRPCMiddlewareInterceptor,
},
}

View File

@ -12,6 +12,7 @@ import (
"github.com/golang/protobuf/proto"
"github.com/lightningnetwork/lnd/lnrpc"
"github.com/lightningnetwork/lnd/lntemp/node"
"github.com/lightningnetwork/lnd/lntest"
"github.com/lightningnetwork/lnd/macaroons"
"github.com/stretchr/testify/assert"
@ -58,7 +59,7 @@ func testMacaroonAuthentication(net *lntest.NetworkHarness, ht *harnessTest) {
[]byte("dummy_root_key"), []byte("0"), "itest",
macaroon.LatestVersion,
)
cleanup, client := macaroonClient(
cleanup, client := macaroonClientOld(
t, testNode, invalidMac,
)
defer cleanup()
@ -75,7 +76,7 @@ func testMacaroonAuthentication(net *lntest.NetworkHarness, ht *harnessTest) {
testNode.ReadMacPath(), defaultTimeout,
)
require.NoError(t, err)
cleanup, client := macaroonClient(
cleanup, client := macaroonClientOld(
t, testNode, readonlyMac,
)
defer cleanup()
@ -96,7 +97,7 @@ func testMacaroonAuthentication(net *lntest.NetworkHarness, ht *harnessTest) {
readonlyMac, macaroons.TimeoutConstraint(-30),
)
require.NoError(t, err)
cleanup, client := macaroonClient(
cleanup, client := macaroonClientOld(
t, testNode, timeoutMac,
)
defer cleanup()
@ -118,7 +119,7 @@ func testMacaroonAuthentication(net *lntest.NetworkHarness, ht *harnessTest) {
),
)
require.NoError(t, err)
cleanup, client := macaroonClient(
cleanup, client := macaroonClientOld(
t, testNode, invalidIPAddrMac,
)
defer cleanup()
@ -141,7 +142,7 @@ func testMacaroonAuthentication(net *lntest.NetworkHarness, ht *harnessTest) {
macaroons.IPLockConstraint("127.0.0.1"),
)
require.NoError(t, err)
cleanup, client := macaroonClient(t, testNode, adminMac)
cleanup, client := macaroonClientOld(t, testNode, adminMac)
defer cleanup()
res, err := client.NewAddress(ctxt, newAddrReq)
require.NoError(t, err, "get new address")
@ -174,7 +175,7 @@ func testMacaroonAuthentication(net *lntest.NetworkHarness, ht *harnessTest) {
customMac := &macaroon.Macaroon{}
err = customMac.UnmarshalBinary(customMacBytes)
require.NoError(t, err)
cleanup, client := macaroonClient(
cleanup, client := macaroonClientOld(
t, testNode, customMac,
)
defer cleanup()
@ -426,7 +427,7 @@ func testBakeMacaroon(net *lntest.NetworkHarness, t *harnessTest) {
newMac, err := readMacaroonFromHex(bakeResp.Macaroon)
require.NoError(t, err)
cleanup, readOnlyClient := macaroonClient(
cleanup, readOnlyClient := macaroonClientOld(
t, testNode, newMac,
)
defer cleanup()
@ -505,7 +506,7 @@ func testBakeMacaroon(net *lntest.NetworkHarness, t *harnessTest) {
testNode.AdminMacPath(), defaultTimeout,
)
require.NoError(tt, err)
cleanup, client := macaroonClient(tt, testNode, adminMac)
cleanup, client := macaroonClientOld(tt, testNode, adminMac)
defer cleanup()
tc.run(ctxt, tt, client)
@ -530,7 +531,7 @@ func testDeleteMacaroonID(net *lntest.NetworkHarness, t *harnessTest) {
testNode.AdminMacPath(), defaultTimeout,
)
require.NoError(t.t, err)
cleanup, client := macaroonClient(t.t, testNode, adminMac)
cleanup, client := macaroonClientOld(t.t, testNode, adminMac)
defer cleanup()
// Record the number of macaroon IDs before creation.
@ -595,7 +596,7 @@ func testDeleteMacaroonID(net *lntest.NetworkHarness, t *harnessTest) {
// Check that the deleted macaroon can no longer access macaroon:read.
deletedMac, err := readMacaroonFromHex(macList[0])
require.NoError(t.t, err)
cleanup, client = macaroonClient(t.t, testNode, deletedMac)
cleanup, client = macaroonClientOld(t.t, testNode, deletedMac)
defer cleanup()
// Because the macaroon is deleted, it will be treated as an invalid one.
@ -717,7 +718,8 @@ func readMacaroonFromHex(macHex string) (*macaroon.Macaroon, error) {
return mac, nil
}
func macaroonClient(t *testing.T, testNode *lntest.HarnessNode,
// TODO(yy): remove.
func macaroonClientOld(t *testing.T, testNode *lntest.HarnessNode,
mac *macaroon.Macaroon) (func(), lnrpc.LightningClient) {
conn, err := testNode.ConnectRPCWithMacaroon(mac)
@ -729,3 +731,18 @@ func macaroonClient(t *testing.T, testNode *lntest.HarnessNode,
}
return cleanup, lnrpc.NewLightningClient(conn)
}
func macaroonClient(t *testing.T, testNode *node.HarnessNode,
mac *macaroon.Macaroon) (func(), lnrpc.LightningClient) {
t.Helper()
conn, err := testNode.ConnectRPCWithMacaroon(mac)
require.NoError(t, err, "connect to alice")
cleanup := func() {
err := conn.Close()
require.NoError(t, err, "close")
}
return cleanup, lnrpc.NewLightningClient(conn)
}

View File

@ -8,7 +8,8 @@ import (
"github.com/btcsuite/btcd/btcutil"
"github.com/lightningnetwork/lnd/lnrpc"
"github.com/lightningnetwork/lnd/lntest"
"github.com/lightningnetwork/lnd/lntemp"
"github.com/lightningnetwork/lnd/lntemp/node"
"github.com/lightningnetwork/lnd/macaroons"
"github.com/lightningnetwork/lnd/zpay32"
"github.com/stretchr/testify/require"
@ -18,133 +19,149 @@ import (
// testRPCMiddlewareInterceptor tests that the RPC middleware interceptor can
// be used correctly and in a safe way.
func testRPCMiddlewareInterceptor(net *lntest.NetworkHarness, t *harnessTest) {
func testRPCMiddlewareInterceptor(ht *lntemp.HarnessTest) {
// Let's first enable the middleware interceptor.
net.Alice.Cfg.ExtraArgs = append(
net.Alice.Cfg.ExtraArgs, "--rpcmiddleware.enable",
)
err := net.RestartNode(net.Alice, nil)
require.NoError(t.t, err)
//
// NOTE: we cannot use standby nodes here as the test messes with
// middleware interceptor. Thus we also skip the calling of cleanup of
// each of the following subtests because no standby nodes are used.
alice := ht.NewNode("alice", []string{"--rpcmiddleware.enable"})
bob := ht.NewNode("bob", nil)
// Let's set up a channel between Alice and Bob, just to get some useful
// data to inspect when doing RPC calls to Alice later.
net.EnsureConnected(t.t, net.Alice, net.Bob)
net.SendCoins(t.t, btcutil.SatoshiPerBitcoin, net.Alice)
_ = openChannelAndAssert(
t, net, net.Alice, net.Bob, lntest.OpenChannelParams{
Amt: 1_234_567,
},
)
ht.EnsureConnected(alice, bob)
ht.FundCoins(btcutil.SatoshiPerBitcoin, alice)
ht.OpenChannel(alice, bob, lntemp.OpenChannelParams{Amt: 1_234_567})
// Load or bake the macaroons that the simulated users will use to
// access the RPC.
readonlyMac, err := net.Alice.ReadMacaroon(
net.Alice.ReadMacPath(), defaultTimeout,
readonlyMac, err := alice.ReadMacaroon(
alice.Cfg.ReadMacPath, defaultTimeout,
)
require.NoError(t.t, err)
adminMac, err := net.Alice.ReadMacaroon(
net.Alice.AdminMacPath(), defaultTimeout,
require.NoError(ht, err)
adminMac, err := alice.ReadMacaroon(
alice.Cfg.AdminMacPath, defaultTimeout,
)
require.NoError(t.t, err)
require.NoError(ht, err)
customCaveatReadonlyMac, err := macaroons.SafeCopyMacaroon(readonlyMac)
require.NoError(t.t, err)
require.NoError(ht, err)
addConstraint := macaroons.CustomConstraint(
"itest-caveat", "itest-value",
)
require.NoError(t.t, addConstraint(customCaveatReadonlyMac))
require.NoError(ht, addConstraint(customCaveatReadonlyMac))
customCaveatAdminMac, err := macaroons.SafeCopyMacaroon(adminMac)
require.NoError(t.t, err)
require.NoError(t.t, addConstraint(customCaveatAdminMac))
require.NoError(ht, err)
require.NoError(ht, addConstraint(customCaveatAdminMac))
// Run all sub-tests now. We can't run anything in parallel because that
// would cause the main test function to exit and the nodes being
// cleaned up.
t.t.Run("registration restrictions", func(tt *testing.T) {
middlewareRegistrationRestrictionTests(tt, net.Alice)
ht.Run("registration restrictions", func(tt *testing.T) {
middlewareRegistrationRestrictionTests(tt, alice)
})
t.t.Run("read-only intercept", func(tt *testing.T) {
ht.Run("read-only intercept", func(tt *testing.T) {
registration := registerMiddleware(
tt, net.Alice, &lnrpc.MiddlewareRegistration{
MiddlewareName: "itest-interceptor",
tt, alice, &lnrpc.MiddlewareRegistration{
MiddlewareName: "itest-interceptor-1",
ReadOnlyMode: true,
}, true,
)
defer registration.cancel()
middlewareInterceptionTest(
tt, net.Alice, net.Bob, registration, readonlyMac,
tt, alice, bob, registration, readonlyMac,
customCaveatReadonlyMac, true,
)
})
// We've manually disconnected Bob from Alice in the previous test, make
// sure they're connected again.
net.EnsureConnected(t.t, net.Alice, net.Bob)
t.t.Run("encumbered macaroon intercept", func(tt *testing.T) {
//
// NOTE: we may get an error here saying "interceptor RPC client quit"
// as it takes some time for the interceptor to fully quit. Thus we
// restart the node here to make sure the old interceptor is removed
// from registration.
ht.RestartNode(alice)
ht.EnsureConnected(alice, bob)
ht.Run("encumbered macaroon intercept", func(tt *testing.T) {
registration := registerMiddleware(
tt, net.Alice, &lnrpc.MiddlewareRegistration{
MiddlewareName: "itest-interceptor",
tt, alice, &lnrpc.MiddlewareRegistration{
MiddlewareName: "itest-interceptor-2",
CustomMacaroonCaveatName: "itest-caveat",
}, true,
)
defer registration.cancel()
middlewareInterceptionTest(
tt, net.Alice, net.Bob, registration,
tt, alice, bob, registration,
customCaveatReadonlyMac, readonlyMac, false,
)
})
// Next, run the response manipulation tests.
net.EnsureConnected(t.t, net.Alice, net.Bob)
t.t.Run("read-only not allowed to manipulate", func(tt *testing.T) {
//
// NOTE: we may get an error here saying "interceptor RPC client quit"
// as it takes some time for the interceptor to fully quit. Thus we
// restart the node here to make sure the old interceptor is removed
// from registration.
ht.RestartNode(alice)
ht.EnsureConnected(alice, bob)
ht.Run("read-only not allowed to manipulate", func(tt *testing.T) {
registration := registerMiddleware(
tt, net.Alice, &lnrpc.MiddlewareRegistration{
MiddlewareName: "itest-interceptor",
tt, alice, &lnrpc.MiddlewareRegistration{
MiddlewareName: "itest-interceptor-3",
ReadOnlyMode: true,
}, true,
)
defer registration.cancel()
middlewareRequestManipulationTest(
tt, net.Alice, registration, adminMac, true,
tt, alice, registration, adminMac, true,
)
middlewareResponseManipulationTest(
tt, net.Alice, net.Bob, registration, readonlyMac, true,
tt, alice, bob, registration, readonlyMac, true,
)
})
net.EnsureConnected(t.t, net.Alice, net.Bob)
t.t.Run("encumbered macaroon manipulate", func(tt *testing.T) {
// NOTE: we may get an error here saying "interceptor RPC client quit"
// as it takes some time for the interceptor to fully quit. Thus we
// restart the node here to make sure the old interceptor is removed
// from registration.
ht.RestartNode(alice)
ht.EnsureConnected(alice, bob)
ht.Run("encumbered macaroon manipulate", func(tt *testing.T) {
registration := registerMiddleware(
tt, net.Alice, &lnrpc.MiddlewareRegistration{
MiddlewareName: "itest-interceptor",
tt, alice, &lnrpc.MiddlewareRegistration{
MiddlewareName: "itest-interceptor-4",
CustomMacaroonCaveatName: "itest-caveat",
}, true,
)
defer registration.cancel()
middlewareRequestManipulationTest(
tt, net.Alice, registration, customCaveatAdminMac,
false,
tt, alice, registration, customCaveatAdminMac, false,
)
middlewareResponseManipulationTest(
tt, net.Alice, net.Bob, registration,
tt, alice, bob, registration,
customCaveatReadonlyMac, false,
)
})
// And finally make sure mandatory middleware is always checked for any
// RPC request.
t.t.Run("mandatory middleware", func(tt *testing.T) {
middlewareMandatoryTest(tt, net.Alice, net)
ht.Run("mandatory middleware", func(tt *testing.T) {
st := ht.Subtest(tt)
middlewareMandatoryTest(st, alice)
})
}
// middlewareRegistrationRestrictionTests tests all restrictions that apply to
// registering a middleware.
func middlewareRegistrationRestrictionTests(t *testing.T,
node *lntest.HarnessNode) {
node *node.HarnessNode) {
testCases := []struct {
registration *lnrpc.MiddlewareRegistration
@ -189,10 +206,12 @@ func middlewareRegistrationRestrictionTests(t *testing.T,
// intercepted. It also makes sure that depending on the mode (read-only or
// custom macaroon caveat) a middleware only gets access to the requests it
// should be allowed access to.
func middlewareInterceptionTest(t *testing.T, node *lntest.HarnessNode,
peer *lntest.HarnessNode, registration *middlewareHarness,
userMac *macaroon.Macaroon, disallowedMac *macaroon.Macaroon,
readOnly bool) {
func middlewareInterceptionTest(t *testing.T,
node, peer *node.HarnessNode, registration *middlewareHarness,
userMac *macaroon.Macaroon,
disallowedMac *macaroon.Macaroon, readOnly bool) {
t.Helper()
// Everything we test here should be executed in a matter of
// milliseconds, so we can use one single timeout context for all calls.
@ -253,10 +272,7 @@ func middlewareInterceptionTest(t *testing.T, node *lntest.HarnessNode,
// Disconnect Bob to trigger a peer event without using Alice's RPC
// interface itself.
_, err = peer.DisconnectPeer(ctxc, &lnrpc.DisconnectPeerRequest{
PubKey: node.PubKeyStr,
})
require.NoError(t, err)
peer.RPC.DisconnectPeer(node.PubKeyStr)
peerEvent, err := resp2.Recv()
require.NoError(t, err)
require.Equal(t, lnrpc.PeerEvent_PEER_OFFLINE, peerEvent.GetType())
@ -330,10 +346,12 @@ func middlewareInterceptionTest(t *testing.T, node *lntest.HarnessNode,
// middlewareResponseManipulationTest tests that unary and streaming responses
// can be intercepted and also manipulated, at least if the middleware didn't
// register for read-only access.
func middlewareResponseManipulationTest(t *testing.T, node *lntest.HarnessNode,
peer *lntest.HarnessNode, registration *middlewareHarness,
func middlewareResponseManipulationTest(t *testing.T,
node, peer *node.HarnessNode, registration *middlewareHarness,
userMac *macaroon.Macaroon, readOnly bool) {
t.Helper()
// Everything we test here should be executed in a matter of
// milliseconds, so we can use one single timeout context for all calls.
ctxb := context.Background()
@ -421,10 +439,7 @@ func middlewareResponseManipulationTest(t *testing.T, node *lntest.HarnessNode,
// Disconnect Bob to trigger a peer event without using Alice's RPC
// interface itself.
_, err = peer.DisconnectPeer(ctxc, &lnrpc.DisconnectPeerRequest{
PubKey: node.PubKeyStr,
})
require.NoError(t, err)
peer.RPC.DisconnectPeer(node.PubKeyStr)
peerEvent, err := resp2.Recv()
require.NoError(t, err)
@ -448,10 +463,12 @@ func middlewareResponseManipulationTest(t *testing.T, node *lntest.HarnessNode,
// middlewareRequestManipulationTest tests that unary and streaming requests
// can be intercepted and also manipulated, at least if the middleware didn't
// register for read-only access.
func middlewareRequestManipulationTest(t *testing.T, node *lntest.HarnessNode,
func middlewareRequestManipulationTest(t *testing.T, node *node.HarnessNode,
registration *middlewareHarness, userMac *macaroon.Macaroon,
readOnly bool) {
t.Helper()
// Everything we test here should be executed in a matter of
// milliseconds, so we can use one single timeout context for all calls.
ctxb := context.Background()
@ -528,54 +545,44 @@ func middlewareRequestManipulationTest(t *testing.T, node *lntest.HarnessNode,
// middlewareMandatoryTest tests that all RPC requests are blocked if there is
// a mandatory middleware declared that's currently not registered.
func middlewareMandatoryTest(t *testing.T, node *lntest.HarnessNode,
net *lntest.NetworkHarness) {
func middlewareMandatoryTest(ht *lntemp.HarnessTest, node *node.HarnessNode) {
// Let's declare our itest interceptor as mandatory but don't register
// it just yet. That should cause all RPC requests to fail, except for
// the registration itself.
node.Cfg.ExtraArgs = append(
node.Cfg.ExtraArgs,
node.Cfg.SkipUnlock = true
ht.RestartNodeWithExtraArgs(node, []string{
"--noseedbackup", "--rpcmiddleware.enable",
"--rpcmiddleware.addmandatory=itest-interceptor",
)
err := net.RestartNodeNoUnlock(node, nil, false)
require.NoError(t, err)
})
// The "wait for node to start" flag of the above restart does too much
// and has a call to GetInfo built in, which will fail in this special
// test case. So we need to do the wait and client setup manually here.
conn, err := node.ConnectRPC(true)
require.NoError(t, err)
conn, err := node.ConnectRPC()
require.NoError(ht, err)
node.InitRPCClients(conn)
err = node.WaitUntilStateReached(lnrpc.WalletState_RPC_ACTIVE)
require.NoError(t, err)
node.LightningClient = lnrpc.NewLightningClient(conn)
err = node.WaitUntilServerActive()
require.NoError(ht, err)
ctxb := context.Background()
ctxc, cancel := context.WithTimeout(ctxb, defaultTimeout)
defer cancel()
// Test a unary request first.
_, err = node.ListChannels(ctxc, &lnrpc.ListChannelsRequest{})
require.Error(t, err)
require.Contains(
t, err.Error(), "middleware 'itest-interceptor' is "+
"currently not registered",
)
_, err = node.RPC.LN.ListChannels(ctxc, &lnrpc.ListChannelsRequest{})
require.Contains(ht, err.Error(), "middleware 'itest-interceptor' is "+
"currently not registered")
// Then a streaming one.
stream, err := node.SubscribeInvoices(ctxc, &lnrpc.InvoiceSubscription{})
require.NoError(t, err)
stream := node.RPC.SubscribeInvoices(&lnrpc.InvoiceSubscription{})
_, err = stream.Recv()
require.Error(t, err)
require.Contains(
t, err.Error(), "middleware 'itest-interceptor' is "+
"currently not registered",
)
require.Error(ht, err)
require.Contains(ht, err.Error(), "middleware 'itest-interceptor' is "+
"currently not registered")
// Now let's register the middleware and try again.
registration := registerMiddleware(
t, node, &lnrpc.MiddlewareRegistration{
ht.T, node, &lnrpc.MiddlewareRegistration{
MiddlewareName: "itest-interceptor",
CustomMacaroonCaveatName: "itest-caveat",
}, true,
@ -584,16 +591,13 @@ func middlewareMandatoryTest(t *testing.T, node *lntest.HarnessNode,
// Both the unary and streaming requests should now be allowed.
time.Sleep(500 * time.Millisecond)
_, err = node.ListChannels(ctxc, &lnrpc.ListChannelsRequest{})
require.NoError(t, err)
_, err = node.SubscribeInvoices(ctxc, &lnrpc.InvoiceSubscription{})
require.NoError(t, err)
node.RPC.ListChannels(&lnrpc.ListChannelsRequest{})
node.RPC.SubscribeInvoices(&lnrpc.InvoiceSubscription{})
// We now shut down the node manually to prevent the test from failing
// because we can't call the stop RPC if we unregister the middleware in
// the defer statement above.
err = net.ShutdownNode(node)
require.NoError(t, err)
// because we can't call the stop RPC if we unregister the middleware
// in the defer statement above.
ht.KillNode(node)
}
// assertInterceptedType makes sure that the intercept message sent by the RPC
@ -648,35 +652,62 @@ type middlewareHarness struct {
// registerMiddleware creates a new middleware harness and sends the initial
// register message to the RPC server.
func registerMiddleware(t *testing.T, node *lntest.HarnessNode,
func registerMiddleware(t *testing.T, node *node.HarnessNode,
registration *lnrpc.MiddlewareRegistration,
waitForRegister bool) *middlewareHarness {
ctxc, cancel := context.WithCancel(context.Background())
t.Helper()
middlewareStream, err := node.RegisterRPCMiddleware(ctxc)
require.NoError(t, err)
middlewareStream, cancel := node.RPC.RegisterRPCMiddleware()
err = middlewareStream.Send(&lnrpc.RPCMiddlewareResponse{
MiddlewareMessage: &lnrpc.RPCMiddlewareResponse_Register{
errChan := make(chan error)
go func() {
msg := &lnrpc.RPCMiddlewareResponse_Register{
Register: registration,
},
}
err := middlewareStream.Send(&lnrpc.RPCMiddlewareResponse{
MiddlewareMessage: msg,
})
require.NoError(t, err)
if waitForRegister {
// Wait for the registration complete message.
regCompleteMsg, err := middlewareStream.Recv()
require.NoError(t, err)
require.True(t, regCompleteMsg.GetRegComplete())
errChan <- err
}()
select {
case <-time.After(defaultTimeout):
require.Fail(t, "registerMiddleware send timeout")
case err := <-errChan:
require.NoError(t, err, "registerMiddleware send failed")
}
return &middlewareHarness{
mh := &middlewareHarness{
t: t,
cancel: cancel,
stream: middlewareStream,
responsesChan: make(chan *lnrpc.RPCMessage),
}
if !waitForRegister {
return mh
}
// Wait for the registration complete message.
msg := make(chan *lnrpc.RPCMiddlewareRequest)
go func() {
regCompleteMsg, err := middlewareStream.Recv()
require.NoError(t, err, "registerMiddleware recv failed")
msg <- regCompleteMsg
}()
select {
case <-time.After(defaultTimeout):
require.Fail(t, "registerMiddleware recv timeout")
case m := <-msg:
require.True(t, m.GetRegComplete())
}
return mh
}
// interceptUnary intercepts a unary call, optionally requesting to replace the

View File

@ -116,10 +116,6 @@ var allTestCases = []*testCase{
name: "wallet import pubkey",
test: testWalletImportPubKey,
},
{
name: "rpc middleware interceptor",
test: testRPCMiddlewareInterceptor,
},
{
name: "wipe forwarding packages",
test: testWipeForwardingPackages,