diff --git a/cmd/lncli/cmd_macaroon.go b/cmd/lncli/cmd_macaroon.go index 83f906b85..8b8848a2b 100644 --- a/cmd/lncli/cmd_macaroon.go +++ b/cmd/lncli/cmd_macaroon.go @@ -10,11 +10,11 @@ import ( "strings" "unicode" - "github.com/golang/protobuf/proto" "github.com/lightningnetwork/lnd/lncfg" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/macaroons" "github.com/urfave/cli" + "google.golang.org/protobuf/proto" "gopkg.in/macaroon-bakery.v2/bakery" "gopkg.in/macaroon.v2" ) @@ -385,7 +385,7 @@ func printMacaroon(ctx *cli.Context) error { return fmt.Errorf("unable to decode macaroon version: %v", err) } - // Prepare everything to be printed in a more human readable format. + // Prepare everything to be printed in a more human-readable format. content := &macaroonContent{ Version: uint16(mac.Version()), Location: mac.Location(), diff --git a/cmd/lncli/cmd_payments.go b/cmd/lncli/cmd_payments.go index 923f65b4c..ec2eebd9a 100644 --- a/cmd/lncli/cmd_payments.go +++ b/cmd/lncli/cmd_payments.go @@ -17,7 +17,6 @@ import ( "github.com/btcsuite/btcd/btcutil" "github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/text" - "github.com/lightninglabs/protobuf-hex-display/jsonpb" "github.com/lightningnetwork/lnd/chainreg" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lnrpc/routerrpc" @@ -952,7 +951,7 @@ func sendToRoute(ctx *cli.Context) error { // format. var route *lnrpc.Route routes := &lnrpc.QueryRoutesResponse{} - err = jsonpb.UnmarshalString(jsonRoutes, routes) + err = lnrpc.ProtoJSONUnmarshalOpts.Unmarshal([]byte(jsonRoutes), routes) if err == nil { if len(routes.Routes) == 0 { return fmt.Errorf("no routes provided") @@ -966,7 +965,9 @@ func sendToRoute(ctx *cli.Context) error { route = routes.Routes[0] } else { routes := &routerrpc.BuildRouteResponse{} - err = jsonpb.UnmarshalString(jsonRoutes, routes) + err = lnrpc.ProtoJSONUnmarshalOpts.Unmarshal( + []byte(jsonRoutes), routes, + ) if err != nil { return fmt.Errorf("unable to unmarshal json string "+ "from incoming array of routes: %v", err) diff --git a/cmd/lncli/cmd_walletunlocker.go b/cmd/lncli/cmd_walletunlocker.go index 45555c430..cbf900f82 100644 --- a/cmd/lncli/cmd_walletunlocker.go +++ b/cmd/lncli/cmd_walletunlocker.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" - "github.com/lightninglabs/protobuf-hex-display/jsonpb" "github.com/lightningnetwork/lnd/lncfg" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lnrpc/walletrpc" @@ -720,7 +719,7 @@ func createWatchOnly(ctx *cli.Context) error { } jsonAccts := &walletrpc.ListAccountsResponse{} - err = jsonpb.Unmarshal(bytes.NewReader(jsonBytes), jsonAccts) + err = lnrpc.ProtoJSONUnmarshalOpts.Unmarshal(jsonBytes, jsonAccts) if err != nil { return fmt.Errorf("error parsing JSON: %v", err) } diff --git a/cmd/lncli/commands.go b/cmd/lncli/commands.go index b0e3c03c8..1f9b131d4 100644 --- a/cmd/lncli/commands.go +++ b/cmd/lncli/commands.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/hex" + "encoding/json" "errors" "fmt" "io" @@ -17,9 +18,6 @@ import ( "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/wire" - "github.com/lightninglabs/protobuf-hex-display/json" - "github.com/lightninglabs/protobuf-hex-display/jsonpb" - "github.com/lightninglabs/protobuf-hex-display/proto" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/routing" "github.com/lightningnetwork/lnd/routing/route" @@ -27,6 +25,7 @@ import ( "github.com/urfave/cli" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" ) // TODO(roasbeef): cli logic for supporting both positional and unix style @@ -71,19 +70,13 @@ func printJSON(resp interface{}) { } func printRespJSON(resp proto.Message) { - jsonMarshaler := &jsonpb.Marshaler{ - EmitDefaults: true, - OrigName: true, - Indent: " ", - } - - jsonStr, err := jsonMarshaler.MarshalToString(resp) + jsonBytes, err := lnrpc.ProtoJSONMarshalOpts.Marshal(resp) if err != nil { fmt.Println("unable to decode response: ", err) return } - fmt.Println(jsonStr) + fmt.Printf("%s\n", jsonBytes) } // actionDecorator is used to add additional information and error handling diff --git a/cmd/lncli/devrpc_active.go b/cmd/lncli/devrpc_active.go index 925b2df18..02cdcdee6 100644 --- a/cmd/lncli/devrpc_active.go +++ b/cmd/lncli/devrpc_active.go @@ -4,11 +4,9 @@ package main import ( - "bytes" "fmt" - "io/ioutil" + "os" - "github.com/lightninglabs/protobuf-hex-display/jsonpb" "github.com/lightningnetwork/lnd/lncfg" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lnrpc/devrpc" @@ -43,14 +41,14 @@ func importGraph(ctx *cli.Context) error { defer cleanUp() jsonFile := lncfg.CleanAndExpandPath(ctx.Args().First()) - jsonBytes, err := ioutil.ReadFile(jsonFile) + jsonBytes, err := os.ReadFile(jsonFile) if err != nil { return fmt.Errorf("error reading JSON from file %v: %v", jsonFile, err) } jsonGraph := &lnrpc.ChannelGraph{} - err = jsonpb.Unmarshal(bytes.NewReader(jsonBytes), jsonGraph) + err = lnrpc.ProtoJSONUnmarshalOpts.Unmarshal(jsonBytes, jsonGraph) if err != nil { return fmt.Errorf("error parsing JSON: %v", err) } diff --git a/go.mod b/go.mod index dc1139bb8..b740b398d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 github.com/go-errors/errors v1.0.1 - github.com/golang/protobuf v1.5.2 github.com/gorilla/websocket v1.4.2 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 @@ -32,7 +31,6 @@ require ( github.com/kkdai/bstream v1.0.0 github.com/lightninglabs/neutrino v0.15.0 github.com/lightninglabs/neutrino/cache v1.1.1 - github.com/lightninglabs/protobuf-hex-display v1.4.3-hex-display github.com/lightningnetwork/lightning-onion v1.2.1-0.20221202012345-ca23184850a1 github.com/lightningnetwork/lnd/cert v1.2.1 github.com/lightningnetwork/lnd/clock v1.1.0 @@ -85,6 +83,7 @@ require ( github.com/fergusstrange/embedded-postgres v1.10.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/uuid v1.3.0 // indirect diff --git a/go.sum b/go.sum index 1d9141fa5..39bcd61ec 100644 --- a/go.sum +++ b/go.sum @@ -395,8 +395,6 @@ github.com/lightninglabs/neutrino v0.15.0 h1:yr3uz36fLAq8hyM0TRUVlef1TRNoWAqpmmN github.com/lightninglabs/neutrino v0.15.0/go.mod h1:pmjwElN/091TErtSE9Vd5W4hpxoG2/+xlb+HoPm9Gug= github.com/lightninglabs/neutrino/cache v1.1.1 h1:TllWOSlkABhpgbWJfzsrdUaDH2fBy/54VSIB4vVqV8M= github.com/lightninglabs/neutrino/cache v1.1.1/go.mod h1:XJNcgdOw1LQnanGjw8Vj44CvguYA25IMKjWFZczwZuo= -github.com/lightninglabs/protobuf-hex-display v1.4.3-hex-display h1:RZJ8H4ueU/aQ9pFtx5wqsuD3B/DezrewJeVwDKKYY8E= -github.com/lightninglabs/protobuf-hex-display v1.4.3-hex-display/go.mod h1:2oKOBU042GKFHrdbgGiKax4xVrFiZu51lhacUZQ9MnE= github.com/lightningnetwork/lightning-onion v1.2.1-0.20221202012345-ca23184850a1 h1:Wm0g70gkcAu2pGpNZwfWPSVOY21j8IyYsNewwK4OkT4= github.com/lightningnetwork/lightning-onion v1.2.1-0.20221202012345-ca23184850a1/go.mod h1:7dDx73ApjEZA0kcknI799m2O5kkpfg4/gr7N092ojNo= github.com/lightningnetwork/lnd/cert v1.2.1 h1:CTrTcU0L66J73oqdRLVfNylZyp1Fh97ZezX6IuzkrqE= diff --git a/itest/lnd_macaroons_test.go b/itest/lnd_macaroons_test.go index b5dad77bb..22767ec93 100644 --- a/itest/lnd_macaroons_test.go +++ b/itest/lnd_macaroons_test.go @@ -9,13 +9,13 @@ import ( "strings" "testing" - "github.com/golang/protobuf/proto" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lntest" "github.com/lightningnetwork/lnd/lntest/node" "github.com/lightningnetwork/lnd/macaroons" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" "gopkg.in/macaroon.v2" ) diff --git a/itest/lnd_rest_api_test.go b/itest/lnd_rest_api_test.go index 3a9674a35..78648b70d 100644 --- a/itest/lnd_rest_api_test.go +++ b/itest/lnd_rest_api_test.go @@ -7,15 +7,12 @@ import ( "encoding/hex" "fmt" "io" - "io/ioutil" "net/http" "regexp" "strings" "testing" "time" - "github.com/golang/protobuf/jsonpb" - "github.com/golang/protobuf/proto" "github.com/gorilla/websocket" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lnrpc/autopilotrpc" @@ -26,6 +23,7 @@ import ( "github.com/lightningnetwork/lnd/lntest" "github.com/lightningnetwork/lnd/lntest/node" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" ) var ( @@ -35,11 +33,6 @@ var ( restClient = &http.Client{ Transport: insecureTransport, } - jsonMarshaler = &jsonpb.Marshaler{ - EmitDefaults: true, - OrigName: true, - Indent: " ", - } urlEnc = base64.URLEncoding webSocketDialer = &websocket.Dialer{ HandshakeTimeout: time.Second, @@ -288,7 +281,9 @@ func wsTestCaseSubscription(ht *lntest.HarnessTest) { // Make sure we can parse the unwrapped message into the // expected proto message. protoMsg := &chainrpc.BlockEpoch{} - err = jsonpb.UnmarshalString(msgStr, protoMsg) + err = lnrpc.RESTJsonUnmarshalOpts.Unmarshal( + []byte(msgStr), protoMsg, + ) if err != nil { errChan <- err return @@ -378,7 +373,9 @@ func wsTestCaseSubscriptionMacaroon(ht *lntest.HarnessTest) { // Make sure we can parse the unwrapped message into the // expected proto message. protoMsg := &chainrpc.BlockEpoch{} - err = jsonpb.UnmarshalString(msgStr, protoMsg) + err = lnrpc.RESTJsonUnmarshalOpts.Unmarshal( + []byte(msgStr), protoMsg, + ) if err != nil { errChan <- err return @@ -474,7 +471,9 @@ func wsTestCaseBiDirectionalSubscription(ht *lntest.HarnessTest) { // Make sure we can parse the unwrapped message into the // expected proto message. protoMsg := &lnrpc.ChannelAcceptRequest{} - err = jsonpb.UnmarshalString(msgStr, protoMsg) + err = lnrpc.RESTJsonUnmarshalOpts.Unmarshal( + []byte(msgStr), protoMsg, + ) if err != nil { select { case errChan <- err: @@ -488,7 +487,8 @@ func wsTestCaseBiDirectionalSubscription(ht *lntest.HarnessTest) { Accept: true, PendingChanId: protoMsg.PendingChanId, } - resMsg, err := jsonMarshaler.MarshalToString(res) + + resMsg, err := lnrpc.RESTJsonMarshalOpts.Marshal(res) if err != nil { select { case errChan <- err: @@ -496,9 +496,7 @@ func wsTestCaseBiDirectionalSubscription(ht *lntest.HarnessTest) { } return } - err = conn.WriteMessage( - websocket.TextMessage, []byte(resMsg), - ) + err = conn.WriteMessage(websocket.TextMessage, resMsg) if err != nil { select { case errChan <- err: @@ -613,7 +611,9 @@ func wsTestPingPongTimeout(ht *lntest.HarnessTest) { // Make sure we can parse the unwrapped message into the // expected proto message. protoMsg := &lnrpc.Invoice{} - err = jsonpb.UnmarshalString(msgStr, protoMsg) + err = lnrpc.RESTJsonUnmarshalOpts.Unmarshal( + []byte(msgStr), protoMsg, + ) if err != nil { select { case errChan <- err: @@ -684,7 +684,7 @@ func invokeGET(node *node.HarnessNode, url string, resp proto.Message) error { return err } - return jsonpb.Unmarshal(bytes.NewReader(rawResp), resp) + return lnrpc.RESTJsonUnmarshalOpts.Unmarshal(rawResp, resp) } // invokePOST calls the given URL with the POST method, request body and @@ -693,19 +693,21 @@ func invokeGET(node *node.HarnessNode, url string, resp proto.Message) error { func invokePOST(node *node.HarnessNode, url string, req, resp proto.Message) error { - // Marshal the request to JSON using the jsonpb marshaler to get correct + // Marshal the request to JSON using the REST marshaler to get correct // field names. - var buf bytes.Buffer - if err := jsonMarshaler.Marshal(&buf, req); err != nil { - return err - } - - _, rawResp, err := makeRequest(node, url, "POST", &buf, nil) + reqBytes, err := lnrpc.RESTJsonMarshalOpts.Marshal(req) if err != nil { return err } - return jsonpb.Unmarshal(bytes.NewReader(rawResp), resp) + _, rawResp, err := makeRequest( + node, url, "POST", bytes.NewReader(reqBytes), nil, + ) + if err != nil { + return err + } + + return lnrpc.RESTJsonUnmarshalOpts.Unmarshal(rawResp, resp) } // makeRequest calls the given URL with the given method, request body and @@ -737,7 +739,7 @@ func makeRequest(node *node.HarnessNode, url, method string, } defer func() { _ = resp.Body.Close() }() - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) return resp.Header, data, err } @@ -766,11 +768,11 @@ func openWebSocket(node *node.HarnessNode, url, method string, defer func() { _ = resp.Body.Close() }() // Send the given request message as the first message on the socket. - reqMsg, err := jsonMarshaler.MarshalToString(req) + reqMsg, err := lnrpc.RESTJsonMarshalOpts.Marshal(req) if err != nil { return nil, err } - err = conn.WriteMessage(websocket.TextMessage, []byte(reqMsg)) + err = conn.WriteMessage(websocket.TextMessage, reqMsg) if err != nil { return nil, err } diff --git a/lnd.go b/lnd.go index f24ea380e..7d52d63da 100644 --- a/lnd.go +++ b/lnd.go @@ -37,7 +37,6 @@ import ( "github.com/lightningnetwork/lnd/watchtower" "google.golang.org/grpc" "google.golang.org/grpc/credentials" - "google.golang.org/protobuf/encoding/protojson" "gopkg.in/macaroon-bakery.v2/bakery" "gopkg.in/macaroon.v2" ) @@ -857,10 +856,8 @@ func startRestProxy(cfg *Config, rpcServer *rpcServer, restDialOpts []grpc.DialO // that the marshaler prints all values, even if they are falsey. customMarshalerOption := proxy.WithMarshalerOption( proxy.MIMEWildcard, &proxy.JSONPb{ - MarshalOptions: protojson.MarshalOptions{ - UseProtoNames: true, - EmitUnpopulated: true, - }, + MarshalOptions: *lnrpc.RESTJsonMarshalOpts, + UnmarshalOptions: *lnrpc.RESTJsonUnmarshalOpts, }, ) mux := proxy.NewServeMux( diff --git a/lnrpc/rpc_utils.go b/lnrpc/rpc_utils.go index fbd14a0dc..53cc1b147 100644 --- a/lnrpc/rpc_utils.go +++ b/lnrpc/rpc_utils.go @@ -7,6 +7,7 @@ import ( "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/lightningnetwork/lnd/lnwallet" + "google.golang.org/protobuf/encoding/protojson" ) const ( @@ -16,6 +17,47 @@ const ( RegisterRPCMiddlewareURI = "/lnrpc.Lightning/RegisterRPCMiddleware" ) +var ( + // ProtoJSONMarshalOpts is a struct that holds the default marshal + // options for marshaling protobuf messages into JSON in a + // human-readable way. This should only be used in the CLI and in + // integration tests. + // + // TODO(guggero): Use custom fork of this library to support hex + // encoding byte slices instead of the default base64 encoding. + ProtoJSONMarshalOpts = &protojson.MarshalOptions{ + EmitUnpopulated: true, + UseProtoNames: true, + Indent: " ", + } + + // ProtoJSONUnmarshalOpts is a struct that holds the default unmarshal + // options for un-marshaling lncli JSON into protobuf messages. This + // should only be used in the CLI and in integration tests. + // + // TODO(guggero): Use custom fork of this library to support hex + // encoding byte slices instead of the default base64 encoding. + ProtoJSONUnmarshalOpts = &protojson.UnmarshalOptions{ + AllowPartial: false, + } + + // RESTJsonMarshalOpts is a struct that holds the default marshal + // options for marshaling protobuf messages into REST JSON in a + // human-readable way. This should be used when interacting with the + // REST proxy only. + RESTJsonMarshalOpts = &protojson.MarshalOptions{ + EmitUnpopulated: true, + UseProtoNames: true, + } + + // RESTJsonUnmarshalOpts is a struct that holds the default unmarshal + // options for un-marshaling REST JSON into protobuf messages. This + // should be used when interacting with the REST proxy only. + RESTJsonUnmarshalOpts = &protojson.UnmarshalOptions{ + AllowPartial: false, + } +) + // RPCTransaction returns a rpc transaction. func RPCTransaction(tx *lnwallet.TransactionDetail) *Transaction { var destAddresses []string