mod+cmd/lncli: use google.golang.org/protobuf package

cmd/lncli/cmd_macaroon.go

@@ -10,11 +10,11 @@ import (
 	"strings"
 	"unicode"
 
-	"github.com/golang/protobuf/proto"
 	"github.com/lightningnetwork/lnd/lncfg"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/macaroons"
 	"github.com/urfave/cli"
+	"google.golang.org/protobuf/proto"
 	"gopkg.in/macaroon-bakery.v2/bakery"
 	"gopkg.in/macaroon.v2"
 )
@@ -385,7 +385,7 @@ func printMacaroon(ctx *cli.Context) error {
 		return fmt.Errorf("unable to decode macaroon version: %v", err)
 	}
 
-	// Prepare everything to be printed in a more human readable format.
+	// Prepare everything to be printed in a more human-readable format.
 	content := &macaroonContent{
 		Version:     uint16(mac.Version()),
 		Location:    mac.Location(),

cmd/lncli/cmd_payments.go

@@ -17,7 +17,6 @@ import (
 	"github.com/btcsuite/btcd/btcutil"
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/jedib0t/go-pretty/v6/text"
-	"github.com/lightninglabs/protobuf-hex-display/jsonpb"
 	"github.com/lightningnetwork/lnd/chainreg"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/lnrpc/routerrpc"
@@ -952,7 +951,7 @@ func sendToRoute(ctx *cli.Context) error {
 	// format.
 	var route *lnrpc.Route
 	routes := &lnrpc.QueryRoutesResponse{}
-	err = jsonpb.UnmarshalString(jsonRoutes, routes)
+	err = lnrpc.ProtoJSONUnmarshalOpts.Unmarshal([]byte(jsonRoutes), routes)
 	if err == nil {
 		if len(routes.Routes) == 0 {
 			return fmt.Errorf("no routes provided")
@@ -966,7 +965,9 @@ func sendToRoute(ctx *cli.Context) error {
 		route = routes.Routes[0]
 	} else {
 		routes := &routerrpc.BuildRouteResponse{}
-		err = jsonpb.UnmarshalString(jsonRoutes, routes)
+		err = lnrpc.ProtoJSONUnmarshalOpts.Unmarshal(
+			[]byte(jsonRoutes), routes,
+		)
 		if err != nil {
 			return fmt.Errorf("unable to unmarshal json string "+
 				"from incoming array of routes: %v", err)

cmd/lncli/cmd_walletunlocker.go

@@ -10,7 +10,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/lightninglabs/protobuf-hex-display/jsonpb"
 	"github.com/lightningnetwork/lnd/lncfg"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/lnrpc/walletrpc"
@@ -720,7 +719,7 @@ func createWatchOnly(ctx *cli.Context) error {
 	}
 
 	jsonAccts := &walletrpc.ListAccountsResponse{}
-	err = jsonpb.Unmarshal(bytes.NewReader(jsonBytes), jsonAccts)
+	err = lnrpc.ProtoJSONUnmarshalOpts.Unmarshal(jsonBytes, jsonAccts)
 	if err != nil {
 		return fmt.Errorf("error parsing JSON: %v", err)
 	}

cmd/lncli/commands.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -17,9 +18,6 @@ import (
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcd/wire"
-	"github.com/lightninglabs/protobuf-hex-display/json"
-	"github.com/lightninglabs/protobuf-hex-display/jsonpb"
-	"github.com/lightninglabs/protobuf-hex-display/proto"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/routing"
 	"github.com/lightningnetwork/lnd/routing/route"
@@ -27,6 +25,7 @@ import (
 	"github.com/urfave/cli"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
 )
 
 // TODO(roasbeef): cli logic for supporting both positional and unix style
@@ -71,19 +70,13 @@ func printJSON(resp interface{}) {
 }
 
 func printRespJSON(resp proto.Message) {
-	jsonMarshaler := &jsonpb.Marshaler{
+	jsonBytes, err := lnrpc.ProtoJSONMarshalOpts.Marshal(resp)
-		EmitDefaults: true,
-		OrigName:     true,
-		Indent:       "    ",
-	}
-
-	jsonStr, err := jsonMarshaler.MarshalToString(resp)
 	if err != nil {
 		fmt.Println("unable to decode response: ", err)
 		return
 	}
 
-	fmt.Println(jsonStr)
+	fmt.Printf("%s\n", jsonBytes)
 }
 
 // actionDecorator is used to add additional information and error handling

cmd/lncli/devrpc_active.go

@@ -4,11 +4,9 @@
 package main
 
 import (
-	"bytes"
 	"fmt"
-	"io/ioutil"
+	"os"
 
-	"github.com/lightninglabs/protobuf-hex-display/jsonpb"
 	"github.com/lightningnetwork/lnd/lncfg"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/lnrpc/devrpc"
@@ -43,14 +41,14 @@ func importGraph(ctx *cli.Context) error {
 	defer cleanUp()
 
 	jsonFile := lncfg.CleanAndExpandPath(ctx.Args().First())
-	jsonBytes, err := ioutil.ReadFile(jsonFile)
+	jsonBytes, err := os.ReadFile(jsonFile)
 	if err != nil {
 		return fmt.Errorf("error reading JSON from file %v: %v",
 			jsonFile, err)
 	}
 
 	jsonGraph := &lnrpc.ChannelGraph{}
-	err = jsonpb.Unmarshal(bytes.NewReader(jsonBytes), jsonGraph)
+	err = lnrpc.ProtoJSONUnmarshalOpts.Unmarshal(jsonBytes, jsonGraph)
 	if err != nil {
 		return fmt.Errorf("error parsing JSON: %v", err)
 	}

go.mod

@@ -18,7 +18,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1
 	github.com/go-errors/errors v1.0.1
-	github.com/golang/protobuf v1.5.2
 	github.com/gorilla/websocket v1.4.2
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
@@ -32,7 +31,6 @@ require (
 	github.com/kkdai/bstream v1.0.0
 	github.com/lightninglabs/neutrino v0.15.0
 	github.com/lightninglabs/neutrino/cache v1.1.1
-	github.com/lightninglabs/protobuf-hex-display v1.4.3-hex-display
 	github.com/lightningnetwork/lightning-onion v1.2.1-0.20221202012345-ca23184850a1
 	github.com/lightningnetwork/lnd/cert v1.2.1
 	github.com/lightningnetwork/lnd/clock v1.1.0
@@ -85,6 +83,7 @@ require (
 	github.com/fergusstrange/embedded-postgres v1.10.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/uuid v1.3.0 // indirect

go.sum

@@ -395,8 +395,6 @@ github.com/lightninglabs/neutrino v0.15.0 h1:yr3uz36fLAq8hyM0TRUVlef1TRNoWAqpmmN
 github.com/lightninglabs/neutrino v0.15.0/go.mod h1:pmjwElN/091TErtSE9Vd5W4hpxoG2/+xlb+HoPm9Gug=
 github.com/lightninglabs/neutrino/cache v1.1.1 h1:TllWOSlkABhpgbWJfzsrdUaDH2fBy/54VSIB4vVqV8M=
 github.com/lightninglabs/neutrino/cache v1.1.1/go.mod h1:XJNcgdOw1LQnanGjw8Vj44CvguYA25IMKjWFZczwZuo=
-github.com/lightninglabs/protobuf-hex-display v1.4.3-hex-display h1:RZJ8H4ueU/aQ9pFtx5wqsuD3B/DezrewJeVwDKKYY8E=
-github.com/lightninglabs/protobuf-hex-display v1.4.3-hex-display/go.mod h1:2oKOBU042GKFHrdbgGiKax4xVrFiZu51lhacUZQ9MnE=
 github.com/lightningnetwork/lightning-onion v1.2.1-0.20221202012345-ca23184850a1 h1:Wm0g70gkcAu2pGpNZwfWPSVOY21j8IyYsNewwK4OkT4=
 github.com/lightningnetwork/lightning-onion v1.2.1-0.20221202012345-ca23184850a1/go.mod h1:7dDx73ApjEZA0kcknI799m2O5kkpfg4/gr7N092ojNo=
 github.com/lightningnetwork/lnd/cert v1.2.1 h1:CTrTcU0L66J73oqdRLVfNylZyp1Fh97ZezX6IuzkrqE=

itest/lnd_macaroons_test.go

@@ -9,13 +9,13 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/golang/protobuf/proto"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/lntest"
 	"github.com/lightningnetwork/lnd/lntest/node"
 	"github.com/lightningnetwork/lnd/macaroons"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
 	"gopkg.in/macaroon.v2"
 )
 

itest/lnd_rest_api_test.go

@@ -7,15 +7,12 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"regexp"
 	"strings"
 	"testing"
 	"time"
 
-	"github.com/golang/protobuf/jsonpb"
-	"github.com/golang/protobuf/proto"
 	"github.com/gorilla/websocket"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/lnrpc/autopilotrpc"
@@ -26,6 +23,7 @@ import (
 	"github.com/lightningnetwork/lnd/lntest"
 	"github.com/lightningnetwork/lnd/lntest/node"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
 )
 
 var (
@@ -35,11 +33,6 @@ var (
 	restClient = &http.Client{
 		Transport: insecureTransport,
 	}
-	jsonMarshaler = &jsonpb.Marshaler{
-		EmitDefaults: true,
-		OrigName:     true,
-		Indent:       "    ",
-	}
 	urlEnc          = base64.URLEncoding
 	webSocketDialer = &websocket.Dialer{
 		HandshakeTimeout: time.Second,
@@ -288,7 +281,9 @@ func wsTestCaseSubscription(ht *lntest.HarnessTest) {
 		// Make sure we can parse the unwrapped message into the
 		// expected proto message.
 		protoMsg := &chainrpc.BlockEpoch{}
-		err = jsonpb.UnmarshalString(msgStr, protoMsg)
+		err = lnrpc.RESTJsonUnmarshalOpts.Unmarshal(
+			[]byte(msgStr), protoMsg,
+		)
 		if err != nil {
 			errChan <- err
 			return
@@ -378,7 +373,9 @@ func wsTestCaseSubscriptionMacaroon(ht *lntest.HarnessTest) {
 		// Make sure we can parse the unwrapped message into the
 		// expected proto message.
 		protoMsg := &chainrpc.BlockEpoch{}
-		err = jsonpb.UnmarshalString(msgStr, protoMsg)
+		err = lnrpc.RESTJsonUnmarshalOpts.Unmarshal(
+			[]byte(msgStr), protoMsg,
+		)
 		if err != nil {
 			errChan <- err
 			return
@@ -474,7 +471,9 @@ func wsTestCaseBiDirectionalSubscription(ht *lntest.HarnessTest) {
 			// Make sure we can parse the unwrapped message into the
 			// expected proto message.
 			protoMsg := &lnrpc.ChannelAcceptRequest{}
-			err = jsonpb.UnmarshalString(msgStr, protoMsg)
+			err = lnrpc.RESTJsonUnmarshalOpts.Unmarshal(
+				[]byte(msgStr), protoMsg,
+			)
 			if err != nil {
 				select {
 				case errChan <- err:
@@ -488,7 +487,8 @@ func wsTestCaseBiDirectionalSubscription(ht *lntest.HarnessTest) {
 				Accept:        true,
 				PendingChanId: protoMsg.PendingChanId,
 			}
-			resMsg, err := jsonMarshaler.MarshalToString(res)
+
+			resMsg, err := lnrpc.RESTJsonMarshalOpts.Marshal(res)
 			if err != nil {
 				select {
 				case errChan <- err:
@@ -496,9 +496,7 @@ func wsTestCaseBiDirectionalSubscription(ht *lntest.HarnessTest) {
 				}
 				return
 			}
-			err = conn.WriteMessage(
+			err = conn.WriteMessage(websocket.TextMessage, resMsg)
-				websocket.TextMessage, []byte(resMsg),
-			)
 			if err != nil {
 				select {
 				case errChan <- err:
@@ -613,7 +611,9 @@ func wsTestPingPongTimeout(ht *lntest.HarnessTest) {
 			// Make sure we can parse the unwrapped message into the
 			// expected proto message.
 			protoMsg := &lnrpc.Invoice{}
-			err = jsonpb.UnmarshalString(msgStr, protoMsg)
+			err = lnrpc.RESTJsonUnmarshalOpts.Unmarshal(
+				[]byte(msgStr), protoMsg,
+			)
 			if err != nil {
 				select {
 				case errChan <- err:
@@ -684,7 +684,7 @@ func invokeGET(node *node.HarnessNode, url string, resp proto.Message) error {
 		return err
 	}
 
-	return jsonpb.Unmarshal(bytes.NewReader(rawResp), resp)
+	return lnrpc.RESTJsonUnmarshalOpts.Unmarshal(rawResp, resp)
 }
 
 // invokePOST calls the given URL with the POST method, request body and
@@ -693,19 +693,21 @@ func invokeGET(node *node.HarnessNode, url string, resp proto.Message) error {
 func invokePOST(node *node.HarnessNode, url string, req,
 	resp proto.Message) error {
 
-	// Marshal the request to JSON using the jsonpb marshaler to get correct
+	// Marshal the request to JSON using the REST marshaler to get correct
 	// field names.
-	var buf bytes.Buffer
+	reqBytes, err := lnrpc.RESTJsonMarshalOpts.Marshal(req)
-	if err := jsonMarshaler.Marshal(&buf, req); err != nil {
-		return err
-	}
-
-	_, rawResp, err := makeRequest(node, url, "POST", &buf, nil)
 	if err != nil {
 		return err
 	}
 
-	return jsonpb.Unmarshal(bytes.NewReader(rawResp), resp)
+	_, rawResp, err := makeRequest(
+		node, url, "POST", bytes.NewReader(reqBytes), nil,
+	)
+	if err != nil {
+		return err
+	}
+
+	return lnrpc.RESTJsonUnmarshalOpts.Unmarshal(rawResp, resp)
 }
 
 // makeRequest calls the given URL with the given method, request body and
@@ -737,7 +739,7 @@ func makeRequest(node *node.HarnessNode, url, method string,
 	}
 	defer func() { _ = resp.Body.Close() }()
 
-	data, err := ioutil.ReadAll(resp.Body)
+	data, err := io.ReadAll(resp.Body)
 	return resp.Header, data, err
 }
 
@@ -766,11 +768,11 @@ func openWebSocket(node *node.HarnessNode, url, method string,
 	defer func() { _ = resp.Body.Close() }()
 
 	// Send the given request message as the first message on the socket.
-	reqMsg, err := jsonMarshaler.MarshalToString(req)
+	reqMsg, err := lnrpc.RESTJsonMarshalOpts.Marshal(req)
 	if err != nil {
 		return nil, err
 	}
-	err = conn.WriteMessage(websocket.TextMessage, []byte(reqMsg))
+	err = conn.WriteMessage(websocket.TextMessage, reqMsg)
 	if err != nil {
 		return nil, err
 	}

lnd.go

@@ -37,7 +37,6 @@ import (
 	"github.com/lightningnetwork/lnd/watchtower"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
-	"google.golang.org/protobuf/encoding/protojson"
 	"gopkg.in/macaroon-bakery.v2/bakery"
 	"gopkg.in/macaroon.v2"
 )
@@ -857,10 +856,8 @@ func startRestProxy(cfg *Config, rpcServer *rpcServer, restDialOpts []grpc.DialO
 	// that the marshaler prints all values, even if they are falsey.
 	customMarshalerOption := proxy.WithMarshalerOption(
 		proxy.MIMEWildcard, &proxy.JSONPb{
-			MarshalOptions: protojson.MarshalOptions{
+			MarshalOptions:   *lnrpc.RESTJsonMarshalOpts,
-				UseProtoNames:   true,
+			UnmarshalOptions: *lnrpc.RESTJsonUnmarshalOpts,
-				EmitUnpopulated: true,
-			},
 		},
 	)
 	mux := proxy.NewServeMux(

lnrpc/rpc_utils.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/lightningnetwork/lnd/lnwallet"
+	"google.golang.org/protobuf/encoding/protojson"
 )
 
 const (
@@ -16,6 +17,47 @@ const (
 	RegisterRPCMiddlewareURI = "/lnrpc.Lightning/RegisterRPCMiddleware"
 )
 
+var (
+	// ProtoJSONMarshalOpts is a struct that holds the default marshal
+	// options for marshaling protobuf messages into JSON in a
+	// human-readable way. This should only be used in the CLI and in
+	// integration tests.
+	//
+	// TODO(guggero): Use custom fork of this library to support hex
+	// encoding byte slices instead of the default base64 encoding.
+	ProtoJSONMarshalOpts = &protojson.MarshalOptions{
+		EmitUnpopulated: true,
+		UseProtoNames:   true,
+		Indent:          "    ",
+	}
+
+	// ProtoJSONUnmarshalOpts is a struct that holds the default unmarshal
+	// options for un-marshaling lncli JSON into protobuf messages. This
+	// should only be used in the CLI and in integration tests.
+	//
+	// TODO(guggero): Use custom fork of this library to support hex
+	// encoding byte slices instead of the default base64 encoding.
+	ProtoJSONUnmarshalOpts = &protojson.UnmarshalOptions{
+		AllowPartial: false,
+	}
+
+	// RESTJsonMarshalOpts is a struct that holds the default marshal
+	// options for marshaling protobuf messages into REST JSON in a
+	// human-readable way. This should be used when interacting with the
+	// REST proxy only.
+	RESTJsonMarshalOpts = &protojson.MarshalOptions{
+		EmitUnpopulated: true,
+		UseProtoNames:   true,
+	}
+
+	// RESTJsonUnmarshalOpts is a struct that holds the default unmarshal
+	// options for un-marshaling REST JSON into protobuf messages. This
+	// should be used when interacting with the REST proxy only.
+	RESTJsonUnmarshalOpts = &protojson.UnmarshalOptions{
+		AllowPartial: false,
+	}
+)
+
 // RPCTransaction returns a rpc transaction.
 func RPCTransaction(tx *lnwallet.TransactionDetail) *Transaction {
 	var destAddresses []string