tlv: add MakeBigSizeRecord to save space

This commit adds a pair of encoder/decoder to take the advantage of the
BigSize format when encoding an uint64 and possibly an uint32.

Often the time an uint64 value is not big enough to fill all the 8
bytes, thus using BigSize can save extra bytes when save it to db. And
for uint32, if we know most of the values do not exceed 65536, we can
also save at least 1 byte using BigSize format.

This commit introduces `MakeBigSizeRecord` that can be used optionally
where db space is a concern.

docs/release-notes/release-notes-0.15.0.md

@@ -175,6 +175,9 @@ then watch it on chain. Taproot script spends are also supported through the
 * [Announce the keysend feature bit in NodeAnnouncement if `--accept-keysend` 
   is set](https://github.com/lightningnetwork/lnd/pull/6414)
 
+* [Add a new method in `tlv` to encode an uint64/uint32 field using `BigSize`
+  format.](https://github.com/lightningnetwork/lnd/pull/6421)
+
 ## RPC Server
 
 * [Add value to the field

tlv/primitive.go

@@ -307,3 +307,58 @@ func DVarBytes(r io.Reader, val interface{}, _ *[8]byte, l uint64) error {
 	}
 	return NewTypeForDecodingErr(val, "[]byte", l, l)
 }
+
+// EBigSize encodes an uint32 or an uint64 using BigSize format. An error is
+// returned if val is not either *uint32 or *uint64.
+func EBigSize(w io.Writer, val interface{}, buf *[8]byte) error {
+	if i, ok := val.(*uint32); ok {
+		return WriteVarInt(w, uint64(*i), buf)
+	}
+
+	if i, ok := val.(*uint64); ok {
+		return WriteVarInt(w, uint64(*i), buf)
+	}
+
+	return NewTypeForEncodingErr(val, "BigSize")
+}
+
+// DBigSize decodes an uint32 or an uint64 using BigSize format. An error is
+// returned if val is not either *uint32 or *uint64.
+func DBigSize(r io.Reader, val interface{}, buf *[8]byte, l uint64) error {
+	if i, ok := val.(*uint32); ok {
+		v, err := ReadVarInt(r, buf)
+		if err != nil {
+			return err
+		}
+		*i = uint32(v)
+		return nil
+	}
+
+	if i, ok := val.(*uint64); ok {
+		v, err := ReadVarInt(r, buf)
+		if err != nil {
+			return err
+		}
+		*i = v
+		return nil
+	}
+
+	return NewTypeForDecodingErr(val, "BigSize", l, 8)
+}
+
+// SizeBigSize returns a SizeFunc that can compute the length of BigSize.
+func SizeBigSize(val interface{}) SizeFunc {
+	var size uint64
+
+	if i, ok := val.(*uint32); ok {
+		size = VarIntSize(uint64(*i))
+	}
+
+	if i, ok := val.(*uint64); ok {
+		size = VarIntSize(uint64(*i))
+	}
+
+	return func() uint64 {
+		return size
+	}
+}

tlv/record.go

@@ -249,3 +249,40 @@ func SortRecords(records []Record) {
 		return records[i].Type() < records[j].Type()
 	})
 }
+
+// MakeBigSizeRecord creates a tlv record using the BigSize format. The only
+// allowed values are uint64 and uint32.
+//
+// NOTE: for uint32, we would only gain space reduction if the encoded value is
+// no greater than 65535, which requires at most 3 bytes to encode.
+func MakeBigSizeRecord(typ Type, val interface{}) Record {
+	var (
+		staticSize uint64
+		sizeFunc   SizeFunc
+		encoder    Encoder
+		decoder    Decoder
+	)
+	switch val.(type) {
+	case *uint32:
+		sizeFunc = SizeBigSize(val)
+		encoder = EBigSize
+		decoder = DBigSize
+
+	case *uint64:
+		sizeFunc = SizeBigSize(val)
+		encoder = EBigSize
+		decoder = DBigSize
+
+	default:
+		panic(fmt.Sprintf("unknown supported compact type: %T", val))
+	}
+
+	return Record{
+		value:      val,
+		typ:        typ,
+		staticSize: staticSize,
+		sizeFunc:   sizeFunc,
+		encoder:    encoder,
+		decoder:    decoder,
+	}
+}

tlv/stream_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/lightningnetwork/lnd/tlv"
+	"github.com/stretchr/testify/require"
 )
 
 type parsedTypeTest struct {
@@ -88,3 +89,164 @@ func testParsedTypes(t *testing.T, test parsedTypeTest) {
 		t.Fatalf("error mismatch on parsed types")
 	}
 }
+
+var (
+	smallValue      = 1
+	smallValueBytes = []byte{
+		// uint32 tlv, value uses 1 byte.
+		0xa, 0x1, 0x1,
+		// uint64 tlv, value uses 1 byte.
+		0xb, 0x1, 0x1,
+	}
+
+	medianValue      = 255
+	medianValueBytes = []byte{
+		// uint32 tlv, value uses 3 byte.
+		0xa, 0x3, 0xfd, 0x0, 0xff,
+		// uint64 tlv, value uses 3 byte.
+		0xb, 0x3, 0xfd, 0x0, 0xff,
+	}
+
+	largeValue      = 65536
+	largeValueBytes = []byte{
+		// uint32 tlv, value uses 5 byte.
+		0xa, 0x5, 0xfe, 0x0, 0x1, 0x0, 0x0,
+		// uint64 tlv, value uses 5 byte.
+		0xb, 0x5, 0xfe, 0x0, 0x1, 0x0, 0x0,
+	}
+)
+
+// TestEncodeBigSizeFormatTlvStream tests that the bigsize encoder works as
+// expected.
+func TestEncodeBigSizeFormatTlvStream(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name          string
+		value         int
+		expectedBytes []byte
+	}{
+		{
+			// Test encode 1, which saves us space.
+			name:          "encode small value",
+			value:         smallValue,
+			expectedBytes: smallValueBytes,
+		},
+		{
+			// Test encode 255, which still saves us space.
+			name:          "encode median value",
+			value:         medianValue,
+			expectedBytes: medianValueBytes,
+		},
+		{
+			// Test encode 65536, which takes more space to encode
+			// an uint32.
+			name:          "encode large value",
+			value:         largeValue,
+			expectedBytes: largeValueBytes,
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			testUint32 := uint32(tc.value)
+			testUint64 := uint64(tc.value)
+			ts := makeBigSizeFormatTlvStream(
+				t, &testUint32, &testUint64,
+			)
+
+			// Encode the tlv stream.
+			buf := bytes.NewBuffer([]byte{})
+			require.NoError(t, ts.Encode(buf))
+
+			// Check the bytes are written as expected.
+			require.Equal(t, tc.expectedBytes, buf.Bytes())
+		})
+	}
+}
+
+// TestDecodeBigSizeFormatTlvStream tests that the bigsize decoder works as
+// expected.
+func TestDecodeBigSizeFormatTlvStream(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name          string
+		bytes         []byte
+		expectedValue int
+	}{
+		{
+			// Test decode 1.
+			name: "decode small value",
+			bytes: []byte{
+				// uint32 tlv, value uses 1 byte.
+				0xa, 0x1, 0x1,
+				// uint64 tlv, value uses 1 byte.
+				0xb, 0x1, 0x1,
+			},
+			expectedValue: smallValue,
+		},
+		{
+			// Test decode 255.
+			name: "decode median value",
+			bytes: []byte{
+				// uint32 tlv, value uses 3 byte.
+				0xa, 0x3, 0xfd, 0x0, 0xff,
+				// uint64 tlv, value uses 3 byte.
+				0xb, 0x3, 0xfd, 0x0, 0xff,
+			},
+			expectedValue: medianValue,
+		},
+		{
+			// Test decode 65536.
+			name: "decode value 65536",
+			bytes: []byte{
+				// uint32 tlv, value uses 5 byte.
+				0xa, 0x5, 0xfe, 0x0, 0x1, 0x0, 0x0,
+				// uint64 tlv, value uses 5 byte.
+				0xb, 0x5, 0xfe, 0x0, 0x1, 0x0, 0x0,
+			},
+			expectedValue: largeValue,
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			var (
+				testUint32 uint32
+				testUint64 uint64
+			)
+			ts := makeBigSizeFormatTlvStream(
+				t, &testUint32, &testUint64,
+			)
+
+			// Decode the tlv stream.
+			buf := bytes.NewBuffer(tc.bytes)
+			require.NoError(t, ts.Decode(buf))
+
+			// Check the values are written as expected.
+			require.EqualValues(t, tc.expectedValue, testUint32)
+			require.EqualValues(t, tc.expectedValue, testUint64)
+		})
+	}
+}
+
+func makeBigSizeFormatTlvStream(t *testing.T, vUint32 *uint32,
+	vUint64 *uint64) *tlv.Stream {
+
+	const (
+		typeUint32 tlv.Type = 10
+		typeUint64 tlv.Type = 11
+	)
+
+	// Create a dummy tlv stream for testing.
+	ts, err := tlv.NewStream(
+		tlv.MakeBigSizeRecord(typeUint32, vUint32),
+		tlv.MakeBigSizeRecord(typeUint64, vUint64),
+	)
+	require.NoError(t, err)
+
+	return ts
+}