rlp/rlpgen: RLP encoder code generator (#24251)

This change adds a code generator tool for creating EncodeRLP method
implementations. The generated methods will behave identically to the
reflect-based encoder, but run faster because there is no reflection overhead.

Package rlp now provides the EncoderBuffer type for incremental encoding. This
is used by generated code, but the new methods can also be useful for
hand-written encoders.

There is also experimental support for generating DecodeRLP, and some new
methods have been added to the existing Stream type to support this. Creating
decoders with rlpgen is not recommended at this time because the generated
methods create very poor error reporting.

More detail about package rlp changes:

* rlp: externalize struct field processing / validation

This adds a new package, rlp/internal/rlpstruct, in preparation for the
RLP encoder generator.

I think the struct field rules are subtle enough to warrant extracting
this into their own package, even though it means that a bunch of
adapter code is needed for converting to/from rlpstruct.Type.

* rlp: add more decoder methods (for rlpgen)

This adds new methods on rlp.Stream:

- Uint64, Uint32, Uint16, Uint8, BigInt
- ReadBytes for decoding into []byte
- MoreDataInList - useful for optional list elements

* rlp: expose encoder buffer (for rlpgen)

This exposes the internal encoder buffer type for use in EncodeRLP
implementations.

The new EncoderBuffer type is a sort-of 'opaque handle' for a pointer to
encBuffer. It is implemented this way to ensure the global encBuffer pool
is handled correctly.
pull/24414/head
Felix Lange 3 years ago committed by GitHub
parent 4335bbbf0a
commit 9b93564e21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      go.mod
  2. 2
      go.sum
  3. 180
      rlp/decode.go
  4. 43
      rlp/decode_test.go
  5. 352
      rlp/encbuffer.go
  6. 45
      rlp/encbuffer_example_test.go
  7. 362
      rlp/encode.go
  8. 3
      rlp/encode_test.go
  9. 10
      rlp/encoder_example_test.go
  10. 213
      rlp/internal/rlpstruct/rlpstruct.go
  11. 735
      rlp/rlpgen/gen.go
  12. 92
      rlp/rlpgen/gen_test.go
  13. 148
      rlp/rlpgen/main.go
  14. 10
      rlp/rlpgen/testdata/bigint.in.txt
  15. 49
      rlp/rlpgen/testdata/bigint.out.txt
  16. 30
      rlp/rlpgen/testdata/nil.in.txt
  17. 289
      rlp/rlpgen/testdata/nil.out.txt
  18. 17
      rlp/rlpgen/testdata/optional.in.txt
  19. 153
      rlp/rlpgen/testdata/optional.out.txt
  20. 11
      rlp/rlpgen/testdata/rawvalue.in.txt
  21. 64
      rlp/rlpgen/testdata/rawvalue.out.txt
  22. 10
      rlp/rlpgen/testdata/uints.in.txt
  23. 53
      rlp/rlpgen/testdata/uints.out.txt
  24. 98
      rlp/rlpgen/types.go
  25. 181
      rlp/typecache.go

@ -67,6 +67,7 @@ require (
golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912
golang.org/x/text v0.3.6 golang.org/x/text v0.3.6
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
golang.org/x/tools v0.1.0
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce
gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6
gopkg.in/urfave/cli.v1 v1.20.0 gopkg.in/urfave/cli.v1 v1.20.0

@ -460,6 +460,7 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -580,6 +581,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

@ -27,6 +27,8 @@ import (
"reflect" "reflect"
"strings" "strings"
"sync" "sync"
"github.com/ethereum/go-ethereum/rlp/internal/rlpstruct"
) )
//lint:ignore ST1012 EOL is not an error. //lint:ignore ST1012 EOL is not an error.
@ -148,7 +150,7 @@ var (
bigInt = reflect.TypeOf(big.Int{}) bigInt = reflect.TypeOf(big.Int{})
) )
func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) { func makeDecoder(typ reflect.Type, tags rlpstruct.Tags) (dec decoder, err error) {
kind := typ.Kind() kind := typ.Kind()
switch { switch {
case typ == rawValueType: case typ == rawValueType:
@ -220,55 +222,20 @@ func decodeBigIntNoPtr(s *Stream, val reflect.Value) error {
} }
func decodeBigInt(s *Stream, val reflect.Value) error { func decodeBigInt(s *Stream, val reflect.Value) error {
var buffer []byte
kind, size, err := s.Kind()
switch {
case err != nil:
return wrapStreamError(err, val.Type())
case kind == List:
return wrapStreamError(ErrExpectedString, val.Type())
case kind == Byte:
buffer = s.uintbuf[:1]
buffer[0] = s.byteval
s.kind = -1 // re-arm Kind
case size == 0:
// Avoid zero-length read.
s.kind = -1
case size <= uint64(len(s.uintbuf)):
// For integers smaller than s.uintbuf, allocating a buffer
// can be avoided.
buffer = s.uintbuf[:size]
if err := s.readFull(buffer); err != nil {
return wrapStreamError(err, val.Type())
}
// Reject inputs where single byte encoding should have been used.
if size == 1 && buffer[0] < 128 {
return wrapStreamError(ErrCanonSize, val.Type())
}
default:
// For large integers, a temporary buffer is needed.
buffer = make([]byte, size)
if err := s.readFull(buffer); err != nil {
return wrapStreamError(err, val.Type())
}
}
// Reject leading zero bytes.
if len(buffer) > 0 && buffer[0] == 0 {
return wrapStreamError(ErrCanonInt, val.Type())
}
// Set the integer bytes.
i := val.Interface().(*big.Int) i := val.Interface().(*big.Int)
if i == nil { if i == nil {
i = new(big.Int) i = new(big.Int)
val.Set(reflect.ValueOf(i)) val.Set(reflect.ValueOf(i))
} }
i.SetBytes(buffer)
err := s.decodeBigInt(i)
if err != nil {
return wrapStreamError(err, val.Type())
}
return nil return nil
} }
func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) { func makeListDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) {
etype := typ.Elem() etype := typ.Elem()
if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) { if etype.Kind() == reflect.Uint8 && !reflect.PtrTo(etype).Implements(decoderInterface) {
if typ.Kind() == reflect.Array { if typ.Kind() == reflect.Array {
@ -276,7 +243,7 @@ func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) {
} }
return decodeByteSlice, nil return decodeByteSlice, nil
} }
etypeinfo := theTC.infoWhileGenerating(etype, tags{}) etypeinfo := theTC.infoWhileGenerating(etype, rlpstruct.Tags{})
if etypeinfo.decoderErr != nil { if etypeinfo.decoderErr != nil {
return nil, etypeinfo.decoderErr return nil, etypeinfo.decoderErr
} }
@ -286,7 +253,7 @@ func makeListDecoder(typ reflect.Type, tag tags) (decoder, error) {
dec = func(s *Stream, val reflect.Value) error { dec = func(s *Stream, val reflect.Value) error {
return decodeListArray(s, val, etypeinfo.decoder) return decodeListArray(s, val, etypeinfo.decoder)
} }
case tag.tail: case tag.Tail:
// A slice with "tail" tag can occur as the last field // A slice with "tail" tag can occur as the last field
// of a struct and is supposed to swallow all remaining // of a struct and is supposed to swallow all remaining
// list elements. The struct decoder already called s.List, // list elements. The struct decoder already called s.List,
@ -451,16 +418,16 @@ func zeroFields(structval reflect.Value, fields []field) {
} }
// makePtrDecoder creates a decoder that decodes into the pointer's element type. // makePtrDecoder creates a decoder that decodes into the pointer's element type.
func makePtrDecoder(typ reflect.Type, tag tags) (decoder, error) { func makePtrDecoder(typ reflect.Type, tag rlpstruct.Tags) (decoder, error) {
etype := typ.Elem() etype := typ.Elem()
etypeinfo := theTC.infoWhileGenerating(etype, tags{}) etypeinfo := theTC.infoWhileGenerating(etype, rlpstruct.Tags{})
switch { switch {
case etypeinfo.decoderErr != nil: case etypeinfo.decoderErr != nil:
return nil, etypeinfo.decoderErr return nil, etypeinfo.decoderErr
case !tag.nilOK: case !tag.NilOK:
return makeSimplePtrDecoder(etype, etypeinfo), nil return makeSimplePtrDecoder(etype, etypeinfo), nil
default: default:
return makeNilPtrDecoder(etype, etypeinfo, tag.nilKind), nil return makeNilPtrDecoder(etype, etypeinfo, tag), nil
} }
} }
@ -481,9 +448,13 @@ func makeSimplePtrDecoder(etype reflect.Type, etypeinfo *typeinfo) decoder {
// values are decoded into a value of the element type, just like makePtrDecoder does. // values are decoded into a value of the element type, just like makePtrDecoder does.
// //
// This decoder is used for pointer-typed struct fields with struct tag "nil". // This decoder is used for pointer-typed struct fields with struct tag "nil".
func makeNilPtrDecoder(etype reflect.Type, etypeinfo *typeinfo, nilKind Kind) decoder { func makeNilPtrDecoder(etype reflect.Type, etypeinfo *typeinfo, ts rlpstruct.Tags) decoder {
typ := reflect.PtrTo(etype) typ := reflect.PtrTo(etype)
nilPtr := reflect.Zero(typ) nilPtr := reflect.Zero(typ)
// Determine the value kind that results in nil pointer.
nilKind := typeNilKind(etype, ts)
return func(s *Stream, val reflect.Value) (err error) { return func(s *Stream, val reflect.Value) (err error) {
kind, size, err := s.Kind() kind, size, err := s.Kind()
if err != nil { if err != nil {
@ -659,6 +630,37 @@ func (s *Stream) Bytes() ([]byte, error) {
} }
} }
// ReadBytes decodes the next RLP value and stores the result in b.
// The value size must match len(b) exactly.
func (s *Stream) ReadBytes(b []byte) error {
kind, size, err := s.Kind()
if err != nil {
return err
}
switch kind {
case Byte:
if len(b) != 1 {
return fmt.Errorf("input value has wrong size 1, want %d", len(b))
}
b[0] = s.byteval
s.kind = -1 // rearm Kind
return nil
case String:
if uint64(len(b)) != size {
return fmt.Errorf("input value has wrong size %d, want %d", size, len(b))
}
if err = s.readFull(b); err != nil {
return err
}
if size == 1 && b[0] < 128 {
return ErrCanonSize
}
return nil
default:
return ErrExpectedString
}
}
// Raw reads a raw encoded value including RLP type information. // Raw reads a raw encoded value including RLP type information.
func (s *Stream) Raw() ([]byte, error) { func (s *Stream) Raw() ([]byte, error) {
kind, size, err := s.Kind() kind, size, err := s.Kind()
@ -687,10 +689,31 @@ func (s *Stream) Raw() ([]byte, error) {
// Uint reads an RLP string of up to 8 bytes and returns its contents // Uint reads an RLP string of up to 8 bytes and returns its contents
// as an unsigned integer. If the input does not contain an RLP string, the // as an unsigned integer. If the input does not contain an RLP string, the
// returned error will be ErrExpectedString. // returned error will be ErrExpectedString.
//
// Deprecated: use s.Uint64 instead.
func (s *Stream) Uint() (uint64, error) { func (s *Stream) Uint() (uint64, error) {
return s.uint(64) return s.uint(64)
} }
func (s *Stream) Uint64() (uint64, error) {
return s.uint(64)
}
func (s *Stream) Uint32() (uint32, error) {
i, err := s.uint(32)
return uint32(i), err
}
func (s *Stream) Uint16() (uint16, error) {
i, err := s.uint(16)
return uint16(i), err
}
func (s *Stream) Uint8() (uint8, error) {
i, err := s.uint(8)
return uint8(i), err
}
func (s *Stream) uint(maxbits int) (uint64, error) { func (s *Stream) uint(maxbits int) (uint64, error) {
kind, size, err := s.Kind() kind, size, err := s.Kind()
if err != nil { if err != nil {
@ -781,6 +804,65 @@ func (s *Stream) ListEnd() error {
return nil return nil
} }
// MoreDataInList reports whether the current list context contains
// more data to be read.
func (s *Stream) MoreDataInList() bool {
_, listLimit := s.listLimit()
return listLimit > 0
}
// BigInt decodes an arbitrary-size integer value.
func (s *Stream) BigInt() (*big.Int, error) {
i := new(big.Int)
if err := s.decodeBigInt(i); err != nil {
return nil, err
}
return i, nil
}
func (s *Stream) decodeBigInt(dst *big.Int) error {
var buffer []byte
kind, size, err := s.Kind()
switch {
case err != nil:
return err
case kind == List:
return ErrExpectedString
case kind == Byte:
buffer = s.uintbuf[:1]
buffer[0] = s.byteval
s.kind = -1 // re-arm Kind
case size == 0:
// Avoid zero-length read.
s.kind = -1
case size <= uint64(len(s.uintbuf)):
// For integers smaller than s.uintbuf, allocating a buffer
// can be avoided.
buffer = s.uintbuf[:size]
if err := s.readFull(buffer); err != nil {
return err
}
// Reject inputs where single byte encoding should have been used.
if size == 1 && buffer[0] < 128 {
return ErrCanonSize
}
default:
// For large integers, a temporary buffer is needed.
buffer = make([]byte, size)
if err := s.readFull(buffer); err != nil {
return err
}
}
// Reject leading zero bytes.
if len(buffer) > 0 && buffer[0] == 0 {
return ErrCanonInt
}
// Set the integer bytes.
dst.SetBytes(buffer)
return nil
}
// Decode decodes a value and stores the result in the value pointed // Decode decodes a value and stores the result in the value pointed
// to by val. Please see the documentation for the Decode function // to by val. Please see the documentation for the Decode function
// to learn about the decoding rules. // to learn about the decoding rules.

@ -286,6 +286,47 @@ func TestStreamRaw(t *testing.T) {
} }
} }
func TestStreamReadBytes(t *testing.T) {
tests := []struct {
input string
size int
err string
}{
// kind List
{input: "C0", size: 1, err: "rlp: expected String or Byte"},
// kind Byte
{input: "04", size: 0, err: "input value has wrong size 1, want 0"},
{input: "04", size: 1},
{input: "04", size: 2, err: "input value has wrong size 1, want 2"},
// kind String
{input: "820102", size: 0, err: "input value has wrong size 2, want 0"},
{input: "820102", size: 1, err: "input value has wrong size 2, want 1"},
{input: "820102", size: 2},
{input: "820102", size: 3, err: "input value has wrong size 2, want 3"},
}
for _, test := range tests {
test := test
name := fmt.Sprintf("input_%s/size_%d", test.input, test.size)
t.Run(name, func(t *testing.T) {
s := NewStream(bytes.NewReader(unhex(test.input)), 0)
b := make([]byte, test.size)
err := s.ReadBytes(b)
if test.err == "" {
if err != nil {
t.Errorf("unexpected error %q", err)
}
} else {
if err == nil {
t.Errorf("expected error, got nil")
} else if err.Error() != test.err {
t.Errorf("wrong error %q", err)
}
}
})
}
}
func TestDecodeErrors(t *testing.T) { func TestDecodeErrors(t *testing.T) {
r := bytes.NewReader(nil) r := bytes.NewReader(nil)
@ -990,7 +1031,7 @@ func TestInvalidOptionalField(t *testing.T) {
v interface{} v interface{}
err string err string
}{ }{
{v: new(invalid1), err: `rlp: struct field rlp.invalid1.B needs "optional" tag`}, {v: new(invalid1), err: `rlp: invalid struct tag "" for rlp.invalid1.B (must be optional because preceding field "A" is optional)`},
{v: new(invalid2), err: `rlp: invalid struct tag "optional" for rlp.invalid2.T (also has "tail" tag)`}, {v: new(invalid2), err: `rlp: invalid struct tag "optional" for rlp.invalid2.T (also has "tail" tag)`},
{v: new(invalid3), err: `rlp: invalid struct tag "tail" for rlp.invalid3.T (also has "optional" tag)`}, {v: new(invalid3), err: `rlp: invalid struct tag "tail" for rlp.invalid3.T (also has "optional" tag)`},
} }

@ -0,0 +1,352 @@
package rlp
import (
"io"
"math/big"
"reflect"
"sync"
)
type encBuffer struct {
str []byte // string data, contains everything except list headers
lheads []listhead // all list headers
lhsize int // sum of sizes of all encoded list headers
sizebuf [9]byte // auxiliary buffer for uint encoding
}
// The global encBuffer pool.
var encBufferPool = sync.Pool{
New: func() interface{} { return new(encBuffer) },
}
func getEncBuffer() *encBuffer {
buf := encBufferPool.Get().(*encBuffer)
buf.reset()
return buf
}
func (buf *encBuffer) reset() {
buf.lhsize = 0
buf.str = buf.str[:0]
buf.lheads = buf.lheads[:0]
}
// size returns the length of the encoded data.
func (buf *encBuffer) size() int {
return len(buf.str) + buf.lhsize
}
// toBytes creates the encoder output.
func (w *encBuffer) toBytes() []byte {
out := make([]byte, w.size())
strpos := 0
pos := 0
for _, head := range w.lheads {
// write string data before header
n := copy(out[pos:], w.str[strpos:head.offset])
pos += n
strpos += n
// write the header
enc := head.encode(out[pos:])
pos += len(enc)
}
// copy string data after the last list header
copy(out[pos:], w.str[strpos:])
return out
}
// toWriter writes the encoder output to w.
func (buf *encBuffer) toWriter(w io.Writer) (err error) {
strpos := 0
for _, head := range buf.lheads {
// write string data before header
if head.offset-strpos > 0 {
n, err := w.Write(buf.str[strpos:head.offset])
strpos += n
if err != nil {
return err
}
}
// write the header
enc := head.encode(buf.sizebuf[:])
if _, err = w.Write(enc); err != nil {
return err
}
}
if strpos < len(buf.str) {
// write string data after the last list header
_, err = w.Write(buf.str[strpos:])
}
return err
}
// Write implements io.Writer and appends b directly to the output.
func (buf *encBuffer) Write(b []byte) (int, error) {
buf.str = append(buf.str, b...)
return len(b), nil
}
// writeBool writes b as the integer 0 (false) or 1 (true).
func (buf *encBuffer) writeBool(b bool) {
if b {
buf.str = append(buf.str, 0x01)
} else {
buf.str = append(buf.str, 0x80)
}
}
func (buf *encBuffer) writeUint64(i uint64) {
if i == 0 {
buf.str = append(buf.str, 0x80)
} else if i < 128 {
// fits single byte
buf.str = append(buf.str, byte(i))
} else {
s := putint(buf.sizebuf[1:], i)
buf.sizebuf[0] = 0x80 + byte(s)
buf.str = append(buf.str, buf.sizebuf[:s+1]...)
}
}
func (buf *encBuffer) writeBytes(b []byte) {
if len(b) == 1 && b[0] <= 0x7F {
// fits single byte, no string header
buf.str = append(buf.str, b[0])
} else {
buf.encodeStringHeader(len(b))
buf.str = append(buf.str, b...)
}
}
// wordBytes is the number of bytes in a big.Word
const wordBytes = (32 << (uint64(^big.Word(0)) >> 63)) / 8
// writeBigInt writes i as an integer.
func (w *encBuffer) writeBigInt(i *big.Int) {
bitlen := i.BitLen()
if bitlen <= 64 {
w.writeUint64(i.Uint64())
return
}
// Integer is larger than 64 bits, encode from i.Bits().
// The minimal byte length is bitlen rounded up to the next
// multiple of 8, divided by 8.
length := ((bitlen + 7) & -8) >> 3
w.encodeStringHeader(length)
w.str = append(w.str, make([]byte, length)...)
index := length
buf := w.str[len(w.str)-length:]
for _, d := range i.Bits() {
for j := 0; j < wordBytes && index > 0; j++ {
index--
buf[index] = byte(d)
d >>= 8
}
}
}
// list adds a new list header to the header stack. It returns the index of the header.
// Call listEnd with this index after encoding the content of the list.
func (buf *encBuffer) list() int {
buf.lheads = append(buf.lheads, listhead{offset: len(buf.str), size: buf.lhsize})
return len(buf.lheads) - 1
}
func (buf *encBuffer) listEnd(index int) {
lh := &buf.lheads[index]
lh.size = buf.size() - lh.offset - lh.size
if lh.size < 56 {
buf.lhsize++ // length encoded into kind tag
} else {
buf.lhsize += 1 + intsize(uint64(lh.size))
}
}
func (buf *encBuffer) encode(val interface{}) error {
rval := reflect.ValueOf(val)
writer, err := cachedWriter(rval.Type())
if err != nil {
return err
}
return writer(rval, buf)
}
func (buf *encBuffer) encodeStringHeader(size int) {
if size < 56 {
buf.str = append(buf.str, 0x80+byte(size))
} else {
sizesize := putint(buf.sizebuf[1:], uint64(size))
buf.sizebuf[0] = 0xB7 + byte(sizesize)
buf.str = append(buf.str, buf.sizebuf[:sizesize+1]...)
}
}
// encReader is the io.Reader returned by EncodeToReader.
// It releases its encbuf at EOF.
type encReader struct {
buf *encBuffer // the buffer we're reading from. this is nil when we're at EOF.
lhpos int // index of list header that we're reading
strpos int // current position in string buffer
piece []byte // next piece to be read
}
func (r *encReader) Read(b []byte) (n int, err error) {
for {
if r.piece = r.next(); r.piece == nil {
// Put the encode buffer back into the pool at EOF when it
// is first encountered. Subsequent calls still return EOF
// as the error but the buffer is no longer valid.
if r.buf != nil {
encBufferPool.Put(r.buf)
r.buf = nil
}
return n, io.EOF
}
nn := copy(b[n:], r.piece)
n += nn
if nn < len(r.piece) {
// piece didn't fit, see you next time.
r.piece = r.piece[nn:]
return n, nil
}
r.piece = nil
}
}
// next returns the next piece of data to be read.
// it returns nil at EOF.
func (r *encReader) next() []byte {
switch {
case r.buf == nil:
return nil
case r.piece != nil:
// There is still data available for reading.
return r.piece
case r.lhpos < len(r.buf.lheads):
// We're before the last list header.
head := r.buf.lheads[r.lhpos]
sizebefore := head.offset - r.strpos
if sizebefore > 0 {
// String data before header.
p := r.buf.str[r.strpos:head.offset]
r.strpos += sizebefore
return p
}
r.lhpos++
return head.encode(r.buf.sizebuf[:])
case r.strpos < len(r.buf.str):
// String data at the end, after all list headers.
p := r.buf.str[r.strpos:]
r.strpos = len(r.buf.str)
return p
default:
return nil
}
}
// EncoderBuffer is a buffer for incremental encoding.
//
// The zero value is NOT ready for use. To get a usable buffer,
// create it using NewEncoderBuffer or call Reset.
type EncoderBuffer struct {
buf *encBuffer
dst io.Writer
ownBuffer bool
}
// NewEncoderBuffer creates an encoder buffer.
func NewEncoderBuffer(dst io.Writer) EncoderBuffer {
var w EncoderBuffer
w.Reset(dst)
return w
}
// Reset truncates the buffer and sets the output destination.
func (w *EncoderBuffer) Reset(dst io.Writer) {
if w.buf != nil && !w.ownBuffer {
panic("can't Reset derived EncoderBuffer")
}
// If the destination writer has an *encBuffer, use it.
// Note that w.ownBuffer is left false here.
if dst != nil {
if outer, ok := dst.(*encBuffer); ok {
*w = EncoderBuffer{outer, nil, false}
return
}
if outer, ok := dst.(EncoderBuffer); ok {
*w = EncoderBuffer{outer.buf, nil, false}
return
}
}
// Get a fresh buffer.
if w.buf == nil {
w.buf = encBufferPool.Get().(*encBuffer)
w.ownBuffer = true
}
w.buf.reset()
w.dst = dst
}
// Flush writes encoded RLP data to the output writer. This can only be called once.
// If you want to re-use the buffer after Flush, you must call Reset.
func (w *EncoderBuffer) Flush() error {
var err error
if w.dst != nil {
err = w.buf.toWriter(w.dst)
}
// Release the internal buffer.
if w.ownBuffer {
encBufferPool.Put(w.buf)
}
*w = EncoderBuffer{}
return err
}
// ToBytes returns the encoded bytes.
func (w *EncoderBuffer) ToBytes() []byte {
return w.buf.toBytes()
}
// Write appends b directly to the encoder output.
func (w EncoderBuffer) Write(b []byte) (int, error) {
return w.buf.Write(b)
}
// WriteBool writes b as the integer 0 (false) or 1 (true).
func (w EncoderBuffer) WriteBool(b bool) {
w.buf.writeBool(b)
}
// WriteUint64 encodes an unsigned integer.
func (w EncoderBuffer) WriteUint64(i uint64) {
w.buf.writeUint64(i)
}
// WriteBigInt encodes a big.Int as an RLP string.
// Note: Unlike with Encode, the sign of i is ignored.
func (w EncoderBuffer) WriteBigInt(i *big.Int) {
w.buf.writeBigInt(i)
}
// WriteBytes encodes b as an RLP string.
func (w EncoderBuffer) WriteBytes(b []byte) {
w.buf.writeBytes(b)
}
// List starts a list. It returns an internal index. Call EndList with
// this index after encoding the content to finish the list.
func (w EncoderBuffer) List() int {
return w.buf.list()
}
// ListEnd finishes the given list.
func (w EncoderBuffer) ListEnd(index int) {
w.buf.listEnd(index)
}

@ -0,0 +1,45 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rlp_test
import (
"bytes"
"fmt"
"github.com/ethereum/go-ethereum/rlp"
)
func ExampleEncoderBuffer() {
var w bytes.Buffer
// Encode [4, [5, 6]] to w.
buf := rlp.NewEncoderBuffer(&w)
l1 := buf.List()
buf.WriteUint64(4)
l2 := buf.List()
buf.WriteUint64(5)
buf.WriteUint64(6)
buf.ListEnd(l2)
buf.ListEnd(l1)
if err := buf.Flush(); err != nil {
panic(err)
}
fmt.Printf("%X\n", w.Bytes())
// Output:
// C404C20506
}

@ -17,11 +17,13 @@
package rlp package rlp
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"math/big" "math/big"
"reflect" "reflect"
"sync"
"github.com/ethereum/go-ethereum/rlp/internal/rlpstruct"
) )
var ( var (
@ -31,6 +33,8 @@ var (
EmptyList = []byte{0xC0} EmptyList = []byte{0xC0}
) )
var ErrNegativeBigInt = errors.New("rlp: cannot encode negative big.Int")
// Encoder is implemented by types that require custom // Encoder is implemented by types that require custom
// encoding rules or want to encode private fields. // encoding rules or want to encode private fields.
type Encoder interface { type Encoder interface {
@ -51,30 +55,33 @@ type Encoder interface {
// //
// Please see package-level documentation of encoding rules. // Please see package-level documentation of encoding rules.
func Encode(w io.Writer, val interface{}) error { func Encode(w io.Writer, val interface{}) error {
if outer, ok := w.(*encbuf); ok { // Optimization: reuse *encBuffer when called by EncodeRLP.
// Encode was called by some type's EncodeRLP. if buf, ok := w.(*encBuffer); ok {
// Avoid copying by writing to the outer encbuf directly. return buf.encode(val)
return outer.encode(val) }
} if ebuf, ok := w.(EncoderBuffer); ok {
eb := encbufPool.Get().(*encbuf) return ebuf.buf.encode(val)
defer encbufPool.Put(eb) }
eb.reset()
if err := eb.encode(val); err != nil { buf := getEncBuffer()
defer encBufferPool.Put(buf)
if err := buf.encode(val); err != nil {
return err return err
} }
return eb.toWriter(w) return buf.toWriter(w)
} }
// EncodeToBytes returns the RLP encoding of val. // EncodeToBytes returns the RLP encoding of val.
// Please see package-level documentation for the encoding rules. // Please see package-level documentation for the encoding rules.
func EncodeToBytes(val interface{}) ([]byte, error) { func EncodeToBytes(val interface{}) ([]byte, error) {
eb := encbufPool.Get().(*encbuf) buf := getEncBuffer()
defer encbufPool.Put(eb) defer encBufferPool.Put(buf)
eb.reset()
if err := eb.encode(val); err != nil { if err := buf.encode(val); err != nil {
return nil, err return nil, err
} }
return eb.toBytes(), nil return buf.toBytes(), nil
} }
// EncodeToReader returns a reader from which the RLP encoding of val // EncodeToReader returns a reader from which the RLP encoding of val
@ -83,12 +90,15 @@ func EncodeToBytes(val interface{}) ([]byte, error) {
// //
// Please see the documentation of Encode for the encoding rules. // Please see the documentation of Encode for the encoding rules.
func EncodeToReader(val interface{}) (size int, r io.Reader, err error) { func EncodeToReader(val interface{}) (size int, r io.Reader, err error) {
eb := encbufPool.Get().(*encbuf) buf := getEncBuffer()
eb.reset() if err := buf.encode(val); err != nil {
if err := eb.encode(val); err != nil { encBufferPool.Put(buf)
return 0, nil, err return 0, nil, err
} }
return eb.size(), &encReader{buf: eb}, nil // Note: can't put the reader back into the pool here
// because it is held by encReader. The reader puts it
// back when it has been fully consumed.
return buf.size(), &encReader{buf: buf}, nil
} }
type listhead struct { type listhead struct {
@ -123,207 +133,10 @@ func puthead(buf []byte, smalltag, largetag byte, size uint64) int {
return sizesize + 1 return sizesize + 1
} }
type encbuf struct {
str []byte // string data, contains everything except list headers
lheads []listhead // all list headers
lhsize int // sum of sizes of all encoded list headers
sizebuf [9]byte // auxiliary buffer for uint encoding
}
// encbufs are pooled.
var encbufPool = sync.Pool{
New: func() interface{} { return new(encbuf) },
}
func (w *encbuf) reset() {
w.lhsize = 0
w.str = w.str[:0]
w.lheads = w.lheads[:0]
}
// encbuf implements io.Writer so it can be passed it into EncodeRLP.
func (w *encbuf) Write(b []byte) (int, error) {
w.str = append(w.str, b...)
return len(b), nil
}
func (w *encbuf) encode(val interface{}) error {
rval := reflect.ValueOf(val)
writer, err := cachedWriter(rval.Type())
if err != nil {
return err
}
return writer(rval, w)
}
func (w *encbuf) encodeStringHeader(size int) {
if size < 56 {
w.str = append(w.str, 0x80+byte(size))
} else {
sizesize := putint(w.sizebuf[1:], uint64(size))
w.sizebuf[0] = 0xB7 + byte(sizesize)
w.str = append(w.str, w.sizebuf[:sizesize+1]...)
}
}
func (w *encbuf) encodeString(b []byte) {
if len(b) == 1 && b[0] <= 0x7F {
// fits single byte, no string header
w.str = append(w.str, b[0])
} else {
w.encodeStringHeader(len(b))
w.str = append(w.str, b...)
}
}
func (w *encbuf) encodeUint(i uint64) {
if i == 0 {
w.str = append(w.str, 0x80)
} else if i < 128 {
// fits single byte
w.str = append(w.str, byte(i))
} else {
s := putint(w.sizebuf[1:], i)
w.sizebuf[0] = 0x80 + byte(s)
w.str = append(w.str, w.sizebuf[:s+1]...)
}
}
// list adds a new list header to the header stack. It returns the index
// of the header. The caller must call listEnd with this index after encoding
// the content of the list.
func (w *encbuf) list() int {
w.lheads = append(w.lheads, listhead{offset: len(w.str), size: w.lhsize})
return len(w.lheads) - 1
}
func (w *encbuf) listEnd(index int) {
lh := &w.lheads[index]
lh.size = w.size() - lh.offset - lh.size
if lh.size < 56 {
w.lhsize++ // length encoded into kind tag
} else {
w.lhsize += 1 + intsize(uint64(lh.size))
}
}
func (w *encbuf) size() int {
return len(w.str) + w.lhsize
}
func (w *encbuf) toBytes() []byte {
out := make([]byte, w.size())
strpos := 0
pos := 0
for _, head := range w.lheads {
// write string data before header
n := copy(out[pos:], w.str[strpos:head.offset])
pos += n
strpos += n
// write the header
enc := head.encode(out[pos:])
pos += len(enc)
}
// copy string data after the last list header
copy(out[pos:], w.str[strpos:])
return out
}
func (w *encbuf) toWriter(out io.Writer) (err error) {
strpos := 0
for _, head := range w.lheads {
// write string data before header
if head.offset-strpos > 0 {
n, err := out.Write(w.str[strpos:head.offset])
strpos += n
if err != nil {
return err
}
}
// write the header
enc := head.encode(w.sizebuf[:])
if _, err = out.Write(enc); err != nil {
return err
}
}
if strpos < len(w.str) {
// write string data after the last list header
_, err = out.Write(w.str[strpos:])
}
return err
}
// encReader is the io.Reader returned by EncodeToReader.
// It releases its encbuf at EOF.
type encReader struct {
buf *encbuf // the buffer we're reading from. this is nil when we're at EOF.
lhpos int // index of list header that we're reading
strpos int // current position in string buffer
piece []byte // next piece to be read
}
func (r *encReader) Read(b []byte) (n int, err error) {
for {
if r.piece = r.next(); r.piece == nil {
// Put the encode buffer back into the pool at EOF when it
// is first encountered. Subsequent calls still return EOF
// as the error but the buffer is no longer valid.
if r.buf != nil {
encbufPool.Put(r.buf)
r.buf = nil
}
return n, io.EOF
}
nn := copy(b[n:], r.piece)
n += nn
if nn < len(r.piece) {
// piece didn't fit, see you next time.
r.piece = r.piece[nn:]
return n, nil
}
r.piece = nil
}
}
// next returns the next piece of data to be read.
// it returns nil at EOF.
func (r *encReader) next() []byte {
switch {
case r.buf == nil:
return nil
case r.piece != nil:
// There is still data available for reading.
return r.piece
case r.lhpos < len(r.buf.lheads):
// We're before the last list header.
head := r.buf.lheads[r.lhpos]
sizebefore := head.offset - r.strpos
if sizebefore > 0 {
// String data before header.
p := r.buf.str[r.strpos:head.offset]
r.strpos += sizebefore
return p
}
r.lhpos++
return head.encode(r.buf.sizebuf[:])
case r.strpos < len(r.buf.str):
// String data at the end, after all list headers.
p := r.buf.str[r.strpos:]
r.strpos = len(r.buf.str)
return p
default:
return nil
}
}
var encoderInterface = reflect.TypeOf(new(Encoder)).Elem() var encoderInterface = reflect.TypeOf(new(Encoder)).Elem()
// makeWriter creates a writer function for the given type. // makeWriter creates a writer function for the given type.
func makeWriter(typ reflect.Type, ts tags) (writer, error) { func makeWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) {
kind := typ.Kind() kind := typ.Kind()
switch { switch {
case typ == rawValueType: case typ == rawValueType:
@ -357,71 +170,45 @@ func makeWriter(typ reflect.Type, ts tags) (writer, error) {
} }
} }
func writeRawValue(val reflect.Value, w *encbuf) error { func writeRawValue(val reflect.Value, w *encBuffer) error {
w.str = append(w.str, val.Bytes()...) w.str = append(w.str, val.Bytes()...)
return nil return nil
} }
func writeUint(val reflect.Value, w *encbuf) error { func writeUint(val reflect.Value, w *encBuffer) error {
w.encodeUint(val.Uint()) w.writeUint64(val.Uint())
return nil return nil
} }
func writeBool(val reflect.Value, w *encbuf) error { func writeBool(val reflect.Value, w *encBuffer) error {
if val.Bool() { w.writeBool(val.Bool())
w.str = append(w.str, 0x01)
} else {
w.str = append(w.str, 0x80)
}
return nil return nil
} }
func writeBigIntPtr(val reflect.Value, w *encbuf) error { func writeBigIntPtr(val reflect.Value, w *encBuffer) error {
ptr := val.Interface().(*big.Int) ptr := val.Interface().(*big.Int)
if ptr == nil { if ptr == nil {
w.str = append(w.str, 0x80) w.str = append(w.str, 0x80)
return nil return nil
} }
return writeBigInt(ptr, w) if ptr.Sign() == -1 {
return ErrNegativeBigInt
}
w.writeBigInt(ptr)
return nil
} }
func writeBigIntNoPtr(val reflect.Value, w *encbuf) error { func writeBigIntNoPtr(val reflect.Value, w *encBuffer) error {
i := val.Interface().(big.Int) i := val.Interface().(big.Int)
return writeBigInt(&i, w)
}
// wordBytes is the number of bytes in a big.Word
const wordBytes = (32 << (uint64(^big.Word(0)) >> 63)) / 8
func writeBigInt(i *big.Int, w *encbuf) error {
if i.Sign() == -1 { if i.Sign() == -1 {
return fmt.Errorf("rlp: cannot encode negative *big.Int") return ErrNegativeBigInt
}
bitlen := i.BitLen()
if bitlen <= 64 {
w.encodeUint(i.Uint64())
return nil
}
// Integer is larger than 64 bits, encode from i.Bits().
// The minimal byte length is bitlen rounded up to the next
// multiple of 8, divided by 8.
length := ((bitlen + 7) & -8) >> 3
w.encodeStringHeader(length)
w.str = append(w.str, make([]byte, length)...)
index := length
buf := w.str[len(w.str)-length:]
for _, d := range i.Bits() {
for j := 0; j < wordBytes && index > 0; j++ {
index--
buf[index] = byte(d)
d >>= 8
}
} }
w.writeBigInt(&i)
return nil return nil
} }
func writeBytes(val reflect.Value, w *encbuf) error { func writeBytes(val reflect.Value, w *encBuffer) error {
w.encodeString(val.Bytes()) w.writeBytes(val.Bytes())
return nil return nil
} }
@ -433,7 +220,7 @@ func makeByteArrayWriter(typ reflect.Type) writer {
return writeLengthOneByteArray return writeLengthOneByteArray
default: default:
length := typ.Len() length := typ.Len()
return func(val reflect.Value, w *encbuf) error { return func(val reflect.Value, w *encBuffer) error {
if !val.CanAddr() { if !val.CanAddr() {
// Getting the byte slice of val requires it to be addressable. Make it // Getting the byte slice of val requires it to be addressable. Make it
// addressable by copying. // addressable by copying.
@ -449,12 +236,12 @@ func makeByteArrayWriter(typ reflect.Type) writer {
} }
} }
func writeLengthZeroByteArray(val reflect.Value, w *encbuf) error { func writeLengthZeroByteArray(val reflect.Value, w *encBuffer) error {
w.str = append(w.str, 0x80) w.str = append(w.str, 0x80)
return nil return nil
} }
func writeLengthOneByteArray(val reflect.Value, w *encbuf) error { func writeLengthOneByteArray(val reflect.Value, w *encBuffer) error {
b := byte(val.Index(0).Uint()) b := byte(val.Index(0).Uint())
if b <= 0x7f { if b <= 0x7f {
w.str = append(w.str, b) w.str = append(w.str, b)
@ -464,7 +251,7 @@ func writeLengthOneByteArray(val reflect.Value, w *encbuf) error {
return nil return nil
} }
func writeString(val reflect.Value, w *encbuf) error { func writeString(val reflect.Value, w *encBuffer) error {
s := val.String() s := val.String()
if len(s) == 1 && s[0] <= 0x7f { if len(s) == 1 && s[0] <= 0x7f {
// fits single byte, no string header // fits single byte, no string header
@ -476,7 +263,7 @@ func writeString(val reflect.Value, w *encbuf) error {
return nil return nil
} }
func writeInterface(val reflect.Value, w *encbuf) error { func writeInterface(val reflect.Value, w *encBuffer) error {
if val.IsNil() { if val.IsNil() {
// Write empty list. This is consistent with the previous RLP // Write empty list. This is consistent with the previous RLP
// encoder that we had and should therefore avoid any // encoder that we had and should therefore avoid any
@ -492,17 +279,17 @@ func writeInterface(val reflect.Value, w *encbuf) error {
return writer(eval, w) return writer(eval, w)
} }
func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) { func makeSliceWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) {
etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) etypeinfo := theTC.infoWhileGenerating(typ.Elem(), rlpstruct.Tags{})
if etypeinfo.writerErr != nil { if etypeinfo.writerErr != nil {
return nil, etypeinfo.writerErr return nil, etypeinfo.writerErr
} }
var wfn writer var wfn writer
if ts.tail { if ts.Tail {
// This is for struct tail slices. // This is for struct tail slices.
// w.list is not called for them. // w.list is not called for them.
wfn = func(val reflect.Value, w *encbuf) error { wfn = func(val reflect.Value, w *encBuffer) error {
vlen := val.Len() vlen := val.Len()
for i := 0; i < vlen; i++ { for i := 0; i < vlen; i++ {
if err := etypeinfo.writer(val.Index(i), w); err != nil { if err := etypeinfo.writer(val.Index(i), w); err != nil {
@ -513,7 +300,7 @@ func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) {
} }
} else { } else {
// This is for regular slices and arrays. // This is for regular slices and arrays.
wfn = func(val reflect.Value, w *encbuf) error { wfn = func(val reflect.Value, w *encBuffer) error {
vlen := val.Len() vlen := val.Len()
if vlen == 0 { if vlen == 0 {
w.str = append(w.str, 0xC0) w.str = append(w.str, 0xC0)
@ -547,7 +334,7 @@ func makeStructWriter(typ reflect.Type) (writer, error) {
firstOptionalField := firstOptionalField(fields) firstOptionalField := firstOptionalField(fields)
if firstOptionalField == len(fields) { if firstOptionalField == len(fields) {
// This is the writer function for structs without any optional fields. // This is the writer function for structs without any optional fields.
writer = func(val reflect.Value, w *encbuf) error { writer = func(val reflect.Value, w *encBuffer) error {
lh := w.list() lh := w.list()
for _, f := range fields { for _, f := range fields {
if err := f.info.writer(val.Field(f.index), w); err != nil { if err := f.info.writer(val.Field(f.index), w); err != nil {
@ -560,7 +347,7 @@ func makeStructWriter(typ reflect.Type) (writer, error) {
} else { } else {
// If there are any "optional" fields, the writer needs to perform additional // If there are any "optional" fields, the writer needs to perform additional
// checks to determine the output list length. // checks to determine the output list length.
writer = func(val reflect.Value, w *encbuf) error { writer = func(val reflect.Value, w *encBuffer) error {
lastField := len(fields) - 1 lastField := len(fields) - 1
for ; lastField >= firstOptionalField; lastField-- { for ; lastField >= firstOptionalField; lastField-- {
if !val.Field(fields[lastField].index).IsZero() { if !val.Field(fields[lastField].index).IsZero() {
@ -580,33 +367,18 @@ func makeStructWriter(typ reflect.Type) (writer, error) {
return writer, nil return writer, nil
} }
// nilEncoding returns the encoded value of a nil pointer. func makePtrWriter(typ reflect.Type, ts rlpstruct.Tags) (writer, error) {
func nilEncoding(typ reflect.Type, ts tags) uint8 { nilEncoding := byte(0xC0)
var nilKind Kind if typeNilKind(typ.Elem(), ts) == String {
if ts.nilOK { nilEncoding = 0x80
nilKind = ts.nilKind // use struct tag if provided
} else {
nilKind = defaultNilKind(typ.Elem())
} }
switch nilKind { etypeinfo := theTC.infoWhileGenerating(typ.Elem(), rlpstruct.Tags{})
case String:
return 0x80
case List:
return 0xC0
default:
panic(fmt.Errorf("rlp: invalid nil kind %d", nilKind))
}
}
func makePtrWriter(typ reflect.Type, ts tags) (writer, error) {
etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{})
if etypeinfo.writerErr != nil { if etypeinfo.writerErr != nil {
return nil, etypeinfo.writerErr return nil, etypeinfo.writerErr
} }
nilEncoding := nilEncoding(typ, ts)
writer := func(val reflect.Value, w *encbuf) error { writer := func(val reflect.Value, w *encBuffer) error {
if ev := val.Elem(); ev.IsValid() { if ev := val.Elem(); ev.IsValid() {
return etypeinfo.writer(ev, w) return etypeinfo.writer(ev, w)
} }
@ -618,11 +390,11 @@ func makePtrWriter(typ reflect.Type, ts tags) (writer, error) {
func makeEncoderWriter(typ reflect.Type) writer { func makeEncoderWriter(typ reflect.Type) writer {
if typ.Implements(encoderInterface) { if typ.Implements(encoderInterface) {
return func(val reflect.Value, w *encbuf) error { return func(val reflect.Value, w *encBuffer) error {
return val.Interface().(Encoder).EncodeRLP(w) return val.Interface().(Encoder).EncodeRLP(w)
} }
} }
w := func(val reflect.Value, w *encbuf) error { w := func(val reflect.Value, w *encBuffer) error {
if !val.CanAddr() { if !val.CanAddr() {
// package json simply doesn't call MarshalJSON for this case, but encodes the // package json simply doesn't call MarshalJSON for this case, but encodes the
// value as if it didn't implement the interface. We don't want to handle it that // value as if it didn't implement the interface. We don't want to handle it that

@ -145,7 +145,8 @@ var encTests = []encTest{
{val: *big.NewInt(0xFFFFFF), output: "83FFFFFF"}, {val: *big.NewInt(0xFFFFFF), output: "83FFFFFF"},
// negative ints are not supported // negative ints are not supported
{val: big.NewInt(-1), error: "rlp: cannot encode negative *big.Int"}, {val: big.NewInt(-1), error: "rlp: cannot encode negative big.Int"},
{val: *big.NewInt(-1), error: "rlp: cannot encode negative big.Int"},
// byte arrays // byte arrays
{val: [0]byte{}, output: "80"}, {val: [0]byte{}, output: "80"},

@ -14,11 +14,13 @@
// You should have received a copy of the GNU Lesser General Public License // You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package rlp package rlp_test
import ( import (
"fmt" "fmt"
"io" "io"
"github.com/ethereum/go-ethereum/rlp"
) )
type MyCoolType struct { type MyCoolType struct {
@ -28,16 +30,16 @@ type MyCoolType struct {
// EncodeRLP writes x as RLP list [a, b] that omits the Name field. // EncodeRLP writes x as RLP list [a, b] that omits the Name field.
func (x *MyCoolType) EncodeRLP(w io.Writer) (err error) { func (x *MyCoolType) EncodeRLP(w io.Writer) (err error) {
return Encode(w, []uint{x.a, x.b}) return rlp.Encode(w, []uint{x.a, x.b})
} }
func ExampleEncoder() { func ExampleEncoder() {
var t *MyCoolType // t is nil pointer to MyCoolType var t *MyCoolType // t is nil pointer to MyCoolType
bytes, _ := EncodeToBytes(t) bytes, _ := rlp.EncodeToBytes(t)
fmt.Printf("%v → %X\n", t, bytes) fmt.Printf("%v → %X\n", t, bytes)
t = &MyCoolType{Name: "foobar", a: 5, b: 6} t = &MyCoolType{Name: "foobar", a: 5, b: 6}
bytes, _ = EncodeToBytes(t) bytes, _ = rlp.EncodeToBytes(t)
fmt.Printf("%v → %X\n", t, bytes) fmt.Printf("%v → %X\n", t, bytes)
// Output: // Output:

@ -0,0 +1,213 @@
// Copyright 2021 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package rlpstruct implements struct processing for RLP encoding/decoding.
//
// In particular, this package handles all rules around field filtering,
// struct tags and nil value determination.
package rlpstruct
import (
"fmt"
"reflect"
"strings"
)
// Field represents a struct field.
type Field struct {
Name string
Index int
Exported bool
Type Type
Tag string
}
// Type represents the attributes of a Go type.
type Type struct {
Name string
Kind reflect.Kind
IsEncoder bool // whether type implements rlp.Encoder
IsDecoder bool // whether type implements rlp.Decoder
Elem *Type // non-nil for Kind values of Ptr, Slice, Array
}
// defaultNilValue determines whether a nil pointer to t encodes/decodes
// as an empty string or empty list.
func (t Type) DefaultNilValue() NilKind {
k := t.Kind
if isUint(k) || k == reflect.String || k == reflect.Bool || isByteArray(t) {
return NilKindString
}
return NilKindList
}
// NilKind is the RLP value encoded in place of nil pointers.
type NilKind uint8
const (
NilKindString NilKind = 0x80
NilKindList NilKind = 0xC0
)
// Tags represents struct tags.
type Tags struct {
// rlp:"nil" controls whether empty input results in a nil pointer.
// nilKind is the kind of empty value allowed for the field.
NilKind NilKind
NilOK bool
// rlp:"optional" allows for a field to be missing in the input list.
// If this is set, all subsequent fields must also be optional.
Optional bool
// rlp:"tail" controls whether this field swallows additional list elements. It can
// only be set for the last field, which must be of slice type.
Tail bool
// rlp:"-" ignores fields.
Ignored bool
}
// TagError is raised for invalid struct tags.
type TagError struct {
StructType string
// These are set by this package.
Field string
Tag string
Err string
}
func (e TagError) Error() string {
field := "field " + e.Field
if e.StructType != "" {
field = e.StructType + "." + e.Field
}
return fmt.Sprintf("rlp: invalid struct tag %q for %s (%s)", e.Tag, field, e.Err)
}
// ProcessFields filters the given struct fields, returning only fields
// that should be considered for encoding/decoding.
func ProcessFields(allFields []Field) ([]Field, []Tags, error) {
lastPublic := lastPublicField(allFields)
// Gather all exported fields and their tags.
var fields []Field
var tags []Tags
for _, field := range allFields {
if !field.Exported {
continue
}
ts, err := parseTag(field, lastPublic)
if err != nil {
return nil, nil, err
}
if ts.Ignored {
continue
}
fields = append(fields, field)
tags = append(tags, ts)
}
// Verify optional field consistency. If any optional field exists,
// all fields after it must also be optional. Note: optional + tail
// is supported.
var anyOptional bool
var firstOptionalName string
for i, ts := range tags {
name := fields[i].Name
if ts.Optional || ts.Tail {
if !anyOptional {
firstOptionalName = name
}
anyOptional = true
} else {
if anyOptional {
msg := fmt.Sprintf("must be optional because preceding field %q is optional", firstOptionalName)
return nil, nil, TagError{Field: name, Err: msg}
}
}
}
return fields, tags, nil
}
func parseTag(field Field, lastPublic int) (Tags, error) {
name := field.Name
tag := reflect.StructTag(field.Tag)
var ts Tags
for _, t := range strings.Split(tag.Get("rlp"), ",") {
switch t = strings.TrimSpace(t); t {
case "":
// empty tag is allowed for some reason
case "-":
ts.Ignored = true
case "nil", "nilString", "nilList":
ts.NilOK = true
if field.Type.Kind != reflect.Ptr {
return ts, TagError{Field: name, Tag: t, Err: "field is not a pointer"}
}
switch t {
case "nil":
ts.NilKind = field.Type.Elem.DefaultNilValue()
case "nilString":
ts.NilKind = NilKindString
case "nilList":
ts.NilKind = NilKindList
}
case "optional":
ts.Optional = true
if ts.Tail {
return ts, TagError{Field: name, Tag: t, Err: `also has "tail" tag`}
}
case "tail":
ts.Tail = true
if field.Index != lastPublic {
return ts, TagError{Field: name, Tag: t, Err: "must be on last field"}
}
if ts.Optional {
return ts, TagError{Field: name, Tag: t, Err: `also has "optional" tag`}
}
if field.Type.Kind != reflect.Slice {
return ts, TagError{Field: name, Tag: t, Err: "field type is not slice"}
}
default:
return ts, TagError{Field: name, Tag: t, Err: "unknown tag"}
}
}
return ts, nil
}
func lastPublicField(fields []Field) int {
last := 0
for _, f := range fields {
if f.Exported {
last = f.Index
}
}
return last
}
func isUint(k reflect.Kind) bool {
return k >= reflect.Uint && k <= reflect.Uintptr
}
func isByte(typ Type) bool {
return typ.Kind == reflect.Uint8 && !typ.IsEncoder
}
func isByteArray(typ Type) bool {
return (typ.Kind == reflect.Slice || typ.Kind == reflect.Array) && isByte(*typ.Elem)
}

@ -0,0 +1,735 @@
package main
import (
"bytes"
"fmt"
"go/format"
"go/types"
"sort"
"github.com/ethereum/go-ethereum/rlp/internal/rlpstruct"
)
// buildContext keeps the data needed for make*Op.
type buildContext struct {
topType *types.Named // the type we're creating methods for
encoderIface *types.Interface
decoderIface *types.Interface
rawValueType *types.Named
typeToStructCache map[types.Type]*rlpstruct.Type
}
func newBuildContext(packageRLP *types.Package) *buildContext {
enc := packageRLP.Scope().Lookup("Encoder").Type().Underlying()
dec := packageRLP.Scope().Lookup("Decoder").Type().Underlying()
rawv := packageRLP.Scope().Lookup("RawValue").Type()
return &buildContext{
typeToStructCache: make(map[types.Type]*rlpstruct.Type),
encoderIface: enc.(*types.Interface),
decoderIface: dec.(*types.Interface),
rawValueType: rawv.(*types.Named),
}
}
func (bctx *buildContext) isEncoder(typ types.Type) bool {
return types.Implements(typ, bctx.encoderIface)
}
func (bctx *buildContext) isDecoder(typ types.Type) bool {
return types.Implements(typ, bctx.decoderIface)
}
// typeToStructType converts typ to rlpstruct.Type.
func (bctx *buildContext) typeToStructType(typ types.Type) *rlpstruct.Type {
if prev := bctx.typeToStructCache[typ]; prev != nil {
return prev // short-circuit for recursive types.
}
// Resolve named types to their underlying type, but keep the name.
name := types.TypeString(typ, nil)
for {
utype := typ.Underlying()
if utype == typ {
break
}
typ = utype
}
// Create the type and store it in cache.
t := &rlpstruct.Type{
Name: name,
Kind: typeReflectKind(typ),
IsEncoder: bctx.isEncoder(typ),
IsDecoder: bctx.isDecoder(typ),
}
bctx.typeToStructCache[typ] = t
// Assign element type.
switch typ.(type) {
case *types.Array, *types.Slice, *types.Pointer:
etype := typ.(interface{ Elem() types.Type }).Elem()
t.Elem = bctx.typeToStructType(etype)
}
return t
}
// genContext is passed to the gen* methods of op when generating
// the output code. It tracks packages to be imported by the output
// file and assigns unique names of temporary variables.
type genContext struct {
inPackage *types.Package
imports map[string]struct{}
tempCounter int
}
func newGenContext(inPackage *types.Package) *genContext {
return &genContext{
inPackage: inPackage,
imports: make(map[string]struct{}),
}
}
func (ctx *genContext) temp() string {
v := fmt.Sprintf("_tmp%d", ctx.tempCounter)
ctx.tempCounter++
return v
}
func (ctx *genContext) resetTemp() {
ctx.tempCounter = 0
}
func (ctx *genContext) addImport(path string) {
if path == ctx.inPackage.Path() {
return // avoid importing the package that we're generating in.
}
// TODO: renaming?
ctx.imports[path] = struct{}{}
}
// importsList returns all packages that need to be imported.
func (ctx *genContext) importsList() []string {
imp := make([]string, 0, len(ctx.imports))
for k := range ctx.imports {
imp = append(imp, k)
}
sort.Strings(imp)
return imp
}
// qualify is the types.Qualifier used for printing types.
func (ctx *genContext) qualify(pkg *types.Package) string {
if pkg.Path() == ctx.inPackage.Path() {
return ""
}
ctx.addImport(pkg.Path())
// TODO: renaming?
return pkg.Name()
}
type op interface {
// genWrite creates the encoder. The generated code should write v,
// which is any Go expression, to the rlp.EncoderBuffer 'w'.
genWrite(ctx *genContext, v string) string
// genDecode creates the decoder. The generated code should read
// a value from the rlp.Stream 'dec' and store it to dst.
genDecode(ctx *genContext) (string, string)
}
// basicOp handles basic types bool, uint*, string.
type basicOp struct {
typ types.Type
writeMethod string // calle write the value
writeArgType types.Type // parameter type of writeMethod
decMethod string
decResultType types.Type // return type of decMethod
decUseBitSize bool // if true, result bit size is appended to decMethod
}
func (*buildContext) makeBasicOp(typ *types.Basic) (op, error) {
op := basicOp{typ: typ}
kind := typ.Kind()
switch {
case kind == types.Bool:
op.writeMethod = "WriteBool"
op.writeArgType = types.Typ[types.Bool]
op.decMethod = "Bool"
op.decResultType = types.Typ[types.Bool]
case kind >= types.Uint8 && kind <= types.Uint64:
op.writeMethod = "WriteUint64"
op.writeArgType = types.Typ[types.Uint64]
op.decMethod = "Uint"
op.decResultType = typ
op.decUseBitSize = true
case kind == types.String:
op.writeMethod = "WriteString"
op.writeArgType = types.Typ[types.String]
op.decMethod = "String"
op.decResultType = types.Typ[types.String]
default:
return nil, fmt.Errorf("unhandled basic type: %v", typ)
}
return op, nil
}
func (*buildContext) makeByteSliceOp(typ *types.Slice) op {
if !isByte(typ.Elem()) {
panic("non-byte slice type in makeByteSliceOp")
}
bslice := types.NewSlice(types.Typ[types.Uint8])
return basicOp{
typ: typ,
writeMethod: "WriteBytes",
writeArgType: bslice,
decMethod: "Bytes",
decResultType: bslice,
}
}
func (bctx *buildContext) makeRawValueOp() op {
bslice := types.NewSlice(types.Typ[types.Uint8])
return basicOp{
typ: bctx.rawValueType,
writeMethod: "Write",
writeArgType: bslice,
decMethod: "Raw",
decResultType: bslice,
}
}
func (op basicOp) writeNeedsConversion() bool {
return !types.AssignableTo(op.typ, op.writeArgType)
}
func (op basicOp) decodeNeedsConversion() bool {
return !types.AssignableTo(op.decResultType, op.typ)
}
func (op basicOp) genWrite(ctx *genContext, v string) string {
if op.writeNeedsConversion() {
v = fmt.Sprintf("%s(%s)", op.writeArgType, v)
}
return fmt.Sprintf("w.%s(%s)\n", op.writeMethod, v)
}
func (op basicOp) genDecode(ctx *genContext) (string, string) {
var (
resultV = ctx.temp()
result = resultV
method = op.decMethod
)
if op.decUseBitSize {
// Note: For now, this only works for platform-independent integer
// sizes. makeBasicOp forbids the platform-dependent types.
var sizes types.StdSizes
method = fmt.Sprintf("%s%d", op.decMethod, sizes.Sizeof(op.typ)*8)
}
// Call the decoder method.
var b bytes.Buffer
fmt.Fprintf(&b, "%s, err := dec.%s()\n", resultV, method)
fmt.Fprintf(&b, "if err != nil { return err }\n")
if op.decodeNeedsConversion() {
conv := ctx.temp()
fmt.Fprintf(&b, "%s := %s(%s)\n", conv, types.TypeString(op.typ, ctx.qualify), resultV)
result = conv
}
return result, b.String()
}
// byteArrayOp handles [...]byte.
type byteArrayOp struct {
typ types.Type
name types.Type // name != typ for named byte array types (e.g. common.Address)
}
func (bctx *buildContext) makeByteArrayOp(name *types.Named, typ *types.Array) byteArrayOp {
nt := types.Type(name)
if name == nil {
nt = typ
}
return byteArrayOp{typ, nt}
}
func (op byteArrayOp) genWrite(ctx *genContext, v string) string {
return fmt.Sprintf("w.WriteBytes(%s[:])\n", v)
}
func (op byteArrayOp) genDecode(ctx *genContext) (string, string) {
var resultV = ctx.temp()
var b bytes.Buffer
fmt.Fprintf(&b, "var %s %s\n", resultV, types.TypeString(op.name, ctx.qualify))
fmt.Fprintf(&b, "if err := dec.ReadBytes(%s[:]); err != nil { return err }\n", resultV)
return resultV, b.String()
}
// bigIntNoPtrOp handles non-pointer big.Int.
// This exists because big.Int has it's own decoder operation on rlp.Stream,
// but the decode method returns *big.Int, so it needs to be dereferenced.
type bigIntOp struct {
pointer bool
}
func (op bigIntOp) genWrite(ctx *genContext, v string) string {
var b bytes.Buffer
fmt.Fprintf(&b, "if %s.Sign() == -1 {\n", v)
fmt.Fprintf(&b, " return rlp.ErrNegativeBigInt\n")
fmt.Fprintf(&b, "}\n")
dst := v
if !op.pointer {
dst = "&" + v
}
fmt.Fprintf(&b, "w.WriteBigInt(%s)\n", dst)
// Wrap with nil check.
if op.pointer {
code := b.String()
b.Reset()
fmt.Fprintf(&b, "if %s == nil {\n", v)
fmt.Fprintf(&b, " w.Write(rlp.EmptyString)")
fmt.Fprintf(&b, "} else {\n")
fmt.Fprint(&b, code)
fmt.Fprintf(&b, "}\n")
}
return b.String()
}
func (op bigIntOp) genDecode(ctx *genContext) (string, string) {
var resultV = ctx.temp()
var b bytes.Buffer
fmt.Fprintf(&b, "%s, err := dec.BigInt()\n", resultV)
fmt.Fprintf(&b, "if err != nil { return err }\n")
result := resultV
if !op.pointer {
result = "(*" + resultV + ")"
}
return result, b.String()
}
// encoderDecoderOp handles rlp.Encoder and rlp.Decoder.
// In order to be used with this, the type must implement both interfaces.
// This restriction may be lifted in the future by creating separate ops for
// encoding and decoding.
type encoderDecoderOp struct {
typ types.Type
}
func (op encoderDecoderOp) genWrite(ctx *genContext, v string) string {
return fmt.Sprintf("if err := %s.EncodeRLP(w); err != nil { return err }\n", v)
}
func (op encoderDecoderOp) genDecode(ctx *genContext) (string, string) {
// DecodeRLP must have pointer receiver, and this is verified in makeOp.
etyp := op.typ.(*types.Pointer).Elem()
var resultV = ctx.temp()
var b bytes.Buffer
fmt.Fprintf(&b, "%s := new(%s)\n", resultV, types.TypeString(etyp, ctx.qualify))
fmt.Fprintf(&b, "if err := %s.DecodeRLP(dec); err != nil { return err }\n", resultV)
return resultV, b.String()
}
// ptrOp handles pointer types.
type ptrOp struct {
elemTyp types.Type
elem op
nilOK bool
nilValue rlpstruct.NilKind
}
func (bctx *buildContext) makePtrOp(elemTyp types.Type, tags rlpstruct.Tags) (op, error) {
elemOp, err := bctx.makeOp(nil, elemTyp, rlpstruct.Tags{})
if err != nil {
return nil, err
}
op := ptrOp{elemTyp: elemTyp, elem: elemOp}
// Determine nil value.
if tags.NilOK {
op.nilOK = true
op.nilValue = tags.NilKind
} else {
styp := bctx.typeToStructType(elemTyp)
op.nilValue = styp.DefaultNilValue()
}
return op, nil
}
func (op ptrOp) genWrite(ctx *genContext, v string) string {
// Note: in writer functions, accesses to v are read-only, i.e. v is any Go
// expression. To make all accesses work through the pointer, we substitute
// v with (*v). This is required for most accesses including `v`, `call(v)`,
// and `v[index]` on slices.
//
// For `v.field` and `v[:]` on arrays, the dereference operation is not required.
var vv string
_, isStruct := op.elem.(structOp)
_, isByteArray := op.elem.(byteArrayOp)
if isStruct || isByteArray {
vv = v
} else {
vv = fmt.Sprintf("(*%s)", v)
}
var b bytes.Buffer
fmt.Fprintf(&b, "if %s == nil {\n", v)
fmt.Fprintf(&b, " w.Write([]byte{0x%X})\n", op.nilValue)
fmt.Fprintf(&b, "} else {\n")
fmt.Fprintf(&b, " %s", op.elem.genWrite(ctx, vv))
fmt.Fprintf(&b, "}\n")
return b.String()
}
func (op ptrOp) genDecode(ctx *genContext) (string, string) {
result, code := op.elem.genDecode(ctx)
if !op.nilOK {
// If nil pointers are not allowed, we can just decode the element.
return "&" + result, code
}
// nil is allowed, so check the kind and size first.
// If size is zero and kind matches the nilKind of the type,
// the value decodes as a nil pointer.
var (
resultV = ctx.temp()
kindV = ctx.temp()
sizeV = ctx.temp()
wantKind string
)
if op.nilValue == rlpstruct.NilKindList {
wantKind = "rlp.List"
} else {
wantKind = "rlp.String"
}
var b bytes.Buffer
fmt.Fprintf(&b, "var %s %s\n", resultV, types.TypeString(types.NewPointer(op.elemTyp), ctx.qualify))
fmt.Fprintf(&b, "if %s, %s, err := dec.Kind(); err != nil {\n", kindV, sizeV)
fmt.Fprintf(&b, " return err\n")
fmt.Fprintf(&b, "} else if %s != 0 || %s != %s {\n", sizeV, kindV, wantKind)
fmt.Fprint(&b, code)
fmt.Fprintf(&b, " %s = &%s\n", resultV, result)
fmt.Fprintf(&b, "}\n")
return resultV, b.String()
}
// structOp handles struct types.
type structOp struct {
named *types.Named
typ *types.Struct
fields []*structField
optionalFields []*structField
}
type structField struct {
name string
typ types.Type
elem op
}
func (bctx *buildContext) makeStructOp(named *types.Named, typ *types.Struct) (op, error) {
// Convert fields to []rlpstruct.Field.
var allStructFields []rlpstruct.Field
for i := 0; i < typ.NumFields(); i++ {
f := typ.Field(i)
allStructFields = append(allStructFields, rlpstruct.Field{
Name: f.Name(),
Exported: f.Exported(),
Index: i,
Tag: typ.Tag(i),
Type: *bctx.typeToStructType(f.Type()),
})
}
// Filter/validate fields.
fields, tags, err := rlpstruct.ProcessFields(allStructFields)
if err != nil {
return nil, err
}
// Create field ops.
var op = structOp{named: named, typ: typ}
for i, field := range fields {
// Advanced struct tags are not supported yet.
tag := tags[i]
if err := checkUnsupportedTags(field.Name, tag); err != nil {
return nil, err
}
typ := typ.Field(field.Index).Type()
elem, err := bctx.makeOp(nil, typ, tags[i])
if err != nil {
return nil, fmt.Errorf("field %s: %v", field.Name, err)
}
f := &structField{name: field.Name, typ: typ, elem: elem}
if tag.Optional {
op.optionalFields = append(op.optionalFields, f)
} else {
op.fields = append(op.fields, f)
}
}
return op, nil
}
func checkUnsupportedTags(field string, tag rlpstruct.Tags) error {
if tag.Tail {
return fmt.Errorf(`field %s has unsupported struct tag "tail"`, field)
}
return nil
}
func (op structOp) genWrite(ctx *genContext, v string) string {
var b bytes.Buffer
var listMarker = ctx.temp()
fmt.Fprintf(&b, "%s := w.List()\n", listMarker)
for _, field := range op.fields {
selector := v + "." + field.name
fmt.Fprint(&b, field.elem.genWrite(ctx, selector))
}
op.writeOptionalFields(&b, ctx, v)
fmt.Fprintf(&b, "w.ListEnd(%s)\n", listMarker)
return b.String()
}
func (op structOp) writeOptionalFields(b *bytes.Buffer, ctx *genContext, v string) {
if len(op.optionalFields) == 0 {
return
}
// First check zero-ness of all optional fields.
var zeroV = make([]string, len(op.optionalFields))
for i, field := range op.optionalFields {
selector := v + "." + field.name
zeroV[i] = ctx.temp()
fmt.Fprintf(b, "%s := %s\n", zeroV[i], nonZeroCheck(selector, field.typ, ctx.qualify))
}
// Now write the fields.
for i, field := range op.optionalFields {
selector := v + "." + field.name
cond := ""
for j := i; j < len(op.optionalFields); j++ {
if j > i {
cond += " || "
}
cond += zeroV[j]
}
fmt.Fprintf(b, "if %s {\n", cond)
fmt.Fprint(b, field.elem.genWrite(ctx, selector))
fmt.Fprintf(b, "}\n")
}
}
func (op structOp) genDecode(ctx *genContext) (string, string) {
// Get the string representation of the type.
// Here, named types are handled separately because the output
// would contain a copy of the struct definition otherwise.
var typeName string
if op.named != nil {
typeName = types.TypeString(op.named, ctx.qualify)
} else {
typeName = types.TypeString(op.typ, ctx.qualify)
}
// Create struct object.
var resultV = ctx.temp()
var b bytes.Buffer
fmt.Fprintf(&b, "var %s %s\n", resultV, typeName)
// Decode fields.
fmt.Fprintf(&b, "{\n")
fmt.Fprintf(&b, "if _, err := dec.List(); err != nil { return err }\n")
for _, field := range op.fields {
result, code := field.elem.genDecode(ctx)
fmt.Fprintf(&b, "// %s:\n", field.name)
fmt.Fprint(&b, code)
fmt.Fprintf(&b, "%s.%s = %s\n", resultV, field.name, result)
}
op.decodeOptionalFields(&b, ctx, resultV)
fmt.Fprintf(&b, "if err := dec.ListEnd(); err != nil { return err }\n")
fmt.Fprintf(&b, "}\n")
return resultV, b.String()
}
func (op structOp) decodeOptionalFields(b *bytes.Buffer, ctx *genContext, resultV string) {
var suffix bytes.Buffer
for _, field := range op.optionalFields {
result, code := field.elem.genDecode(ctx)
fmt.Fprintf(b, "// %s:\n", field.name)
fmt.Fprintf(b, "if dec.MoreDataInList() {\n")
fmt.Fprint(b, code)
fmt.Fprintf(b, "%s.%s = %s\n", resultV, field.name, result)
fmt.Fprintf(&suffix, "}\n")
}
suffix.WriteTo(b)
}
// sliceOp handles slice types.
type sliceOp struct {
typ *types.Slice
elemOp op
}
func (bctx *buildContext) makeSliceOp(typ *types.Slice) (op, error) {
elemOp, err := bctx.makeOp(nil, typ.Elem(), rlpstruct.Tags{})
if err != nil {
return nil, err
}
return sliceOp{typ: typ, elemOp: elemOp}, nil
}
func (op sliceOp) genWrite(ctx *genContext, v string) string {
var (
listMarker = ctx.temp() // holds return value of w.List()
iterElemV = ctx.temp() // iteration variable
elemCode = op.elemOp.genWrite(ctx, iterElemV)
)
var b bytes.Buffer
fmt.Fprintf(&b, "%s := w.List()\n", listMarker)
fmt.Fprintf(&b, "for _, %s := range %s {\n", iterElemV, v)
fmt.Fprint(&b, elemCode)
fmt.Fprintf(&b, "}\n")
fmt.Fprintf(&b, "w.ListEnd(%s)\n", listMarker)
return b.String()
}
func (op sliceOp) genDecode(ctx *genContext) (string, string) {
var sliceV = ctx.temp() // holds the output slice
elemResult, elemCode := op.elemOp.genDecode(ctx)
var b bytes.Buffer
fmt.Fprintf(&b, "var %s %s\n", sliceV, types.TypeString(op.typ, ctx.qualify))
fmt.Fprintf(&b, "if _, err := dec.List(); err != nil { return err }\n")
fmt.Fprintf(&b, "for dec.MoreDataInList() {\n")
fmt.Fprintf(&b, " %s", elemCode)
fmt.Fprintf(&b, " %s = append(%s, %s)\n", sliceV, sliceV, elemResult)
fmt.Fprintf(&b, "}\n")
fmt.Fprintf(&b, "if err := dec.ListEnd(); err != nil { return err }\n")
return sliceV, b.String()
}
func (bctx *buildContext) makeOp(name *types.Named, typ types.Type, tags rlpstruct.Tags) (op, error) {
switch typ := typ.(type) {
case *types.Named:
if isBigInt(typ) {
return bigIntOp{}, nil
}
if typ == bctx.rawValueType {
return bctx.makeRawValueOp(), nil
}
if bctx.isDecoder(typ) {
return nil, fmt.Errorf("type %v implements rlp.Decoder with non-pointer receiver", typ)
}
// TODO: same check for encoder?
return bctx.makeOp(typ, typ.Underlying(), tags)
case *types.Pointer:
if isBigInt(typ.Elem()) {
return bigIntOp{pointer: true}, nil
}
// Encoder/Decoder interfaces.
if bctx.isEncoder(typ) {
if bctx.isDecoder(typ) {
return encoderDecoderOp{typ}, nil
}
return nil, fmt.Errorf("type %v implements rlp.Encoder but not rlp.Decoder", typ)
}
if bctx.isDecoder(typ) {
return nil, fmt.Errorf("type %v implements rlp.Decoder but not rlp.Encoder", typ)
}
// Default pointer handling.
return bctx.makePtrOp(typ.Elem(), tags)
case *types.Basic:
return bctx.makeBasicOp(typ)
case *types.Struct:
return bctx.makeStructOp(name, typ)
case *types.Slice:
etyp := typ.Elem()
if isByte(etyp) && !bctx.isEncoder(etyp) {
return bctx.makeByteSliceOp(typ), nil
}
return bctx.makeSliceOp(typ)
case *types.Array:
etyp := typ.Elem()
if isByte(etyp) && !bctx.isEncoder(etyp) {
return bctx.makeByteArrayOp(name, typ), nil
}
return nil, fmt.Errorf("unhandled array type: %v", typ)
default:
return nil, fmt.Errorf("unhandled type: %v", typ)
}
}
// generateDecoder generates the DecodeRLP method on 'typ'.
func generateDecoder(ctx *genContext, typ string, op op) []byte {
ctx.resetTemp()
ctx.addImport(pathOfPackageRLP)
result, code := op.genDecode(ctx)
var b bytes.Buffer
fmt.Fprintf(&b, "func (obj *%s) DecodeRLP(dec *rlp.Stream) error {\n", typ)
fmt.Fprint(&b, code)
fmt.Fprintf(&b, " *obj = %s\n", result)
fmt.Fprintf(&b, " return nil\n")
fmt.Fprintf(&b, "}\n")
return b.Bytes()
}
// generateEncoder generates the EncodeRLP method on 'typ'.
func generateEncoder(ctx *genContext, typ string, op op) []byte {
ctx.resetTemp()
ctx.addImport("io")
ctx.addImport(pathOfPackageRLP)
var b bytes.Buffer
fmt.Fprintf(&b, "func (obj *%s) EncodeRLP(_w io.Writer) error {\n", typ)
fmt.Fprintf(&b, " w := rlp.NewEncoderBuffer(_w)\n")
fmt.Fprint(&b, op.genWrite(ctx, "obj"))
fmt.Fprintf(&b, " return w.Flush()\n")
fmt.Fprintf(&b, "}\n")
return b.Bytes()
}
func (bctx *buildContext) generate(typ *types.Named, encoder, decoder bool) ([]byte, error) {
bctx.topType = typ
pkg := typ.Obj().Pkg()
op, err := bctx.makeOp(nil, typ, rlpstruct.Tags{})
if err != nil {
return nil, err
}
var (
ctx = newGenContext(pkg)
encSource []byte
decSource []byte
)
if encoder {
encSource = generateEncoder(ctx, typ.Obj().Name(), op)
}
if decoder {
decSource = generateDecoder(ctx, typ.Obj().Name(), op)
}
var b bytes.Buffer
fmt.Fprintf(&b, "package %s\n\n", pkg.Name())
for _, imp := range ctx.importsList() {
fmt.Fprintf(&b, "import %q\n", imp)
}
if encoder {
fmt.Fprintln(&b)
b.Write(encSource)
}
if decoder {
fmt.Fprintln(&b)
b.Write(decSource)
}
source := b.Bytes()
// fmt.Println(string(source))
return format.Source(source)
}

@ -0,0 +1,92 @@
package main
import (
"bytes"
"fmt"
"go/ast"
"go/importer"
"go/parser"
"go/token"
"go/types"
"io/ioutil"
"os"
"path/filepath"
"testing"
)
// Package RLP is loaded only once and reused for all tests.
var (
testFset = token.NewFileSet()
testImporter = importer.ForCompiler(testFset, "source", nil).(types.ImporterFrom)
testPackageRLP *types.Package
)
func init() {
cwd, err := os.Getwd()
if err != nil {
panic(err)
}
testPackageRLP, err = testImporter.ImportFrom(pathOfPackageRLP, cwd, 0)
if err != nil {
panic(fmt.Errorf("can't load package RLP: %v", err))
}
}
var tests = []string{"uints", "nil", "rawvalue", "optional", "bigint"}
func TestOutput(t *testing.T) {
for _, test := range tests {
test := test
t.Run(test, func(t *testing.T) {
inputFile := filepath.Join("testdata", test+".in.txt")
outputFile := filepath.Join("testdata", test+".out.txt")
bctx, typ, err := loadTestSource(inputFile, "Test")
if err != nil {
t.Fatal("error loading test source:", err)
}
output, err := bctx.generate(typ, true, true)
if err != nil {
t.Fatal("error in generate:", err)
}
// Set this environment variable to regenerate the test outputs.
if os.Getenv("WRITE_TEST_FILES") != "" {
ioutil.WriteFile(outputFile, output, 0644)
}
// Check if output matches.
wantOutput, err := ioutil.ReadFile(outputFile)
if err != nil {
t.Fatal("error loading expected test output:", err)
}
if !bytes.Equal(output, wantOutput) {
t.Fatal("output mismatch:\n", string(output))
}
})
}
}
func loadTestSource(file string, typeName string) (*buildContext, *types.Named, error) {
// Load the test input.
content, err := ioutil.ReadFile(file)
if err != nil {
return nil, nil, err
}
f, err := parser.ParseFile(testFset, file, content, 0)
if err != nil {
return nil, nil, err
}
conf := types.Config{Importer: testImporter}
pkg, err := conf.Check("test", testFset, []*ast.File{f}, nil)
if err != nil {
return nil, nil, err
}
// Find the test struct.
bctx := newBuildContext(testPackageRLP)
typ, err := lookupStructType(pkg.Scope(), typeName)
if err != nil {
return nil, nil, fmt.Errorf("can't find type %s: %v", typeName, err)
}
return bctx, typ, nil
}

@ -0,0 +1,148 @@
// Copyright 2021 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"errors"
"flag"
"fmt"
"go/types"
"io/ioutil"
"os"
"golang.org/x/tools/go/packages"
)
const pathOfPackageRLP = "github.com/ethereum/go-ethereum/rlp"
func main() {
var (
pkgdir = flag.String("dir", ".", "input package")
output = flag.String("out", "-", "output file (default is stdout)")
genEncoder = flag.Bool("encoder", true, "generate EncodeRLP?")
genDecoder = flag.Bool("decoder", false, "generate DecodeRLP?")
typename = flag.String("type", "", "type to generate methods for")
)
flag.Parse()
cfg := Config{
Dir: *pkgdir,
Type: *typename,
GenerateEncoder: *genEncoder,
GenerateDecoder: *genDecoder,
}
code, err := cfg.process()
if err != nil {
fatal(err)
}
if *output == "-" {
os.Stdout.Write(code)
} else if err := ioutil.WriteFile(*output, code, 0644); err != nil {
fatal(err)
}
}
func fatal(args ...interface{}) {
fmt.Fprintln(os.Stderr, args...)
os.Exit(1)
}
type Config struct {
Dir string // input package directory
Type string
GenerateEncoder bool
GenerateDecoder bool
}
// process generates the Go code.
func (cfg *Config) process() (code []byte, err error) {
// Load packages.
pcfg := &packages.Config{
Mode: packages.NeedName | packages.NeedTypes | packages.NeedImports | packages.NeedDeps,
Dir: cfg.Dir,
BuildFlags: []string{"-tags", "norlpgen"},
}
ps, err := packages.Load(pcfg, pathOfPackageRLP, ".")
if err != nil {
return nil, err
}
if len(ps) == 0 {
return nil, fmt.Errorf("no Go package found in %s", cfg.Dir)
}
packages.PrintErrors(ps)
// Find the packages that were loaded.
var (
pkg *types.Package
packageRLP *types.Package
)
for _, p := range ps {
if len(p.Errors) > 0 {
return nil, fmt.Errorf("package %s has errors", p.PkgPath)
}
if p.PkgPath == pathOfPackageRLP {
packageRLP = p.Types
} else {
pkg = p.Types
}
}
bctx := newBuildContext(packageRLP)
// Find the type and generate.
typ, err := lookupStructType(pkg.Scope(), cfg.Type)
if err != nil {
return nil, fmt.Errorf("can't find %s in %s: %v", typ, pkg, err)
}
code, err = bctx.generate(typ, cfg.GenerateEncoder, cfg.GenerateDecoder)
if err != nil {
return nil, err
}
// Add build comments.
// This is done here to avoid processing these lines with gofmt.
var header bytes.Buffer
fmt.Fprint(&header, "// Code generated by rlpgen. DO NOT EDIT.\n\n")
fmt.Fprint(&header, "//go:build !norlpgen\n")
fmt.Fprint(&header, "// +build !norlpgen\n\n")
return append(header.Bytes(), code...), nil
}
func lookupStructType(scope *types.Scope, name string) (*types.Named, error) {
typ, err := lookupType(scope, name)
if err != nil {
return nil, err
}
_, ok := typ.Underlying().(*types.Struct)
if !ok {
return nil, errors.New("not a struct type")
}
return typ, nil
}
func lookupType(scope *types.Scope, name string) (*types.Named, error) {
obj := scope.Lookup(name)
if obj == nil {
return nil, errors.New("no such identifier")
}
typ, ok := obj.(*types.TypeName)
if !ok {
return nil, errors.New("not a type")
}
return typ.Type().(*types.Named), nil
}

@ -0,0 +1,10 @@
// -*- mode: go -*-
package test
import "math/big"
type Test struct {
Int *big.Int
IntNoPtr big.Int
}

@ -0,0 +1,49 @@
package test
import "github.com/ethereum/go-ethereum/rlp"
import "io"
func (obj *Test) EncodeRLP(_w io.Writer) error {
w := rlp.NewEncoderBuffer(_w)
_tmp0 := w.List()
if obj.Int == nil {
w.Write(rlp.EmptyString)
} else {
if obj.Int.Sign() == -1 {
return rlp.ErrNegativeBigInt
}
w.WriteBigInt(obj.Int)
}
if obj.IntNoPtr.Sign() == -1 {
return rlp.ErrNegativeBigInt
}
w.WriteBigInt(&obj.IntNoPtr)
w.ListEnd(_tmp0)
return w.Flush()
}
func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
var _tmp0 Test
{
if _, err := dec.List(); err != nil {
return err
}
// Int:
_tmp1, err := dec.BigInt()
if err != nil {
return err
}
_tmp0.Int = _tmp1
// IntNoPtr:
_tmp2, err := dec.BigInt()
if err != nil {
return err
}
_tmp0.IntNoPtr = (*_tmp2)
if err := dec.ListEnd(); err != nil {
return err
}
}
*obj = _tmp0
return nil
}

@ -0,0 +1,30 @@
// -*- mode: go -*-
package test
type Aux struct{
A uint32
}
type Test struct{
Uint8 *byte `rlp:"nil"`
Uint8List *byte `rlp:"nilList"`
Uint32 *uint32 `rlp:"nil"`
Uint32List *uint32 `rlp:"nilList"`
Uint64 *uint64 `rlp:"nil"`
Uint64List *uint64 `rlp:"nilList"`
String *string `rlp:"nil"`
StringList *string `rlp:"nilList"`
ByteArray *[3]byte `rlp:"nil"`
ByteArrayList *[3]byte `rlp:"nilList"`
ByteSlice *[]byte `rlp:"nil"`
ByteSliceList *[]byte `rlp:"nilList"`
Struct *Aux `rlp:"nil"`
StructString *Aux `rlp:"nilString"`
}

@ -0,0 +1,289 @@
package test
import "github.com/ethereum/go-ethereum/rlp"
import "io"
func (obj *Test) EncodeRLP(_w io.Writer) error {
w := rlp.NewEncoderBuffer(_w)
_tmp0 := w.List()
if obj.Uint8 == nil {
w.Write([]byte{0x80})
} else {
w.WriteUint64(uint64((*obj.Uint8)))
}
if obj.Uint8List == nil {
w.Write([]byte{0xC0})
} else {
w.WriteUint64(uint64((*obj.Uint8List)))
}
if obj.Uint32 == nil {
w.Write([]byte{0x80})
} else {
w.WriteUint64(uint64((*obj.Uint32)))
}
if obj.Uint32List == nil {
w.Write([]byte{0xC0})
} else {
w.WriteUint64(uint64((*obj.Uint32List)))
}
if obj.Uint64 == nil {
w.Write([]byte{0x80})
} else {
w.WriteUint64((*obj.Uint64))
}
if obj.Uint64List == nil {
w.Write([]byte{0xC0})
} else {
w.WriteUint64((*obj.Uint64List))
}
if obj.String == nil {
w.Write([]byte{0x80})
} else {
w.WriteString((*obj.String))
}
if obj.StringList == nil {
w.Write([]byte{0xC0})
} else {
w.WriteString((*obj.StringList))
}
if obj.ByteArray == nil {
w.Write([]byte{0x80})
} else {
w.WriteBytes(obj.ByteArray[:])
}
if obj.ByteArrayList == nil {
w.Write([]byte{0xC0})
} else {
w.WriteBytes(obj.ByteArrayList[:])
}
if obj.ByteSlice == nil {
w.Write([]byte{0x80})
} else {
w.WriteBytes((*obj.ByteSlice))
}
if obj.ByteSliceList == nil {
w.Write([]byte{0xC0})
} else {
w.WriteBytes((*obj.ByteSliceList))
}
if obj.Struct == nil {
w.Write([]byte{0xC0})
} else {
_tmp1 := w.List()
w.WriteUint64(uint64(obj.Struct.A))
w.ListEnd(_tmp1)
}
if obj.StructString == nil {
w.Write([]byte{0x80})
} else {
_tmp2 := w.List()
w.WriteUint64(uint64(obj.StructString.A))
w.ListEnd(_tmp2)
}
w.ListEnd(_tmp0)
return w.Flush()
}
func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
var _tmp0 Test
{
if _, err := dec.List(); err != nil {
return err
}
// Uint8:
var _tmp2 *byte
if _tmp3, _tmp4, err := dec.Kind(); err != nil {
return err
} else if _tmp4 != 0 || _tmp3 != rlp.String {
_tmp1, err := dec.Uint8()
if err != nil {
return err
}
_tmp2 = &_tmp1
}
_tmp0.Uint8 = _tmp2
// Uint8List:
var _tmp6 *byte
if _tmp7, _tmp8, err := dec.Kind(); err != nil {
return err
} else if _tmp8 != 0 || _tmp7 != rlp.List {
_tmp5, err := dec.Uint8()
if err != nil {
return err
}
_tmp6 = &_tmp5
}
_tmp0.Uint8List = _tmp6
// Uint32:
var _tmp10 *uint32
if _tmp11, _tmp12, err := dec.Kind(); err != nil {
return err
} else if _tmp12 != 0 || _tmp11 != rlp.String {
_tmp9, err := dec.Uint32()
if err != nil {
return err
}
_tmp10 = &_tmp9
}
_tmp0.Uint32 = _tmp10
// Uint32List:
var _tmp14 *uint32
if _tmp15, _tmp16, err := dec.Kind(); err != nil {
return err
} else if _tmp16 != 0 || _tmp15 != rlp.List {
_tmp13, err := dec.Uint32()
if err != nil {
return err
}
_tmp14 = &_tmp13
}
_tmp0.Uint32List = _tmp14
// Uint64:
var _tmp18 *uint64
if _tmp19, _tmp20, err := dec.Kind(); err != nil {
return err
} else if _tmp20 != 0 || _tmp19 != rlp.String {
_tmp17, err := dec.Uint64()
if err != nil {
return err
}
_tmp18 = &_tmp17
}
_tmp0.Uint64 = _tmp18
// Uint64List:
var _tmp22 *uint64
if _tmp23, _tmp24, err := dec.Kind(); err != nil {
return err
} else if _tmp24 != 0 || _tmp23 != rlp.List {
_tmp21, err := dec.Uint64()
if err != nil {
return err
}
_tmp22 = &_tmp21
}
_tmp0.Uint64List = _tmp22
// String:
var _tmp26 *string
if _tmp27, _tmp28, err := dec.Kind(); err != nil {
return err
} else if _tmp28 != 0 || _tmp27 != rlp.String {
_tmp25, err := dec.String()
if err != nil {
return err
}
_tmp26 = &_tmp25
}
_tmp0.String = _tmp26
// StringList:
var _tmp30 *string
if _tmp31, _tmp32, err := dec.Kind(); err != nil {
return err
} else if _tmp32 != 0 || _tmp31 != rlp.List {
_tmp29, err := dec.String()
if err != nil {
return err
}
_tmp30 = &_tmp29
}
_tmp0.StringList = _tmp30
// ByteArray:
var _tmp34 *[3]byte
if _tmp35, _tmp36, err := dec.Kind(); err != nil {
return err
} else if _tmp36 != 0 || _tmp35 != rlp.String {
var _tmp33 [3]byte
if err := dec.ReadBytes(_tmp33[:]); err != nil {
return err
}
_tmp34 = &_tmp33
}
_tmp0.ByteArray = _tmp34
// ByteArrayList:
var _tmp38 *[3]byte
if _tmp39, _tmp40, err := dec.Kind(); err != nil {
return err
} else if _tmp40 != 0 || _tmp39 != rlp.List {
var _tmp37 [3]byte
if err := dec.ReadBytes(_tmp37[:]); err != nil {
return err
}
_tmp38 = &_tmp37
}
_tmp0.ByteArrayList = _tmp38
// ByteSlice:
var _tmp42 *[]byte
if _tmp43, _tmp44, err := dec.Kind(); err != nil {
return err
} else if _tmp44 != 0 || _tmp43 != rlp.String {
_tmp41, err := dec.Bytes()
if err != nil {
return err
}
_tmp42 = &_tmp41
}
_tmp0.ByteSlice = _tmp42
// ByteSliceList:
var _tmp46 *[]byte
if _tmp47, _tmp48, err := dec.Kind(); err != nil {
return err
} else if _tmp48 != 0 || _tmp47 != rlp.List {
_tmp45, err := dec.Bytes()
if err != nil {
return err
}
_tmp46 = &_tmp45
}
_tmp0.ByteSliceList = _tmp46
// Struct:
var _tmp51 *Aux
if _tmp52, _tmp53, err := dec.Kind(); err != nil {
return err
} else if _tmp53 != 0 || _tmp52 != rlp.List {
var _tmp49 Aux
{
if _, err := dec.List(); err != nil {
return err
}
// A:
_tmp50, err := dec.Uint32()
if err != nil {
return err
}
_tmp49.A = _tmp50
if err := dec.ListEnd(); err != nil {
return err
}
}
_tmp51 = &_tmp49
}
_tmp0.Struct = _tmp51
// StructString:
var _tmp56 *Aux
if _tmp57, _tmp58, err := dec.Kind(); err != nil {
return err
} else if _tmp58 != 0 || _tmp57 != rlp.String {
var _tmp54 Aux
{
if _, err := dec.List(); err != nil {
return err
}
// A:
_tmp55, err := dec.Uint32()
if err != nil {
return err
}
_tmp54.A = _tmp55
if err := dec.ListEnd(); err != nil {
return err
}
}
_tmp56 = &_tmp54
}
_tmp0.StructString = _tmp56
if err := dec.ListEnd(); err != nil {
return err
}
}
*obj = _tmp0
return nil
}

@ -0,0 +1,17 @@
// -*- mode: go -*-
package test
type Aux struct {
A uint64
}
type Test struct {
Uint64 uint64 `rlp:"optional"`
Pointer *uint64 `rlp:"optional"`
String string `rlp:"optional"`
Slice []uint64 `rlp:"optional"`
Array [3]byte `rlp:"optional"`
NamedStruct Aux `rlp:"optional"`
AnonStruct struct{ A string } `rlp:"optional"`
}

@ -0,0 +1,153 @@
package test
import "github.com/ethereum/go-ethereum/rlp"
import "io"
func (obj *Test) EncodeRLP(_w io.Writer) error {
w := rlp.NewEncoderBuffer(_w)
_tmp0 := w.List()
_tmp1 := obj.Uint64 != 0
_tmp2 := obj.Pointer != nil
_tmp3 := obj.String != ""
_tmp4 := len(obj.Slice) > 0
_tmp5 := obj.Array != ([3]byte{})
_tmp6 := obj.NamedStruct != (Aux{})
_tmp7 := obj.AnonStruct != (struct{ A string }{})
if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 {
w.WriteUint64(obj.Uint64)
}
if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 {
if obj.Pointer == nil {
w.Write([]byte{0x80})
} else {
w.WriteUint64((*obj.Pointer))
}
}
if _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 {
w.WriteString(obj.String)
}
if _tmp4 || _tmp5 || _tmp6 || _tmp7 {
_tmp8 := w.List()
for _, _tmp9 := range obj.Slice {
w.WriteUint64(_tmp9)
}
w.ListEnd(_tmp8)
}
if _tmp5 || _tmp6 || _tmp7 {
w.WriteBytes(obj.Array[:])
}
if _tmp6 || _tmp7 {
_tmp10 := w.List()
w.WriteUint64(obj.NamedStruct.A)
w.ListEnd(_tmp10)
}
if _tmp7 {
_tmp11 := w.List()
w.WriteString(obj.AnonStruct.A)
w.ListEnd(_tmp11)
}
w.ListEnd(_tmp0)
return w.Flush()
}
func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
var _tmp0 Test
{
if _, err := dec.List(); err != nil {
return err
}
// Uint64:
if dec.MoreDataInList() {
_tmp1, err := dec.Uint64()
if err != nil {
return err
}
_tmp0.Uint64 = _tmp1
// Pointer:
if dec.MoreDataInList() {
_tmp2, err := dec.Uint64()
if err != nil {
return err
}
_tmp0.Pointer = &_tmp2
// String:
if dec.MoreDataInList() {
_tmp3, err := dec.String()
if err != nil {
return err
}
_tmp0.String = _tmp3
// Slice:
if dec.MoreDataInList() {
var _tmp4 []uint64
if _, err := dec.List(); err != nil {
return err
}
for dec.MoreDataInList() {
_tmp5, err := dec.Uint64()
if err != nil {
return err
}
_tmp4 = append(_tmp4, _tmp5)
}
if err := dec.ListEnd(); err != nil {
return err
}
_tmp0.Slice = _tmp4
// Array:
if dec.MoreDataInList() {
var _tmp6 [3]byte
if err := dec.ReadBytes(_tmp6[:]); err != nil {
return err
}
_tmp0.Array = _tmp6
// NamedStruct:
if dec.MoreDataInList() {
var _tmp7 Aux
{
if _, err := dec.List(); err != nil {
return err
}
// A:
_tmp8, err := dec.Uint64()
if err != nil {
return err
}
_tmp7.A = _tmp8
if err := dec.ListEnd(); err != nil {
return err
}
}
_tmp0.NamedStruct = _tmp7
// AnonStruct:
if dec.MoreDataInList() {
var _tmp9 struct{ A string }
{
if _, err := dec.List(); err != nil {
return err
}
// A:
_tmp10, err := dec.String()
if err != nil {
return err
}
_tmp9.A = _tmp10
if err := dec.ListEnd(); err != nil {
return err
}
}
_tmp0.AnonStruct = _tmp9
}
}
}
}
}
}
}
if err := dec.ListEnd(); err != nil {
return err
}
}
*obj = _tmp0
return nil
}

@ -0,0 +1,11 @@
// -*- mode: go -*-
package test
import "github.com/ethereum/go-ethereum/rlp"
type Test struct {
RawValue rlp.RawValue
PointerToRawValue *rlp.RawValue
SliceOfRawValue []rlp.RawValue
}

@ -0,0 +1,64 @@
package test
import "github.com/ethereum/go-ethereum/rlp"
import "io"
func (obj *Test) EncodeRLP(_w io.Writer) error {
w := rlp.NewEncoderBuffer(_w)
_tmp0 := w.List()
w.Write(obj.RawValue)
if obj.PointerToRawValue == nil {
w.Write([]byte{0x80})
} else {
w.Write((*obj.PointerToRawValue))
}
_tmp1 := w.List()
for _, _tmp2 := range obj.SliceOfRawValue {
w.Write(_tmp2)
}
w.ListEnd(_tmp1)
w.ListEnd(_tmp0)
return w.Flush()
}
func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
var _tmp0 Test
{
if _, err := dec.List(); err != nil {
return err
}
// RawValue:
_tmp1, err := dec.Raw()
if err != nil {
return err
}
_tmp0.RawValue = _tmp1
// PointerToRawValue:
_tmp2, err := dec.Raw()
if err != nil {
return err
}
_tmp0.PointerToRawValue = &_tmp2
// SliceOfRawValue:
var _tmp3 []rlp.RawValue
if _, err := dec.List(); err != nil {
return err
}
for dec.MoreDataInList() {
_tmp4, err := dec.Raw()
if err != nil {
return err
}
_tmp3 = append(_tmp3, _tmp4)
}
if err := dec.ListEnd(); err != nil {
return err
}
_tmp0.SliceOfRawValue = _tmp3
if err := dec.ListEnd(); err != nil {
return err
}
}
*obj = _tmp0
return nil
}

@ -0,0 +1,10 @@
// -*- mode: go -*-
package test
type Test struct{
A uint8
B uint16
C uint32
D uint64
}

@ -0,0 +1,53 @@
package test
import "github.com/ethereum/go-ethereum/rlp"
import "io"
func (obj *Test) EncodeRLP(_w io.Writer) error {
w := rlp.NewEncoderBuffer(_w)
_tmp0 := w.List()
w.WriteUint64(uint64(obj.A))
w.WriteUint64(uint64(obj.B))
w.WriteUint64(uint64(obj.C))
w.WriteUint64(obj.D)
w.ListEnd(_tmp0)
return w.Flush()
}
func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
var _tmp0 Test
{
if _, err := dec.List(); err != nil {
return err
}
// A:
_tmp1, err := dec.Uint8()
if err != nil {
return err
}
_tmp0.A = _tmp1
// B:
_tmp2, err := dec.Uint16()
if err != nil {
return err
}
_tmp0.B = _tmp2
// C:
_tmp3, err := dec.Uint32()
if err != nil {
return err
}
_tmp0.C = _tmp3
// D:
_tmp4, err := dec.Uint64()
if err != nil {
return err
}
_tmp0.D = _tmp4
if err := dec.ListEnd(); err != nil {
return err
}
}
*obj = _tmp0
return nil
}

@ -0,0 +1,98 @@
package main
import (
"fmt"
"go/types"
"reflect"
)
// typeReflectKind gives the reflect.Kind that represents typ.
func typeReflectKind(typ types.Type) reflect.Kind {
switch typ := typ.(type) {
case *types.Basic:
k := typ.Kind()
if k >= types.Bool && k <= types.Complex128 {
// value order matches for Bool..Complex128
return reflect.Bool + reflect.Kind(k-types.Bool)
}
if k == types.String {
return reflect.String
}
if k == types.UnsafePointer {
return reflect.UnsafePointer
}
panic(fmt.Errorf("unhandled BasicKind %v", k))
case *types.Array:
return reflect.Array
case *types.Chan:
return reflect.Chan
case *types.Interface:
return reflect.Interface
case *types.Map:
return reflect.Map
case *types.Pointer:
return reflect.Ptr
case *types.Signature:
return reflect.Func
case *types.Slice:
return reflect.Slice
case *types.Struct:
return reflect.Struct
default:
panic(fmt.Errorf("unhandled type %T", typ))
}
}
// nonZeroCheck returns the expression that checks whether 'v' is a non-zero value of type 'vtyp'.
func nonZeroCheck(v string, vtyp types.Type, qualify types.Qualifier) string {
// Resolve type name.
typ := resolveUnderlying(vtyp)
switch typ := typ.(type) {
case *types.Basic:
k := typ.Kind()
switch {
case k == types.Bool:
return v
case k >= types.Uint && k <= types.Complex128:
return fmt.Sprintf("%s != 0", v)
case k == types.String:
return fmt.Sprintf(`%s != ""`, v)
default:
panic(fmt.Errorf("unhandled BasicKind %v", k))
}
case *types.Array, *types.Struct:
return fmt.Sprintf("%s != (%s{})", v, types.TypeString(vtyp, qualify))
case *types.Interface, *types.Pointer, *types.Signature:
return fmt.Sprintf("%s != nil", v)
case *types.Slice, *types.Map:
return fmt.Sprintf("len(%s) > 0", v)
default:
panic(fmt.Errorf("unhandled type %T", typ))
}
}
// isBigInt checks whether 'typ' is "math/big".Int.
func isBigInt(typ types.Type) bool {
named, ok := typ.(*types.Named)
if !ok {
return false
}
name := named.Obj()
return name.Pkg().Path() == "math/big" && name.Name() == "Int"
}
// isByte checks whether the underlying type of 'typ' is uint8.
func isByte(typ types.Type) bool {
basic, ok := resolveUnderlying(typ).(*types.Basic)
return ok && basic.Kind() == types.Uint8
}
func resolveUnderlying(typ types.Type) types.Type {
for {
t := typ.Underlying()
if t == typ {
return t
}
typ = t
}
}

@ -19,9 +19,10 @@ package rlp
import ( import (
"fmt" "fmt"
"reflect" "reflect"
"strings"
"sync" "sync"
"sync/atomic" "sync/atomic"
"github.com/ethereum/go-ethereum/rlp/internal/rlpstruct"
) )
// typeinfo is an entry in the type cache. // typeinfo is an entry in the type cache.
@ -32,35 +33,16 @@ type typeinfo struct {
writerErr error // error from makeWriter writerErr error // error from makeWriter
} }
// tags represents struct tags.
type tags struct {
// rlp:"nil" controls whether empty input results in a nil pointer.
// nilKind is the kind of empty value allowed for the field.
nilKind Kind
nilOK bool
// rlp:"optional" allows for a field to be missing in the input list.
// If this is set, all subsequent fields must also be optional.
optional bool
// rlp:"tail" controls whether this field swallows additional list elements. It can
// only be set for the last field, which must be of slice type.
tail bool
// rlp:"-" ignores fields.
ignored bool
}
// typekey is the key of a type in typeCache. It includes the struct tags because // typekey is the key of a type in typeCache. It includes the struct tags because
// they might generate a different decoder. // they might generate a different decoder.
type typekey struct { type typekey struct {
reflect.Type reflect.Type
tags rlpstruct.Tags
} }
type decoder func(*Stream, reflect.Value) error type decoder func(*Stream, reflect.Value) error
type writer func(reflect.Value, *encbuf) error type writer func(reflect.Value, *encBuffer) error
var theTC = newTypeCache() var theTC = newTypeCache()
@ -95,10 +77,10 @@ func (c *typeCache) info(typ reflect.Type) *typeinfo {
} }
// Not in the cache, need to generate info for this type. // Not in the cache, need to generate info for this type.
return c.generate(typ, tags{}) return c.generate(typ, rlpstruct.Tags{})
} }
func (c *typeCache) generate(typ reflect.Type, tags tags) *typeinfo { func (c *typeCache) generate(typ reflect.Type, tags rlpstruct.Tags) *typeinfo {
c.mu.Lock() c.mu.Lock()
defer c.mu.Unlock() defer c.mu.Unlock()
@ -122,7 +104,7 @@ func (c *typeCache) generate(typ reflect.Type, tags tags) *typeinfo {
return info return info
} }
func (c *typeCache) infoWhileGenerating(typ reflect.Type, tags tags) *typeinfo { func (c *typeCache) infoWhileGenerating(typ reflect.Type, tags rlpstruct.Tags) *typeinfo {
key := typekey{typ, tags} key := typekey{typ, tags}
if info := c.next[key]; info != nil { if info := c.next[key]; info != nil {
return info return info
@ -144,35 +126,40 @@ type field struct {
// structFields resolves the typeinfo of all public fields in a struct type. // structFields resolves the typeinfo of all public fields in a struct type.
func structFields(typ reflect.Type) (fields []field, err error) { func structFields(typ reflect.Type) (fields []field, err error) {
var ( // Convert fields to rlpstruct.Field.
lastPublic = lastPublicField(typ) var allStructFields []rlpstruct.Field
anyOptional = false
)
for i := 0; i < typ.NumField(); i++ { for i := 0; i < typ.NumField(); i++ {
if f := typ.Field(i); f.PkgPath == "" { // exported rf := typ.Field(i)
tags, err := parseStructTag(typ, i, lastPublic) allStructFields = append(allStructFields, rlpstruct.Field{
Name: rf.Name,
Index: i,
Exported: rf.PkgPath == "",
Tag: string(rf.Tag),
Type: *rtypeToStructType(rf.Type, nil),
})
}
// Filter/validate fields.
structFields, structTags, err := rlpstruct.ProcessFields(allStructFields)
if err != nil { if err != nil {
if tagErr, ok := err.(rlpstruct.TagError); ok {
tagErr.StructType = typ.String()
return nil, tagErr
}
return nil, err return nil, err
} }
// Skip rlp:"-" fields. // Resolve typeinfo.
if tags.ignored { for i, sf := range structFields {
continue typ := typ.Field(sf.Index).Type
} tags := structTags[i]
// If any field has the "optional" tag, subsequent fields must also have it. info := theTC.infoWhileGenerating(typ, tags)
if tags.optional || tags.tail { fields = append(fields, field{sf.Index, info, tags.Optional})
anyOptional = true
} else if anyOptional {
return nil, fmt.Errorf(`rlp: struct field %v.%s needs "optional" tag`, typ, f.Name)
}
info := theTC.infoWhileGenerating(f.Type, tags)
fields = append(fields, field{i, info, tags.optional})
}
} }
return fields, nil return fields, nil
} }
// anyOptionalFields returns the index of the first field with "optional" tag. // firstOptionalField returns the index of the first field with "optional" tag.
func firstOptionalField(fields []field) int { func firstOptionalField(fields []field) int {
for i, f := range fields { for i, f := range fields {
if f.optional { if f.optional {
@ -192,82 +179,56 @@ func (e structFieldError) Error() string {
return fmt.Sprintf("%v (struct field %v.%s)", e.err, e.typ, e.typ.Field(e.field).Name) return fmt.Sprintf("%v (struct field %v.%s)", e.err, e.typ, e.typ.Field(e.field).Name)
} }
type structTagError struct { func (i *typeinfo) generate(typ reflect.Type, tags rlpstruct.Tags) {
typ reflect.Type i.decoder, i.decoderErr = makeDecoder(typ, tags)
field, tag, err string i.writer, i.writerErr = makeWriter(typ, tags)
}
func (e structTagError) Error() string {
return fmt.Sprintf("rlp: invalid struct tag %q for %v.%s (%s)", e.tag, e.typ, e.field, e.err)
} }
func parseStructTag(typ reflect.Type, fi, lastPublic int) (tags, error) { // rtypeToStructType converts typ to rlpstruct.Type.
f := typ.Field(fi) func rtypeToStructType(typ reflect.Type, rec map[reflect.Type]*rlpstruct.Type) *rlpstruct.Type {
var ts tags k := typ.Kind()
for _, t := range strings.Split(f.Tag.Get("rlp"), ",") { if k == reflect.Invalid {
switch t = strings.TrimSpace(t); t { panic("invalid kind")
case "":
case "-":
ts.ignored = true
case "nil", "nilString", "nilList":
ts.nilOK = true
if f.Type.Kind() != reflect.Ptr {
return ts, structTagError{typ, f.Name, t, "field is not a pointer"}
}
switch t {
case "nil":
ts.nilKind = defaultNilKind(f.Type.Elem())
case "nilString":
ts.nilKind = String
case "nilList":
ts.nilKind = List
}
case "optional":
ts.optional = true
if ts.tail {
return ts, structTagError{typ, f.Name, t, `also has "tail" tag`}
}
case "tail":
ts.tail = true
if fi != lastPublic {
return ts, structTagError{typ, f.Name, t, "must be on last field"}
}
if ts.optional {
return ts, structTagError{typ, f.Name, t, `also has "optional" tag`}
}
if f.Type.Kind() != reflect.Slice {
return ts, structTagError{typ, f.Name, t, "field type is not slice"}
} }
default:
return ts, fmt.Errorf("rlp: unknown struct tag %q on %v.%s", t, typ, f.Name) if prev := rec[typ]; prev != nil {
return prev // short-circuit for recursive types
} }
if rec == nil {
rec = make(map[reflect.Type]*rlpstruct.Type)
} }
return ts, nil
}
func lastPublicField(typ reflect.Type) int { t := &rlpstruct.Type{
last := 0 Name: typ.String(),
for i := 0; i < typ.NumField(); i++ { Kind: k,
if typ.Field(i).PkgPath == "" { IsEncoder: typ.Implements(encoderInterface),
last = i IsDecoder: typ.Implements(decoderInterface),
} }
rec[typ] = t
if k == reflect.Array || k == reflect.Slice || k == reflect.Ptr {
t.Elem = rtypeToStructType(typ.Elem(), rec)
} }
return last return t
} }
func (i *typeinfo) generate(typ reflect.Type, tags tags) { // typeNilKind gives the RLP value kind for nil pointers to 'typ'.
i.decoder, i.decoderErr = makeDecoder(typ, tags) func typeNilKind(typ reflect.Type, tags rlpstruct.Tags) Kind {
i.writer, i.writerErr = makeWriter(typ, tags) styp := rtypeToStructType(typ, nil)
}
// defaultNilKind determines whether a nil pointer to typ encodes/decodes var nk rlpstruct.NilKind
// as an empty string or empty list. if tags.NilOK {
func defaultNilKind(typ reflect.Type) Kind { nk = tags.NilKind
k := typ.Kind() } else {
if isUint(k) || k == reflect.String || k == reflect.Bool || isByteArray(typ) { nk = styp.DefaultNilValue()
return String
} }
switch nk {
case rlpstruct.NilKindString:
return String
case rlpstruct.NilKindList:
return List return List
default:
panic("invalid nil kind value")
}
} }
func isUint(k reflect.Kind) bool { func isUint(k reflect.Kind) bool {
@ -277,7 +238,3 @@ func isUint(k reflect.Kind) bool {
func isByte(typ reflect.Type) bool { func isByte(typ reflect.Type) bool {
return typ.Kind() == reflect.Uint8 && !typ.Implements(encoderInterface) return typ.Kind() == reflect.Uint8 && !typ.Implements(encoderInterface)
} }
func isByteArray(typ reflect.Type) bool {
return (typ.Kind() == reflect.Slice || typ.Kind() == reflect.Array) && isByte(typ.Elem())
}

Loading…
Cancel
Save