tests/fuzzers: fuzzbuzz fuzzers for keystore, rlp, trie, whisper (#19910)

* fuzzers: fuzzers for keystore, rlp, trie, whisper (cred to @guidovranken)

* fuzzers: move fuzzers to testdata

* testdata/fuzzers: documentation

* testdata/fuzzers: corpus for rlp

* tests/fuzzers: fixup

fuzzbuzz.yaml

@@ -0,0 +1,36 @@
+# bmt  keystore  rlp  trie  whisperv6
+
+base: ubuntu:16.04
+targets:
+  - name: rlp
+    language: go
+    version: "1.13"
+    corpus: ./fuzzers/rlp/corpus
+    harness:
+      function: Fuzz
+      package: github.com/ethereum/go-ethereum/tests/fuzzers/rlp
+      checkout: github.com/ethereum/go-ethereum/
+  - name: keystore
+    language: go
+    version: "1.13"
+    corpus: ./fuzzers/keystore/corpus
+    harness:
+      function: Fuzz
+      package: github.com/ethereum/go-ethereum/tests/fuzzers/keystore
+      checkout: github.com/ethereum/go-ethereum/
+  - name: trie
+    language: go
+    version: "1.13"
+    corpus: ./fuzzers/trie/corpus
+    harness:
+      function: Fuzz
+      package: github.com/ethereum/go-ethereum/tests/fuzzers/trie
+      checkout: github.com/ethereum/go-ethereum/
+  - name: whisperv6
+    language: go
+    version: "1.13"
+    corpus: ./fuzzers/whisperv6/corpus
+    harness:
+      function: Fuzz
+      package: github.com/ethereum/go-ethereum/tests/fuzzers/whisperv6
+      checkout: github.com/ethereum/go-ethereum/

tests/fuzzers/README.txt

@@ -0,0 +1,45 @@
+## Fuzzers
+
+To run a fuzzer locally, you need [go-fuzz](https://github.com/dvyukov/go-fuzz) installed. 
+
+First build a fuzzing-binary out of the selected package:
+
+```
+(cd ./rlp && CGO_ENABLED=0 go-fuzz-build .)
+```
+That command should generate a `rlp-fuzz.zip` in the `rlp/` directory. If you are already in that directory, you can do
+
+```
+[user@work rlp]$ go-fuzz
+2019/11/26 13:36:54 workers: 6, corpus: 3 (3s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 0, uptime: 3s
+2019/11/26 13:36:57 workers: 6, corpus: 3 (6s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 1054, uptime: 6s
+2019/11/26 13:37:00 workers: 6, corpus: 3 (9s ago), crashers: 0, restarts: 1/8358, execs: 25074 (2786/sec), cover: 1054, uptime: 9s
+2019/11/26 13:37:03 workers: 6, corpus: 3 (12s ago), crashers: 0, restarts: 1/8497, execs: 50986 (4249/sec), cover: 1054, uptime: 12s
+2019/11/26 13:37:06 workers: 6, corpus: 3 (15s ago), crashers: 0, restarts: 1/9330, execs: 74640 (4976/sec), cover: 1054, uptime: 15s
+2019/11/26 13:37:09 workers: 6, corpus: 3 (18s ago), crashers: 0, restarts: 1/9948, execs: 99482 (5527/sec), cover: 1054, uptime: 18s
+2019/11/26 13:37:12 workers: 6, corpus: 3 (21s ago), crashers: 0, restarts: 1/9428, execs: 122568 (5836/sec), cover: 1054, uptime: 21s
+2019/11/26 13:37:15 workers: 6, corpus: 3 (24s ago), crashers: 0, restarts: 1/9676, execs: 145152 (6048/sec), cover: 1054, uptime: 24s
+2019/11/26 13:37:18 workers: 6, corpus: 3 (27s ago), crashers: 0, restarts: 1/9855, execs: 167538 (6205/sec), cover: 1054, uptime: 27s
+2019/11/26 13:37:21 workers: 6, corpus: 3 (30s ago), crashers: 0, restarts: 1/9645, execs: 192901 (6430/sec), cover: 1054, uptime: 30s
+2019/11/26 13:37:24 workers: 6, corpus: 3 (33s ago), crashers: 0, restarts: 1/9967, execs: 219294 (6645/sec), cover: 1054, uptime: 33s
+
+```
+Otherwise: 
+```
+go-fuzz -bin ./rlp/rlp-fuzz.zip
+```
+
+### Notes
+
+Once a 'crasher' is found, the fuzzer tries to avoid reporting the same vector twice, so stores the fault in the `suppressions` folder. Thus, if you 
+e.g. make changes to fix a bug, you should _remove_ all data from the `suppressions`-folder, to verify that the issue is indeed resolved. 
+
+Also, if you have only one and the same exit-point for multiple different types of test, the suppression can make the fuzzer hide differnent types of errors. So make 
+sure that each type of failure is unique (for an example, see the rlp fuzzer, where a counter `i` is used to differentiate between failures: 
+
+```golang
+		if !bytes.Equal(input, output) {
+			panic(fmt.Sprintf("case %d: encode-decode is not equal, \ninput : %x\noutput: %x", i, input, output))
+		}
+```
+

tests/fuzzers/keystore/corpus/0176eaf52ed014ec5c91cf4afa070dd3fd469077-1

@@ -0,0 +1 @@
+ns©›,²Ô

tests/fuzzers/keystore/keystore-fuzzer.go

@@ -0,0 +1,37 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package keystore
+
+import (
+	"os"
+
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+)
+
+func Fuzz(input []byte) int {
+	ks := keystore.NewKeyStore("/tmp/ks", keystore.LightScryptN, keystore.LightScryptP)
+
+	a, err := ks.NewAccount(string(input))
+	if err != nil {
+		panic(err)
+	}
+	if err := ks.Unlock(a, string(input)); err != nil {
+		panic(err)
+	}
+	os.Remove(a.URL.Path)
+	return 0
+}

tests/fuzzers/rlp/corpus/r.bin

@@ -0,0 +1 @@
+Ë€€€À€ÀÃÀÀÀÀ

tests/fuzzers/rlp/corpus/transaction.rlp

@@ -0,0 +1,2 @@
+øN
ƒ“à€€€‚
+• <EFBFBD>aùËåÀP?-'´{ÏЋD<EFBFBD>Y¯<EFBFBD>³fÆj\ÿE÷ ~ì<EFBFBD>•ÒçF?1(íij6<EFBFBD>@Év±LÀÝÚ‘‘

tests/fuzzers/rlp/rlp_fuzzer.go

@@ -0,0 +1,127 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rlp
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+func decodeEncode(input []byte, val interface{}, i int) {
+	if err := rlp.DecodeBytes(input, val); err == nil {
+		output, err := rlp.EncodeToBytes(val)
+		if err != nil {
+			panic(err)
+		}
+		if !bytes.Equal(input, output) {
+			panic(fmt.Sprintf("case %d: encode-decode is not equal, \ninput : %x\noutput: %x", i, input, output))
+		}
+	}
+}
+
+func Fuzz(input []byte) int {
+	var i int
+	{
+		if len(input) > 0 {
+			rlp.Split(input)
+		}
+	}
+	{
+		if len(input) > 0 {
+			if elems, _, err := rlp.SplitList(input); err == nil {
+				rlp.CountValues(elems)
+			}
+		}
+	}
+
+	{
+		rlp.NewStream(bytes.NewReader(input), 0).Decode(new(interface{}))
+	}
+
+	{
+		decodeEncode(input, new(interface{}), i)
+		i++
+	}
+	{
+		var v struct {
+			Int    uint
+			String string
+			Bytes  []byte
+		}
+		decodeEncode(input, &v, i)
+		i++
+	}
+
+	{
+		type Types struct {
+			Bool  bool
+			Raw   rlp.RawValue
+			Slice []*Types
+			Iface []interface{}
+		}
+		var v Types
+		decodeEncode(input, &v, i)
+		i++
+	}
+	{
+		type AllTypes struct {
+			Int    uint
+			String string
+			Bytes  []byte
+			Bool   bool
+			Raw    rlp.RawValue
+			Slice  []*AllTypes
+			Array  [3]*AllTypes
+			Iface  []interface{}
+		}
+		var v AllTypes
+		decodeEncode(input, &v, i)
+		i++
+	}
+	{
+		decodeEncode(input, [10]byte{}, i)
+		i++
+	}
+	{
+		var v struct {
+			Byte [10]byte
+			Rool [10]bool
+		}
+		decodeEncode(input, &v, i)
+		i++
+	}
+	{
+		var h types.Header
+		decodeEncode(input, &h, i)
+		i++
+		var b types.Block
+		decodeEncode(input, &b, i)
+		i++
+		var t types.Transaction
+		decodeEncode(input, &t, i)
+		i++
+		var txs types.Transactions
+		decodeEncode(input, &txs, i)
+		i++
+		var rs types.Receipts
+		decodeEncode(input, &rs, i)
+	}
+	return 0
+}

tests/fuzzers/trie/corpus/data

@@ -0,0 +1 @@
+asdlfkjasf23oiejfasdfadkfqlkjfasdlkfjalwk4jfalsdkfjawlefkjsadlfkjasldkfjwalefkjasdlfkjM

tests/fuzzers/trie/trie-fuzzer.go

@@ -0,0 +1,189 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/trie"
+)
+
+// randTest performs random trie operations.
+// Instances of this test are created by Generate.
+type randTest []randTestStep
+
+type randTestStep struct {
+	op    int
+	key   []byte // for opUpdate, opDelete, opGet
+	value []byte // for opUpdate
+	err   error  // for debugging
+}
+
+type proofDb struct{}
+
+func (proofDb) Put(key []byte, value []byte) error {
+	return nil
+}
+
+func (proofDb) Delete(key []byte) error {
+	return nil
+}
+
+const (
+	opUpdate = iota
+	opDelete
+	opGet
+	opCommit
+	opHash
+	opReset
+	opItercheckhash
+	opProve
+	opMax // boundary value, not an actual op
+)
+
+type dataSource struct {
+	input  []byte
+	reader *bytes.Reader
+}
+
+func newDataSource(input []byte) *dataSource {
+	return &dataSource{
+		input, bytes.NewReader(input),
+	}
+}
+func (ds *dataSource) ReadByte() byte {
+	if b, err := ds.reader.ReadByte(); err != nil {
+		return 0
+	} else {
+		return b
+	}
+}
+func (ds *dataSource) Read(buf []byte) (int, error) {
+	return ds.reader.Read(buf)
+}
+func (ds *dataSource) Ended() bool {
+	return ds.reader.Len() == 0
+}
+
+func Generate(input []byte) randTest {
+
+	var allKeys [][]byte
+	r := newDataSource(input)
+	genKey := func() []byte {
+
+		if len(allKeys) < 2 || r.ReadByte() < 0x0f {
+			// new key
+			key := make([]byte, r.ReadByte()%50)
+			r.Read(key)
+			allKeys = append(allKeys, key)
+			return key
+		}
+		// use existing key
+		return allKeys[int(r.ReadByte())%len(allKeys)]
+	}
+
+	var steps randTest
+
+	for i := 0; !r.Ended(); i++ {
+
+		step := randTestStep{op: int(r.ReadByte()) % opMax}
+		switch step.op {
+		case opUpdate:
+			step.key = genKey()
+			step.value = make([]byte, 8)
+			binary.BigEndian.PutUint64(step.value, uint64(i))
+		case opGet, opDelete, opProve:
+			step.key = genKey()
+		}
+		steps = append(steps, step)
+		if len(steps) > 500 {
+			break
+		}
+	}
+
+	return steps
+}
+
+func Fuzz(input []byte) int {
+	program := Generate(input)
+	if len(program) == 0 {
+		return -1
+	}
+	if err := runRandTest(program); err != nil {
+		panic(err)
+	}
+	return 0
+}
+
+func runRandTest(rt randTest) error {
+
+	triedb := trie.NewDatabase(memorydb.New())
+
+	tr, _ := trie.New(common.Hash{}, triedb)
+	values := make(map[string]string) // tracks content of the trie
+
+	for i, step := range rt {
+		switch step.op {
+		case opUpdate:
+			tr.Update(step.key, step.value)
+			values[string(step.key)] = string(step.value)
+		case opDelete:
+			tr.Delete(step.key)
+			delete(values, string(step.key))
+		case opGet:
+			v := tr.Get(step.key)
+			want := values[string(step.key)]
+			if string(v) != want {
+				rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want)
+			}
+		case opCommit:
+			_, rt[i].err = tr.Commit(nil)
+		case opHash:
+			tr.Hash()
+		case opReset:
+			hash, err := tr.Commit(nil)
+			if err != nil {
+				return err
+			}
+			newtr, err := trie.New(hash, triedb)
+			if err != nil {
+				return err
+			}
+			tr = newtr
+		case opItercheckhash:
+			checktr, _ := trie.New(common.Hash{}, triedb)
+			it := trie.NewIterator(tr.NodeIterator(nil))
+			for it.Next() {
+				checktr.Update(it.Key, it.Value)
+			}
+			if tr.Hash() != checktr.Hash() {
+				return fmt.Errorf("hash mismatch in opItercheckhash")
+			}
+		case opProve:
+			rt[i].err = tr.Prove(step.key, 0, proofDb{})
+		}
+		// Abort the test on error.
+		if rt[i].err != nil {
+			return rt[i].err
+		}
+	}
+	return nil
+}

tests/fuzzers/whisperv6/whisper-fuzzer.go

@@ -0,0 +1,90 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package whisperv6
+
+import (
+	"bytes"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/whisper/whisperv6"
+)
+
+type MessageParams struct {
+	Topic    whisperv6.TopicType
+	WorkTime uint32
+	TTL      uint32
+	KeySym   []byte
+	Payload  []byte
+}
+
+//export fuzzer_entry
+func Fuzz(input []byte) int {
+
+	var paramsDecoded MessageParams
+	err := rlp.DecodeBytes(input, &paramsDecoded)
+	if err != nil {
+		return 0
+	}
+	var params whisperv6.MessageParams
+	params.KeySym = make([]byte, 32)
+	if len(paramsDecoded.KeySym) <= 32 {
+		copy(params.KeySym, paramsDecoded.KeySym)
+	}
+	if input[0] == 255 {
+		params.PoW = 0.01
+		params.WorkTime = 1
+	} else {
+		params.PoW = 0
+		params.WorkTime = 0
+	}
+	params.TTL = paramsDecoded.TTL
+	params.Payload = paramsDecoded.Payload
+	text := make([]byte, 0, 512)
+	text = append(text, params.Payload...)
+	params.Topic = paramsDecoded.Topic
+	params.Src, err = crypto.GenerateKey()
+	if err != nil {
+		return 0
+	}
+	msg, err := whisperv6.NewSentMessage(&params)
+	if err != nil {
+		panic(err)
+		//return
+	}
+	env, err := msg.Wrap(&params)
+	if err != nil {
+		panic(err)
+	}
+	decrypted, err := env.OpenSymmetric(params.KeySym)
+	if err != nil {
+		panic(err)
+	}
+	if !decrypted.ValidateAndParse() {
+		panic("ValidateAndParse failed")
+	}
+	if !bytes.Equal(text, decrypted.Payload) {
+		panic("text != decrypted.Payload")
+	}
+	if len(decrypted.Signature) != 65 {
+		panic("Unexpected signature length")
+	}
+	if !whisperv6.IsPubKeyEqual(decrypted.Src, &params.Src.PublicKey) {
+		panic("Unexpected public key")
+	}
+	return 0
+}