diff --git a/.gitignore b/.gitignore index 3016facee0..1ee8b83022 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ build/_vendor/pkg # used by the Makefile /build/_workspace/ +/build/cache/ /build/bin/ /geth*.zip diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000000..44fce8bea3 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,45 @@ +# This file configures github.com/golangci/golangci-lint. + +run: + timeout: 2m + tests: true + # default is true. Enables skipping of directories: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs-use-default: true + +linters: + disable-all: true + enable: + - deadcode + - goconst + - goimports + - gosimple + - govet + - ineffassign + - misspell + # - staticcheck + - unconvert + # - unused + - varcheck + +linters-settings: + gofmt: + simplify: true + goconst: + min-len: 3 # minimum length of string constant + min-occurrences: 6 # minimum number of occurrences + +issues: + exclude-rules: + - path: crypto/blake2b/ + linters: + - deadcode + - path: crypto/bn256/cloudflare + linters: + - deadcode + - path: p2p/discv5/ + linters: + - deadcode + - path: core/vm/instructions_test.go + linters: + - goconst diff --git a/.travis.yml b/.travis.yml index 5366c691b7..9d62046848 100644 --- a/.travis.yml +++ b/.travis.yml @@ -87,12 +87,9 @@ jobs: - fakeroot - python-bzrlib - python-paramiko - cache: - directories: - - $HOME/.gobundle script: - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts - - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " -goversion 1.13.4 -gohash 95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 -gobundle $HOME/.gobundle/go.tar.gz + - go run build/ci.go debsrc -goversion 1.13.4 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " # This builder does the Linux Azure uploads - stage: build diff --git a/build/checksums.txt b/build/checksums.txt new file mode 100644 index 0000000000..bb814e3392 --- /dev/null +++ b/build/checksums.txt @@ -0,0 +1,19 @@ +# This file contains sha256 checksums of optional build dependencies. + +95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 go1.13.4.src.tar.gz + +1fcbc9e36f4319eeed02beb8cfd1b3d425ffc2f90ddf09a80f18d5064c51e0cb golangci-lint-1.21.0-linux-386.tar.gz +267b4066e67139a38d29499331a002d6a29ad5be7aafc83db3b1e88f1b027f90 golangci-lint-1.21.0-linux-armv6.tar.gz +a602c1f25f90e46e621019cff0a8cb3f4e1837011f3537f15e730d6a9ebf507b golangci-lint-1.21.0-freebsd-armv7.tar.gz +2c861f8dc56b560474aa27cab0c075991628cc01af3451e27ac82f5d10d5106b golangci-lint-1.21.0-linux-amd64.tar.gz +a1c39e055280e755acaa906e7abfc20b99a5c28be8af541c57fbc44abbb20dde golangci-lint-1.21.0-linux-arm64.tar.gz +a8f8bda8c6a4136acf858091077830b1e83ad5612606cb69d5dced869ce00bd8 golangci-lint-1.21.0-linux-ppc64le.tar.gz +0a8a8c3bc660ccbca668897ab520f7ee9878f16cc8e4dd24fe46236ceec97ba3 golangci-lint-1.21.0-freebsd-armv6.tar.gz +699b07f45e216571f54002bcbd83b511c4801464a422162158e299587b095b18 golangci-lint-1.21.0-freebsd-amd64.tar.gz +980fb4993942154bb5c8129ea3b86de09574fe81b24384ebb58cd7a9d2f04483 golangci-lint-1.21.0-linux-armv7.tar.gz +f15b689088a47f20d5d3c1d945e9ee7c6238f2b84ea468b5f886cf8713dce62e golangci-lint-1.21.0-windows-386.zip +2e40ded7adcf11e59013cb15c24438b15a86526ca241edfcfdf1abd73a5280a8 golangci-lint-1.21.0-windows-amd64.zip +6052c7cfea4d6dc2fc722f6c12792a5ec087420198db495afffbc22052653bf7 golangci-lint-1.21.0-freebsd-386.tar.gz +ca00b8eacf9af14a71b908b4149606c762aa5c0eac781e74ca0abedfdfdf6c8c golangci-lint-1.21.0-linux-s390x.tar.gz +1365455940c342f95718159d89d66ad2eef19f0846c3e87023e915a3527b929f golangci-lint-1.21.0-darwin-386.tar.gz +2b2713ec5007e67883aa501eebb81f22abfab0cf0909134ba90f60a066db3760 golangci-lint-1.21.0-darwin-amd64.tar.gz diff --git a/build/ci.go b/build/ci.go index 55a632d3b2..f4d336b647 100644 --- a/build/ci.go +++ b/build/ci.go @@ -58,7 +58,6 @@ import ( "strings" "time" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/internal/build" "github.com/ethereum/go-ethereum/params" ) @@ -331,7 +330,7 @@ func doTest(cmdline []string) { // Test a single package at a time. CI builders are slow // and some tests run into timeouts under load. gotest := goTool("test", buildFlags(env)...) - gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m", "--short") + gotest.Args = append(gotest.Args, "-p", "1", "-timeout", "5m") if *coverage { gotest.Args = append(gotest.Args, "-covermode=atomic", "-cover") } @@ -340,39 +339,38 @@ func doTest(cmdline []string) { build.MustRun(gotest) } -// runs gometalinter on requested packages +// doLint runs golangci-lint on requested packages. func doLint(cmdline []string) { + var ( + cachedir = flag.String("cachedir", "./build/cache", "directory for caching golangci-lint binary.") + ) flag.CommandLine.Parse(cmdline) - packages := []string{"./..."} if len(flag.CommandLine.Args()) > 0 { packages = flag.CommandLine.Args() } - // Get metalinter and install all supported linters - build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2")) - build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install") - - // Run fast linters batched together - configs := []string{ - "--vendor", - "--tests", - "--deadline=2m", - "--disable-all", - "--enable=goimports", - "--enable=varcheck", - "--enable=vet", - "--enable=gofmt", - "--enable=misspell", - "--enable=goconst", - "--min-occurrences=6", // for goconst - } - build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) - - // Run slow linters one by one - for _, linter := range []string{"unconvert", "gosimple"} { - configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter} - build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...) + + linter := downloadLinter(*cachedir) + lflags := []string{"run", "--config", ".golangci.yml"} + build.MustRunCommand(linter, append(lflags, packages...)...) + fmt.Println("You have achieved perfection.") +} + +// downloadLinter downloads and unpacks golangci-lint. +func downloadLinter(cachedir string) string { + const version = "1.21.0" + + csdb := build.MustLoadChecksums("build/checksums.txt") + base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) + url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s.tar.gz", version, base) + archivePath := filepath.Join(cachedir, base+".tar.gz") + if err := csdb.DownloadFile(url, archivePath); err != nil { + log.Fatal(err) + } + if err := build.ExtractTarballArchive(archivePath, cachedir); err != nil { + log.Fatal(err) } + return filepath.Join(cachedir, base, "golangci-lint") } // Release Packaging @@ -476,8 +474,7 @@ func maybeSkipArchive(env build.Environment) { func doDebianSource(cmdline []string) { var ( goversion = flag.String("goversion", "", `Go version to build with (will be included in the source package)`) - gobundle = flag.String("gobundle", "/tmp/go.tar.gz", `Filesystem path to cache the downloaded Go bundles at`) - gohash = flag.String("gohash", "", `SHA256 checksum of the Go sources requested to build with`) + cachedir = flag.String("cachedir", "./build/cache", `Filesystem path to cache the downloaded Go bundles at`) signer = flag.String("signer", "", `Signing key name, also used as package author`) upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) @@ -495,24 +492,25 @@ func doDebianSource(cmdline []string) { gpg.Stdin = bytes.NewReader(key) build.MustRun(gpg) } - // Download and verify the Go source package - if err := build.EnsureGoSources(*goversion, hexutil.MustDecode("0x"+*gohash), *gobundle); err != nil { - log.Fatalf("Failed to ensure Go source package: %v", err) - } - // Create Debian packages and upload them + + // Download and verify the Go source package. + gobundle := downloadGoSources(*goversion, *cachedir) + + // Create Debian packages and upload them. for _, pkg := range debPackages { for distro, goboot := range debDistroGoBoots { - // Prepare the debian package with the go-ethereum sources + // Prepare the debian package with the go-ethereum sources. meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) pkgdir := stageDebianSource(*workdir, meta) - // Ship the Go sources along so we have a proper thing to build with - if err := build.ExtractTarballArchive(*gobundle, pkgdir); err != nil { + // Add Go source code. + if err := build.ExtractTarballArchive(gobundle, pkgdir); err != nil { log.Fatalf("Failed to extract Go sources: %v", err) } if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { log.Fatalf("Failed to rename Go source folder: %v", err) } + // Run the packaging and upload to the PPA debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz") debuild.Dir = pkgdir @@ -534,6 +532,17 @@ func doDebianSource(cmdline []string) { } } +func downloadGoSources(version string, cachedir string) string { + csdb := build.MustLoadChecksums("build/checksums.txt") + file := fmt.Sprintf("go%s.src.tar.gz", version) + url := "https://dl.google.com/go/" + file + dst := filepath.Join(cachedir, file) + if err := csdb.DownloadFile(url, dst); err != nil { + log.Fatal(err) + } + return dst +} + func ppaUpload(workdir, ppa, sshUser string, files []string) { p := strings.Split(ppa, "/") if len(p) != 2 { diff --git a/cmd/clef/main.go b/cmd/clef/main.go index d34f8c28d2..9172b49021 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -760,21 +760,19 @@ func testExternalUI(api *core.SignerAPI) { api.UI.ShowInfo("Please approve the next request for signing a clique header") time.Sleep(delay) cliqueHeader := types.Header{ - common.HexToHash("0000H45H"), - common.HexToHash("0000H45H"), - common.HexToAddress("0000H45H"), - common.HexToHash("0000H00H"), - common.HexToHash("0000H45H"), - common.HexToHash("0000H45H"), - types.Bloom{}, - big.NewInt(1337), - big.NewInt(1337), - 1338, - 1338, - 1338, - []byte("Extra data Extra data Extra data Extra data Extra data Extra data Extra data Extra data"), - common.HexToHash("0x0000H45H"), - types.BlockNonce{}, + ParentHash: common.HexToHash("0000H45H"), + UncleHash: common.HexToHash("0000H45H"), + Coinbase: common.HexToAddress("0000H45H"), + Root: common.HexToHash("0000H00H"), + TxHash: common.HexToHash("0000H45H"), + ReceiptHash: common.HexToHash("0000H45H"), + Difficulty: big.NewInt(1337), + Number: big.NewInt(1337), + GasLimit: 1338, + GasUsed: 1338, + Time: 1338, + Extra: []byte("Extra data Extra data Extra data Extra data Extra data Extra data Extra data Extra data"), + MixDigest: common.HexToHash("0x0000H45H"), } cliqueRlp, err := rlp.EncodeToBytes(cliqueHeader) if err != nil { @@ -938,7 +936,7 @@ func GenDoc(ctx *cli.Context) { "of the work in canonicalizing and making sense of the data, and it's up to the UI to present" + "the user with the contents of the `message`" sighash, msg := accounts.TextAndHash([]byte("hello world")) - messages := []*core.NameValueType{{"message", msg, accounts.MimetypeTextPlain}} + messages := []*core.NameValueType{{Name: "message", Value: msg, Typ: accounts.MimetypeTextPlain}} add("SignDataRequest", desc, &core.SignDataRequest{ Address: common.NewMixedcaseAddress(a), @@ -969,8 +967,8 @@ func GenDoc(ctx *cli.Context) { add("SignTxRequest", desc, &core.SignTxRequest{ Meta: meta, Callinfo: []core.ValidationInfo{ - {"Warning", "Something looks odd, show this message as a warning"}, - {"Info", "User should see this aswell"}, + {Typ: "Warning", Message: "Something looks odd, show this message as a warning"}, + {Typ: "Info", Message: "User should see this as well"}, }, Transaction: core.SendTxArgs{ Data: &data, @@ -1036,16 +1034,21 @@ func GenDoc(ctx *cli.Context) { &core.ListRequest{ Meta: meta, Accounts: []accounts.Account{ - {a, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/a"}}, - {b, accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/b"}}}, + {Address: a, URL: accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/a"}}, + {Address: b, URL: accounts.URL{Scheme: "keystore", Path: "/path/to/keyfile/b"}}}, }) add("ListResponse", "Response to list request. The response contains a list of all addresses to show to the caller. "+ "Note: the UI is free to respond with any address the caller, regardless of whether it exists or not", &core.ListResponse{ Accounts: []accounts.Account{ - {common.HexToAddress("0xcowbeef000000cowbeef00000000000000000c0w"), accounts.URL{Path: ".. ignored .."}}, - {common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"), accounts.URL{}}, + { + Address: common.HexToAddress("0xcowbeef000000cowbeef00000000000000000c0w"), + URL: accounts.URL{Path: ".. ignored .."}, + }, + { + Address: common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"), + }, }}) } diff --git a/cmd/utils/customflags.go b/cmd/utils/customflags.go index 66ebf9ab04..17dcba5f74 100644 --- a/cmd/utils/customflags.go +++ b/cmd/utils/customflags.go @@ -185,28 +185,6 @@ func GlobalBig(ctx *cli.Context, name string) *big.Int { return (*big.Int)(val.(*bigValue)) } -func prefixFor(name string) (prefix string) { - if len(name) == 1 { - prefix = "-" - } else { - prefix = "--" - } - - return -} - -func prefixedNames(fullName string) (prefixed string) { - parts := strings.Split(fullName, ",") - for i, name := range parts { - name = strings.Trim(name, " ") - prefixed += prefixFor(name) + name - if i < len(parts)-1 { - prefixed += ", " - } - } - return -} - // Expands a file path // 1. replace tilde with users home dir // 2. expands embedded environment variables diff --git a/core/blockchain.go b/core/blockchain.go index 9fb02b1482..d19fe6dd16 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -42,7 +42,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru" ) var ( diff --git a/core/forkid/forkid.go b/core/forkid/forkid.go index 1e2d7a7441..e433db4460 100644 --- a/core/forkid/forkid.go +++ b/core/forkid/forkid.go @@ -186,13 +186,6 @@ func newFilter(config *params.ChainConfig, genesis common.Hash, headfn func() ui } } -// checksum calculates the IEEE CRC32 checksum of a block number. -func checksum(fork uint64) uint32 { - var blob [8]byte - binary.BigEndian.PutUint64(blob[:], fork) - return crc32.ChecksumIEEE(blob[:]) -} - // checksumUpdate calculates the next IEEE CRC32 checksum based on the previous // one and a fork block number (equivalent to CRC32(original-blob || fork)). func checksumUpdate(hash uint32, fork uint64) uint32 { diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 7de1081517..e4a22b3768 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -41,14 +41,6 @@ func getChunk(size int, b int) []byte { return data } -func print(t *testing.T, f *freezerTable, item uint64) { - a, err := f.Retrieve(item) - if err != nil { - t.Fatal(err) - } - t.Logf("db[%d] = %x\n", item, a) -} - // TestFreezerBasics test initializing a freezertable from scratch, writing to the table, // and reading it back. func TestFreezerBasics(t *testing.T) { diff --git a/core/state/state_object_test.go b/core/state/state_object_test.go index e86d3b9943..42fd778025 100644 --- a/core/state/state_object_test.go +++ b/core/state/state_object_test.go @@ -18,10 +18,7 @@ package state import ( "bytes" - "fmt" - "math/rand" "testing" - "time" "github.com/ethereum/go-ethereum/common" ) @@ -35,9 +32,7 @@ func BenchmarkCutOriginal(b *testing.B) { func BenchmarkCutsetterFn(b *testing.B) { value := common.HexToHash("0x01") - cutSetFn := func(r rune) bool { - return int32(r) == int32(0) - } + cutSetFn := func(r rune) bool { return r == 0 } for i := 0; i < b.N; i++ { bytes.TrimLeftFunc(value[:], cutSetFn) } @@ -49,22 +44,3 @@ func BenchmarkCutCustomTrim(b *testing.B) { common.TrimLeftZeroes(value[:]) } } - -func xTestFuzzCutter(t *testing.T) { - rand.Seed(time.Now().Unix()) - for { - v := make([]byte, 20) - zeroes := rand.Intn(21) - rand.Read(v[zeroes:]) - exp := bytes.TrimLeft(v[:], "\x00") - got := common.TrimLeftZeroes(v) - if !bytes.Equal(exp, got) { - - fmt.Printf("Input %x\n", v) - fmt.Printf("Exp %x\n", exp) - fmt.Printf("Got %x\n", got) - t.Fatalf("Error") - } - //break - } -} diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index b12df39057..bdc62d1777 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -232,7 +232,9 @@ func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcas // utility function to fill the json-file with testcases // Enable this test to generate the 'testcases_xx.json' files -func xTestWriteExpectedValues(t *testing.T) { +func TestWriteExpectedValues(t *testing.T) { + t.Skip("Enable this test to create json test cases.") + for name, method := range twoOpMethods { data, err := json.Marshal(getResult(commonParams, method)) if err != nil { @@ -243,7 +245,6 @@ func xTestWriteExpectedValues(t *testing.T) { t.Fatal(err) } } - t.Fatal("This test should not be activated") } // TestJsonTestcases runs through all the testcases defined as json-files diff --git a/crypto/bn256/bn256_fast.go b/crypto/bn256/bn256_fast.go index 5c081493b0..14b5965393 100644 --- a/crypto/bn256/bn256_fast.go +++ b/crypto/bn256/bn256_fast.go @@ -7,17 +7,19 @@ // Package bn256 implements the Optimal Ate pairing over a 256-bit Barreto-Naehrig curve. package bn256 -import "github.com/ethereum/go-ethereum/crypto/bn256/cloudflare" +import ( + bn256cf "github.com/ethereum/go-ethereum/crypto/bn256/cloudflare" +) // G1 is an abstract cyclic group. The zero value is suitable for use as the // output of an operation, but cannot be used as an input. -type G1 = bn256.G1 +type G1 = bn256cf.G1 // G2 is an abstract cyclic group. The zero value is suitable for use as the // output of an operation, but cannot be used as an input. -type G2 = bn256.G2 +type G2 = bn256cf.G2 // PairingCheck calculates the Optimal Ate pairing for a set of points. func PairingCheck(a []*G1, b []*G2) bool { - return bn256.PairingCheck(a, b) + return bn256cf.PairingCheck(a, b) } diff --git a/crypto/ecies/ecies_test.go b/crypto/ecies/ecies_test.go index 2def505d0c..b465f076f4 100644 --- a/crypto/ecies/ecies_test.go +++ b/crypto/ecies/ecies_test.go @@ -66,22 +66,6 @@ func cmpParams(p1, p2 *ECIESParams) bool { p1.BlockSize == p2.BlockSize } -// cmpPublic returns true if the two public keys represent the same pojnt. -func cmpPublic(pub1, pub2 PublicKey) bool { - if pub1.X == nil || pub1.Y == nil { - fmt.Println(ErrInvalidPublicKey.Error()) - return false - } - if pub2.X == nil || pub2.Y == nil { - fmt.Println(ErrInvalidPublicKey.Error()) - return false - } - pub1Out := elliptic.Marshal(pub1.Curve, pub1.X, pub1.Y) - pub2Out := elliptic.Marshal(pub2.Curve, pub2.X, pub2.Y) - - return bytes.Equal(pub1Out, pub2Out) -} - // Validate the ECDH component. func TestSharedKey(t *testing.T) { prv1, err := GenerateKey(rand.Reader, DefaultCurve, nil) diff --git a/internal/build/archive.go b/internal/build/archive.go index 8571edd5a5..a00258d999 100644 --- a/internal/build/archive.go +++ b/internal/build/archive.go @@ -213,11 +213,10 @@ func ExtractTarballArchive(archive string, dest string) error { target := filepath.Join(dest, header.Name) switch header.Typeflag { - case tar.TypeDir: - if err := os.MkdirAll(target, 0755); err != nil { + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { return err } - case tar.TypeReg: file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) if err != nil { return err diff --git a/internal/build/download.go b/internal/build/download.go new file mode 100644 index 0000000000..c506800295 --- /dev/null +++ b/internal/build/download.go @@ -0,0 +1,149 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package build + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "path/filepath" + "strings" +) + +// ChecksumDB keeps file checksums. +type ChecksumDB struct { + allChecksums []string +} + +// MustLoadChecksums loads a file containing checksums. +func MustLoadChecksums(file string) *ChecksumDB { + content, err := ioutil.ReadFile(file) + if err != nil { + log.Fatal("can't load checksum file: " + err.Error()) + } + return &ChecksumDB{strings.Split(string(content), "\n")} +} + +// Verify checks whether the given file is valid according to the checksum database. +func (db *ChecksumDB) Verify(path string) error { + fd, err := os.Open(path) + if err != nil { + return err + } + defer fd.Close() + + h := sha256.New() + if _, err := io.Copy(h, bufio.NewReader(fd)); err != nil { + return err + } + fileHash := hex.EncodeToString(h.Sum(nil)) + if !db.findHash(filepath.Base(path), fileHash) { + return fmt.Errorf("invalid file hash %s", fileHash) + } + return nil +} + +func (db *ChecksumDB) findHash(basename, hash string) bool { + want := hash + " " + basename + for _, line := range db.allChecksums { + if strings.TrimSpace(line) == want { + return true + } + } + return false +} + +// DownloadFile downloads a file and verifies its checksum. +func (db *ChecksumDB) DownloadFile(url, dstPath string) error { + if err := db.Verify(dstPath); err == nil { + fmt.Printf("%s is up-to-date\n", dstPath) + return nil + } + fmt.Printf("%s is stale\n", dstPath) + fmt.Printf("downloading from %s\n", url) + + resp, err := http.Get(url) + if err != nil || resp.StatusCode != http.StatusOK { + return fmt.Errorf("download error: code %d, err %v", resp.StatusCode, err) + } + defer resp.Body.Close() + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + return err + } + fd, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return err + } + dst := newDownloadWriter(fd, resp.ContentLength) + _, err = io.Copy(dst, resp.Body) + dst.Close() + if err != nil { + return err + } + + return db.Verify(dstPath) +} + +type downloadWriter struct { + file *os.File + dstBuf *bufio.Writer + size int64 + written int64 + lastpct int64 +} + +func newDownloadWriter(dst *os.File, size int64) *downloadWriter { + return &downloadWriter{ + file: dst, + dstBuf: bufio.NewWriter(dst), + size: size, + } +} + +func (w *downloadWriter) Write(buf []byte) (int, error) { + n, err := w.dstBuf.Write(buf) + + // Report progress. + w.written += int64(n) + pct := w.written * 10 / w.size * 10 + if pct != w.lastpct { + if w.lastpct != 0 { + fmt.Print("...") + } + fmt.Print(pct, "%") + w.lastpct = pct + } + return n, err +} + +func (w *downloadWriter) Close() error { + if w.lastpct > 0 { + fmt.Println() // Finish the progress line. + } + flushErr := w.dstBuf.Flush() + closeErr := w.file.Close() + if flushErr != nil { + return flushErr + } + return closeErr +} diff --git a/internal/build/gosrc.go b/internal/build/gosrc.go deleted file mode 100644 index c85e469680..0000000000 --- a/internal/build/gosrc.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package build - -import ( - "bytes" - "crypto/sha256" - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "strings" -) - -// EnsureGoSources ensures that path contains a file with the given SHA256 hash, -// and if not, it downloads a fresh Go source package from upstream and replaces -// path with it (if the hash matches). -func EnsureGoSources(version string, hash []byte, path string) error { - // Sanity check the destination path to ensure we don't do weird things - if !strings.HasSuffix(path, ".tar.gz") { - return fmt.Errorf("destination path (%s) must end with .tar.gz", path) - } - // If the file exists, validate it's hash - if archive, err := ioutil.ReadFile(path); err == nil { // Go sources are ~20MB, it's fine to read all - hasher := sha256.New() - hasher.Write(archive) - have := hasher.Sum(nil) - - if bytes.Equal(have, hash) { - fmt.Printf("Go %s [%x] available at %s\n", version, hash, path) - return nil - } - fmt.Printf("Go %s hash mismatch (have %x, want %x) at %s, deleting old archive\n", version, have, hash, path) - if err := os.Remove(path); err != nil { - return err - } - } - // Archive missing or bad hash, download a new one - fmt.Printf("Downloading Go %s [want %x] into %s\n", version, hash, path) - - res, err := http.Get(fmt.Sprintf("https://dl.google.com/go/go%s.src.tar.gz", version)) - if err != nil || res.StatusCode != http.StatusOK { - return fmt.Errorf("failed to access Go sources: code %d, err %v", res.StatusCode, err) - } - defer res.Body.Close() - - archive, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - // Sanity check the downloaded archive, save if checks out - hasher := sha256.New() - hasher.Write(archive) - - if have := hasher.Sum(nil); !bytes.Equal(have, hash) { - return fmt.Errorf("downloaded Go %s hash mismatch (have %x, want %x)", version, have, hash) - } - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return err - } - if err := ioutil.WriteFile(path, archive, 0644); err != nil { - return err - } - fmt.Printf("Downloaded Go %s [%x] into %s\n", version, hash, path) - return nil -} diff --git a/les/api_test.go b/les/api_test.go index 660af8eeec..06a519b62b 100644 --- a/les/api_test.go +++ b/les/api_test.go @@ -326,7 +326,8 @@ func testRequest(ctx context.Context, t *testing.T, client *rpc.Client) bool { var res string var addr common.Address rand.Read(addr[:]) - c, _ := context.WithTimeout(ctx, time.Second*12) + c, cancel := context.WithTimeout(ctx, time.Second*12) + defer cancel() err := client.CallContext(c, &res, "eth_getBalance", addr, "latest") if err != nil { t.Log("request error:", err) diff --git a/les/clientpool.go b/les/clientpool.go index 0169dc706b..b553783acd 100644 --- a/les/clientpool.go +++ b/les/clientpool.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" - "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru" ) const ( diff --git a/les/protocol.go b/les/protocol.go index 5fdf32b74a..36af88aea6 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -235,5 +235,3 @@ func (hn *hashOrNumber) DecodeRLP(s *rlp.Stream) error { type CodeData []struct { Value []byte } - -type proofsData [][]rlp.RawValue diff --git a/p2p/nat/nat.go b/p2p/nat/nat.go index 8fad921c48..504b7b074a 100644 --- a/p2p/nat/nat.go +++ b/p2p/nat/nat.go @@ -26,7 +26,7 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "github.com/jackpal/go-nat-pmp" + natpmp "github.com/jackpal/go-nat-pmp" ) // An implementation of nat.Interface can map local ports to ports diff --git a/p2p/nat/natpmp.go b/p2p/nat/natpmp.go index 8ba9714728..7f85543f8e 100644 --- a/p2p/nat/natpmp.go +++ b/p2p/nat/natpmp.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/jackpal/go-nat-pmp" + natpmp "github.com/jackpal/go-nat-pmp" ) // natPMPClient adapts the NAT-PMP protocol implementation so it conforms to diff --git a/p2p/simulations/events.go b/p2p/simulations/events.go index 984c2e088f..d0d03794ed 100644 --- a/p2p/simulations/events.go +++ b/p2p/simulations/events.go @@ -73,8 +73,7 @@ func NewEvent(v interface{}) *Event { switch v := v.(type) { case *Node: event.Type = EventTypeNode - node := *v - event.Node = &node + event.Node = v.copy() case *Conn: event.Type = EventTypeConn conn := *v diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index 84f6ce2a51..e88999f48b 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -420,16 +420,8 @@ type expectEvents struct { } func (t *expectEvents) nodeEvent(id string, up bool) *Event { - node := Node{ - Config: &adapters.NodeConfig{ - ID: enode.HexID(id), - }, - up: up, - } - return &Event{ - Type: EventTypeNode, - Node: &node, - } + config := &adapters.NodeConfig{ID: enode.HexID(id)} + return &Event{Type: EventTypeNode, Node: newNode(nil, config, up)} } func (t *expectEvents) connEvent(one, other string, up bool) *Event { @@ -450,7 +442,7 @@ loop: for { select { case event := <-t.events: - t.Logf("received %s event: %s", event.Type, event) + t.Logf("received %s event: %v", event.Type, event) if event.Type != EventTypeMsg || event.Msg.Received { continue loop @@ -486,7 +478,7 @@ func (t *expectEvents) expect(events ...*Event) { for { select { case event := <-t.events: - t.Logf("received %s event: %s", event.Type, event) + t.Logf("received %s event: %v", event.Type, event) expected := events[i] if event.Type != expected.Type { diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 58fd9a28b0..ef5451e77e 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -119,10 +119,7 @@ func (net *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error) if err != nil { return nil, err } - node := &Node{ - Node: adapterNode, - Config: conf, - } + node := newNode(adapterNode, conf, false) log.Trace("Node created", "id", conf.ID) nodeIndex := len(net.Nodes) @@ -448,7 +445,7 @@ func (net *Network) GetNodeIDs(excludeIDs ...enode.ID) []enode.ID { } func (net *Network) getNodeIDs(excludeIDs []enode.ID) []enode.ID { - // Get all curent nodeIDs + // Get all current nodeIDs nodeIDs := make([]enode.ID, 0, len(net.nodeMap)) for id := range net.nodeMap { nodeIDs = append(nodeIDs, id) @@ -735,7 +732,16 @@ type Node struct { // up tracks whether or not the node is running up bool - upMu sync.RWMutex + upMu *sync.RWMutex +} + +func newNode(an adapters.Node, ac *adapters.NodeConfig, up bool) *Node { + return &Node{Node: an, Config: ac, up: up, upMu: new(sync.RWMutex)} +} + +func (n *Node) copy() *Node { + configCpy := *n.Config + return newNode(n.Node, &configCpy, n.Up()) } // Up returns whether the node is currently up (online) @@ -787,22 +793,19 @@ func (n *Node) MarshalJSON() ([]byte, error) { }) } -// UnmarshalJSON implements json.Unmarshaler interface so that we don't lose -// Node.up status. IMPORTANT: The implementation is incomplete; we lose -// p2p.NodeInfo. +// UnmarshalJSON implements json.Unmarshaler interface so that we don't lose Node.up +// status. IMPORTANT: The implementation is incomplete; we lose p2p.NodeInfo. func (n *Node) UnmarshalJSON(raw []byte) error { // TODO: How should we turn back NodeInfo into n.Node? // Ticket: https://github.com/ethersphere/go-ethereum/issues/1177 - node := struct { + var node struct { Config *adapters.NodeConfig `json:"config,omitempty"` Up bool `json:"up"` - }{} + } if err := json.Unmarshal(raw, &node); err != nil { return err } - - n.SetUp(node.Up) - n.Config = node.Config + *n = *newNode(nil, node.Config, node.Up) return nil } @@ -899,7 +902,7 @@ func (net *Network) snapshot(addServices []string, removeServices []string) (*Sn Nodes: make([]NodeSnapshot, len(net.Nodes)), } for i, node := range net.Nodes { - snap.Nodes[i] = NodeSnapshot{Node: *node} + snap.Nodes[i] = NodeSnapshot{Node: *node.copy()} if !node.Up() { continue } diff --git a/p2p/simulations/network_test.go b/p2p/simulations/network_test.go index f504b9a698..d0cb592036 100644 --- a/p2p/simulations/network_test.go +++ b/p2p/simulations/network_test.go @@ -758,27 +758,22 @@ func benchmarkMinimalServiceTmp(b *testing.B) { } func TestNode_UnmarshalJSON(t *testing.T) { - t.Run( - "test unmarshal of Node up field", - func(t *testing.T) { - runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONUpField()) - }, - ) - t.Run( - "test unmarshal of Node Config field", - func(t *testing.T) { - runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONConfigField()) - }, - ) + t.Run("up_field", func(t *testing.T) { + runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONUpField()) + }) + t.Run("config_field", func(t *testing.T) { + runNodeUnmarshalJSON(t, casesNodeUnmarshalJSONConfigField()) + }) } func runNodeUnmarshalJSON(t *testing.T, tests []nodeUnmarshalTestCase) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var got Node - if err := got.UnmarshalJSON([]byte(tt.marshaled)); err != nil { + var got *Node + if err := json.Unmarshal([]byte(tt.marshaled), &got); err != nil { expectErrorMessageToContain(t, err, tt.wantErr) + got = nil } expectNodeEquality(t, got, tt.want) }) @@ -788,7 +783,7 @@ func runNodeUnmarshalJSON(t *testing.T, tests []nodeUnmarshalTestCase) { type nodeUnmarshalTestCase struct { name string marshaled string - want Node + want *Node wantErr string } @@ -812,7 +807,7 @@ func expectErrorMessageToContain(t *testing.T, got error, want string) { } } -func expectNodeEquality(t *testing.T, got Node, want Node) { +func expectNodeEquality(t *testing.T, got, want *Node) { t.Helper() if !reflect.DeepEqual(got, want) { t.Errorf("Node.UnmarshalJSON() = %v, want %v", got, want) @@ -824,23 +819,17 @@ func casesNodeUnmarshalJSONUpField() []nodeUnmarshalTestCase { { name: "empty json", marshaled: "{}", - want: Node{ - up: false, - }, + want: newNode(nil, nil, false), }, { name: "a stopped node", marshaled: "{\"up\": false}", - want: Node{ - up: false, - }, + want: newNode(nil, nil, false), }, { name: "a running node", marshaled: "{\"up\": true}", - want: Node{ - up: true, - }, + want: newNode(nil, nil, true), }, { name: "invalid JSON value on valid key", @@ -867,26 +856,17 @@ func casesNodeUnmarshalJSONConfigField() []nodeUnmarshalTestCase { { name: "Config field is omitted", marshaled: "{}", - want: Node{ - Config: nil, - }, + want: newNode(nil, nil, false), }, { name: "Config field is nil", - marshaled: "{\"config\": nil}", - want: Node{ - Config: nil, - }, + marshaled: "{\"config\": null}", + want: newNode(nil, nil, false), }, { name: "a non default Config field", marshaled: "{\"config\":{\"name\":\"node_ecdd0\",\"port\":44665}}", - want: Node{ - Config: &adapters.NodeConfig{ - Name: "node_ecdd0", - Port: 44665, - }, - }, + want: newNode(nil, &adapters.NodeConfig{Name: "node_ecdd0", Port: 44665}, false), }, } } diff --git a/rlp/decode.go b/rlp/decode.go index 524395915d..a001866b9d 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -152,7 +152,6 @@ func makeDecoder(typ reflect.Type, tags tags) (dec decoder, err error) { switch { case typ == rawValueType: return decodeRawValue, nil - return decodeDecoder, nil case typ.AssignableTo(reflect.PtrTo(bigInt)): return decodeBigInt, nil case typ.AssignableTo(bigInt): diff --git a/signer/core/signed_data_test.go b/signer/core/signed_data_test.go index 6b0da45530..e5f4780890 100644 --- a/signer/core/signed_data_test.go +++ b/signer/core/signed_data_test.go @@ -358,7 +358,7 @@ func TestJsonFiles(t *testing.T) { continue } var typedData core.TypedData - err = json.Unmarshal([]byte(data), &typedData) + err = json.Unmarshal(data, &typedData) if err != nil { t.Errorf("Test %d, file %v, json unmarshalling failed: %v", i, fInfo.Name(), err) continue @@ -390,7 +390,7 @@ func TestFuzzerFiles(t *testing.T) { continue } var typedData core.TypedData - err = json.Unmarshal([]byte(data), &typedData) + err = json.Unmarshal(data, &typedData) if err != nil { t.Errorf("Test %d, file %v, json unmarshalling failed: %v", i, fInfo.Name(), err) continue diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index 39c2abf041..012e217710 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -262,13 +262,12 @@ func TestWhisperIdentityManagement(t *testing.T) { func TestWhisperSymKeyManagement(t *testing.T) { InitSingleTest() - var err error - var k1, k2 []byte - w := New(&DefaultConfig) - id1 := string("arbitrary-string-1") - id2 := string("arbitrary-string-2") - - id1, err = w.GenerateSymKey() + var ( + k1, k2 []byte + w = New(&DefaultConfig) + id2 = string("arbitrary-string-2") + ) + id1, err := w.GenerateSymKey() if err != nil { t.Fatalf("failed GenerateSymKey with seed %d: %s.", seed, err) }