From 057af8c5c842714feff675faeab089e497ec1739 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Fri, 1 Dec 2017 10:32:14 +0100 Subject: [PATCH 01/94] swarm: add CLI --ens-endpoint flag (#14386) Implement a CLI flag that can be repeated to allow multiple ENS resolvers for different TLDs. --- cmd/swarm/main.go | 103 ++++++++++++++---------------- cmd/swarm/main_test.go | 141 +++++++++++++++++++++++++++++++++++++++++ swarm/api/api.go | 64 +++++++++++++++++++ swarm/api/api_test.go | 117 ++++++++++++++++++++++++++++++++++ swarm/swarm.go | 97 +++++++++++++++++++++++++--- 5 files changed, 459 insertions(+), 63 deletions(-) create mode 100644 cmd/swarm/main_test.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 603fd9b941..98a0352dca 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -17,11 +17,9 @@ package main import ( - "context" "crypto/ecdsa" "fmt" "io/ioutil" - "math/big" "os" "os/signal" "runtime" @@ -29,14 +27,13 @@ import ( "strconv" "strings" "syscall" - "time" + "unicode" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console" - "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/internal/debug" @@ -45,7 +42,6 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" "gopkg.in/urfave/cli.v1" @@ -101,6 +97,10 @@ var ( Name: "sync", Usage: "Swarm Syncing enabled (default true)", } + EnsEndpointsFlag = cli.StringSliceFlag{ + Name: "ens-endpoint", + Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", + } EnsAPIFlag = cli.StringFlag{ Name: "ens-api", Usage: "URL of the Ethereum API provider to use for ENS record lookups", @@ -323,6 +323,7 @@ DEPRECATED: use 'swarm db clean'. utils.PasswordFileFlag, // bzzd-specific flags CorsStringFlag, + EnsEndpointsFlag, EnsAPIFlag, EnsAddrFlag, SwarmConfigPathFlag, @@ -416,38 +417,6 @@ func bzzd(ctx *cli.Context) error { return nil } -// detectEnsAddr determines the ENS contract address by getting both the -// version and genesis hash using the client and matching them to either -// mainnet or testnet addresses -func detectEnsAddr(client *rpc.Client) (common.Address, error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - var version string - if err := client.CallContext(ctx, &version, "net_version"); err != nil { - return common.Address{}, err - } - - block, err := ethclient.NewClient(client).BlockByNumber(ctx, big.NewInt(0)) - if err != nil { - return common.Address{}, err - } - - switch { - - case version == "1" && block.Hash() == params.MainnetGenesisHash: - log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress) - return ens.MainNetAddress, nil - - case version == "3" && block.Hash() == params.TestnetGenesisHash: - log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress) - return ens.TestNetAddress, nil - - default: - return common.Address{}, fmt.Errorf("unknown version and genesis hash: %s %s", version, block.Hash()) - } -} - func registerBzzService(ctx *cli.Context, stack *node.Node) { prvkey := getAccount(ctx, stack) @@ -476,6 +445,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { utils.Fatalf("SWAP is enabled but --swap-api is not set") } + ensEndpoints := ctx.GlobalStringSlice(EnsEndpointsFlag.Name) ensapi := ctx.GlobalString(EnsAPIFlag.Name) ensAddr := ctx.GlobalString(EnsAddrFlag.Name) @@ -491,34 +461,59 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } } - var ensClient *ethclient.Client + ensClientConfigs := []swarm.ENSClientConfig{} if ensapi != "" { - log.Info("connecting to ENS API", "url", ensapi) - client, err := rpc.Dial(ensapi) - if err != nil { - return nil, fmt.Errorf("error connecting to ENS API %s: %s", ensapi, err) - } - ensClient = ethclient.NewClient(client) - - if ensAddr != "" { - bzzconfig.EnsRoot = common.HexToAddress(ensAddr) - } else { - ensAddr, err := detectEnsAddr(client) - if err == nil { - bzzconfig.EnsRoot = ensAddr - } else { - log.Warn(fmt.Sprintf("could not determine ENS contract address, using default %s", bzzconfig.EnsRoot), "err", err) + ensClientConfigs = append(ensClientConfigs, swarm.ENSClientConfig{ + Endpoint: ensapi, + ContractAddress: ensAddr, + }) + } + if ensEndpoints != nil { + for _, s := range ensEndpoints { + if s != "" { + ensClientConfigs = append(ensClientConfigs, parseFlagEnsEndpoint(s)) } } } + if len(ensClientConfigs) == 0 { + log.Warn("No ENS, please specify non-empty --ens-api or --ens-endpoint to use domain name resolution") + } - return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, swapEnabled, syncEnabled, cors) + return swarm.NewSwarm(ctx, swapClient, ensClientConfigs, bzzconfig, swapEnabled, syncEnabled, cors) } if err := stack.Register(boot); err != nil { utils.Fatalf("Failed to register the Swarm service: %v", err) } } +func parseFlagEnsEndpoint(s string) swarm.ENSClientConfig { + isAllLetterString := func(s string) bool { + for _, r := range s { + if !unicode.IsLetter(r) { + return false + } + } + return true + } + endpoint := s + var addr, tld string + if i := strings.Index(endpoint, ":"); i > 0 { + if isAllLetterString(endpoint[:i]) && len(endpoint) > i+2 && endpoint[i+1:i+3] != "//" { + tld = endpoint[:i] + endpoint = endpoint[i+1:] + } + } + if i := strings.Index(endpoint, "@"); i > 0 { + addr = endpoint[:i] + endpoint = endpoint[i+1:] + } + return swarm.ENSClientConfig{ + Endpoint: endpoint, + ContractAddress: addr, + TLD: tld, + } +} + func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey { keyid := ctx.GlobalString(SwarmAccountFlag.Name) diff --git a/cmd/swarm/main_test.go b/cmd/swarm/main_test.go new file mode 100644 index 0000000000..054034434b --- /dev/null +++ b/cmd/swarm/main_test.go @@ -0,0 +1,141 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see
. + +package main + +import ( + "testing" + + "github.com/ethereum/go-ethereum/swarm" +) + +func TestParseFlagEnsEndpoint(t *testing.T) { + for _, x := range []struct { + description string + value string + config swarm.ENSClientConfig + }{ + { + description: "IPC endpoint", + value: "/data/testnet/geth.ipc", + config: swarm.ENSClientConfig{ + Endpoint: "/data/testnet/geth.ipc", + }, + }, + { + description: "HTTP endpoint", + value: "http://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "http://127.0.0.1:1234", + }, + }, + { + description: "WS endpoint", + value: "ws://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "ws://127.0.0.1:1234", + }, + }, + { + description: "IPC Endpoint and TLD", + value: "test:/data/testnet/geth.ipc", + config: swarm.ENSClientConfig{ + Endpoint: "/data/testnet/geth.ipc", + TLD: "test", + }, + }, + { + description: "HTTP endpoint and TLD", + value: "test:http://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "http://127.0.0.1:1234", + TLD: "test", + }, + }, + { + description: "WS endpoint and TLD", + value: "test:ws://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "ws://127.0.0.1:1234", + TLD: "test", + }, + }, + { + description: "IPC Endpoint and contract address", + value: "314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + config: swarm.ENSClientConfig{ + Endpoint: "/data/testnet/geth.ipc", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + }, + }, + { + description: "HTTP endpoint and contract address", + value: "314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "http://127.0.0.1:1234", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + }, + }, + { + description: "WS endpoint and contract address", + value: "314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "ws://127.0.0.1:1234", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + }, + }, + { + description: "IPC Endpoint, TLD and contract address", + value: "test:314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + config: swarm.ENSClientConfig{ + Endpoint: "/data/testnet/geth.ipc", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + TLD: "test", + }, + }, + { + description: "HTTP endpoint, TLD and contract address", + value: "eth:314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "http://127.0.0.1:1234", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + TLD: "eth", + }, + }, + { + description: "WS endpoint, TLD and contract address", + value: "eth:314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234", + config: swarm.ENSClientConfig{ + Endpoint: "ws://127.0.0.1:1234", + ContractAddress: "314159265dD8dbb310642f98f50C066173C1259b", + TLD: "eth", + }, + }, + } { + t.Run(x.description, func(t *testing.T) { + config := parseFlagEnsEndpoint(x.value) + if config.Endpoint != x.config.Endpoint { + t.Errorf("expected Endpoint %q, got %q", x.config.Endpoint, config.Endpoint) + } + if config.ContractAddress != x.config.ContractAddress { + t.Errorf("expected ContractAddress %q, got %q", x.config.ContractAddress, config.ContractAddress) + } + if config.TLD != x.config.TLD { + t.Errorf("expected TLD %q, got %q", x.config.TLD, config.TLD) + } + }) + } +} diff --git a/swarm/api/api.go b/swarm/api/api.go index 79de29a1cf..d0c3754e00 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -17,6 +17,7 @@ package api import ( + "errors" "fmt" "io" "net/http" @@ -40,6 +41,69 @@ type Resolver interface { Resolve(string) (common.Hash, error) } +// errNoResolver is returned by MultiResolver.Resolve if no resolver +// can be found for the address. +var errNoResolver = errors.New("no resolver") + +// MultiResolver is used to resolve URL addresses based on their TLDs. +// Each TLD can have multiple resolvers, and the resoluton from the +// first one in the sequence will be returned. +type MultiResolver struct { + resolvers map[string][]Resolver +} + +// MultiResolverOption sets options for MultiResolver and is used as +// arguments for its constructor. +type MultiResolverOption func(*MultiResolver) + +// MultiResolverOptionWithResolver adds a Resolver to a list of resolvers +// for a specific TLD. If TLD is an empty string, the resolver will be added +// to the list of default resolver, the ones that will be used for resolution +// of addresses which do not have their TLD resolver specified. +func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption { + return func(m *MultiResolver) { + if _, ok := m.resolvers[tld]; !ok { + m.resolvers[tld] = []Resolver{} + } + m.resolvers[tld] = append(m.resolvers[tld], r) + } +} + +// NewMultiResolver creates a new instance of MultiResolver. +func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { + m = &MultiResolver{ + resolvers: map[string][]Resolver{}, + } + for _, o := range opts { + o(m) + } + return m +} + +// Resolve resolves address by choosing a Resolver by TLD. +// If there are more default Resolvers, or for a specific TLD, +// the Hash from the the first one which does not return error +// will be returned. +func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { + rs, _ := m.resolvers[""] + if i := strings.LastIndex(addr, "."); i >= 0 { + rstld, ok := m.resolvers[addr[i+1:]] + if ok { + rs = rstld + } + } + if rs == nil { + return h, errNoResolver + } + for _, r := range rs { + h, err = r.Resolve(addr) + if err == nil { + return + } + } + return +} + /* Api implements webserver/file system related content storage and retrieval on top of the dpa diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index f9caed27f5..36cbdb83c5 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -237,3 +237,120 @@ func TestAPIResolve(t *testing.T) { }) } } + +func TestMultiResolver(t *testing.T) { + doesntResolve := newTestResolver("") + + ethAddr := "swarm.eth" + ethHash := "0x2222222222222222222222222222222222222222222222222222222222222222" + ethResolve := newTestResolver(ethHash) + + testAddr := "swarm.test" + testHash := "0x1111111111111111111111111111111111111111111111111111111111111111" + testResolve := newTestResolver(testHash) + + tests := []struct { + desc string + r Resolver + addr string + result string + err error + }{ + { + desc: "No resolvers, returns error", + r: NewMultiResolver(), + err: errNoResolver, + }, + { + desc: "One default resolver, returns resolved address", + r: NewMultiResolver(MultiResolverOptionWithResolver(ethResolve, "")), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Two default resolvers, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, ""), + MultiResolverOptionWithResolver(ethResolve, ""), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Two default resolvers, first doesn't resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, ""), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Default resolver doesn't resolve, tld resolver resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Three TLD resolvers, third resolves, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, "eth"), + MultiResolverOptionWithResolver(doesntResolve, "eth"), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "One TLD resolver doesn't resolve, returns error", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "One defautl and one TLD resolver, all doesn't resolve, returns error", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(doesntResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + err: errors.New(`DNS name not found: "swarm.eth"`), + }, + { + desc: "Two TLD resolvers, both resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, "eth"), + MultiResolverOptionWithResolver(testResolve, "test"), + ), + addr: testAddr, + result: testHash, + }, + } + for _, x := range tests { + t.Run(x.desc, func(t *testing.T) { + res, err := x.r.Resolve(x.addr) + if err == nil { + if x.err != nil { + t.Fatalf("expected error %q, got result %q", x.err, res.Hex()) + } + if res.Hex() != x.result { + t.Fatalf("expected result %q, got %q", x.result, res.Hex()) + } + } else { + if x.err == nil { + t.Fatalf("expected no error, got %q", err) + } + if err.Error() != x.err.Error() { + t.Fatalf("expected error %q, got %q", x.err, err) + } + } + }) + } +} diff --git a/swarm/swarm.go b/swarm/swarm.go index 9db15325ae..9b6abc5b96 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -21,7 +21,9 @@ import ( "context" "crypto/ecdsa" "fmt" + "math/big" "net" + "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" @@ -33,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/api" httpapi "github.com/ethereum/go-ethereum/swarm/api/http" @@ -76,7 +79,7 @@ func (self *Swarm) API() *SwarmAPI { // creates a new swarm service instance // implements node.Service -func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string) (self *Swarm, err error) { +func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClientConfigs []ENSClientConfig, config *api.Config, swapEnabled, syncEnabled bool, cors string) (self *Swarm, err error) { if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) { return nil, fmt.Errorf("empty public key") } @@ -133,18 +136,22 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) log.Debug(fmt.Sprintf("-> Content Store API")) - // set up high level api - transactOpts := bind.NewKeyedTransactor(self.privateKey) - - if ensClient == nil { - log.Warn("No ENS, please specify non-empty --ens-api to use domain name resolution") - } else { - self.dns, err = ens.NewENS(transactOpts, config.EnsRoot, ensClient) + if len(ensClientConfigs) == 1 { + self.dns, err = newEnsClient(ensClientConfigs[0].Endpoint, ensClientConfigs[0].ContractAddress, config) if err != nil { return nil, err } + } else if len(ensClientConfigs) > 1 { + opts := []api.MultiResolverOption{} + for _, c := range ensClientConfigs { + r, err := newEnsClient(c.Endpoint, c.ContractAddress, config) + if err != nil { + return nil, err + } + opts = append(opts, api.MultiResolverOptionWithResolver(r, c.TLD)) + } + self.dns = api.NewMultiResolver(opts...) } - log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar @ address %v", config.EnsRoot.Hex())) self.api = api.NewApi(self.dpa, self.dns) // Manifests for Smart Hosting @@ -156,6 +163,78 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e return self, nil } +// ENSClientConfig holds information to construct ENS resolver. If TLD is non-empty, +// the resolver will be used only for that TLD. If ContractAddress is empty, +// it will be discovered by the client, or used one from the configuration. +type ENSClientConfig struct { + Endpoint string + ContractAddress string + TLD string +} + +// newEnsClient creates a new ENS client for that is a consumer of +// a ENS API on a specific endpoint. It is used as a helper function +// for creating multiple resolvers in NewSwarm function. +func newEnsClient(endpoint, addr string, config *api.Config) (*ens.ENS, error) { + log.Info("connecting to ENS API", "url", endpoint) + client, err := rpc.Dial(endpoint) + if err != nil { + return nil, fmt.Errorf("error connecting to ENS API %s: %s", endpoint, err) + } + ensClient := ethclient.NewClient(client) + + ensRoot := config.EnsRoot + if addr != "" { + ensRoot = common.HexToAddress(addr) + } else { + a, err := detectEnsAddr(client) + if err == nil { + ensRoot = a + } else { + log.Warn(fmt.Sprintf("could not determine ENS contract address, using default %s", ensRoot), "err", err) + } + } + transactOpts := bind.NewKeyedTransactor(config.Swap.PrivateKey()) + dns, err := ens.NewENS(transactOpts, ensRoot, ensClient) + if err != nil { + return nil, err + } + log.Debug(fmt.Sprintf("-> Swarm Domain Name Registrar %v @ address %v", endpoint, ensRoot.Hex())) + return dns, err +} + +// detectEnsAddr determines the ENS contract address by getting both the +// version and genesis hash using the client and matching them to either +// mainnet or testnet addresses +func detectEnsAddr(client *rpc.Client) (common.Address, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var version string + if err := client.CallContext(ctx, &version, "net_version"); err != nil { + return common.Address{}, err + } + + block, err := ethclient.NewClient(client).BlockByNumber(ctx, big.NewInt(0)) + if err != nil { + return common.Address{}, err + } + + switch { + + case version == "1" && block.Hash() == params.MainnetGenesisHash: + log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress) + return ens.MainNetAddress, nil + + case version == "3" && block.Hash() == params.TestnetGenesisHash: + log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress) + return ens.TestNetAddress, nil + + default: + return common.Address{}, fmt.Errorf("unknown version and genesis hash: %s %s", version, block.Hash()) + } +} + /* Start is called when the stack is started * starts the network kademlia hive peer management From 7898e0d585ac16ca80ddef3ef6933cc6d12ba576 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 12:44:24 +0100 Subject: [PATCH 02/94] swarm: multiple --ens-api flags Allow multiple --ens-api flags to be specified with value format [tld:][contract-addr@]url. Backward compatibility with only one --ens-api flag and --ens-addr flag is preserved and conflict cases are handled: - multiple --ens-api with --ens-addr returns an error - single --ens-api with contract address and --ens-addr with different contract address returns an error Previously implemented --ens-endpoint is removed. Its functionality is replaced with multiple --ens-api flags. --- cmd/swarm/main.go | 48 ++++++++++++++++++++++++++---------------- cmd/swarm/main_test.go | 4 ++-- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 98a0352dca..70cc42425b 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -97,14 +97,9 @@ var ( Name: "sync", Usage: "Swarm Syncing enabled (default true)", } - EnsEndpointsFlag = cli.StringSliceFlag{ - Name: "ens-endpoint", - Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", - } - EnsAPIFlag = cli.StringFlag{ + EnsAPIFlag = cli.StringSliceFlag{ Name: "ens-api", - Usage: "URL of the Ethereum API provider to use for ENS record lookups", - Value: node.DefaultIPCEndpoint("geth"), + Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", } EnsAddrFlag = cli.StringFlag{ Name: "ens-addr", @@ -323,7 +318,6 @@ DEPRECATED: use 'swarm db clean'. utils.PasswordFileFlag, // bzzd-specific flags CorsStringFlag, - EnsEndpointsFlag, EnsAPIFlag, EnsAddrFlag, SwarmConfigPathFlag, @@ -445,8 +439,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { utils.Fatalf("SWAP is enabled but --swap-api is not set") } - ensEndpoints := ctx.GlobalStringSlice(EnsEndpointsFlag.Name) - ensapi := ctx.GlobalString(EnsAPIFlag.Name) + ensAPIs := ctx.GlobalStringSlice(EnsAPIFlag.Name) ensAddr := ctx.GlobalString(EnsAddrFlag.Name) cors := ctx.GlobalString(CorsStringFlag.Name) @@ -462,21 +455,40 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } ensClientConfigs := []swarm.ENSClientConfig{} - if ensapi != "" { + switch len(ensAPIs) { + case 0: ensClientConfigs = append(ensClientConfigs, swarm.ENSClientConfig{ - Endpoint: ensapi, + Endpoint: node.DefaultIPCEndpoint("geth"), ContractAddress: ensAddr, }) - } - if ensEndpoints != nil { - for _, s := range ensEndpoints { + case 1: + // Check if only one --ens-api is specified in order to use --ens-addr value + // to preserve the backward compatibility with single --ens-api flag. + // Multiple + c := parseFlagEnsAPI(ensAPIs[0]) + if ensAddr != "" { + // If contract address is specified in both cases, check for conflict. + if c.ContractAddress != "" && ensAddr != c.ContractAddress { + utils.Fatalf("--ens-addr flag in conflict with --ens-api flag contract address") + } + c.ContractAddress = ensAddr + } + ensClientConfigs = append(ensClientConfigs, c) + default: + // Backward compatibility with single --ens-api flag and --ens-addr is preserved. + // Check for case where multiple --ens-api flags are set with --ens-addr where + // the specified contract address is not clear to which api belongs. + if ensAddr != "" { + utils.Fatalf("--ens-addr flag can not be used with multiple --ens-api flags") + } + for _, s := range ensAPIs { if s != "" { - ensClientConfigs = append(ensClientConfigs, parseFlagEnsEndpoint(s)) + ensClientConfigs = append(ensClientConfigs, parseFlagEnsAPI(s)) } } } if len(ensClientConfigs) == 0 { - log.Warn("No ENS, please specify non-empty --ens-api or --ens-endpoint to use domain name resolution") + log.Warn("No ENS, please specify non-empty --ens-api to use domain name resolution") } return swarm.NewSwarm(ctx, swapClient, ensClientConfigs, bzzconfig, swapEnabled, syncEnabled, cors) @@ -486,7 +498,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } } -func parseFlagEnsEndpoint(s string) swarm.ENSClientConfig { +func parseFlagEnsAPI(s string) swarm.ENSClientConfig { isAllLetterString := func(s string) bool { for _, r := range s { if !unicode.IsLetter(r) { diff --git a/cmd/swarm/main_test.go b/cmd/swarm/main_test.go index 054034434b..f815f3387d 100644 --- a/cmd/swarm/main_test.go +++ b/cmd/swarm/main_test.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/swarm" ) -func TestParseFlagEnsEndpoint(t *testing.T) { +func TestParseFlagEnsAPI(t *testing.T) { for _, x := range []struct { description string value string @@ -126,7 +126,7 @@ func TestParseFlagEnsEndpoint(t *testing.T) { }, } { t.Run(x.description, func(t *testing.T) { - config := parseFlagEnsEndpoint(x.value) + config := parseFlagEnsAPI(x.value) if config.Endpoint != x.config.Endpoint { t.Errorf("expected Endpoint %q, got %q", x.config.Endpoint, config.Endpoint) } From b33a051a487bed2a010b62a6f2dc531157341013 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:20:29 +0100 Subject: [PATCH 03/94] swarm: add comment for parseFlagEnsAPI and fix a mistake in comment in code --- cmd/swarm/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 70cc42425b..d9744c011d 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -464,7 +464,6 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { case 1: // Check if only one --ens-api is specified in order to use --ens-addr value // to preserve the backward compatibility with single --ens-api flag. - // Multiple c := parseFlagEnsAPI(ensAPIs[0]) if ensAddr != "" { // If contract address is specified in both cases, check for conflict. @@ -498,6 +497,9 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } } +// parseFlagEnsAPI parses EnsAPIFlag according to format +// [tld:][contract-addr@]url and returns ENSClientConfig structure +// with endpoint, contract address and TLD. func parseFlagEnsAPI(s string) swarm.ENSClientConfig { isAllLetterString := func(s string) bool { for _, r := range s { From 15ad6f27da1bbcb4219f0786d3d08d52c27a4385 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:28:11 +0100 Subject: [PATCH 04/94] swarm: check if "--ens-api ''" is specified in order to disable ENS --- cmd/swarm/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index d9744c011d..a937a91191 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -462,6 +462,10 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { ContractAddress: ensAddr, }) case 1: + // Check if "--ens-api ''" is specified in order to disable ENS. + if ensAPIs[0] == "" { + break + } // Check if only one --ens-api is specified in order to use --ens-addr value // to preserve the backward compatibility with single --ens-api flag. c := parseFlagEnsAPI(ensAPIs[0]) From 34edbc8868b84e15d7102a1164c7af8ef4873882 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:29:37 +0100 Subject: [PATCH 05/94] swarm/api: remove unneeded assignment in MultiResolverOptionWithResolver --- swarm/api/api.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index d0c3754e00..3682ab3092 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -62,9 +62,6 @@ type MultiResolverOption func(*MultiResolver) // of addresses which do not have their TLD resolver specified. func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption { return func(m *MultiResolver) { - if _, ok := m.resolvers[tld]; !ok { - m.resolvers[tld] = []Resolver{} - } m.resolvers[tld] = append(m.resolvers[tld], r) } } From a758b5cf7ac3111c053bc1275ca849f7aae080e1 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:31:00 +0100 Subject: [PATCH 06/94] swarm/api: initialize map with make to comply with the convention --- swarm/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 3682ab3092..d6e9d37327 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -69,7 +69,7 @@ func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption // NewMultiResolver creates a new instance of MultiResolver. func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { m = &MultiResolver{ - resolvers: map[string][]Resolver{}, + resolvers: make(map[string][]Resolver), } for _, o := range opts { o(m) From 3732c15faa83e60024ef46b3def0c105c6912c47 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:32:33 +0100 Subject: [PATCH 07/94] swarm/api: remove unneeded blank assignement --- swarm/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index d6e9d37327..8bd3866434 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -82,7 +82,7 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // the Hash from the the first one which does not return error // will be returned. func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { - rs, _ := m.resolvers[""] + rs := m.resolvers[""] if i := strings.LastIndex(addr, "."); i >= 0 { rstld, ok := m.resolvers[addr[i+1:]] if ok { From e451b65faef95b159c38a607f716866e7bd3310f Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:41:21 +0100 Subject: [PATCH 08/94] swarm: deprecate --ens-addr CLI flag with a warning message --- cmd/swarm/main.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index a937a91191..48cf032a24 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -101,10 +101,6 @@ var ( Name: "ens-api", Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", } - EnsAddrFlag = cli.StringFlag{ - Name: "ens-addr", - Usage: "ENS contract address (default is detected as testnet or mainnet using --ens-api)", - } SwarmApiFlag = cli.StringFlag{ Name: "bzzapi", Usage: "Swarm HTTP endpoint", @@ -140,6 +136,10 @@ var ( Name: "ethapi", Usage: "DEPRECATED: please use --ens-api and --swap-api", } + DeprecatedEnsAddrFlag = cli.StringFlag{ + Name: "ens-addr", + Usage: "DEPRECATED: ENS contract address, please use --ens-api with contract address according to its format", + } ) var defaultNodeConfig = node.DefaultConfig @@ -319,7 +319,6 @@ DEPRECATED: use 'swarm db clean'. // bzzd-specific flags CorsStringFlag, EnsAPIFlag, - EnsAddrFlag, SwarmConfigPathFlag, SwarmSwapEnabledFlag, SwarmSwapAPIFlag, @@ -338,6 +337,7 @@ DEPRECATED: use 'swarm db clean'. SwarmUploadMimeType, //deprecated flags DeprecatedEthAPIFlag, + DeprecatedEnsAddrFlag, } app.Flags = append(app.Flags, debug.Flags...) app.Before = func(ctx *cli.Context) error { @@ -440,7 +440,11 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { } ensAPIs := ctx.GlobalStringSlice(EnsAPIFlag.Name) - ensAddr := ctx.GlobalString(EnsAddrFlag.Name) + ensAddr := ctx.GlobalString(DeprecatedEnsAddrFlag.Name) + + if ensAddr != "" { + log.Warn("--ens-addr is no longer a valid command line flag, please use --ens-api to specify contract address.") + } cors := ctx.GlobalString(CorsStringFlag.Name) From 1dc19de5da64962a98a37bbc7b93a3895d2eb6e6 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:56:52 +0100 Subject: [PATCH 09/94] swarm/api: use path.Ext instead strings.LastIndex in MultiResolver.Resolve --- swarm/api/api.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 8bd3866434..2d9db022e4 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "net/http" + "path" "regexp" "strings" "sync" @@ -83,8 +84,8 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // will be returned. func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { rs := m.resolvers[""] - if i := strings.LastIndex(addr, "."); i >= 0 { - rstld, ok := m.resolvers[addr[i+1:]] + if ext := path.Ext(addr); ext != "" { + rstld, ok := m.resolvers[ext[1:]] if ok { rs = rstld } From c0a4d9e1e64a09a19484c8c12e24505d9bacbd57 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 18 Dec 2017 16:22:39 +0100 Subject: [PATCH 10/94] cmd/swarm, swarm: disable ENS API by default Specifying ENS API CLI flag, env variable or configuration field is required for ENS resolving. Backward compatibility is preserved with --ens-api="" CLI flag value. --- cmd/swarm/config.go | 23 +++++------------------ swarm/api/config.go | 2 -- swarm/swarm.go | 23 +++++++---------------- 3 files changed, 12 insertions(+), 36 deletions(-) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 9931b12a21..c66a2a4fe5 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -195,17 +195,13 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) } - //EnsAPIs can be set to "", so can't check for empty string, as it is allowed! if ctx.GlobalIsSet(EnsAPIFlag.Name) { ensAPIs := ctx.GlobalStringSlice(EnsAPIFlag.Name) - // Disable ENS resolver if --ens-api="" is specified + // preserve backward compatibility to disable ENS with --ens-api="" if len(ensAPIs) == 1 && ensAPIs[0] == "" { - currentConfig.EnsDisabled = true - currentConfig.EnsAPIs = nil - } else { - currentConfig.EnsDisabled = false - currentConfig.EnsAPIs = ensAPIs + ensAPIs = nil } + currentConfig.EnsAPIs = ensAPIs } if ensaddr := ctx.GlobalString(DeprecatedEnsAddrFlag.Name); ensaddr != "" { @@ -275,17 +271,8 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) } - //EnsAPIs can be set to "", so can't check for empty string, as it is allowed - if ensapi, exists := os.LookupEnv(SWARM_ENV_ENS_API); exists { - ensAPIs := strings.Split(ensapi, ",") - // Disable ENS resolver if SWARM_ENS_API="" is specified - if len(ensAPIs) == 0 { - currentConfig.EnsDisabled = true - currentConfig.EnsAPIs = nil - } else { - currentConfig.EnsDisabled = false - currentConfig.EnsAPIs = ensAPIs - } + if ensapi := os.Getenv(SWARM_ENV_ENS_API); ensapi != "" { + currentConfig.EnsAPIs = strings.Split(ensapi, ",") } if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" { diff --git a/swarm/api/config.go b/swarm/api/config.go index 3f1a2a0742..6b224140a4 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -48,7 +48,6 @@ type Config struct { *network.SyncParams Contract common.Address EnsRoot common.Address - EnsDisabled bool EnsAPIs []string Path string ListenAddr string @@ -78,7 +77,6 @@ func NewDefaultConfig() (self *Config) { Path: node.DefaultDataDir(), EnsAPIs: nil, EnsRoot: ens.TestNetAddress, - EnsDisabled: false, NetworkId: network.NetworkId, SwapEnabled: false, SyncEnabled: true, diff --git a/swarm/swarm.go b/swarm/swarm.go index ae6805dfda..3c77d6eab6 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -138,26 +138,17 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) log.Debug(fmt.Sprintf("-> Content Store API")) - if !config.EnsDisabled { - if len(config.EnsAPIs) == 0 { - // ENS is enabled and has no specific configuration, - // use defaults - self.dns, err = newEnsClient(node.DefaultIPCEndpoint("geth"), config.EnsRoot, config) + if len(config.EnsAPIs) > 0 { + opts := []api.MultiResolverOption{} + for _, c := range config.EnsAPIs { + tld, endpoint, addr := parseEnsAPIAddress(c) + r, err := newEnsClient(endpoint, addr, config) if err != nil { return nil, err } - } else { - opts := []api.MultiResolverOption{} - for _, c := range config.EnsAPIs { - tld, endpoint, addr := parseEnsAPIAddress(c) - r, err := newEnsClient(endpoint, addr, config) - if err != nil { - return nil, err - } - opts = append(opts, api.MultiResolverOptionWithResolver(r, tld)) - } - self.dns = api.NewMultiResolver(opts...) + opts = append(opts, api.MultiResolverOptionWithResolver(r, tld)) } + self.dns = api.NewMultiResolver(opts...) } self.api = api.NewApi(self.dpa, self.dns) From 0d6a735a72340130acd6b7e536dad5d8bee40d84 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 18 Dec 2017 23:07:48 +0100 Subject: [PATCH 11/94] swarm/api: implement NoResolverError with information about TLD MultiResolver needs to provide information about TLD that has no resolver configured for. --- swarm/api/api.go | 26 ++++++++++++++++++++------ swarm/api/api_test.go | 10 +++++++++- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 2d9db022e4..c143d8a3f9 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -17,7 +17,6 @@ package api import ( - "errors" "fmt" "io" "net/http" @@ -42,9 +41,22 @@ type Resolver interface { Resolve(string) (common.Hash, error) } -// errNoResolver is returned by MultiResolver.Resolve if no resolver +// NoResolverError is returned by MultiResolver.Resolve if no resolver // can be found for the address. -var errNoResolver = errors.New("no resolver") +type NoResolverError struct { + TLD string +} + +func NewNoResolverError(tld string) *NoResolverError { + return &NoResolverError{TLD: tld} +} + +func (e *NoResolverError) Error() string { + if e.TLD == "" { + return "no ENS resolver" + } + return fmt.Sprintf("no ENS endpoint configured to resolve .%s TLD names", e.TLD) +} // MultiResolver is used to resolve URL addresses based on their TLDs. // Each TLD can have multiple resolvers, and the resoluton from the @@ -84,14 +96,16 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // will be returned. func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { rs := m.resolvers[""] - if ext := path.Ext(addr); ext != "" { - rstld, ok := m.resolvers[ext[1:]] + tld := path.Ext(addr) + if tld != "" { + tld = tld[1:] + rstld, ok := m.resolvers[tld] if ok { rs = rstld } } if rs == nil { - return h, errNoResolver + return h, NewNoResolverError(tld) } for _, r := range rs { h, err = r.Resolve(addr) diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 36cbdb83c5..dbe7c1dac4 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -259,7 +259,7 @@ func TestMultiResolver(t *testing.T) { { desc: "No resolvers, returns error", r: NewMultiResolver(), - err: errNoResolver, + err: NewNoResolverError(""), }, { desc: "One default resolver, returns resolved address", @@ -332,6 +332,14 @@ func TestMultiResolver(t *testing.T) { addr: testAddr, result: testHash, }, + { + desc: "One TLD resolver, no default resolver, returns error for different TLD", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: testAddr, + err: NewNoResolverError("test"), + }, } for _, x := range tests { t.Run(x.desc, func(t *testing.T) { From dd5ae4fd8e28d1ce618668d213e81781ef59f067 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Tue, 19 Dec 2017 11:47:26 +0100 Subject: [PATCH 12/94] cmd/swarm: add validation for EnsAPIs configuration parameter --- cmd/swarm/config.go | 35 +++++++++++++++ cmd/swarm/config_test.go | 95 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index c66a2a4fe5..566b00f488 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -102,6 +102,8 @@ func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) { config = envVarsOverride(config) //override settings provided by command line config = cmdLineOverride(config, ctx) + //validate configuration parameters + err = validateConfig(config) return } @@ -319,6 +321,39 @@ func checkDeprecated(ctx *cli.Context) { } } +//validate configuration parameters +func validateConfig(cfg *bzzapi.Config) (err error) { + for _, ensAPI := range cfg.EnsAPIs { + if ensAPI != "" { + if err := validateEnsAPIs(ensAPI); err != nil { + return fmt.Errorf("invalid format [tld:][contract-addr@]url for ENS API endpoint configuration %q: %v", ensAPI, err) + } + } + } + return nil +} + +//validate EnsAPIs configuration parameter +func validateEnsAPIs(s string) (err error) { + // missing contract address + if strings.HasPrefix(s, "@") { + return errors.New("missing contract address") + } + // missing url + if strings.HasSuffix(s, "@") { + return errors.New("missing url") + } + // missing tld + if strings.HasPrefix(s, ":") { + return errors.New("missing tld") + } + // missing url + if strings.HasSuffix(s, ":") { + return errors.New("missing url") + } + return nil +} + //print a Config as string func printConfig(config *bzzapi.Config) string { out, err := tomlSettings.Marshal(&config) diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go index 166980d148..9bf584f50c 100644 --- a/cmd/swarm/config_test.go +++ b/cmd/swarm/config_test.go @@ -457,3 +457,98 @@ func TestCmdLineOverridesFile(t *testing.T) { node.Shutdown() } + +func TestValidateConfig(t *testing.T) { + for _, c := range []struct { + cfg *api.Config + err string + }{ + { + cfg: &api.Config{EnsAPIs: []string{ + "/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "http://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "ws://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "test:/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "test:ws://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "test:314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "eth:314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "eth:314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:12344", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "eth:", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"eth:\": missing url", + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"314159265dD8dbb310642f98f50C066173C1259b@\": missing url", + }, + { + cfg: &api.Config{EnsAPIs: []string{ + ":314159265dD8dbb310642f98f50C066173C1259", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \":314159265dD8dbb310642f98f50C066173C1259\": missing tld", + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "@/data/testnet/geth.ipc", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"@/data/testnet/geth.ipc\": missing contract address", + }, + } { + err := validateConfig(c.cfg) + if c.err != "" && err.Error() != c.err { + t.Errorf("expected error %q, got %q", c.err, err) + } + if c.err == "" && err != nil { + t.Errorf("unexpected error %q", err) + } + } +} From 820cf09c98706f71d4b02b6c25e62db15830f3fb Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Tue, 19 Dec 2017 23:05:23 +0100 Subject: [PATCH 13/94] cmd/swarm: return error early in buildConfig function --- cmd/swarm/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 566b00f488..a3c03c00d5 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -98,6 +98,9 @@ func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) { config = bzzapi.NewDefaultConfig() //first load settings from config file (if provided) config, err = configFileOverride(config, ctx) + if err != nil { + return nil, err + } //override settings provided by environment variables config = envVarsOverride(config) //override settings provided by command line From e7cd627d93d43ff52d13c66b336e12b96d9bfdef Mon Sep 17 00:00:00 2001 From: croath
Date: Fri, 29 Dec 2017 19:46:40 +0800 Subject: [PATCH 14/94] accounts/abi: fix for one output interface crashing --- accounts/abi/argument.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index ad17fbf2b1..65f79c9d4c 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -169,6 +169,16 @@ func (arguments Arguments) unpackAtomic(v interface{}, output []byte) error { if err != nil { return err } + + // if we reach this part, there is only one output member from the contract event. + // for mobile, the result type is always a slice. + if reflect.Slice == value.Kind() && value.Len() >= 1 { + //check if it's not a byte slice + if reflect.TypeOf([]byte{}) != value.Type() { + value = value.Index(0).Elem() + } + } + return set(value, reflect.ValueOf(marshalledValue), arg) } From 88e67c552e5da4d114d80be3ff471f4f2ed030f5 Mon Sep 17 00:00:00 2001 From: croath
Date: Sun, 31 Dec 2017 19:12:55 +0800 Subject: [PATCH 15/94] accounts/abi: add a test case for unpacking mobile interfaces --- accounts/abi/unpack_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index 3910950732..a7901896ab 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -287,6 +287,42 @@ func TestUnpack(t *testing.T) { } } +var unpackMobileTests = []unpackTest{ + { + def: `[{"type": "uint256"}]`, + enc: "0000000000000000000000000000000000000000000000000000000000000001", + want: big.NewInt(1), + }, +} + +func TestUnpackMobileOnly(t *testing.T) { + for i, test := range unpackMobileTests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def) + abi, err := JSON(strings.NewReader(def)) + if err != nil { + t.Fatalf("invalid ABI definition %s: %v", def, err) + } + encb, err := hex.DecodeString(test.enc) + if err != nil { + t.Fatalf("invalid hex: %s" + test.enc) + } + outptr := reflect.New(reflect.TypeOf(test.want)) + results := make([]interface{}, 1) + copy(results, []interface{}{outptr.Interface()}) + err = abi.Unpack(&results, "method", encb) + if err := test.checkError(err); err != nil { + t.Errorf("test %d (%v) failed: %v", i, test.def, err) + return + } + out := outptr.Elem().Interface() + if !reflect.DeepEqual(test.want, out) { + t.Errorf("test %d (%v) failed: expected %v, got %v", i, test.def, test.want, out) + } + }) + } +} + type methodMultiOutput struct { Int *big.Int String string From 407339085fd6fffd15032572bc6ceb472504a451 Mon Sep 17 00:00:00 2001 From: zelig
Date: Tue, 2 Jan 2018 23:30:09 +0100 Subject: [PATCH 16/94] p2p/protocols, p2p/testing: protocol abstraction and testing --- p2p/protocols/protocol.go | 311 ++++++++++++++++++++++++++ p2p/protocols/protocol_test.go | 389 +++++++++++++++++++++++++++++++++ p2p/testing/peerpool.go | 67 ++++++ p2p/testing/protocolsession.go | 206 +++++++++++++++++ p2p/testing/protocoltester.go | 200 +++++++++++++++++ 5 files changed, 1173 insertions(+) create mode 100644 p2p/protocols/protocol.go create mode 100644 p2p/protocols/protocol_test.go create mode 100644 p2p/testing/peerpool.go create mode 100644 p2p/testing/protocolsession.go create mode 100644 p2p/testing/protocoltester.go diff --git a/p2p/protocols/protocol.go b/p2p/protocols/protocol.go new file mode 100644 index 0000000000..9914c99587 --- /dev/null +++ b/p2p/protocols/protocol.go @@ -0,0 +1,311 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see
. + +/* +Package protocols is an extension to p2p. It offers a user friendly simple way to define +devp2p subprotocols by abstracting away code standardly shared by protocols. + +* automate assigments of code indexes to messages +* automate RLP decoding/encoding based on reflecting +* provide the forever loop to read incoming messages +* standardise error handling related to communication +* standardised handshake negotiation +* TODO: automatic generation of wire protocol specification for peers + +*/ +package protocols + +import ( + "context" + "fmt" + "reflect" + "sync" + + "github.com/ethereum/go-ethereum/p2p" +) + +// error codes used by this protocol scheme +const ( + ErrMsgTooLong = iota + ErrDecode + ErrWrite + ErrInvalidMsgCode + ErrInvalidMsgType + ErrHandshake + ErrNoHandler + ErrHandler +) + +// error description strings associated with the codes +var errorToString = map[int]string{ + ErrMsgTooLong: "Message too long", + ErrDecode: "Invalid message (RLP error)", + ErrWrite: "Error sending message", + ErrInvalidMsgCode: "Invalid message code", + ErrInvalidMsgType: "Invalid message type", + ErrHandshake: "Handshake error", + ErrNoHandler: "No handler registered error", + ErrHandler: "Message handler error", +} + +/* +Error implements the standard go error interface. +Use: + + errorf(code, format, params ...interface{}) + +Prints as: + +
:
+ +where description is given by code in errorToString +and details is fmt.Sprintf(format, params...) + +exported field Code can be checked +*/ +type Error struct { + Code int + message string + format string + params []interface{} +} + +func (e Error) Error() (message string) { + if len(e.message) == 0 { + name, ok := errorToString[e.Code] + if !ok { + panic("invalid message code") + } + e.message = name + if e.format != "" { + e.message += ": " + fmt.Sprintf(e.format, e.params...) + } + } + return e.message +} + +func errorf(code int, format string, params ...interface{}) *Error { + return &Error{ + Code: code, + format: format, + params: params, + } +} + +// Spec is a protocol specification including its name and version as well as +// the types of messages which are exchanged +type Spec struct { + // Name is the name of the protocol, often a three-letter word + Name string + + // Version is the version number of the protocol + Version uint + + // MaxMsgSize is the maximum accepted length of the message payload + MaxMsgSize uint32 + + // Messages is a list of message data types which this protocol uses, with + // each message type being sent with its array index as the code (so + // [&foo{}, &bar{}, &baz{}] would send foo, bar and baz with codes + // 0, 1 and 2 respectively) + // each message must have a single unique data type + Messages []interface{} + + initOnce sync.Once + codes map[reflect.Type]uint64 + types map[uint64]reflect.Type +} + +func (s *Spec) init() { + s.initOnce.Do(func() { + s.codes = make(map[reflect.Type]uint64, len(s.Messages)) + s.types = make(map[uint64]reflect.Type, len(s.Messages)) + for i, msg := range s.Messages { + code := uint64(i) + typ := reflect.TypeOf(msg) + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + s.codes[typ] = code + s.types[code] = typ + } + }) +} + +// Length returns the number of message types in the protocol +func (s *Spec) Length() uint64 { + return uint64(len(s.Messages)) +} + +// GetCode returns the message code of a type, and boolean second argument is +// false if the message type is not found +func (s *Spec) GetCode(msg interface{}) (uint64, bool) { + s.init() + typ := reflect.TypeOf(msg) + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + } + code, ok := s.codes[typ] + return code, ok +} + +// NewMsg construct a new message type given the code +func (s *Spec) NewMsg(code uint64) (interface{}, bool) { + s.init() + typ, ok := s.types[code] + if !ok { + return nil, false + } + return reflect.New(typ).Interface(), true +} + +// Peer represents a remote peer or protocol instance that is running on a peer connection with +// a remote peer +type Peer struct { + *p2p.Peer // the p2p.Peer object representing the remote + rw p2p.MsgReadWriter // p2p.MsgReadWriter to send messages to and read messages from + spec *Spec +} + +// NewPeer constructs a new peer +// this constructor is called by the p2p.Protocol#Run function +// the first two arguments are the arguments passed to p2p.Protocol.Run function +// the third argument is the Spec describing the protocol +func NewPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *Spec) *Peer { + return &Peer{ + Peer: p, + rw: rw, + spec: spec, + } +} + +// Run starts the forever loop that handles incoming messages +// called within the p2p.Protocol#Run function +// the handler argument is a function which is called for each message received +// from the remote peer, a returned error causes the loop to exit +// resulting in disconnection +func (p *Peer) Run(handler func(msg interface{}) error) error { + for { + if err := p.handleIncoming(handler); err != nil { + return err + } + } +} + +// Drop disconnects a peer. +// TODO: may need to implement protocol drop only? don't want to kick off the peer +// if they are useful for other protocols +func (p *Peer) Drop(err error) { + p.Disconnect(p2p.DiscSubprotocolError) +} + +// Send takes a message, encodes it in RLP, finds the right message code and sends the +// message off to the peer +// this low level call will be wrapped by libraries providing routed or broadcast sends +// but often just used to forward and push messages to directly connected peers +func (p *Peer) Send(msg interface{}) error { + code, found := p.spec.GetCode(msg) + if !found { + return errorf(ErrInvalidMsgType, "%v", code) + } + return p2p.Send(p.rw, code, msg) +} + +// handleIncoming(code) +// is called each cycle of the main forever loop that dispatches incoming messages +// if this returns an error the loop returns and the peer is disconnected with the error +// this generic handler +// * checks message size, +// * checks for out-of-range message codes, +// * handles decoding with reflection, +// * call handlers as callbacks +func (p *Peer) handleIncoming(handle func(msg interface{}) error) error { + msg, err := p.rw.ReadMsg() + if err != nil { + return err + } + // make sure that the payload has been fully consumed + defer msg.Discard() + + if msg.Size > p.spec.MaxMsgSize { + return errorf(ErrMsgTooLong, "%v > %v", msg.Size, p.spec.MaxMsgSize) + } + + val, ok := p.spec.NewMsg(msg.Code) + if !ok { + return errorf(ErrInvalidMsgCode, "%v", msg.Code) + } + if err := msg.Decode(val); err != nil { + return errorf(ErrDecode, "<= %v: %v", msg, err) + } + + // call the registered handler callbacks + // a registered callback take the decoded message as argument as an interface + // which the handler is supposed to cast to the appropriate type + // it is entirely safe not to check the cast in the handler since the handler is + // chosen based on the proper type in the first place + if err := handle(val); err != nil { + return errorf(ErrHandler, "(msg code %v): %v", msg.Code, err) + } + return nil +} + +// Handshake negotiates a handshake on the peer connection +// * arguments +// * context +// * the local handshake to be sent to the remote peer +// * funcion to be called on the remote handshake (can be nil) +// * expects a remote handshake back of the same type +// * the dialing peer needs to send the handshake first and then waits for remote +// * the listening peer waits for the remote handshake and then sends it +// returns the remote handshake and an error +func (p *Peer) Handshake(ctx context.Context, hs interface{}, verify func(interface{}) error) (rhs interface{}, err error) { + if _, ok := p.spec.GetCode(hs); !ok { + return nil, errorf(ErrHandshake, "unknown handshake message type: %T", hs) + } + errc := make(chan error, 2) + handle := func(msg interface{}) error { + rhs = msg + if verify != nil { + return verify(rhs) + } + return nil + } + send := func() { errc <- p.Send(hs) } + receive := func() { errc <- p.handleIncoming(handle) } + + go func() { + if p.Inbound() { + receive() + send() + } else { + send() + receive() + } + }() + + for i := 0; i < 2; i++ { + select { + case err = <-errc: + case <-ctx.Done(): + err = ctx.Err() + } + if err != nil { + return nil, errorf(ErrHandshake, err.Error()) + } + } + return rhs, nil +} diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go new file mode 100644 index 0000000000..b041a650a1 --- /dev/null +++ b/p2p/protocols/protocol_test.go @@ -0,0 +1,389 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see
. + +package protocols + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + p2ptest "github.com/ethereum/go-ethereum/p2p/testing" +) + +// handshake message type +type hs0 struct { + C uint +} + +// message to kill/drop the peer with nodeID +type kill struct { + C discover.NodeID +} + +// message to drop connection +type drop struct { +} + +/// protoHandshake represents module-independent aspects of the protocol and is +// the first message peers send and receive as part the initial exchange +type protoHandshake struct { + Version uint // local and remote peer should have identical version + NetworkID string // local and remote peer should have identical network id +} + +// checkProtoHandshake verifies local and remote protoHandshakes match +func checkProtoHandshake(testVersion uint, testNetworkID string) func(interface{}) error { + return func(rhs interface{}) error { + remote := rhs.(*protoHandshake) + if remote.NetworkID != testNetworkID { + return fmt.Errorf("%s (!= %s)", remote.NetworkID, testNetworkID) + } + + if remote.Version != testVersion { + return fmt.Errorf("%d (!= %d)", remote.Version, testVersion) + } + return nil + } +} + +// newProtocol sets up a protocol +// the run function here demonstrates a typical protocol using peerPool, handshake +// and messages registered to handlers +func newProtocol(pp *p2ptest.TestPeerPool) func(*p2p.Peer, p2p.MsgReadWriter) error { + spec := &Spec{ + Name: "test", + Version: 42, + MaxMsgSize: 10 * 1024, + Messages: []interface{}{ + protoHandshake{}, + hs0{}, + kill{}, + drop{}, + }, + } + return func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + peer := NewPeer(p, rw, spec) + + // initiate one-off protohandshake and check validity + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + phs := &protoHandshake{42, "420"} + hsCheck := checkProtoHandshake(phs.Version, phs.NetworkID) + _, err := peer.Handshake(ctx, phs, hsCheck) + if err != nil { + return err + } + + lhs := &hs0{42} + // module handshake demonstrating a simple repeatable exchange of same-type message + hs, err := peer.Handshake(ctx, lhs, nil) + if err != nil { + return err + } + + if rmhs := hs.(*hs0); rmhs.C > lhs.C { + return fmt.Errorf("handshake mismatch remote %v > local %v", rmhs.C, lhs.C) + } + + handle := func(msg interface{}) error { + switch msg := msg.(type) { + + case *protoHandshake: + return errors.New("duplicate handshake") + + case *hs0: + rhs := msg + if rhs.C > lhs.C { + return fmt.Errorf("handshake mismatch remote %v > local %v", rhs.C, lhs.C) + } + lhs.C += rhs.C + return peer.Send(lhs) + + case *kill: + // demonstrates use of peerPool, killing another peer connection as a response to a message + id := msg.C + pp.Get(id).Drop(errors.New("killed")) + return nil + + case *drop: + // for testing we can trigger self induced disconnect upon receiving drop message + return errors.New("dropped") + + default: + return fmt.Errorf("unknown message type: %T", msg) + } + } + + pp.Add(peer) + defer pp.Remove(peer) + return peer.Run(handle) + } +} + +func protocolTester(t *testing.T, pp *p2ptest.TestPeerPool) *p2ptest.ProtocolTester { + conf := adapters.RandomNodeConfig() + return p2ptest.NewProtocolTester(t, conf.ID, 2, newProtocol(pp)) +} + +func protoHandshakeExchange(id discover.NodeID, proto *protoHandshake) []p2ptest.Exchange { + + return []p2ptest.Exchange{ + p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: id, + }, + }, + }, + p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 0, + Msg: proto, + Peer: id, + }, + }, + }, + } +} + +func runProtoHandshake(t *testing.T, proto *protoHandshake, errs ...error) { + pp := p2ptest.NewTestPeerPool() + s := protocolTester(t, pp) + // TODO: make this more than one handshake + id := s.IDs[0] + if err := s.TestExchanges(protoHandshakeExchange(id, proto)...); err != nil { + t.Fatal(err) + } + var disconnects []*p2ptest.Disconnect + for i, err := range errs { + disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err}) + } + if err := s.TestDisconnected(disconnects...); err != nil { + t.Fatal(err) + } +} + +func TestProtoHandshakeVersionMismatch(t *testing.T) { + runProtoHandshake(t, &protoHandshake{41, "420"}, errorf(ErrHandshake, errorf(ErrHandler, "(msg code 0): 41 (!= 42)").Error())) +} + +func TestProtoHandshakeNetworkIDMismatch(t *testing.T) { + runProtoHandshake(t, &protoHandshake{42, "421"}, errorf(ErrHandshake, errorf(ErrHandler, "(msg code 0): 421 (!= 420)").Error())) +} + +func TestProtoHandshakeSuccess(t *testing.T) { + runProtoHandshake(t, &protoHandshake{42, "420"}) +} + +func moduleHandshakeExchange(id discover.NodeID, resp uint) []p2ptest.Exchange { + + return []p2ptest.Exchange{ + p2ptest.Exchange{ + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 1, + Msg: &hs0{42}, + Peer: id, + }, + }, + }, + p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 1, + Msg: &hs0{resp}, + Peer: id, + }, + }, + }, + } +} + +func runModuleHandshake(t *testing.T, resp uint, errs ...error) { + pp := p2ptest.NewTestPeerPool() + s := protocolTester(t, pp) + id := s.IDs[0] + if err := s.TestExchanges(protoHandshakeExchange(id, &protoHandshake{42, "420"})...); err != nil { + t.Fatal(err) + } + if err := s.TestExchanges(moduleHandshakeExchange(id, resp)...); err != nil { + t.Fatal(err) + } + var disconnects []*p2ptest.Disconnect + for i, err := range errs { + disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err}) + } + if err := s.TestDisconnected(disconnects...); err != nil { + t.Fatal(err) + } +} + +func TestModuleHandshakeError(t *testing.T) { + runModuleHandshake(t, 43, fmt.Errorf("handshake mismatch remote 43 > local 42")) +} + +func TestModuleHandshakeSuccess(t *testing.T) { + runModuleHandshake(t, 42) +} + +// testing complex interactions over multiple peers, relaying, dropping +func testMultiPeerSetup(a, b discover.NodeID) []p2ptest.Exchange { + + return []p2ptest.Exchange{ + p2ptest.Exchange{ + Label: "primary handshake", + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: a, + }, + p2ptest.Expect{ + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: b, + }, + }, + }, + p2ptest.Exchange{ + Label: "module handshake", + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: a, + }, + p2ptest.Trigger{ + Code: 0, + Msg: &protoHandshake{42, "420"}, + Peer: b, + }, + }, + Expects: []p2ptest.Expect{ + p2ptest.Expect{ + Code: 1, + Msg: &hs0{42}, + Peer: a, + }, + p2ptest.Expect{ + Code: 1, + Msg: &hs0{42}, + Peer: b, + }, + }, + }, + + p2ptest.Exchange{Label: "alternative module handshake", Triggers: []p2ptest.Trigger{p2ptest.Trigger{Code: 1, Msg: &hs0{41}, Peer: a}, + p2ptest.Trigger{Code: 1, Msg: &hs0{41}, Peer: b}}}, + p2ptest.Exchange{Label: "repeated module handshake", Triggers: []p2ptest.Trigger{p2ptest.Trigger{Code: 1, Msg: &hs0{1}, Peer: a}}}, + p2ptest.Exchange{Label: "receiving repeated module handshake", Expects: []p2ptest.Expect{p2ptest.Expect{Code: 1, Msg: &hs0{43}, Peer: a}}}} +} + +func runMultiplePeers(t *testing.T, peer int, errs ...error) { + pp := p2ptest.NewTestPeerPool() + s := protocolTester(t, pp) + + if err := s.TestExchanges(testMultiPeerSetup(s.IDs[0], s.IDs[1])...); err != nil { + t.Fatal(err) + } + // after some exchanges of messages, we can test state changes + // here this is simply demonstrated by the peerPool + // after the handshake negotiations peers must be added to the pool + // time.Sleep(1) + tick := time.NewTicker(10 * time.Millisecond) + timeout := time.NewTimer(1 * time.Second) +WAIT: + for { + select { + case <-tick.C: + if pp.Has(s.IDs[0]) { + break WAIT + } + case <-timeout.C: + t.Fatal("timeout") + } + } + if !pp.Has(s.IDs[1]) { + t.Fatalf("missing peer test-1: %v (%v)", pp, s.IDs) + } + + // peer 0 sends kill request for peer with index
+ err := s.TestExchanges(p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 2, + Msg: &kill{s.IDs[peer]}, + Peer: s.IDs[0], + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + // the peer not killed sends a drop request + err = s.TestExchanges(p2ptest.Exchange{ + Triggers: []p2ptest.Trigger{ + p2ptest.Trigger{ + Code: 3, + Msg: &drop{}, + Peer: s.IDs[(peer+1)%2], + }, + }, + }) + + if err != nil { + t.Fatal(err) + } + + // check the actual discconnect errors on the individual peers + var disconnects []*p2ptest.Disconnect + for i, err := range errs { + disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err}) + } + if err := s.TestDisconnected(disconnects...); err != nil { + t.Fatal(err) + } + // test if disconnected peers have been removed from peerPool + if pp.Has(s.IDs[peer]) { + t.Fatalf("peer test-%v not dropped: %v (%v)", peer, pp, s.IDs) + } + +} + +func TestMultiplePeersDropSelf(t *testing.T) { + runMultiplePeers(t, 0, + fmt.Errorf("subprotocol error"), + fmt.Errorf("Message handler error: (msg code 3): dropped"), + ) +} + +func TestMultiplePeersDropOther(t *testing.T) { + runMultiplePeers(t, 1, + fmt.Errorf("Message handler error: (msg code 3): dropped"), + fmt.Errorf("subprotocol error"), + ) +} diff --git a/p2p/testing/peerpool.go b/p2p/testing/peerpool.go new file mode 100644 index 0000000000..45c6e61425 --- /dev/null +++ b/p2p/testing/peerpool.go @@ -0,0 +1,67 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see
. + +package testing + +import ( + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/discover" +) + +type TestPeer interface { + ID() discover.NodeID + Drop(error) +} + +// TestPeerPool is an example peerPool to demonstrate registration of peer connections +type TestPeerPool struct { + lock sync.Mutex + peers map[discover.NodeID]TestPeer +} + +func NewTestPeerPool() *TestPeerPool { + return &TestPeerPool{peers: make(map[discover.NodeID]TestPeer)} +} + +func (self *TestPeerPool) Add(p TestPeer) { + self.lock.Lock() + defer self.lock.Unlock() + log.Trace(fmt.Sprintf("pp add peer %v", p.ID())) + self.peers[p.ID()] = p + +} + +func (self *TestPeerPool) Remove(p TestPeer) { + self.lock.Lock() + defer self.lock.Unlock() + delete(self.peers, p.ID()) +} + +func (self *TestPeerPool) Has(id discover.NodeID) bool { + self.lock.Lock() + defer self.lock.Unlock() + _, ok := self.peers[id] + return ok +} + +func (self *TestPeerPool) Get(id discover.NodeID) TestPeer { + self.lock.Lock() + defer self.lock.Unlock() + return self.peers[id] +} diff --git a/p2p/testing/protocolsession.go b/p2p/testing/protocolsession.go new file mode 100644 index 0000000000..a779aeebbe --- /dev/null +++ b/p2p/testing/protocolsession.go @@ -0,0 +1,206 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see
. + +package testing + +import ( + "errors" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" +) + +// ProtocolSession is a quasi simulation of a pivot node running +// a service and a number of dummy peers that can send (trigger) or +// receive (expect) messages +type ProtocolSession struct { + Server *p2p.Server + IDs []discover.NodeID + adapter *adapters.SimAdapter + events chan *p2p.PeerEvent +} + +// Exchange is the basic units of protocol tests +// the triggers and expects in the arrays are run immediately and asynchronously +// thus one cannot have multiple expects for the SAME peer with DIFFERENT message types +// because it's unpredictable which expect will receive which message +// (with expect #1 and #2, messages might be sent #2 and #1, and both expects will complain about wrong message code) +// an exchange is defined on a session +type Exchange struct { + Label string + Triggers []Trigger + Expects []Expect +} + +// Trigger is part of the exchange, incoming message for the pivot node +// sent by a peer +type Trigger struct { + Msg interface{} // type of message to be sent + Code uint64 // code of message is given + Peer discover.NodeID // the peer to send the message to + Timeout time.Duration // timeout duration for the sending +} + +// Expect is part of an exchange, outgoing message from the pivot node +// received by a peer +type Expect struct { + Msg interface{} // type of message to expect + Code uint64 // code of message is now given + Peer discover.NodeID // the peer that expects the message + Timeout time.Duration // timeout duration for receiving +} + +// Disconnect represents a disconnect event, used and checked by TestDisconnected +type Disconnect struct { + Peer discover.NodeID // discconnected peer + Error error // disconnect reason +} + +// trigger sends messages from peers +func (self *ProtocolSession) trigger(trig Trigger) error { + simNode, ok := self.adapter.GetNode(trig.Peer) + if !ok { + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(self.IDs)) + } + mockNode, ok := simNode.Services()[0].(*mockNode) + if !ok { + return fmt.Errorf("trigger: peer %v is not a mock", trig.Peer) + } + + errc := make(chan error) + + go func() { + errc <- mockNode.Trigger(&trig) + }() + + t := trig.Timeout + if t == time.Duration(0) { + t = 1000 * time.Millisecond + } + select { + case err := <-errc: + return err + case <-time.After(t): + return fmt.Errorf("timout expecting %v to send to peer %v", trig.Msg, trig.Peer) + } +} + +// expect checks an expectation of a message sent out by the pivot node +func (self *ProtocolSession) expect(exp Expect) error { + if exp.Msg == nil { + return errors.New("no message to expect") + } + simNode, ok := self.adapter.GetNode(exp.Peer) + if !ok { + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", exp.Peer, len(self.IDs)) + } + mockNode, ok := simNode.Services()[0].(*mockNode) + if !ok { + return fmt.Errorf("trigger: peer %v is not a mock", exp.Peer) + } + + errc := make(chan error) + go func() { + errc <- mockNode.Expect(&exp) + }() + + t := exp.Timeout + if t == time.Duration(0) { + t = 2000 * time.Millisecond + } + select { + case err := <-errc: + return err + case <-time.After(t): + return fmt.Errorf("timout expecting %v sent to peer %v", exp.Msg, exp.Peer) + } +} + +// TestExchanges tests a series of exchanges against the session +func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { + // launch all triggers of this exchanges + + for _, e := range exchanges { + errc := make(chan error, len(e.Triggers)+len(e.Expects)) + for _, trig := range e.Triggers { + errc <- self.trigger(trig) + } + + // each expectation is spawned in separate go-routine + // expectations of an exchange are conjunctive but unordered, i.e., + // only all of them arriving constitutes a pass + // each expectation is meant to be for a different peer, otherwise they are expected to panic + // testing of an exchange blocks until all expectations are decided + // an expectation is decided if + // expected message arrives OR + // an unexpected message arrives (panic) + // times out on their individual timeout + for _, ex := range e.Expects { + // expect msg spawned to separate go routine + go func(exp Expect) { + errc <- self.expect(exp) + }(ex) + } + + // time out globally or finish when all expectations satisfied + timeout := time.After(5 * time.Second) + for i := 0; i < len(e.Triggers)+len(e.Expects); i++ { + select { + case err := <-errc: + if err != nil { + return fmt.Errorf("exchange failed with: %v", err) + } + case <-timeout: + return fmt.Errorf("exchange %v: '%v' timed out", i, e.Label) + } + } + } + return nil +} + +// TestDisconnected tests the disconnections given as arguments +// the disconnect structs describe what disconnect error is expected on which peer +func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error { + expects := make(map[discover.NodeID]error) + for _, disconnect := range disconnects { + expects[disconnect.Peer] = disconnect.Error + } + + timeout := time.After(time.Second) + for len(expects) > 0 { + select { + case event := <-self.events: + if event.Type != p2p.PeerEventTypeDrop { + continue + } + expectErr, ok := expects[event.Peer] + if !ok { + continue + } + + if !(expectErr == nil && event.Error == "" || expectErr != nil && expectErr.Error() == event.Error) { + return fmt.Errorf("unexpected error on peer %v. expected '%v', got '%v'", event.Peer, expectErr, event.Error) + } + delete(expects, event.Peer) + case <-timeout: + return fmt.Errorf("timed out waiting for peers to disconnect") + } + } + return nil +} diff --git a/p2p/testing/protocoltester.go b/p2p/testing/protocoltester.go new file mode 100644 index 0000000000..ea5b106ff8 --- /dev/null +++ b/p2p/testing/protocoltester.go @@ -0,0 +1,200 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see
. + +/* +the p2p/testing package provides a unit test scheme to check simple +protocol message exchanges with one pivot node and a number of dummy peers +The pivot test node runs a node.Service, the dummy peers run a mock node +that can be used to send and receive messages +*/ + +package testing + +import ( + "fmt" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rpc" +) + +// ProtocolTester is the tester environment used for unit testing protocol +// message exchanges. It uses p2p/simulations framework +type ProtocolTester struct { + *ProtocolSession + network *simulations.Network +} + +// NewProtocolTester constructs a new ProtocolTester +// it takes as argument the pivot node id, the number of dummy peers and the +// protocol run function called on a peer connection by the p2p server +func NewProtocolTester(t *testing.T, id discover.NodeID, n int, run func(*p2p.Peer, p2p.MsgReadWriter) error) *ProtocolTester { + services := adapters.Services{ + "test": func(ctx *adapters.ServiceContext) (node.Service, error) { + return &testNode{run}, nil + }, + "mock": func(ctx *adapters.ServiceContext) (node.Service, error) { + return newMockNode(), nil + }, + } + adapter := adapters.NewSimAdapter(services) + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{}) + if _, err := net.NewNodeWithConfig(&adapters.NodeConfig{ + ID: id, + EnableMsgEvents: true, + Services: []string{"test"}, + }); err != nil { + panic(err.Error()) + } + if err := net.Start(id); err != nil { + panic(err.Error()) + } + + node := net.GetNode(id).Node.(*adapters.SimNode) + peers := make([]*adapters.NodeConfig, n) + peerIDs := make([]discover.NodeID, n) + for i := 0; i < n; i++ { + peers[i] = adapters.RandomNodeConfig() + peers[i].Services = []string{"mock"} + peerIDs[i] = peers[i].ID + } + events := make(chan *p2p.PeerEvent, 1000) + node.SubscribeEvents(events) + ps := &ProtocolSession{ + Server: node.Server(), + IDs: peerIDs, + adapter: adapter, + events: events, + } + self := &ProtocolTester{ + ProtocolSession: ps, + network: net, + } + + self.Connect(id, peers...) + + return self +} + +// Stop stops the p2p server +func (self *ProtocolTester) Stop() error { + self.Server.Stop() + return nil +} + +// Connect brings up the remote peer node and connects it using the +// p2p/simulations network connection with the in memory network adapter +func (self *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) { + for _, peer := range peers { + log.Trace(fmt.Sprintf("start node %v", peer.ID)) + if _, err := self.network.NewNodeWithConfig(peer); err != nil { + panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) + } + if err := self.network.Start(peer.ID); err != nil { + panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err)) + } + log.Trace(fmt.Sprintf("connect to %v", peer.ID)) + if err := self.network.Connect(selfID, peer.ID); err != nil { + panic(fmt.Sprintf("error connecting to peer %v: %v", peer.ID, err)) + } + } + +} + +// testNode wraps a protocol run function and implements the node.Service +// interface +type testNode struct { + run func(*p2p.Peer, p2p.MsgReadWriter) error +} + +func (t *testNode) Protocols() []p2p.Protocol { + return []p2p.Protocol{{ + Length: 100, + Run: t.run, + }} +} + +func (t *testNode) APIs() []rpc.API { + return nil +} + +func (t *testNode) Start(server *p2p.Server) error { + return nil +} + +func (t *testNode) Stop() error { + return nil +} + +// mockNode is a testNode which doesn't actually run a protocol, instead +// exposing channels so that tests can manually trigger and expect certain +// messages +type mockNode struct { + testNode + + trigger chan *Trigger + expect chan *Expect + err chan error + stop chan struct{} + stopOnce sync.Once +} + +func newMockNode() *mockNode { + mock := &mockNode{ + trigger: make(chan *Trigger), + expect: make(chan *Expect), + err: make(chan error), + stop: make(chan struct{}), + } + mock.testNode.run = mock.Run + return mock +} + +// Run is a protocol run function which just loops waiting for tests to +// instruct it to either trigger or expect a message from the peer +func (m *mockNode) Run(peer *p2p.Peer, rw p2p.MsgReadWriter) error { + for { + select { + case trig := <-m.trigger: + m.err <- p2p.Send(rw, trig.Code, trig.Msg) + case exp := <-m.expect: + m.err <- p2p.ExpectMsg(rw, exp.Code, exp.Msg) + case <-m.stop: + return nil + } + } +} + +func (m *mockNode) Trigger(trig *Trigger) error { + m.trigger <- trig + return <-m.err +} + +func (m *mockNode) Expect(exp *Expect) error { + m.expect <- exp + return <-m.err +} + +func (m *mockNode) Stop() error { + m.stopOnce.Do(func() { close(m.stop) }) + return nil +} From a6787a6308d2109006b036c8a6a331afa938912d Mon Sep 17 00:00:00 2001 From: croath
Date: Tue, 23 Jan 2018 19:10:23 +0800 Subject: [PATCH 17/94] accounts/abi: fix the `output` at the beginning instead of making a workaround --- accounts/abi/argument.go | 10 ---------- accounts/abi/unpack_test.go | 36 ------------------------------------ mobile/bind.go | 18 +++++++++++++----- 3 files changed, 13 insertions(+), 51 deletions(-) diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 65f79c9d4c..ad17fbf2b1 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -169,16 +169,6 @@ func (arguments Arguments) unpackAtomic(v interface{}, output []byte) error { if err != nil { return err } - - // if we reach this part, there is only one output member from the contract event. - // for mobile, the result type is always a slice. - if reflect.Slice == value.Kind() && value.Len() >= 1 { - //check if it's not a byte slice - if reflect.TypeOf([]byte{}) != value.Type() { - value = value.Index(0).Elem() - } - } - return set(value, reflect.ValueOf(marshalledValue), arg) } diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index a7901896ab..3910950732 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -287,42 +287,6 @@ func TestUnpack(t *testing.T) { } } -var unpackMobileTests = []unpackTest{ - { - def: `[{"type": "uint256"}]`, - enc: "0000000000000000000000000000000000000000000000000000000000000001", - want: big.NewInt(1), - }, -} - -func TestUnpackMobileOnly(t *testing.T) { - for i, test := range unpackMobileTests { - t.Run(strconv.Itoa(i), func(t *testing.T) { - def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def) - abi, err := JSON(strings.NewReader(def)) - if err != nil { - t.Fatalf("invalid ABI definition %s: %v", def, err) - } - encb, err := hex.DecodeString(test.enc) - if err != nil { - t.Fatalf("invalid hex: %s" + test.enc) - } - outptr := reflect.New(reflect.TypeOf(test.want)) - results := make([]interface{}, 1) - copy(results, []interface{}{outptr.Interface()}) - err = abi.Unpack(&results, "method", encb) - if err := test.checkError(err); err != nil { - t.Errorf("test %d (%v) failed: %v", i, test.def, err) - return - } - out := outptr.Elem().Interface() - if !reflect.DeepEqual(test.want, out) { - t.Errorf("test %d (%v) failed: expected %v, got %v", i, test.def, test.want, out) - } - }) - } -} - type methodMultiOutput struct { Int *big.Int String string diff --git a/mobile/bind.go b/mobile/bind.go index 7b79bedafd..e8164d523e 100644 --- a/mobile/bind.go +++ b/mobile/bind.go @@ -154,12 +154,20 @@ func (c *BoundContract) GetDeployer() *Transaction { // Call invokes the (constant) contract method with params as input values and // sets the output to result. func (c *BoundContract) Call(opts *CallOpts, out *Interfaces, method string, args *Interfaces) error { - results := make([]interface{}, len(out.objects)) - copy(results, out.objects) - if err := c.contract.Call(&opts.opts, &results, method, args.objects...); err != nil { - return err + if len(out.objects) == 1 { + result := out.objects[0] + if err := c.contract.Call(&opts.opts, result, method, args.objects...); err != nil { + return err + } + out.objects[0] = result + } else { + results := make([]interface{}, len(out.objects)) + copy(results, out.objects) + if err := c.contract.Call(&opts.opts, &results, method, args.objects...); err != nil { + return err + } + copy(out.objects, results) } - copy(out.objects, results) return nil } From 32301a4d6b3a9684e954057e7cdb15998764122b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?=
Date: Fri, 16 Feb 2018 17:05:08 +0200 Subject: [PATCH 18/94] p2p/discover: validate bond against lastpong, not db presence --- p2p/discover/udp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discover/udp.go b/p2p/discover/udp.go index e40de2c36f..5cc0b3d74b 100644 --- a/p2p/discover/udp.go +++ b/p2p/discover/udp.go @@ -613,7 +613,7 @@ func (req *findnode) handle(t *udp, from *net.UDPAddr, fromID NodeID, mac []byte if expired(req.Expiration) { return errExpired } - if t.db.node(fromID) == nil { + if age := time.Since(t.db.lastPong(fromID)); age > nodeDBNodeExpiration { // No bond exists, we don't process the packet. This prevents // an attack vector where the discovery protocol could be used // to amplify traffic in a DDOS attack. A malicious actor From aeedec4078647c22244552803c88521391224ab1 Mon Sep 17 00:00:00 2001 From: Felix Lange
Date: Fri, 16 Feb 2018 20:16:23 +0100 Subject: [PATCH 19/94] p2p/discover: s/lastPong/bondTime/, update TestUDP_findnode I forgot to change the check in udp.go when I changed Table.bond to be based on lastPong instead of node presence in db. Rename lastPong to bondTime and add hasBond so it's clearer what this DB key is used for now. --- p2p/discover/database.go | 17 +++++++++++------ p2p/discover/database_test.go | 18 +++++++++--------- p2p/discover/table.go | 6 +++--- p2p/discover/udp.go | 2 +- p2p/discover/udp_test.go | 8 ++------ 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/p2p/discover/database.go b/p2p/discover/database.go index b136609f24..6f98de9b42 100644 --- a/p2p/discover/database.go +++ b/p2p/discover/database.go @@ -257,7 +257,7 @@ func (db *nodeDB) expireNodes() error { } // Skip the node if not expired yet (and not self) if !bytes.Equal(id[:], db.self[:]) { - if seen := db.lastPong(id); seen.After(threshold) { + if seen := db.bondTime(id); seen.After(threshold) { continue } } @@ -278,13 +278,18 @@ func (db *nodeDB) updateLastPing(id NodeID, instance time.Time) error { return db.storeInt64(makeKey(id, nodeDBDiscoverPing), instance.Unix()) } -// lastPong retrieves the time of the last successful contact from remote node. -func (db *nodeDB) lastPong(id NodeID) time.Time { +// bondTime retrieves the time of the last successful pong from remote node. +func (db *nodeDB) bondTime(id NodeID) time.Time { return time.Unix(db.fetchInt64(makeKey(id, nodeDBDiscoverPong)), 0) } -// updateLastPong updates the last time a remote node successfully contacted. -func (db *nodeDB) updateLastPong(id NodeID, instance time.Time) error { +// hasBond reports whether the given node is considered bonded. +func (db *nodeDB) hasBond(id NodeID) bool { + return time.Since(db.bondTime(id)) < nodeDBNodeExpiration +} + +// updateBondTime updates the last pong time of a node. +func (db *nodeDB) updateBondTime(id NodeID, instance time.Time) error { return db.storeInt64(makeKey(id, nodeDBDiscoverPong), instance.Unix()) } @@ -327,7 +332,7 @@ seek: if n.ID == db.self { continue seek } - if now.Sub(db.lastPong(n.ID)) > maxAge { + if now.Sub(db.bondTime(n.ID)) > maxAge { continue seek } for i := range nodes { diff --git a/p2p/discover/database_test.go b/p2p/discover/database_test.go index be972fd2c3..c4fa44d099 100644 --- a/p2p/discover/database_test.go +++ b/p2p/discover/database_test.go @@ -125,13 +125,13 @@ func TestNodeDBFetchStore(t *testing.T) { t.Errorf("ping: value mismatch: have %v, want %v", stored, inst) } // Check fetch/store operations on a node pong object - if stored := db.lastPong(node.ID); stored.Unix() != 0 { + if stored := db.bondTime(node.ID); stored.Unix() != 0 { t.Errorf("pong: non-existing object: %v", stored) } - if err := db.updateLastPong(node.ID, inst); err != nil { + if err := db.updateBondTime(node.ID, inst); err != nil { t.Errorf("pong: failed to update: %v", err) } - if stored := db.lastPong(node.ID); stored.Unix() != inst.Unix() { + if stored := db.bondTime(node.ID); stored.Unix() != inst.Unix() { t.Errorf("pong: value mismatch: have %v, want %v", stored, inst) } // Check fetch/store operations on a node findnode-failure object @@ -224,8 +224,8 @@ func TestNodeDBSeedQuery(t *testing.T) { if err := db.updateNode(seed.node); err != nil { t.Fatalf("node %d: failed to insert: %v", i, err) } - if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil { - t.Fatalf("node %d: failed to insert lastPong: %v", i, err) + if err := db.updateBondTime(seed.node.ID, seed.pong); err != nil { + t.Fatalf("node %d: failed to insert bondTime: %v", i, err) } } @@ -332,8 +332,8 @@ func TestNodeDBExpiration(t *testing.T) { if err := db.updateNode(seed.node); err != nil { t.Fatalf("node %d: failed to insert: %v", i, err) } - if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil { - t.Fatalf("node %d: failed to update pong: %v", i, err) + if err := db.updateBondTime(seed.node.ID, seed.pong); err != nil { + t.Fatalf("node %d: failed to update bondTime: %v", i, err) } } // Expire some of them, and check the rest @@ -365,8 +365,8 @@ func TestNodeDBSelfExpiration(t *testing.T) { if err := db.updateNode(seed.node); err != nil { t.Fatalf("node %d: failed to insert: %v", i, err) } - if err := db.updateLastPong(seed.node.ID, seed.pong); err != nil { - t.Fatalf("node %d: failed to update pong: %v", i, err) + if err := db.updateBondTime(seed.node.ID, seed.pong); err != nil { + t.Fatalf("node %d: failed to update bondTime: %v", i, err) } } // Expire the nodes and make sure self has been evacuated too diff --git a/p2p/discover/table.go b/p2p/discover/table.go index 17c9db7774..6509326e69 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -455,7 +455,7 @@ func (tab *Table) loadSeedNodes(bond bool) { } for i := range seeds { seed := seeds[i] - age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.lastPong(seed.ID)) }} + age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.bondTime(seed.ID)) }} log.Debug("Found seed node in database", "id", seed.ID, "addr", seed.addr(), "age", age) tab.add(seed) } @@ -596,7 +596,7 @@ func (tab *Table) bond(pinged bool, id NodeID, addr *net.UDPAddr, tcpPort uint16 } // Start bonding if we haven't seen this node for a while or if it failed findnode too often. node, fails := tab.db.node(id), tab.db.findFails(id) - age := time.Since(tab.db.lastPong(id)) + age := time.Since(tab.db.bondTime(id)) var result error if fails > 0 || age > nodeDBNodeExpiration { log.Trace("Starting bonding ping/pong", "id", id, "known", node != nil, "failcount", fails, "age", age) @@ -663,7 +663,7 @@ func (tab *Table) ping(id NodeID, addr *net.UDPAddr) error { if err := tab.net.ping(id, addr); err != nil { return err } - tab.db.updateLastPong(id, time.Now()) + tab.db.updateBondTime(id, time.Now()) return nil } diff --git a/p2p/discover/udp.go b/p2p/discover/udp.go index 5cc0b3d74b..524c6e4988 100644 --- a/p2p/discover/udp.go +++ b/p2p/discover/udp.go @@ -613,7 +613,7 @@ func (req *findnode) handle(t *udp, from *net.UDPAddr, fromID NodeID, mac []byte if expired(req.Expiration) { return errExpired } - if age := time.Since(t.db.lastPong(fromID)); age > nodeDBNodeExpiration { + if !t.db.hasBond(fromID) { // No bond exists, we don't process the packet. This prevents // an attack vector where the discovery protocol could be used // to amplify traffic in a DDOS attack. A malicious actor diff --git a/p2p/discover/udp_test.go b/p2p/discover/udp_test.go index 3ffa5c4dd1..db9804f7bc 100644 --- a/p2p/discover/udp_test.go +++ b/p2p/discover/udp_test.go @@ -247,12 +247,8 @@ func TestUDP_findnode(t *testing.T) { // ensure there's a bond with the test node, // findnode won't be accepted otherwise. - test.table.db.updateNode(NewNode( - PubkeyID(&test.remotekey.PublicKey), - test.remoteaddr.IP, - uint16(test.remoteaddr.Port), - 99, - )) + test.table.db.updateBondTime(PubkeyID(&test.remotekey.PublicKey), time.Now()) + // check that closest neighbors are returned. test.packetIn(nil, findnodePacket, &findnode{Target: testTarget, Expiration: futureExp}) expected := test.table.closest(targetHash, bucketSize) From 0a7cbd915a65357613d0dbdd37d6430975e110a8 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov
Date: Sat, 17 Feb 2018 14:35:59 +0100 Subject: [PATCH 20/94] vendor: update rjeczalik/notify so that it compiles on go1.10 --- .../rjeczalik/notify/watcher_fsevents_cgo.go | 6 +++--- .../rjeczalik/notify/watcher_fsevents_go1.10.go | 9 --------- .../rjeczalik/notify/watcher_fsevents_go1.9.go | 14 -------------- vendor/vendor.json | 6 +++--- 4 files changed, 6 insertions(+), 29 deletions(-) delete mode 100644 vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go delete mode 100644 vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go index 2248a1b129..a2b332a2e0 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go @@ -48,7 +48,7 @@ var wg sync.WaitGroup // used to wait until the runloop starts // started and is ready via the wg. It also serves purpose of a dummy source, // thanks to it the runloop does not return as it also has at least one source // registered. -var source = C.CFRunLoopSourceCreate(refZero, 0, &C.CFRunLoopSourceContext{ +var source = C.CFRunLoopSourceCreate(nil, 0, &C.CFRunLoopSourceContext{ perform: (C.CFRunLoopPerformCallBack)(C.gosource), }) @@ -162,8 +162,8 @@ func (s *stream) Start() error { return nil } wg.Wait() - p := C.CFStringCreateWithCStringNoCopy(refZero, C.CString(s.path), C.kCFStringEncodingUTF8, refZero) - path := C.CFArrayCreate(refZero, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) + p := C.CFStringCreateWithCStringNoCopy(nil, C.CString(s.path), C.kCFStringEncodingUTF8, nil) + path := C.CFArrayCreate(nil, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) ctx := C.FSEventStreamContext{} ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags) if ref == nilstream { diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go deleted file mode 100644 index 0edd3782f5..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2017 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,go1.10 - -package notify - -const refZero = 0 diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go deleted file mode 100644 index b81c3c1859..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.9.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2017 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,cgo,!go1.10 - -package notify - -/* -#include
-*/ -import "C" - -var refZero = (*C.struct___CFAllocator)(nil) diff --git a/vendor/vendor.json b/vendor/vendor.json index 4360223298..e938ce5e64 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -322,10 +322,10 @@ "revisionTime": "2016-11-28T21:05:44Z" }, { - "checksumSHA1": "1ESHllhZOIBg7MnlGHUdhz047bI=", + "checksumSHA1": "28UVHMmHx0iqO0XiJsjx+fwILyI=", "path": "github.com/rjeczalik/notify", - "revision": "27b537f07230b3f917421af6dcf044038dbe57e2", - "revisionTime": "2018-01-03T13:19:05Z" + "revision": "c31e5f2cb22b3e4ef3f882f413847669bf2652b9", + "revisionTime": "2018-02-03T14:01:15Z" }, { "checksumSHA1": "5uqO4ITTDMklKi3uNaE/D9LQ5nM=", From e07603bbc4d474667a0e724e9baefd8b7c330a61 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 12 Feb 2018 18:40:25 +0100 Subject: [PATCH 21/94] p2p/testing: check for all expectations in TestExchanges Handle all expectations in ProtocolSession.TestExchanges in any order that are received. --- p2p/testing/protocolsession.go | 178 +++++++++++++++++++++++---------- p2p/testing/protocoltester.go | 79 ++++++++++++++- 2 files changed, 200 insertions(+), 57 deletions(-) diff --git a/p2p/testing/protocolsession.go b/p2p/testing/protocolsession.go index a779aeebbe..361285f06e 100644 --- a/p2p/testing/protocolsession.go +++ b/p2p/testing/protocolsession.go @@ -19,13 +19,17 @@ package testing import ( "errors" "fmt" + "sync" "time" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" ) +var errTimedOut = errors.New("timed out") + // ProtocolSession is a quasi simulation of a pivot node running // a service and a number of dummy peers that can send (trigger) or // receive (expect) messages @@ -46,6 +50,7 @@ type Exchange struct { Label string Triggers []Trigger Expects []Expect + Timeout time.Duration } // Trigger is part of the exchange, incoming message for the pivot node @@ -102,76 +107,145 @@ func (self *ProtocolSession) trigger(trig Trigger) error { } // expect checks an expectation of a message sent out by the pivot node -func (self *ProtocolSession) expect(exp Expect) error { - if exp.Msg == nil { - return errors.New("no message to expect") - } - simNode, ok := self.adapter.GetNode(exp.Peer) - if !ok { - return fmt.Errorf("trigger: peer %v does not exist (1- %v)", exp.Peer, len(self.IDs)) +func (self *ProtocolSession) expect(exps []Expect) error { + // construct a map of expectations for each node + peerExpects := make(map[discover.NodeID][]Expect) + for _, exp := range exps { + if exp.Msg == nil { + return errors.New("no message to expect") + } + peerExpects[exp.Peer] = append(peerExpects[exp.Peer], exp) } - mockNode, ok := simNode.Services()[0].(*mockNode) - if !ok { - return fmt.Errorf("trigger: peer %v is not a mock", exp.Peer) + + // construct a map of mockNodes for each node + mockNodes := make(map[discover.NodeID]*mockNode) + for nodeID := range peerExpects { + simNode, ok := self.adapter.GetNode(nodeID) + if !ok { + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", nodeID, len(self.IDs)) + } + mockNode, ok := simNode.Services()[0].(*mockNode) + if !ok { + return fmt.Errorf("trigger: peer %v is not a mock", nodeID) + } + mockNodes[nodeID] = mockNode } + // done chanell cancels all created goroutines when function returns + done := make(chan struct{}) + defer close(done) + // errc catches the first error from errc := make(chan error) + + wg := &sync.WaitGroup{} + wg.Add(len(mockNodes)) + for nodeID, mockNode := range mockNodes { + nodeID := nodeID + mockNode := mockNode + go func() { + defer wg.Done() + + // Sum all Expect timeouts to give the maximum + // time for all expectations to finish. + // mockNode.Expect checks all received messages against + // a list of expected messages and timeout for each + // of them can not be checked separately. + var t time.Duration + for _, exp := range peerExpects[nodeID] { + if exp.Timeout == time.Duration(0) { + t += 2000 * time.Millisecond + } else { + t += exp.Timeout + } + } + alarm := time.NewTimer(t) + defer alarm.Stop() + + // expectErrc is used to check if error returned + // from mockNode.Expect is not nil and to send it to + // errc only in that case. + // done channel will be closed when function + expectErrc := make(chan error) + go func() { + select { + case expectErrc <- mockNode.Expect(peerExpects[nodeID]...): + case <-done: + case <-alarm.C: + } + }() + + select { + case err := <-expectErrc: + if err != nil { + select { + case errc <- err: + case <-done: + case <-alarm.C: + errc <- errTimedOut + } + } + case <-done: + case <-alarm.C: + errc <- errTimedOut + } + + }() + } + go func() { - errc <- mockNode.Expect(&exp) + wg.Wait() + // close errc when all goroutines finish to return nill err from errc + close(errc) }() - t := exp.Timeout - if t == time.Duration(0) { - t = 2000 * time.Millisecond - } - select { - case err := <-errc: - return err - case <-time.After(t): - return fmt.Errorf("timout expecting %v sent to peer %v", exp.Msg, exp.Peer) - } + return <-errc } // TestExchanges tests a series of exchanges against the session func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { - // launch all triggers of this exchanges + for i, e := range exchanges { + if err := self.testExchange(e); err != nil { + return fmt.Errorf("exchange #%d %q: %v", i, e.Label, err) + } + log.Trace(fmt.Sprintf("exchange #%d %q: run successfully", i, e.Label)) + } + return nil +} + +// testExchange tests a single Exchange. +// Default timeout value is 2 seconds. +func (self *ProtocolSession) testExchange(e Exchange) error { + errc := make(chan error) + done := make(chan struct{}) + defer close(done) - for _, e := range exchanges { - errc := make(chan error, len(e.Triggers)+len(e.Expects)) + go func() { for _, trig := range e.Triggers { - errc <- self.trigger(trig) + err := self.trigger(trig) + if err != nil { + errc <- err + return + } } - // each expectation is spawned in separate go-routine - // expectations of an exchange are conjunctive but unordered, i.e., - // only all of them arriving constitutes a pass - // each expectation is meant to be for a different peer, otherwise they are expected to panic - // testing of an exchange blocks until all expectations are decided - // an expectation is decided if - // expected message arrives OR - // an unexpected message arrives (panic) - // times out on their individual timeout - for _, ex := range e.Expects { - // expect msg spawned to separate go routine - go func(exp Expect) { - errc <- self.expect(exp) - }(ex) + select { + case errc <- self.expect(e.Expects): + case <-done: } + }() - // time out globally or finish when all expectations satisfied - timeout := time.After(5 * time.Second) - for i := 0; i < len(e.Triggers)+len(e.Expects); i++ { - select { - case err := <-errc: - if err != nil { - return fmt.Errorf("exchange failed with: %v", err) - } - case <-timeout: - return fmt.Errorf("exchange %v: '%v' timed out", i, e.Label) - } - } + // time out globally or finish when all expectations satisfied + t := e.Timeout + if t == 0 { + t = 2000 * time.Millisecond + } + alarm := time.NewTimer(t) + select { + case err := <-errc: + return err + case <-alarm.C: + return errTimedOut } - return nil } // TestDisconnected tests the disconnections given as arguments diff --git a/p2p/testing/protocoltester.go b/p2p/testing/protocoltester.go index ea5b106ff8..a797412d60 100644 --- a/p2p/testing/protocoltester.go +++ b/p2p/testing/protocoltester.go @@ -24,7 +24,11 @@ that can be used to send and receive messages package testing import ( + "bytes" "fmt" + "io" + "io/ioutil" + "strings" "sync" "testing" @@ -34,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ) @@ -152,7 +157,7 @@ type mockNode struct { testNode trigger chan *Trigger - expect chan *Expect + expect chan []Expect err chan error stop chan struct{} stopOnce sync.Once @@ -161,7 +166,7 @@ type mockNode struct { func newMockNode() *mockNode { mock := &mockNode{ trigger: make(chan *Trigger), - expect: make(chan *Expect), + expect: make(chan []Expect), err: make(chan error), stop: make(chan struct{}), } @@ -176,8 +181,8 @@ func (m *mockNode) Run(peer *p2p.Peer, rw p2p.MsgReadWriter) error { select { case trig := <-m.trigger: m.err <- p2p.Send(rw, trig.Code, trig.Msg) - case exp := <-m.expect: - m.err <- p2p.ExpectMsg(rw, exp.Code, exp.Msg) + case exps := <-m.expect: + m.err <- expectMsgs(rw, exps) case <-m.stop: return nil } @@ -189,7 +194,7 @@ func (m *mockNode) Trigger(trig *Trigger) error { return <-m.err } -func (m *mockNode) Expect(exp *Expect) error { +func (m *mockNode) Expect(exp ...Expect) error { m.expect <- exp return <-m.err } @@ -198,3 +203,67 @@ func (m *mockNode) Stop() error { m.stopOnce.Do(func() { close(m.stop) }) return nil } + +func expectMsgs(rw p2p.MsgReadWriter, exps []Expect) error { + matched := make([]bool, len(exps)) + for { + msg, err := rw.ReadMsg() + if err != nil { + if err == io.EOF { + break + } + return err + } + actualContent, err := ioutil.ReadAll(msg.Payload) + if err != nil { + return err + } + var found bool + for i, exp := range exps { + if exp.Code == msg.Code && bytes.Equal(actualContent, mustEncodeMsg(exp.Msg)) { + if matched[i] { + return fmt.Errorf("message #%d received two times", i) + } + matched[i] = true + found = true + break + } + } + if !found { + expected := make([]string, 0) + for i, exp := range exps { + if matched[i] { + continue + } + expected = append(expected, fmt.Sprintf("code %d payload %x", exp.Code, mustEncodeMsg(exp.Msg))) + } + return fmt.Errorf("unexpected message code %d payload %x, expected %s", msg.Code, actualContent, strings.Join(expected, " or ")) + } + done := true + for _, m := range matched { + if !m { + done = false + break + } + } + if done { + return nil + } + } + for i, m := range matched { + if !m { + return fmt.Errorf("expected message #%d not received", i) + } + } + return nil +} + +// mustEncodeMsg uses rlp to encode a message. +// In case of error it panics. +func mustEncodeMsg(msg interface{}) []byte { + contentEnc, err := rlp.EncodeToBytes(msg) + if err != nil { + panic("content encode error: " + err.Error()) + } + return contentEnc +} From 1bdde620dad3c5fccb8e5de19fe7f42b1209a156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?=
Date: Mon, 19 Feb 2018 09:41:30 +0100 Subject: [PATCH 22/94] les: fix light fetcher database race (#16103) * les: fix light fetcher database race * les: lightFetcher comments --- les/fetcher.go | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/les/fetcher.go b/les/fetcher.go index 9d224176fc..e12a2c78a2 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -36,24 +36,26 @@ const ( maxNodeCount = 20 // maximum number of fetcherTreeNode entries remembered for each peer ) -// lightFetcher +// lightFetcher implements retrieval of newly announced headers. It also provides a peerHasBlock function for the +// ODR system to ensure that we only request data related to a certain block from peers who have already processed +// and announced that block. type lightFetcher struct { pm *ProtocolManager odr *LesOdr chain *light.LightChain + lock sync.Mutex // lock protects access to the fetcher's internal state variables except sent requests maxConfirmedTd *big.Int peers map[*peer]*fetcherPeerInfo lastUpdateStats *updateStatsEntry + syncing bool + syncDone chan *peer - lock sync.Mutex // qwerqwerqwe - deliverChn chan fetchResponse - reqMu sync.RWMutex + reqMu sync.RWMutex // reqMu protects access to sent header fetch requests requested map[uint64]fetchRequest + deliverChn chan fetchResponse timeoutChn chan uint64 requestChn chan bool // true if initiated from outside - syncing bool - syncDone chan *peer } // fetcherPeerInfo holds fetcher-specific information about each active peer @@ -560,8 +562,13 @@ func (f *lightFetcher) checkAnnouncedHeaders(fp *fetcherPeerInfo, headers []*typ return true } // we ran out of recently delivered headers but have not reached a node known by this peer yet, continue matching - td = f.chain.GetTd(header.ParentHash, header.Number.Uint64()-1) - header = f.chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) + hash, number := header.ParentHash, header.Number.Uint64()-1 + td = f.chain.GetTd(hash, number) + header = f.chain.GetHeader(hash, number) + if header == nil || td == nil { + log.Error("Missing parent of validated header", "hash", hash, "number", number) + return false + } } else { header = headers[i] td = tds[i] @@ -645,13 +652,18 @@ func (f *lightFetcher) checkKnownNode(p *peer, n *fetcherTreeNode) bool { if td == nil { return false } + header := f.chain.GetHeader(n.hash, n.number) + // check the availability of both header and td because reads are not protected by chain db mutex + // Note: returning false is always safe here + if header == nil { + return false + } fp := f.peers[p] if fp == nil { p.Log().Debug("Unknown peer to check known nodes") return false } - header := f.chain.GetHeader(n.hash, n.number) if !f.checkAnnouncedHeaders(fp, []*types.Header{header}, []*big.Int{td}) { p.Log().Debug("Inconsistent announcement") go f.pm.removePeer(p.id) From 1e67410e88d2685bc54611a7c9f75c327b553ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?=
Date: Mon, 19 Feb 2018 11:35:31 +0200 Subject: [PATCH 23/94] params: release Geth v1.8.1 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 2775859349..86dae73430 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 1 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 1 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 46a5532ac5ef06e5d7f77a4af19ffd2e234dcb07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?=
Date: Mon, 19 Feb 2018 11:38:36 +0200 Subject: [PATCH 24/94] VERSION, params: begin v1.8.2 release cycle --- VERSION | 2 +- params/version.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index a8fdfda1c7..53adb84c82 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.1 +1.8.2 diff --git a/params/version.go b/params/version.go index 86dae73430..c0437cef02 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 1 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 2 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 5603715c06998ff9ed30eb134c4398ad1439ef48 Mon Sep 17 00:00:00 2001 From: steve greensill
Date: Tue, 20 Feb 2018 10:38:27 +0100 Subject: [PATCH 25/94] README: add goreportcard.com badge to Readme (#16133) * README: add goreportcard.com badge to Readme * README: fix double github.com --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 527ea048a3..aad14ed7d8 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Official golang implementation of the Ethereum protocol. [](https://godoc.org/github.com/ethereum/go-ethereum) +[](https://goreportcard.com/report/github.com/ethereum/go-ethereum) [](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) Automated builds are available for stable releases and the unstable master branch. From 1ede68355db6adbf468d198a8d1ecb0ad1a3ea31 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende
Date: Thu, 28 Dec 2017 11:17:45 +0100 Subject: [PATCH 26/94] accounts/abi: add another unpack interface --- accounts/abi/abi.go | 6 +- accounts/abi/abi_test.go | 6 +- accounts/abi/argument.go | 59 +++++- accounts/abi/unpack_test.go | 2 +- accounts/abi/unpackv2_test.go | 336 ++++++++++++++++++++++++++++++++++ 5 files changed, 395 insertions(+), 14 deletions(-) create mode 100644 accounts/abi/unpackv2_test.go diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index abcb403db2..32f041890b 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -136,11 +136,11 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { // MethodById looks up a method by the 4-byte id // returns nil if none found -func (abi *ABI) MethodById(sigdata []byte) *Method { +func (abi *ABI) MethodById(sigdata []byte) (*Method, error){ for _, method := range abi.Methods { if bytes.Equal(method.Id(), sigdata[:4]) { - return &method + return &method, nil } } - return nil + return nil, fmt.Errorf("ABI spec does not contain method signature in data: 0x%x", sigdata[:4]) } diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index 2d43b631c2..3bef6add5c 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -689,7 +689,11 @@ func TestABI_MethodById(t *testing.T) { } for name, m := range abi.Methods { a := fmt.Sprintf("%v", m) - b := fmt.Sprintf("%v", abi.MethodById(m.Id())) + m2,err := abi.MethodById(m.Id()) + if err != nil { + t.Fatal(err) + } + b := fmt.Sprintf("%v", m2) if a != b { t.Errorf("Method %v (id %v) not 'findable' by id in ABI", name, common.ToHex(m.Id())) } diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 04ca6150ad..b9b5371211 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -67,6 +67,16 @@ func (arguments Arguments) LengthNonIndexed() int { return out } +func (arguments Arguments) NonIndexed() Arguments{ + var ret []Argument + for _,arg := range arguments{ + if !arg.Indexed{ + ret = append(ret, arg) + } + } + return ret +} + // isTuple returns true for non-atomic constructs, like (uint,uint) or uint[] func (arguments Arguments) isTuple() bool { return len(arguments) > 1 @@ -114,14 +124,9 @@ func (arguments Arguments) unpackTuple(v interface{}, output []byte) error { // `j` counts the number of complex types. // both `i` and `j` are used to to correctly compute `data` offset. - i, j := -1, 0 - for _, arg := range arguments { + j := 0 + for i, arg := range arguments.NonIndexed() { - if arg.Indexed { - // can't read, continue - continue - } - i++ marshalledValue, err := toGoType((i+j)*32, arg.Type, output) if err != nil { return err @@ -178,7 +183,6 @@ func (arguments Arguments) unpackAtomic(v interface{}, output []byte) error { } value := valueOf.Elem() - marshalledValue, err := toGoType(0, arg.Type, output) if err != nil { return err @@ -186,7 +190,44 @@ func (arguments Arguments) unpackAtomic(v interface{}, output []byte) error { return set(value, reflect.ValueOf(marshalledValue), arg) } -// Unpack performs the operation Go format -> Hexdata +// UnpackValues can be used to unpack ABI-encoded hexdata according to the ABI-specification, +// without supplying a struct to unpack into. Instead, this method returns a list containing the +// values. An atomic argument will be a list with one element. +func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error){ + + retval := make([]interface{},0,arguments.LengthNonIndexed()) + + virtualArgs := 0 + + for index,arg:= range arguments.NonIndexed(){ + + marshalledValue, err := toGoType((index + virtualArgs) * 32, arg.Type, data) + + if arg.Type.T == ArrayTy { + //If we have a static array, like [3]uint256, these are coded as + // just like uint256,uint256,uint256. + // This means that we need to add two 'virtual' arguments when + // we count the index from now on + + virtualArgs += arg.Type.Size - 1 + } + + if err != nil{ + return nil, err + } + retval = append(retval, marshalledValue) + } + return retval, nil +} + +// UnpackValues performs the operation Go format -> Hexdata +// It is the semantic opposite of UnpackValues +func (arguments Arguments) PackValues(args []interface{}) ([]byte, error) { + return arguments.Pack(args...) +} + + +// Pack performs the operation Go format -> Hexdata func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { // Make sure arguments match up and pack them abiArgs := arguments diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index a65426a30b..e9f9108125 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -130,7 +130,7 @@ var unpackTests = []unpackTest{ { def: `[{"type": "bytes32"}]`, enc: "0100000000000000000000000000000000000000000000000000000000000000", - want: common.HexToHash("0100000000000000000000000000000000000000000000000000000000000000"), + want: [32]byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, }, { def: `[{"type": "function"}]`, diff --git a/accounts/abi/unpackv2_test.go b/accounts/abi/unpackv2_test.go new file mode 100644 index 0000000000..d0074ff7b1 --- /dev/null +++ b/accounts/abi/unpackv2_test.go @@ -0,0 +1,336 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see
. + +package abi + +import ( + "bytes" + "encoding/hex" + "fmt" + "math/big" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestUnpackV2(t *testing.T) { + for i, test := range unpackTests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def) + abi, err := JSON(strings.NewReader(def)) + if err != nil { + t.Fatalf("invalid ABI definition %s: %v", def, err) + } + encb, err := hex.DecodeString(test.enc) + if err != nil { + t.Fatalf("invalid hex: %s" + test.enc) + } + out, err := abi.Methods["method"].Outputs.UnpackValues(encb) + + if err != nil { + t.Fatal(err) + } + if len(test.err) != 0 { + // The new stuff doesn't have these types of errors + return + } + if !reflect.DeepEqual(test.want, out[0]) { + t.Errorf("test %d (%v) failed: expected %v, got %v", i, test.def, test.want, out[0]) + } + }) + } +} + + +func TestMultiReturnWithArrayV2(t *testing.T) { + const definition = `[{"name" : "multi", "outputs": [{"type": "uint64[3]"}, {"type": "uint64"}]}]` + abi, err := JSON(strings.NewReader(definition)) + if err != nil { + t.Fatal(err) + } + buff := new(bytes.Buffer) + buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000007")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006")) + + out, err := abi.Methods["multi"].Outputs.UnpackValues(buff.Bytes()) + + ret1Exp := [3]uint64{9, 8, 7} + ret2Exp := uint64(6) + + if !reflect.DeepEqual(out[0], ret1Exp) { + t.Error("array result", out[0], "!= Expected", ret1Exp) + } + if out[1] != ret2Exp { + t.Error("int result", out[1], "!= Expected", ret2Exp) + } +} + +func TestUnmarshalV2(t *testing.T) { + const definition = `[ + { "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] }, + { "name" : "bool", "constant" : false, "outputs": [ { "type": "bool" } ] }, + { "name" : "bytes", "constant" : false, "outputs": [ { "type": "bytes" } ] }, + { "name" : "fixed", "constant" : false, "outputs": [ { "type": "bytes32" } ] }, + { "name" : "multi", "constant" : false, "outputs": [ { "type": "bytes" }, { "type": "bytes" } ] }, + { "name" : "intArraySingle", "constant" : false, "outputs": [ { "type": "uint256[3]" } ] }, + { "name" : "addressSliceSingle", "constant" : false, "outputs": [ { "type": "address[]" } ] }, + { "name" : "addressSliceDouble", "constant" : false, "outputs": [ { "name": "a", "type": "address[]" }, { "name": "b", "type": "address[]" } ] }, + { "name" : "mixedBytes", "constant" : true, "outputs": [ { "name": "a", "type": "bytes" }, { "name": "b", "type": "bytes32" } ] }]` + + abi, err := JSON(strings.NewReader(definition)) + if err != nil { + t.Fatal(err) + } + buff := new(bytes.Buffer) + + // marshall mixed bytes (mixedBytes) + p0Exp := common.Hex2Bytes("01020000000000000000") + p1Exp := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000ddeeff") + + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000ddeeff")) + buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000a")) + buff.Write(common.Hex2Bytes("0102000000000000000000000000000000000000000000000000000000000000")) + + mixedBytes, err := abi.Methods["mixedBytes"].Outputs.UnpackValues(buff.Bytes()) + if err != nil { + t.Error(err) + } else { + p0 := mixedBytes[0].([]byte) + p1 := mixedBytes[1].([32]byte) + if !bytes.Equal(p0, p0Exp) { + t.Errorf("unexpected value unpacked: want %x, got %x", p0Exp, p0) + } + + if !bytes.Equal(p1[:], p1Exp) { + t.Errorf("unexpected value unpacked: want %x, got %x", p1Exp, p1) + } + } + + // marshal int + integer, err := abi.Methods["int"].Outputs.UnpackValues(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) + if err != nil { + t.Error(err) + } + if len(integer) == 0 { + t.Error("Expected one integer") + } + intval := integer[0].(*big.Int) + if intval == nil || intval.Cmp(big.NewInt(1)) != 0 { + t.Error("expected Int to be 1 got", intval) + } + + // marshal bool + boolreturns, err := abi.Methods["bool"].Outputs.UnpackValues(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) + if err != nil { + t.Error(err) + } + boolval := boolreturns[0].(bool) + if !boolval { + t.Error("expected Bool to be true") + } + + // marshal dynamic bytes max length 32 + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + bytesOut := common.RightPadBytes([]byte("hello"), 32) + buff.Write(bytesOut) + + bytesreturns, err := abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) + + if err != nil { + t.Error(err) + } + bytesval := bytesreturns[0].([]byte) + if !bytes.Equal(bytesval, bytesOut) { + t.Errorf("expected %x got %x", bytesOut, bytesval) + } + + // marshall dynamic bytes max length 64 + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) + bytesOut = common.RightPadBytes([]byte("hello"), 64) + buff.Write(bytesOut) + + bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) + if err != nil { + t.Error(err) + } + bytesval = bytesreturns[0].([]byte) + if !bytes.Equal(bytesval, bytesOut) { + t.Errorf("expected %x got %x", bytesOut, bytesval) + } + + // marshall dynamic bytes max length 64 + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000003f")) + bytesOut = common.RightPadBytes([]byte("hello"), 64) + buff.Write(bytesOut) + + bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) + if err != nil { + t.Error(err) + } + bytesval = bytesreturns[0].([]byte) + + if !bytes.Equal(bytesval, bytesOut[:len(bytesOut)-1]) { + t.Errorf("expected %x got %x", bytesOut[:len(bytesOut)-1], bytesval) + } + // marshal dynamic bytes output empty (nil) + bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(nil) + if err == nil { + t.Error("expected error") + } + // marshal dynamic bytes output empty + buff.Reset() + bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) + if err == nil { + t.Error("expected error") + } + + // marshal dynamic bytes length 5 + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000005")) + buff.Write(common.RightPadBytes([]byte("hello"), 32)) + + bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) + if err != nil { + t.Error(err) + } + bytesval = bytesreturns[0].([]byte) + + if !bytes.Equal(bytesval, []byte("hello")) { + t.Errorf("expected %x got %x", bytesOut, bytesval) + } + + // marshal dynamic bytes length 5 + buff.Reset() + buff.Write(common.RightPadBytes([]byte("hello"), 32)) + + hashreturns, err := abi.Methods["fixed"].Outputs.UnpackValues(buff.Bytes()) + if err != nil { + t.Error(err) + } + hashval := hashreturns[0].([32]byte) + + helloHash := common.BytesToHash(common.RightPadBytes([]byte("hello"), 32)) + if common.Hash(hashval) != helloHash { + t.Errorf("Expected %x to equal %x", hashval, helloHash) + } + + // marshal error + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + + bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) + if err == nil { + // Error abi: cannot marshal in to go slice: offset 32 would go over slice boundary (len=64) + t.Error("expected error") + } + bytesreturns, err = abi.Methods["multi"].Outputs.UnpackValues(make([]byte, 64)) + + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003")) + // marshal int array + + intArrayReturns, err := abi.Methods["intArraySingle"].Outputs.UnpackValues(buff.Bytes()) + if err != nil { + t.Error(err) + } + intArray := intArrayReturns[0].([3]*big.Int) + + var testAgainstIntArray = [3]*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)} + + for i, intval := range intArray { + if intval.Cmp(testAgainstIntArray[i]) != 0 { + t.Errorf("expected %v, got %v", testAgainstIntArray[i], intval) + } + } + // marshal address slice + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) // offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // size + buff.Write(common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000")) + + outAddrReturns, err := abi.Methods["addressSliceSingle"].Outputs.UnpackValues(buff.Bytes()) + if err != nil { + t.Fatal("didn't expect error:", err) + } + outAddr := outAddrReturns[0].([]common.Address) + if len(outAddr) != 1 { + t.Fatal("expected 1 item, got", len(outAddr)) + } + + if outAddr[0] != (common.Address{1}) { + t.Errorf("expected %x, got %x", common.Address{1}, outAddr[0]) + } + + // marshal multiple address slice + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) // offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000080")) // offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // size + buff.Write(common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // size + buff.Write(common.Hex2Bytes("0000000000000000000000000200000000000000000000000000000000000000")) + buff.Write(common.Hex2Bytes("0000000000000000000000000300000000000000000000000000000000000000")) + + outAddrStructReturns, err := abi.Methods["addressSliceDouble"].Outputs.UnpackValues(buff.Bytes()) + if err != nil { + t.Fatal("didn't expect error:", err) + } + A := outAddrStructReturns[0].([]common.Address) + B := outAddrStructReturns[1].([]common.Address) + + if len(A) != 1 { + t.Fatal("expected 1 item, got", len(A)) + } + + if A[0] != (common.Address{1}) { + t.Errorf("expected %x, got %x", common.Address{1}, A[0]) + } + + if len(B) != 2 { + t.Fatal("expected 1 item, got", len(B)) + } + + if B[0] != (common.Address{2}) { + t.Errorf("expected %x, got %x", common.Address{2}, B[0]) + } + if B[1] != (common.Address{3}) { + t.Errorf("expected %x, got %x", common.Address{3}, B[1]) + } + + // marshal invalid address slice + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000100")) + + err = abi.Unpack(&outAddr, "addressSliceSingle", buff.Bytes()) + _, err = abi.Methods["addressSliceSingle"].Outputs.UnpackValues(buff.Bytes()) + if err == nil { + t.Fatal("expected error:", err) + } + +} From f0f594d0453c6f53eaeeac6187785daf12044f58 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende
Date: Sat, 30 Dec 2017 16:07:12 +0100 Subject: [PATCH 27/94] accounts/abi: Deduplicate code in unpacker --- accounts/abi/abi.go | 2 +- accounts/abi/abi_test.go | 2 +- accounts/abi/argument.go | 81 ++++++++++++++--------------------- accounts/abi/unpackv2_test.go | 1 - 4 files changed, 33 insertions(+), 53 deletions(-) diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index 32f041890b..fd286c2c22 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -136,7 +136,7 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { // MethodById looks up a method by the 4-byte id // returns nil if none found -func (abi *ABI) MethodById(sigdata []byte) (*Method, error){ +func (abi *ABI) MethodById(sigdata []byte) (*Method, error) { for _, method := range abi.Methods { if bytes.Equal(method.Id(), sigdata[:4]) { return &method, nil diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index 3bef6add5c..e66828240a 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -689,7 +689,7 @@ func TestABI_MethodById(t *testing.T) { } for name, m := range abi.Methods { a := fmt.Sprintf("%v", m) - m2,err := abi.MethodById(m.Id()) + m2, err := abi.MethodById(m.Id()) if err != nil { t.Fatal(err) } diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index b9b5371211..bdd0894f17 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -67,10 +67,10 @@ func (arguments Arguments) LengthNonIndexed() int { return out } -func (arguments Arguments) NonIndexed() Arguments{ +func (arguments Arguments) NonIndexed() Arguments { var ret []Argument - for _,arg := range arguments{ - if !arg.Indexed{ + for _, arg := range arguments { + if !arg.Indexed { ret = append(ret, arg) } } @@ -84,21 +84,27 @@ func (arguments Arguments) isTuple() bool { // Unpack performs the operation hexdata -> Go format func (arguments Arguments) Unpack(v interface{}, data []byte) error { - if arguments.isTuple() { - return arguments.unpackTuple(v, data) - } - return arguments.unpackAtomic(v, data) -} -func (arguments Arguments) unpackTuple(v interface{}, output []byte) error { // make sure the passed value is arguments pointer - valueOf := reflect.ValueOf(v) - if reflect.Ptr != valueOf.Kind() { + if reflect.Ptr != reflect.ValueOf(v).Kind() { return fmt.Errorf("abi: Unpack(non-pointer %T)", v) } + marshalledValues, err := arguments.UnpackValues(data) + if err != nil { + return err + } + + if arguments.isTuple() { + return arguments.unpackTuple(v, marshalledValues) + } + return arguments.unpackAtomic(v, marshalledValues) +} + +func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interface{}) error { + var ( - value = valueOf.Elem() + value = reflect.ValueOf(v).Elem() typ = value.Type() kind = value.Kind() ) @@ -120,25 +126,9 @@ func (arguments Arguments) unpackTuple(v interface{}, output []byte) error { exists[field] = true } } - // `i` counts the nonindexed arguments. - // `j` counts the number of complex types. - // both `i` and `j` are used to to correctly compute `data` offset. - - j := 0 for i, arg := range arguments.NonIndexed() { - marshalledValue, err := toGoType((i+j)*32, arg.Type, output) - if err != nil { - return err - } - - if arg.Type.T == ArrayTy { - // combined index ('i' + 'j') need to be adjusted only by size of array, thus - // we need to decrement 'j' because 'i' was incremented - j += arg.Type.Size - 1 - } - - reflectValue := reflect.ValueOf(marshalledValue) + reflectValue := reflect.ValueOf(marshalledValues[i]) switch kind { case reflect.Struct: @@ -171,37 +161,29 @@ func (arguments Arguments) unpackTuple(v interface{}, output []byte) error { } // unpackAtomic unpacks ( hexdata -> go ) a single value -func (arguments Arguments) unpackAtomic(v interface{}, output []byte) error { - // make sure the passed value is arguments pointer - valueOf := reflect.ValueOf(v) - if reflect.Ptr != valueOf.Kind() { - return fmt.Errorf("abi: Unpack(non-pointer %T)", v) - } - arg := arguments[0] - if arg.Indexed { - return fmt.Errorf("abi: attempting to unpack indexed variable into element.") - } +func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interface{}) error { - value := valueOf.Elem() - marshalledValue, err := toGoType(0, arg.Type, output) - if err != nil { - return err + if len(marshalledValues) != 1 { + return fmt.Errorf("abi: wrong length, expected single value, got %d", len(marshalledValues)) } - return set(value, reflect.ValueOf(marshalledValue), arg) + + elem := reflect.ValueOf(v).Elem() + reflectValue := reflect.ValueOf(marshalledValues[0]) + return set(elem, reflectValue, arguments.NonIndexed()[0]) } // UnpackValues can be used to unpack ABI-encoded hexdata according to the ABI-specification, // without supplying a struct to unpack into. Instead, this method returns a list containing the // values. An atomic argument will be a list with one element. -func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error){ +func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) { - retval := make([]interface{},0,arguments.LengthNonIndexed()) + retval := make([]interface{}, 0, arguments.LengthNonIndexed()) virtualArgs := 0 - for index,arg:= range arguments.NonIndexed(){ + for index, arg := range arguments.NonIndexed() { - marshalledValue, err := toGoType((index + virtualArgs) * 32, arg.Type, data) + marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data) if arg.Type.T == ArrayTy { //If we have a static array, like [3]uint256, these are coded as @@ -212,7 +194,7 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error){ virtualArgs += arg.Type.Size - 1 } - if err != nil{ + if err != nil { return nil, err } retval = append(retval, marshalledValue) @@ -226,7 +208,6 @@ func (arguments Arguments) PackValues(args []interface{}) ([]byte, error) { return arguments.Pack(args...) } - // Pack performs the operation Go format -> Hexdata func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { // Make sure arguments match up and pack them diff --git a/accounts/abi/unpackv2_test.go b/accounts/abi/unpackv2_test.go index d0074ff7b1..364a097625 100644 --- a/accounts/abi/unpackv2_test.go +++ b/accounts/abi/unpackv2_test.go @@ -57,7 +57,6 @@ func TestUnpackV2(t *testing.T) { } } - func TestMultiReturnWithArrayV2(t *testing.T) { const definition = `[{"name" : "multi", "outputs": [{"type": "uint64[3]"}, {"type": "uint64"}]}]` abi, err := JSON(strings.NewReader(definition)) From 08c5d4dd271c58385df94842f1b5700ca6ef181c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende
Date: Sat, 13 Jan 2018 15:12:52 +0100 Subject: [PATCH 28/94] accounts/abi: address review concerns --- accounts/abi/abi.go | 2 +- accounts/abi/abi_test.go | 2 +- accounts/abi/argument.go | 18 +- accounts/abi/unpackv2_test.go | 335 ---------------------------------- 4 files changed, 5 insertions(+), 352 deletions(-) delete mode 100644 accounts/abi/unpackv2_test.go diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index fd286c2c22..254b1f7fb4 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -142,5 +142,5 @@ func (abi *ABI) MethodById(sigdata []byte) (*Method, error) { return &method, nil } } - return nil, fmt.Errorf("ABI spec does not contain method signature in data: 0x%x", sigdata[:4]) + return nil, fmt.Errorf("no method with id: %#x", sigdata[:4]) } diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index e66828240a..35e0094ddd 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -691,7 +691,7 @@ func TestABI_MethodById(t *testing.T) { a := fmt.Sprintf("%v", m) m2, err := abi.MethodById(m.Id()) if err != nil { - t.Fatal(err) + t.Fatalf("Failed to look up ABI method: %v", err) } b := fmt.Sprintf("%v", m2) if a != b { diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index bdd0894f17..f171f4cc63 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -67,6 +67,7 @@ func (arguments Arguments) LengthNonIndexed() int { return out } +// NonIndexed returns the arguments with indexed arguments filtered out func (arguments Arguments) NonIndexed() Arguments { var ret []Argument for _, arg := range arguments { @@ -89,12 +90,10 @@ func (arguments Arguments) Unpack(v interface{}, data []byte) error { if reflect.Ptr != reflect.ValueOf(v).Kind() { return fmt.Errorf("abi: Unpack(non-pointer %T)", v) } - marshalledValues, err := arguments.UnpackValues(data) if err != nil { return err } - if arguments.isTuple() { return arguments.unpackTuple(v, marshalledValues) } @@ -162,11 +161,9 @@ func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interfa // unpackAtomic unpacks ( hexdata -> go ) a single value func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interface{}) error { - if len(marshalledValues) != 1 { return fmt.Errorf("abi: wrong length, expected single value, got %d", len(marshalledValues)) } - elem := reflect.ValueOf(v).Elem() reflectValue := reflect.ValueOf(marshalledValues[0]) return set(elem, reflectValue, arguments.NonIndexed()[0]) @@ -176,24 +173,18 @@ func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interf // without supplying a struct to unpack into. Instead, this method returns a list containing the // values. An atomic argument will be a list with one element. func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) { - retval := make([]interface{}, 0, arguments.LengthNonIndexed()) - virtualArgs := 0 - for index, arg := range arguments.NonIndexed() { - marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data) - if arg.Type.T == ArrayTy { - //If we have a static array, like [3]uint256, these are coded as + // If we have a static array, like [3]uint256, these are coded as // just like uint256,uint256,uint256. // This means that we need to add two 'virtual' arguments when // we count the index from now on virtualArgs += arg.Type.Size - 1 } - if err != nil { return nil, err } @@ -202,7 +193,7 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) { return retval, nil } -// UnpackValues performs the operation Go format -> Hexdata +// PackValues performs the operation Go format -> Hexdata // It is the semantic opposite of UnpackValues func (arguments Arguments) PackValues(args []interface{}) ([]byte, error) { return arguments.Pack(args...) @@ -215,7 +206,6 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { if len(args) != len(abiArgs) { return nil, fmt.Errorf("argument count mismatch: %d for %d", len(args), len(abiArgs)) } - // variable input is the output appended at the end of packed // output. This is used for strings and bytes types input. var variableInput []byte @@ -229,7 +219,6 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { inputOffset += 32 } } - var ret []byte for i, a := range args { input := abiArgs[i] @@ -238,7 +227,6 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { if err != nil { return nil, err } - // check for a slice type (string, bytes, slice) if input.Type.requiresLengthPrefix() { // calculate the offset diff --git a/accounts/abi/unpackv2_test.go b/accounts/abi/unpackv2_test.go deleted file mode 100644 index 364a097625..0000000000 --- a/accounts/abi/unpackv2_test.go +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package abi - -import ( - "bytes" - "encoding/hex" - "fmt" - "math/big" - "reflect" - "strconv" - "strings" - "testing" - - "github.com/ethereum/go-ethereum/common" -) - -func TestUnpackV2(t *testing.T) { - for i, test := range unpackTests { - t.Run(strconv.Itoa(i), func(t *testing.T) { - def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def) - abi, err := JSON(strings.NewReader(def)) - if err != nil { - t.Fatalf("invalid ABI definition %s: %v", def, err) - } - encb, err := hex.DecodeString(test.enc) - if err != nil { - t.Fatalf("invalid hex: %s" + test.enc) - } - out, err := abi.Methods["method"].Outputs.UnpackValues(encb) - - if err != nil { - t.Fatal(err) - } - if len(test.err) != 0 { - // The new stuff doesn't have these types of errors - return - } - if !reflect.DeepEqual(test.want, out[0]) { - t.Errorf("test %d (%v) failed: expected %v, got %v", i, test.def, test.want, out[0]) - } - }) - } -} - -func TestMultiReturnWithArrayV2(t *testing.T) { - const definition = `[{"name" : "multi", "outputs": [{"type": "uint64[3]"}, {"type": "uint64"}]}]` - abi, err := JSON(strings.NewReader(definition)) - if err != nil { - t.Fatal(err) - } - buff := new(bytes.Buffer) - buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000007")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006")) - - out, err := abi.Methods["multi"].Outputs.UnpackValues(buff.Bytes()) - - ret1Exp := [3]uint64{9, 8, 7} - ret2Exp := uint64(6) - - if !reflect.DeepEqual(out[0], ret1Exp) { - t.Error("array result", out[0], "!= Expected", ret1Exp) - } - if out[1] != ret2Exp { - t.Error("int result", out[1], "!= Expected", ret2Exp) - } -} - -func TestUnmarshalV2(t *testing.T) { - const definition = `[ - { "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] }, - { "name" : "bool", "constant" : false, "outputs": [ { "type": "bool" } ] }, - { "name" : "bytes", "constant" : false, "outputs": [ { "type": "bytes" } ] }, - { "name" : "fixed", "constant" : false, "outputs": [ { "type": "bytes32" } ] }, - { "name" : "multi", "constant" : false, "outputs": [ { "type": "bytes" }, { "type": "bytes" } ] }, - { "name" : "intArraySingle", "constant" : false, "outputs": [ { "type": "uint256[3]" } ] }, - { "name" : "addressSliceSingle", "constant" : false, "outputs": [ { "type": "address[]" } ] }, - { "name" : "addressSliceDouble", "constant" : false, "outputs": [ { "name": "a", "type": "address[]" }, { "name": "b", "type": "address[]" } ] }, - { "name" : "mixedBytes", "constant" : true, "outputs": [ { "name": "a", "type": "bytes" }, { "name": "b", "type": "bytes32" } ] }]` - - abi, err := JSON(strings.NewReader(definition)) - if err != nil { - t.Fatal(err) - } - buff := new(bytes.Buffer) - - // marshall mixed bytes (mixedBytes) - p0Exp := common.Hex2Bytes("01020000000000000000") - p1Exp := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000ddeeff") - - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000ddeeff")) - buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000a")) - buff.Write(common.Hex2Bytes("0102000000000000000000000000000000000000000000000000000000000000")) - - mixedBytes, err := abi.Methods["mixedBytes"].Outputs.UnpackValues(buff.Bytes()) - if err != nil { - t.Error(err) - } else { - p0 := mixedBytes[0].([]byte) - p1 := mixedBytes[1].([32]byte) - if !bytes.Equal(p0, p0Exp) { - t.Errorf("unexpected value unpacked: want %x, got %x", p0Exp, p0) - } - - if !bytes.Equal(p1[:], p1Exp) { - t.Errorf("unexpected value unpacked: want %x, got %x", p1Exp, p1) - } - } - - // marshal int - integer, err := abi.Methods["int"].Outputs.UnpackValues(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) - if err != nil { - t.Error(err) - } - if len(integer) == 0 { - t.Error("Expected one integer") - } - intval := integer[0].(*big.Int) - if intval == nil || intval.Cmp(big.NewInt(1)) != 0 { - t.Error("expected Int to be 1 got", intval) - } - - // marshal bool - boolreturns, err := abi.Methods["bool"].Outputs.UnpackValues(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) - if err != nil { - t.Error(err) - } - boolval := boolreturns[0].(bool) - if !boolval { - t.Error("expected Bool to be true") - } - - // marshal dynamic bytes max length 32 - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - bytesOut := common.RightPadBytes([]byte("hello"), 32) - buff.Write(bytesOut) - - bytesreturns, err := abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) - - if err != nil { - t.Error(err) - } - bytesval := bytesreturns[0].([]byte) - if !bytes.Equal(bytesval, bytesOut) { - t.Errorf("expected %x got %x", bytesOut, bytesval) - } - - // marshall dynamic bytes max length 64 - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) - bytesOut = common.RightPadBytes([]byte("hello"), 64) - buff.Write(bytesOut) - - bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) - if err != nil { - t.Error(err) - } - bytesval = bytesreturns[0].([]byte) - if !bytes.Equal(bytesval, bytesOut) { - t.Errorf("expected %x got %x", bytesOut, bytesval) - } - - // marshall dynamic bytes max length 64 - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000003f")) - bytesOut = common.RightPadBytes([]byte("hello"), 64) - buff.Write(bytesOut) - - bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) - if err != nil { - t.Error(err) - } - bytesval = bytesreturns[0].([]byte) - - if !bytes.Equal(bytesval, bytesOut[:len(bytesOut)-1]) { - t.Errorf("expected %x got %x", bytesOut[:len(bytesOut)-1], bytesval) - } - // marshal dynamic bytes output empty (nil) - bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(nil) - if err == nil { - t.Error("expected error") - } - // marshal dynamic bytes output empty - buff.Reset() - bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) - if err == nil { - t.Error("expected error") - } - - // marshal dynamic bytes length 5 - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000005")) - buff.Write(common.RightPadBytes([]byte("hello"), 32)) - - bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) - if err != nil { - t.Error(err) - } - bytesval = bytesreturns[0].([]byte) - - if !bytes.Equal(bytesval, []byte("hello")) { - t.Errorf("expected %x got %x", bytesOut, bytesval) - } - - // marshal dynamic bytes length 5 - buff.Reset() - buff.Write(common.RightPadBytes([]byte("hello"), 32)) - - hashreturns, err := abi.Methods["fixed"].Outputs.UnpackValues(buff.Bytes()) - if err != nil { - t.Error(err) - } - hashval := hashreturns[0].([32]byte) - - helloHash := common.BytesToHash(common.RightPadBytes([]byte("hello"), 32)) - if common.Hash(hashval) != helloHash { - t.Errorf("Expected %x to equal %x", hashval, helloHash) - } - - // marshal error - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - - bytesreturns, err = abi.Methods["bytes"].Outputs.UnpackValues(buff.Bytes()) - if err == nil { - // Error abi: cannot marshal in to go slice: offset 32 would go over slice boundary (len=64) - t.Error("expected error") - } - bytesreturns, err = abi.Methods["multi"].Outputs.UnpackValues(make([]byte, 64)) - - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003")) - // marshal int array - - intArrayReturns, err := abi.Methods["intArraySingle"].Outputs.UnpackValues(buff.Bytes()) - if err != nil { - t.Error(err) - } - intArray := intArrayReturns[0].([3]*big.Int) - - var testAgainstIntArray = [3]*big.Int{big.NewInt(1), big.NewInt(2), big.NewInt(3)} - - for i, intval := range intArray { - if intval.Cmp(testAgainstIntArray[i]) != 0 { - t.Errorf("expected %v, got %v", testAgainstIntArray[i], intval) - } - } - // marshal address slice - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) // offset - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // size - buff.Write(common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000")) - - outAddrReturns, err := abi.Methods["addressSliceSingle"].Outputs.UnpackValues(buff.Bytes()) - if err != nil { - t.Fatal("didn't expect error:", err) - } - outAddr := outAddrReturns[0].([]common.Address) - if len(outAddr) != 1 { - t.Fatal("expected 1 item, got", len(outAddr)) - } - - if outAddr[0] != (common.Address{1}) { - t.Errorf("expected %x, got %x", common.Address{1}, outAddr[0]) - } - - // marshal multiple address slice - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) // offset - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000080")) // offset - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // size - buff.Write(common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // size - buff.Write(common.Hex2Bytes("0000000000000000000000000200000000000000000000000000000000000000")) - buff.Write(common.Hex2Bytes("0000000000000000000000000300000000000000000000000000000000000000")) - - outAddrStructReturns, err := abi.Methods["addressSliceDouble"].Outputs.UnpackValues(buff.Bytes()) - if err != nil { - t.Fatal("didn't expect error:", err) - } - A := outAddrStructReturns[0].([]common.Address) - B := outAddrStructReturns[1].([]common.Address) - - if len(A) != 1 { - t.Fatal("expected 1 item, got", len(A)) - } - - if A[0] != (common.Address{1}) { - t.Errorf("expected %x, got %x", common.Address{1}, A[0]) - } - - if len(B) != 2 { - t.Fatal("expected 1 item, got", len(B)) - } - - if B[0] != (common.Address{2}) { - t.Errorf("expected %x, got %x", common.Address{2}, B[0]) - } - if B[1] != (common.Address{3}) { - t.Errorf("expected %x, got %x", common.Address{3}, B[1]) - } - - // marshal invalid address slice - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000100")) - - err = abi.Unpack(&outAddr, "addressSliceSingle", buff.Bytes()) - _, err = abi.Methods["addressSliceSingle"].Outputs.UnpackValues(buff.Bytes()) - if err == nil { - t.Fatal("expected error:", err) - } - -} From bd6ed23899c8a3b4b6d0db29f0f6298e492cedd6 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende
Date: Sat, 13 Jan 2018 16:03:24 +0100 Subject: [PATCH 29/94] accounts/abi: harden unpacking against malicious input --- accounts/abi/unpack.go | 17 ++++++--- accounts/abi/unpack_test.go | 70 +++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 3342456618..51fb9ab9b6 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -95,6 +95,9 @@ func readFixedBytes(t Type, word []byte) (interface{}, error) { // iteratively unpack elements func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) { + if size < 0 { + return nil, fmt.Errorf("cannot marshal input to array, size is negative (%d)", size) + } if start+32*size > len(output) { return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size) } @@ -181,16 +184,22 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { // interprets a 32 byte slice as an offset and then determines which indice to look to decode the type. func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err error) { - offset := int(binary.BigEndian.Uint64(output[index+24 : index+32])) + offsetBig := big.NewInt(0).SetBytes(output[index : index+32]) + if !offsetBig.IsInt64() { + return 0, 0, fmt.Errorf("abi offset larger than int64: %v", offsetBig) + } + offset := int(offsetBig.Int64()) if offset+32 > len(output) { return 0, 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32) } - length = int(binary.BigEndian.Uint64(output[offset+24 : offset+32])) + lengthBig := big.NewInt(0).SetBytes(output[offset : offset+32]) + if !lengthBig.IsInt64() { + return 0, 0, fmt.Errorf("abi length larger than int64: %v", lengthBig) + } + length = int(lengthBig.Int64()) if offset+32+length > len(output) { return 0, 0, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+length) } start = offset + 32 - - //fmt.Printf("LENGTH PREFIX INFO: \nsize: %v\noffset: %v\nstart: %v\n", length, offset, start) return } diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index e9f9108125..742211244b 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -683,3 +683,73 @@ func TestUnmarshal(t *testing.T) { t.Fatal("expected error:", err) } } + +func TestOOMMaliciousInput(t *testing.T) { + oomTests := []unpackTest{ + { + def: `[{"type": "uint8[]"}]`, + enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset + "0000000000000000000000000000000000000000000000000000000000000003" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + { // Length larger than 64 bits + def: `[{"type": "uint8[]"}]`, + enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset + "00ffffffffffffffffffffffffffffffffffffffffffffff0000000000000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + { // Offset very large (over 64 bits) + def: `[{"type": "uint8[]"}]`, + enc: "00ffffffffffffffffffffffffffffffffffffffffffffff0000000000000020" + // offset + "0000000000000000000000000000000000000000000000000000000000000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + { // Offset very large (below 64 bits) + def: `[{"type": "uint8[]"}]`, + enc: "0000000000000000000000000000000000000000000000007ffffffffff00020" + // offset + "0000000000000000000000000000000000000000000000000000000000000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + { // Offset negative (as 64 bit) + def: `[{"type": "uint8[]"}]`, + enc: "000000000000000000000000000000000000000000000000f000000000000020" + // offset + "0000000000000000000000000000000000000000000000000000000000000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + + { // Negative length + def: `[{"type": "uint8[]"}]`, + enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset + "000000000000000000000000000000000000000000000000f000000000000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + { // Very large length + def: `[{"type": "uint8[]"}]`, + enc: "0000000000000000000000000000000000000000000000000000000000000020" + // offset + "0000000000000000000000000000000000000000000000007fffffffff000002" + // num elems + "0000000000000000000000000000000000000000000000000000000000000001" + // elem 1 + "0000000000000000000000000000000000000000000000000000000000000002", // elem 2 + }, + } + for i, test := range oomTests { + def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def) + abi, err := JSON(strings.NewReader(def)) + if err != nil { + t.Fatalf("invalid ABI definition %s: %v", def, err) + } + encb, err := hex.DecodeString(test.enc) + if err != nil { + t.Fatalf("invalid hex: %s" + test.enc) + } + _, err = abi.Methods["method"].Outputs.UnpackValues(encb) + if err == nil { + t.Fatalf("Expected error on malicious input, test %d", i) + } + } +} From 61f2279bdeac595a4607080028715a8222db6cd4 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende
Date: Fri, 2 Feb 2018 14:03:58 +0100 Subject: [PATCH 30/94] abi: fix missing method on go 1.7/1.8 --- accounts/abi/unpack.go | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 51fb9ab9b6..761c80edfd 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -184,22 +184,32 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { // interprets a 32 byte slice as an offset and then determines which indice to look to decode the type. func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err error) { - offsetBig := big.NewInt(0).SetBytes(output[index : index+32]) - if !offsetBig.IsInt64() { - return 0, 0, fmt.Errorf("abi offset larger than int64: %v", offsetBig) + bigOffsetEnd := big.NewInt(0).SetBytes(output[index : index+32]) + bigOffsetEnd.Add(bigOffsetEnd, common.Big32) + outputLength := big.NewInt(int64(len(output))) + + if bigOffsetEnd.Cmp(outputLength) > 0 { + return 0, 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %v would go over slice boundary (len=%v)", bigOffsetEnd, outputLength) } - offset := int(offsetBig.Int64()) - if offset+32 > len(output) { - return 0, 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32) + + if bigOffsetEnd.BitLen() > 63 { + return 0, 0, fmt.Errorf("abi offset larger than int64: %v", bigOffsetEnd) } - lengthBig := big.NewInt(0).SetBytes(output[offset : offset+32]) - if !lengthBig.IsInt64() { - return 0, 0, fmt.Errorf("abi length larger than int64: %v", lengthBig) + + offsetEnd := int(bigOffsetEnd.Uint64()) + lengthBig := big.NewInt(0).SetBytes(output[offsetEnd-32 : offsetEnd]) + + totalSize := big.NewInt(0) + totalSize.Add(totalSize, bigOffsetEnd) + totalSize.Add(totalSize, lengthBig) + if totalSize.BitLen() > 63 { + return 0, 0, fmt.Errorf("abi length larger than int64: %v", totalSize) } - length = int(lengthBig.Int64()) - if offset+32+length > len(output) { - return 0, 0, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+length) + + if totalSize.Cmp(outputLength) > 0 { + return 0, 0, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %v require %v", outputLength, totalSize) } - start = offset + 32 + start = int(bigOffsetEnd.Uint64()) + length = int(lengthBig.Uint64()) return } From 34d94e22d9c9f3c2373a398269ecbba0139827d2 Mon Sep 17 00:00:00 2001 From: Pedro Pombeiro
Date: Tue, 20 Feb 2018 18:20:33 +0100 Subject: [PATCH 31/94] whisper: Fix race condition in whisperv6/peer.go --- whisper/whisperv6/peer.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/whisper/whisperv6/peer.go b/whisper/whisperv6/peer.go index 4ef0f3c434..6d75290fde 100644 --- a/whisper/whisperv6/peer.go +++ b/whisper/whisperv6/peer.go @@ -19,6 +19,7 @@ package whisperv6 import ( "fmt" "math" + "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -36,6 +37,7 @@ type Peer struct { trusted bool powRequirement float64 + bloomMu sync.Mutex bloomFilter []byte fullNode bool @@ -225,10 +227,14 @@ func (peer *Peer) notifyAboutBloomFilterChange(bloom []byte) error { } func (peer *Peer) bloomMatch(env *Envelope) bool { + peer.bloomMu.Lock() + defer peer.bloomMu.Unlock() return peer.fullNode || bloomFilterMatch(peer.bloomFilter, env.Bloom()) } func (peer *Peer) setBloomFilter(bloom []byte) { + peer.bloomMu.Lock() + defer peer.bloomMu.Unlock() peer.bloomFilter = bloom peer.fullNode = isFullNode(bloom) if peer.fullNode && peer.bloomFilter == nil { From 01507d9b9d872aa6dfaf3ad704b8c2b65e378a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?=
Date: Tue, 20 Feb 2018 14:33:34 +0200 Subject: [PATCH 32/94] cmd, console: support all termination signals --- cmd/geth/consolecmd.go | 3 ++- cmd/utils/cmd.go | 5 +++-- console/console.go | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 9d5cc38a1b..7eca4d59f6 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -22,6 +22,7 @@ import ( "os/signal" "path/filepath" "strings" + "syscall" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/console" @@ -207,7 +208,7 @@ func ephemeralConsole(ctx *cli.Context) error { } // Wait for pending callbacks, but stop for Ctrl-C. abort := make(chan os.Signal, 1) - signal.Notify(abort, os.Interrupt) + signal.Notify(abort, syscall.SIGINT, syscall.SIGTERM) go func() { <-abort diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 53cdf7861c..186d18d8f5 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -25,6 +25,7 @@ import ( "os/signal" "runtime" "strings" + "syscall" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -64,7 +65,7 @@ func StartNode(stack *node.Node) { } go func() { sigc := make(chan os.Signal, 1) - signal.Notify(sigc, os.Interrupt) + signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(sigc) <-sigc log.Info("Got interrupt, shutting down...") @@ -85,7 +86,7 @@ func ImportChain(chain *core.BlockChain, fn string) error { // If a signal is received, the import will stop at the next batch. interrupt := make(chan os.Signal, 1) stop := make(chan struct{}) - signal.Notify(interrupt, os.Interrupt) + signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(interrupt) defer close(interrupt) go func() { diff --git a/console/console.go b/console/console.go index 52fe1f542a..b280d4e65d 100644 --- a/console/console.go +++ b/console/console.go @@ -26,6 +26,7 @@ import ( "regexp" "sort" "strings" + "syscall" "github.com/ethereum/go-ethereum/internal/jsre" "github.com/ethereum/go-ethereum/internal/web3ext" @@ -332,7 +333,7 @@ func (c *Console) Interactive() { }() // Monitor Ctrl-C too in case the input is empty and we need to bail abort := make(chan os.Signal, 1) - signal.Notify(abort, os.Interrupt) + signal.Notify(abort, syscall.SIGINT, syscall.SIGTERM) // Start sending prompts to the user and reading back inputs for { From 14c76371bab105a99c796a652df86df3e3e7d958 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak
Date: Wed, 21 Feb 2018 16:03:26 +0200 Subject: [PATCH 33/94] p2p: when peer is removed remove it also from dial history (#16060) This change removes a peer information from dialing history when peer is removed from static list. It allows to force a server to re-dial concrete peer if it is needed. In our case we are running geth node on mobile devices, and it is common for a network connection to flap on mobile. Almost every time it flaps or network connection is changed from cellular to wifi peers are disconnected with read timeout. And usually it takes 30 seconds (default expiration timeout) to recover connection with static peers after connectivity is restored. This change allows us to reconnect with peers almost immediately and it seems harmless enough. --- p2p/dial.go | 13 +++++++++++++ p2p/dial_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/p2p/dial.go b/p2p/dial.go index f5ff2c2117..d8feceb9f3 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -154,6 +154,9 @@ func (s *dialstate) addStatic(n *discover.Node) { func (s *dialstate) removeStatic(n *discover.Node) { // This removes a task so future attempts to connect will not be made. delete(s.static, n.ID) + // This removes a previous dial timestamp so that application + // can force a server to reconnect with chosen peer immediately. + s.hist.remove(n.ID) } func (s *dialstate) newTasks(nRunning int, peers map[discover.NodeID]*Peer, now time.Time) []task { @@ -390,6 +393,16 @@ func (h dialHistory) min() pastDial { } func (h *dialHistory) add(id discover.NodeID, exp time.Time) { heap.Push(h, pastDial{id, exp}) + +} +func (h *dialHistory) remove(id discover.NodeID) bool { + for i, v := range *h { + if v.id == id { + heap.Remove(h, i) + return true + } + } + return false } func (h dialHistory) contains(id discover.NodeID) bool { for _, v := range h { diff --git a/p2p/dial_test.go b/p2p/dial_test.go index ad18ef9abe..2a7941fc65 100644 --- a/p2p/dial_test.go +++ b/p2p/dial_test.go @@ -515,6 +515,50 @@ func TestDialStateStaticDial(t *testing.T) { }) } +// This test checks that static peers will be redialed immediately if they were re-added to a static list. +func TestDialStaticAfterReset(t *testing.T) { + wantStatic := []*discover.Node{ + {ID: uintID(1)}, + {ID: uintID(2)}, + } + + rounds := []round{ + // Static dials are launched for the nodes that aren't yet connected. + { + peers: nil, + new: []task{ + &dialTask{flags: staticDialedConn, dest: &discover.Node{ID: uintID(1)}}, + &dialTask{flags: staticDialedConn, dest: &discover.Node{ID: uintID(2)}}, + }, + }, + // No new dial tasks, all peers are connected. + { + peers: []*Peer{ + {rw: &conn{flags: staticDialedConn, id: uintID(1)}}, + {rw: &conn{flags: staticDialedConn, id: uintID(2)}}, + }, + done: []task{ + &dialTask{flags: staticDialedConn, dest: &discover.Node{ID: uintID(1)}}, + &dialTask{flags: staticDialedConn, dest: &discover.Node{ID: uintID(2)}}, + }, + new: []task{ + &waitExpireTask{Duration: 30 * time.Second}, + }, + }, + } + dTest := dialtest{ + init: newDialState(wantStatic, nil, fakeTable{}, 0, nil), + rounds: rounds, + } + runDialTest(t, dTest) + for _, n := range wantStatic { + dTest.init.removeStatic(n) + dTest.init.addStatic(n) + } + // without removing peers they will be considered recently dialed + runDialTest(t, dTest) +} + // This test checks that past dials are not retried for some time. func TestDialStateCache(t *testing.T) { wantStatic := []*discover.Node{ From b585f761283957dcd1783a134449a01423145df3 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende
Date: Wed, 21 Feb 2018 15:10:18 +0100 Subject: [PATCH 34/94] ethapi: prevent creating contract if no data is provided (#16108) * ethapi: prevent creating contract if no data is provided * internal/ethapi: downcase error for no data on contract creation --- internal/ethapi/api.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 314086335c..636d0bfe26 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1135,6 +1135,18 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) { return errors.New(`Both "data" and "input" are set and not equal. Please use "input" to pass transaction call data.`) } + if args.To == nil { + // Contract creation + var input []byte + if args.Data != nil { + input = *args.Data + } else if args.Input != nil { + input = *args.Input + } + if len(input) == 0 { + return errors.New(`contract creation without any data provided`) + } + } return nil } From 085d3fbf72c00bb0592e88eb6215602a5778979d Mon Sep 17 00:00:00 2001 From: Nilesh Trivedi
Date: Thu, 22 Feb 2018 00:23:50 +0530 Subject: [PATCH 35/94] cmd/puppeth: Don't allow hyphen in network name. Fixes #16155 --- cmd/puppeth/wizard_intro.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/puppeth/wizard_intro.go b/cmd/puppeth/wizard_intro.go index 84998afc93..60aa0f7ffb 100644 --- a/cmd/puppeth/wizard_intro.go +++ b/cmd/puppeth/wizard_intro.go @@ -59,15 +59,16 @@ func (w *wizard) run() { fmt.Println() // Make sure we have a good network name to work with fmt.Println() + // Docker accepts hyphens in image names, but doesn't like it for container names if w.network == "" { - fmt.Println("Please specify a network name to administer (no spaces, please)") + fmt.Println("Please specify a network name to administer (no spaces or hyphens, please)") for { w.network = w.readString() - if !strings.Contains(w.network, " ") { + if !strings.Contains(w.network, " ") && !strings.Contains(w.network, "-") { fmt.Printf("\nSweet, you can set this via --network=%s next time!\n\n", w.network) break } - log.Error("I also like to live dangerously, still no spaces") + log.Error("I also like to live dangerously, still no spaces or hyphens") } } log.Info("Administering Ethereum network", "name", w.network) From bb5349b15463255d269870fec9839a942eeecc1a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet
Date: Thu, 22 Feb 2018 11:25:07 +0100 Subject: [PATCH 36/94] whisper: Support for v2 has long been discontinued, remove it. (#16153) --- whisper/whisperv2/api.go | 402 ----------------------------- whisper/whisperv2/doc.go | 32 --- whisper/whisperv2/envelope.go | 150 ----------- whisper/whisperv2/envelope_test.go | 158 ------------ whisper/whisperv2/filter.go | 129 --------- whisper/whisperv2/filter_test.go | 215 --------------- whisper/whisperv2/main.go | 106 -------- whisper/whisperv2/message.go | 158 ------------ whisper/whisperv2/message_test.go | 158 ------------ whisper/whisperv2/peer.go | 174 ------------- whisper/whisperv2/peer_test.go | 261 ------------------- whisper/whisperv2/topic.go | 140 ---------- whisper/whisperv2/topic_test.go | 215 --------------- whisper/whisperv2/whisper.go | 378 --------------------------- whisper/whisperv2/whisper_test.go | 216 ---------------- 15 files changed, 2892 deletions(-) delete mode 100644 whisper/whisperv2/api.go delete mode 100644 whisper/whisperv2/doc.go delete mode 100644 whisper/whisperv2/envelope.go delete mode 100644 whisper/whisperv2/envelope_test.go delete mode 100644 whisper/whisperv2/filter.go delete mode 100644 whisper/whisperv2/filter_test.go delete mode 100644 whisper/whisperv2/main.go delete mode 100644 whisper/whisperv2/message.go delete mode 100644 whisper/whisperv2/message_test.go delete mode 100644 whisper/whisperv2/peer.go delete mode 100644 whisper/whisperv2/peer_test.go delete mode 100644 whisper/whisperv2/topic.go delete mode 100644 whisper/whisperv2/topic_test.go delete mode 100644 whisper/whisperv2/whisper.go delete mode 100644 whisper/whisperv2/whisper_test.go diff --git a/whisper/whisperv2/api.go b/whisper/whisperv2/api.go deleted file mode 100644 index 5c6d170959..0000000000 --- a/whisper/whisperv2/api.go +++ /dev/null @@ -1,402 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package whisperv2 - -import ( - "encoding/json" - "fmt" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/crypto" -) - -// PublicWhisperAPI provides the whisper RPC service. -type PublicWhisperAPI struct { - w *Whisper - - messagesMu sync.RWMutex - messages map[hexutil.Uint]*whisperFilter -} - -type whisperOfflineError struct{} - -func (e *whisperOfflineError) Error() string { - return "whisper is offline" -} - -// whisperOffLineErr is returned when the node doesn't offer the shh service. -var whisperOffLineErr = new(whisperOfflineError) - -// NewPublicWhisperAPI create a new RPC whisper service. -func NewPublicWhisperAPI(w *Whisper) *PublicWhisperAPI { - return &PublicWhisperAPI{w: w, messages: make(map[hexutil.Uint]*whisperFilter)} -} - -// Version returns the Whisper version this node offers. -func (s *PublicWhisperAPI) Version() (hexutil.Uint, error) { - if s.w == nil { - return 0, whisperOffLineErr - } - return hexutil.Uint(s.w.Version()), nil -} - -// HasIdentity checks if the the whisper node is configured with the private key -// of the specified public pair. -func (s *PublicWhisperAPI) HasIdentity(identity string) (bool, error) { - if s.w == nil { - return false, whisperOffLineErr - } - return s.w.HasIdentity(crypto.ToECDSAPub(common.FromHex(identity))), nil -} - -// NewIdentity generates a new cryptographic identity for the client, and injects -// it into the known identities for message decryption. -func (s *PublicWhisperAPI) NewIdentity() (string, error) { - if s.w == nil { - return "", whisperOffLineErr - } - - identity := s.w.NewIdentity() - return common.ToHex(crypto.FromECDSAPub(&identity.PublicKey)), nil -} - -type NewFilterArgs struct { - To string - From string - Topics [][][]byte -} - -// NewWhisperFilter creates and registers a new message filter to watch for inbound whisper messages. -func (s *PublicWhisperAPI) NewFilter(args NewFilterArgs) (hexutil.Uint, error) { - if s.w == nil { - return 0, whisperOffLineErr - } - - var id hexutil.Uint - filter := Filter{ - To: crypto.ToECDSAPub(common.FromHex(args.To)), - From: crypto.ToECDSAPub(common.FromHex(args.From)), - Topics: NewFilterTopics(args.Topics...), - Fn: func(message *Message) { - wmsg := NewWhisperMessage(message) - s.messagesMu.RLock() // Only read lock to the filter pool - defer s.messagesMu.RUnlock() - if s.messages[id] != nil { - s.messages[id].insert(wmsg) - } - }, - } - id = hexutil.Uint(s.w.Watch(filter)) - - s.messagesMu.Lock() - s.messages[id] = newWhisperFilter(id, s.w) - s.messagesMu.Unlock() - - return id, nil -} - -// GetFilterChanges retrieves all the new messages matched by a filter since the last retrieval. -func (s *PublicWhisperAPI) GetFilterChanges(filterId hexutil.Uint) []WhisperMessage { - s.messagesMu.RLock() - defer s.messagesMu.RUnlock() - - if s.messages[filterId] != nil { - if changes := s.messages[filterId].retrieve(); changes != nil { - return changes - } - } - return returnWhisperMessages(nil) -} - -// UninstallFilter disables and removes an existing filter. -func (s *PublicWhisperAPI) UninstallFilter(filterId hexutil.Uint) bool { - s.messagesMu.Lock() - defer s.messagesMu.Unlock() - - if _, ok := s.messages[filterId]; ok { - delete(s.messages, filterId) - return true - } - return false -} - -// GetMessages retrieves all the known messages that match a specific filter. -func (s *PublicWhisperAPI) GetMessages(filterId hexutil.Uint) []WhisperMessage { - // Retrieve all the cached messages matching a specific, existing filter - s.messagesMu.RLock() - defer s.messagesMu.RUnlock() - - var messages []*Message - if s.messages[filterId] != nil { - messages = s.messages[filterId].messages() - } - - return returnWhisperMessages(messages) -} - -// returnWhisperMessages converts aNhisper message to a RPC whisper message. -func returnWhisperMessages(messages []*Message) []WhisperMessage { - msgs := make([]WhisperMessage, len(messages)) - for i, msg := range messages { - msgs[i] = NewWhisperMessage(msg) - } - return msgs -} - -type PostArgs struct { - From string `json:"from"` - To string `json:"to"` - Topics [][]byte `json:"topics"` - Payload string `json:"payload"` - Priority int64 `json:"priority"` - TTL int64 `json:"ttl"` -} - -// Post injects a message into the whisper network for distribution. -func (s *PublicWhisperAPI) Post(args PostArgs) (bool, error) { - if s.w == nil { - return false, whisperOffLineErr - } - - // construct whisper message with transmission options - message := NewMessage(common.FromHex(args.Payload)) - options := Options{ - To: crypto.ToECDSAPub(common.FromHex(args.To)), - TTL: time.Duration(args.TTL) * time.Second, - Topics: NewTopics(args.Topics...), - } - - // set sender identity - if len(args.From) > 0 { - if key := s.w.GetIdentity(crypto.ToECDSAPub(common.FromHex(args.From))); key != nil { - options.From = key - } else { - return false, fmt.Errorf("unknown identity to send from: %s", args.From) - } - } - - // Wrap and send the message - pow := time.Duration(args.Priority) * time.Millisecond - envelope, err := message.Wrap(pow, options) - if err != nil { - return false, err - } - - return true, s.w.Send(envelope) -} - -// WhisperMessage is the RPC representation of a whisper message. -type WhisperMessage struct { - ref *Message - - Payload string `json:"payload"` - To string `json:"to"` - From string `json:"from"` - Sent int64 `json:"sent"` - TTL int64 `json:"ttl"` - Hash string `json:"hash"` -} - -func (args *PostArgs) UnmarshalJSON(data []byte) (err error) { - var obj struct { - From string `json:"from"` - To string `json:"to"` - Topics []string `json:"topics"` - Payload string `json:"payload"` - Priority hexutil.Uint64 `json:"priority"` - TTL hexutil.Uint64 `json:"ttl"` - } - - if err := json.Unmarshal(data, &obj); err != nil { - return err - } - - args.From = obj.From - args.To = obj.To - args.Payload = obj.Payload - args.Priority = int64(obj.Priority) // TODO(gluk256): handle overflow - args.TTL = int64(obj.TTL) // ... here too ... - - // decode topic strings - args.Topics = make([][]byte, len(obj.Topics)) - for i, topic := range obj.Topics { - args.Topics[i] = common.FromHex(topic) - } - - return nil -} - -// UnmarshalJSON implements the json.Unmarshaler interface, invoked to convert a -// JSON message blob into a WhisperFilterArgs structure. -func (args *NewFilterArgs) UnmarshalJSON(b []byte) (err error) { - // Unmarshal the JSON message and sanity check - var obj struct { - To interface{} `json:"to"` - From interface{} `json:"from"` - Topics interface{} `json:"topics"` - } - if err := json.Unmarshal(b, &obj); err != nil { - return err - } - - // Retrieve the simple data contents of the filter arguments - if obj.To == nil { - args.To = "" - } else { - argstr, ok := obj.To.(string) - if !ok { - return fmt.Errorf("to is not a string") - } - args.To = argstr - } - if obj.From == nil { - args.From = "" - } else { - argstr, ok := obj.From.(string) - if !ok { - return fmt.Errorf("from is not a string") - } - args.From = argstr - } - // Construct the nested topic array - if obj.Topics != nil { - // Make sure we have an actual topic array - list, ok := obj.Topics.([]interface{}) - if !ok { - return fmt.Errorf("topics is not an array") - } - // Iterate over each topic and handle nil, string or array - topics := make([][]string, len(list)) - for idx, field := range list { - switch value := field.(type) { - case nil: - topics[idx] = []string{} - - case string: - topics[idx] = []string{value} - - case []interface{}: - topics[idx] = make([]string, len(value)) - for i, nested := range value { - switch value := nested.(type) { - case nil: - topics[idx][i] = "" - - case string: - topics[idx][i] = value - - default: - return fmt.Errorf("topic[%d][%d] is not a string", idx, i) - } - } - default: - return fmt.Errorf("topic[%d] not a string or array", idx) - } - } - - topicsDecoded := make([][][]byte, len(topics)) - for i, condition := range topics { - topicsDecoded[i] = make([][]byte, len(condition)) - for j, topic := range condition { - topicsDecoded[i][j] = common.FromHex(topic) - } - } - - args.Topics = topicsDecoded - } - return nil -} - -// whisperFilter is the message cache matching a specific filter, accumulating -// inbound messages until the are requested by the client. -type whisperFilter struct { - id hexutil.Uint // Filter identifier for old message retrieval - ref *Whisper // Whisper reference for old message retrieval - - cache []WhisperMessage // Cache of messages not yet polled - skip map[common.Hash]struct{} // List of retrieved messages to avoid duplication - update time.Time // Time of the last message query - - lock sync.RWMutex // Lock protecting the filter internals -} - -// messages retrieves all the cached messages from the entire pool matching the -// filter, resetting the filter's change buffer. -func (w *whisperFilter) messages() []*Message { - w.lock.Lock() - defer w.lock.Unlock() - - w.cache = nil - w.update = time.Now() - - w.skip = make(map[common.Hash]struct{}) - messages := w.ref.Messages(int(w.id)) - for _, message := range messages { - w.skip[message.Hash] = struct{}{} - } - return messages -} - -// insert injects a new batch of messages into the filter cache. -func (w *whisperFilter) insert(messages ...WhisperMessage) { - w.lock.Lock() - defer w.lock.Unlock() - - for _, message := range messages { - if _, ok := w.skip[message.ref.Hash]; !ok { - w.cache = append(w.cache, messages...) - } - } -} - -// retrieve fetches all the cached messages from the filter. -func (w *whisperFilter) retrieve() (messages []WhisperMessage) { - w.lock.Lock() - defer w.lock.Unlock() - - messages, w.cache = w.cache, nil - w.update = time.Now() - - return -} - -// newWhisperFilter creates a new serialized, poll based whisper topic filter. -func newWhisperFilter(id hexutil.Uint, ref *Whisper) *whisperFilter { - return &whisperFilter{ - id: id, - ref: ref, - update: time.Now(), - skip: make(map[common.Hash]struct{}), - } -} - -// NewWhisperMessage converts an internal message into an API version. -func NewWhisperMessage(message *Message) WhisperMessage { - return WhisperMessage{ - ref: message, - - Payload: common.ToHex(message.Payload), - From: common.ToHex(crypto.FromECDSAPub(message.Recover())), - To: common.ToHex(crypto.FromECDSAPub(message.To)), - Sent: message.Sent.Unix(), - TTL: int64(message.TTL / time.Second), - Hash: common.ToHex(message.Hash.Bytes()), - } -} diff --git a/whisper/whisperv2/doc.go b/whisper/whisperv2/doc.go deleted file mode 100644 index 7252f44b10..0000000000 --- a/whisper/whisperv2/doc.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -/* -Package whisper implements the Whisper PoC-1. - -(https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec) - -Whisper combines aspects of both DHTs and datagram messaging systems (e.g. UDP). -As such it may be likened and compared to both, not dissimilar to the -matter/energy duality (apologies to physicists for the blatant abuse of a -fundamental and beautiful natural principle). - -Whisper is a pure identity-based messaging system. Whisper provides a low-level -(non-application-specific) but easily-accessible API without being based upon -or prejudiced by the low-level hardware attributes and characteristics, -particularly the notion of singular endpoints. -*/ -package whisperv2 diff --git a/whisper/whisperv2/envelope.go b/whisper/whisperv2/envelope.go deleted file mode 100644 index 9f1c682040..0000000000 --- a/whisper/whisperv2/envelope.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -// Contains the Whisper protocol Envelope element. For formal details please see -// the specs at https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec#envelopes. - -package whisperv2 - -import ( - "crypto/ecdsa" - "encoding/binary" - "fmt" - "math/big" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/ecies" - "github.com/ethereum/go-ethereum/rlp" -) - -// Envelope represents a clear-text data packet to transmit through the Whisper -// network. Its contents may or may not be encrypted and signed. -type Envelope struct { - Expiry uint32 // Whisper protocol specifies int32, really should be int64 - TTL uint32 // ^^^^^^ - Topics []Topic - Data []byte - Nonce uint32 - - hash common.Hash // Cached hash of the envelope to avoid rehashing every time -} - -// NewEnvelope wraps a Whisper message with expiration and destination data -// included into an envelope for network forwarding. -func NewEnvelope(ttl time.Duration, topics []Topic, msg *Message) *Envelope { - return &Envelope{ - Expiry: uint32(time.Now().Add(ttl).Unix()), - TTL: uint32(ttl.Seconds()), - Topics: topics, - Data: msg.bytes(), - Nonce: 0, - } -} - -// Seal closes the envelope by spending the requested amount of time as a proof -// of work on hashing the data. -func (self *Envelope) Seal(pow time.Duration) { - d := make([]byte, 64) - copy(d[:32], self.rlpWithoutNonce()) - - finish, bestBit := time.Now().Add(pow).UnixNano(), 0 - for nonce := uint32(0); time.Now().UnixNano() < finish; { - for i := 0; i < 1024; i++ { - binary.BigEndian.PutUint32(d[60:], nonce) - - d := new(big.Int).SetBytes(crypto.Keccak256(d)) - firstBit := math.FirstBitSet(d) - if firstBit > bestBit { - self.Nonce, bestBit = nonce, firstBit - } - nonce++ - } - } -} - -// rlpWithoutNonce returns the RLP encoded envelope contents, except the nonce. -func (self *Envelope) rlpWithoutNonce() []byte { - enc, _ := rlp.EncodeToBytes([]interface{}{self.Expiry, self.TTL, self.Topics, self.Data}) - return enc -} - -// Open extracts the message contained within a potentially encrypted envelope. -func (self *Envelope) Open(key *ecdsa.PrivateKey) (msg *Message, err error) { - // Split open the payload into a message construct - data := self.Data - - message := &Message{ - Flags: data[0], - Sent: time.Unix(int64(self.Expiry-self.TTL), 0), - TTL: time.Duration(self.TTL) * time.Second, - Hash: self.Hash(), - } - data = data[1:] - - if message.Flags&signatureFlag == signatureFlag { - if len(data) < signatureLength { - return nil, fmt.Errorf("unable to open envelope. First bit set but len(data) < len(signature)") - } - message.Signature, data = data[:signatureLength], data[signatureLength:] - } - message.Payload = data - - // Decrypt the message, if requested - if key == nil { - return message, nil - } - err = message.decrypt(key) - switch err { - case nil: - return message, nil - - case ecies.ErrInvalidPublicKey: // Payload isn't encrypted - return message, err - - default: - return nil, fmt.Errorf("unable to open envelope, decrypt failed: %v", err) - } -} - -// Hash returns the SHA3 hash of the envelope, calculating it if not yet done. -func (self *Envelope) Hash() common.Hash { - if (self.hash == common.Hash{}) { - enc, _ := rlp.EncodeToBytes(self) - self.hash = crypto.Keccak256Hash(enc) - } - return self.hash -} - -// DecodeRLP decodes an Envelope from an RLP data stream. -func (self *Envelope) DecodeRLP(s *rlp.Stream) error { - raw, err := s.Raw() - if err != nil { - return err - } - // The decoding of Envelope uses the struct fields but also needs - // to compute the hash of the whole RLP-encoded envelope. This - // type has the same structure as Envelope but is not an - // rlp.Decoder so we can reuse the Envelope struct definition. - type rlpenv Envelope - if err := rlp.DecodeBytes(raw, (*rlpenv)(self)); err != nil { - return err - } - self.hash = crypto.Keccak256Hash(raw) - return nil -} diff --git a/whisper/whisperv2/envelope_test.go b/whisper/whisperv2/envelope_test.go deleted file mode 100644 index 490ed9f6f8..0000000000 --- a/whisper/whisperv2/envelope_test.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package whisperv2 - -import ( - "bytes" - "testing" - "time" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/ecies" -) - -func TestEnvelopeOpen(t *testing.T) { - payload := []byte("hello world") - message := NewMessage(payload) - - envelope, err := message.Wrap(DefaultPoW, Options{}) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - opened, err := envelope.Open(nil) - if err != nil { - t.Fatalf("failed to open envelope: %v", err) - } - if opened.Flags != message.Flags { - t.Fatalf("flags mismatch: have %d, want %d", opened.Flags, message.Flags) - } - if !bytes.Equal(opened.Signature, message.Signature) { - t.Fatalf("signature mismatch: have 0x%x, want 0x%x", opened.Signature, message.Signature) - } - if !bytes.Equal(opened.Payload, message.Payload) { - t.Fatalf("payload mismatch: have 0x%x, want 0x%x", opened.Payload, message.Payload) - } - if opened.Sent.Unix() != message.Sent.Unix() { - t.Fatalf("send time mismatch: have %v, want %v", opened.Sent, message.Sent) - } - if opened.TTL/time.Second != DefaultTTL/time.Second { - t.Fatalf("message TTL mismatch: have %v, want %v", opened.TTL, DefaultTTL) - } - - if opened.Hash != envelope.Hash() { - t.Fatalf("message hash mismatch: have 0x%x, want 0x%x", opened.Hash, envelope.Hash()) - } -} - -func TestEnvelopeAnonymousOpenUntargeted(t *testing.T) { - payload := []byte("hello envelope") - envelope, err := NewMessage(payload).Wrap(DefaultPoW, Options{}) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - opened, err := envelope.Open(nil) - if err != nil { - t.Fatalf("failed to open envelope: %v", err) - } - if opened.To != nil { - t.Fatalf("recipient mismatch: have 0x%x, want nil", opened.To) - } - if !bytes.Equal(opened.Payload, payload) { - t.Fatalf("payload mismatch: have 0x%x, want 0x%x", opened.Payload, payload) - } -} - -func TestEnvelopeAnonymousOpenTargeted(t *testing.T) { - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to generate test identity: %v", err) - } - - payload := []byte("hello envelope") - envelope, err := NewMessage(payload).Wrap(DefaultPoW, Options{ - To: &key.PublicKey, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - opened, err := envelope.Open(nil) - if err != nil { - t.Fatalf("failed to open envelope: %v", err) - } - if opened.To != nil { - t.Fatalf("recipient mismatch: have 0x%x, want nil", opened.To) - } - if bytes.Equal(opened.Payload, payload) { - t.Fatalf("payload match, should have been encrypted: 0x%x", opened.Payload) - } -} - -func TestEnvelopeIdentifiedOpenUntargeted(t *testing.T) { - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to generate test identity: %v", err) - } - - payload := []byte("hello envelope") - envelope, err := NewMessage(payload).Wrap(DefaultPoW, Options{}) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - opened, err := envelope.Open(key) - switch err { - case nil: - t.Fatalf("envelope opened with bad key: %v", opened) - - case ecies.ErrInvalidPublicKey: - // Ok, key mismatch but opened - - default: - t.Fatalf("failed to open envelope: %v", err) - } - - if opened.To != nil { - t.Fatalf("recipient mismatch: have 0x%x, want nil", opened.To) - } - if !bytes.Equal(opened.Payload, payload) { - t.Fatalf("payload mismatch: have 0x%x, want 0x%x", opened.Payload, payload) - } -} - -func TestEnvelopeIdentifiedOpenTargeted(t *testing.T) { - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to generate test identity: %v", err) - } - - payload := []byte("hello envelope") - envelope, err := NewMessage(payload).Wrap(DefaultPoW, Options{ - To: &key.PublicKey, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - opened, err := envelope.Open(key) - if err != nil { - t.Fatalf("failed to open envelope: %v", err) - } - if opened.To != nil { - t.Fatalf("recipient mismatch: have 0x%x, want nil", opened.To) - } - if !bytes.Equal(opened.Payload, payload) { - t.Fatalf("payload mismatch: have 0x%x, want 0x%x", opened.Payload, payload) - } -} diff --git a/whisper/whisperv2/filter.go b/whisper/whisperv2/filter.go deleted file mode 100644 index 7404859b78..0000000000 --- a/whisper/whisperv2/filter.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -// Contains the message filter for fine grained subscriptions. - -package whisperv2 - -import ( - "crypto/ecdsa" - - "github.com/ethereum/go-ethereum/event/filter" -) - -// Filter is used to subscribe to specific types of whisper messages. -type Filter struct { - To *ecdsa.PublicKey // Recipient of the message - From *ecdsa.PublicKey // Sender of the message - Topics [][]Topic // Topics to filter messages with - Fn func(msg *Message) // Handler in case of a match -} - -// NewFilterTopics creates a 2D topic array used by whisper.Filter from binary -// data elements. -func NewFilterTopics(data ...[][]byte) [][]Topic { - filter := make([][]Topic, len(data)) - for i, condition := range data { - // Handle the special case of condition == [[]byte{}] - if len(condition) == 1 && len(condition[0]) == 0 { - filter[i] = []Topic{} - continue - } - // Otherwise flatten normally - filter[i] = NewTopics(condition...) - } - return filter -} - -// NewFilterTopicsFlat creates a 2D topic array used by whisper.Filter from flat -// binary data elements. -func NewFilterTopicsFlat(data ...[]byte) [][]Topic { - filter := make([][]Topic, len(data)) - for i, element := range data { - // Only add non-wildcard topics - filter[i] = make([]Topic, 0, 1) - if len(element) > 0 { - filter[i] = append(filter[i], NewTopic(element)) - } - } - return filter -} - -// NewFilterTopicsFromStrings creates a 2D topic array used by whisper.Filter -// from textual data elements. -func NewFilterTopicsFromStrings(data ...[]string) [][]Topic { - filter := make([][]Topic, len(data)) - for i, condition := range data { - // Handle the special case of condition == [""] - if len(condition) == 1 && condition[0] == "" { - filter[i] = []Topic{} - continue - } - // Otherwise flatten normally - filter[i] = NewTopicsFromStrings(condition...) - } - return filter -} - -// NewFilterTopicsFromStringsFlat creates a 2D topic array used by whisper.Filter from flat -// binary data elements. -func NewFilterTopicsFromStringsFlat(data ...string) [][]Topic { - filter := make([][]Topic, len(data)) - for i, element := range data { - // Only add non-wildcard topics - filter[i] = make([]Topic, 0, 1) - if element != "" { - filter[i] = append(filter[i], NewTopicFromString(element)) - } - } - return filter -} - -// filterer is the internal, fully initialized filter ready to match inbound -// messages to a variety of criteria. -type filterer struct { - to string // Recipient of the message - from string // Sender of the message - matcher *topicMatcher // Topics to filter messages with - fn func(data interface{}) // Handler in case of a match -} - -// Compare checks if the specified filter matches the current one. -func (self filterer) Compare(f filter.Filter) bool { - filter := f.(filterer) - - // Check the message sender and recipient - if len(self.to) > 0 && self.to != filter.to { - return false - } - if len(self.from) > 0 && self.from != filter.from { - return false - } - // Check the topic filtering - topics := make([]Topic, len(filter.matcher.conditions)) - for i, group := range filter.matcher.conditions { - // Message should contain a single topic entry, extract - for topics[i] = range group { - break - } - } - return self.matcher.Matches(topics) -} - -// Trigger is called when a filter successfully matches an inbound message. -func (self filterer) Trigger(data interface{}) { - self.fn(data) -} diff --git a/whisper/whisperv2/filter_test.go b/whisper/whisperv2/filter_test.go deleted file mode 100644 index ffdfd7b349..0000000000 --- a/whisper/whisperv2/filter_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package whisperv2 - -import ( - "bytes" - - "testing" -) - -var filterTopicsCreationTests = []struct { - topics [][]string - filter [][][4]byte -}{ - { // Simple topic filter - topics: [][]string{ - {"abc", "def", "ghi"}, - {"def"}, - {"ghi", "abc"}, - }, - filter: [][][4]byte{ - {{0x4e, 0x03, 0x65, 0x7a}, {0x34, 0x60, 0x7c, 0x9b}, {0x21, 0x41, 0x7d, 0xf9}}, - {{0x34, 0x60, 0x7c, 0x9b}}, - {{0x21, 0x41, 0x7d, 0xf9}, {0x4e, 0x03, 0x65, 0x7a}}, - }, - }, - { // Wild-carded topic filter - topics: [][]string{ - {"abc", "def", "ghi"}, - {}, - {""}, - {"def"}, - }, - filter: [][][4]byte{ - {{0x4e, 0x03, 0x65, 0x7a}, {0x34, 0x60, 0x7c, 0x9b}, {0x21, 0x41, 0x7d, 0xf9}}, - {}, - {}, - {{0x34, 0x60, 0x7c, 0x9b}}, - }, - }, -} - -var filterTopicsCreationFlatTests = []struct { - topics []string - filter [][][4]byte -}{ - { // Simple topic list - topics: []string{"abc", "def", "ghi"}, - filter: [][][4]byte{ - {{0x4e, 0x03, 0x65, 0x7a}}, - {{0x34, 0x60, 0x7c, 0x9b}}, - {{0x21, 0x41, 0x7d, 0xf9}}, - }, - }, - { // Wild-carded topic list - topics: []string{"abc", "", "ghi"}, - filter: [][][4]byte{ - {{0x4e, 0x03, 0x65, 0x7a}}, - {}, - {{0x21, 0x41, 0x7d, 0xf9}}, - }, - }, -} - -func TestFilterTopicsCreation(t *testing.T) { - // Check full filter creation - for i, tt := range filterTopicsCreationTests { - // Check the textual creation - filter := NewFilterTopicsFromStrings(tt.topics...) - if len(filter) != len(tt.topics) { - t.Errorf("test %d: condition count mismatch: have %v, want %v", i, len(filter), len(tt.topics)) - continue - } - for j, condition := range filter { - if len(condition) != len(tt.filter[j]) { - t.Errorf("test %d, condition %d: size mismatch: have %v, want %v", i, j, len(condition), len(tt.filter[j])) - continue - } - for k := 0; k < len(condition); k++ { - if !bytes.Equal(condition[k][:], tt.filter[j][k][:]) { - t.Errorf("test %d, condition %d, segment %d: filter mismatch: have 0x%x, want 0x%x", i, j, k, condition[k], tt.filter[j][k]) - } - } - } - // Check the binary creation - binary := make([][][]byte, len(tt.topics)) - for j, condition := range tt.topics { - binary[j] = make([][]byte, len(condition)) - for k, segment := range condition { - binary[j][k] = []byte(segment) - } - } - filter = NewFilterTopics(binary...) - if len(filter) != len(tt.topics) { - t.Errorf("test %d: condition count mismatch: have %v, want %v", i, len(filter), len(tt.topics)) - continue - } - for j, condition := range filter { - if len(condition) != len(tt.filter[j]) { - t.Errorf("test %d, condition %d: size mismatch: have %v, want %v", i, j, len(condition), len(tt.filter[j])) - continue - } - for k := 0; k < len(condition); k++ { - if !bytes.Equal(condition[k][:], tt.filter[j][k][:]) { - t.Errorf("test %d, condition %d, segment %d: filter mismatch: have 0x%x, want 0x%x", i, j, k, condition[k], tt.filter[j][k]) - } - } - } - } - // Check flat filter creation - for i, tt := range filterTopicsCreationFlatTests { - // Check the textual creation - filter := NewFilterTopicsFromStringsFlat(tt.topics...) - if len(filter) != len(tt.topics) { - t.Errorf("test %d: condition count mismatch: have %v, want %v", i, len(filter), len(tt.topics)) - continue - } - for j, condition := range filter { - if len(condition) != len(tt.filter[j]) { - t.Errorf("test %d, condition %d: size mismatch: have %v, want %v", i, j, len(condition), len(tt.filter[j])) - continue - } - for k := 0; k < len(condition); k++ { - if !bytes.Equal(condition[k][:], tt.filter[j][k][:]) { - t.Errorf("test %d, condition %d, segment %d: filter mismatch: have 0x%x, want 0x%x", i, j, k, condition[k], tt.filter[j][k]) - } - } - } - // Check the binary creation - binary := make([][]byte, len(tt.topics)) - for j, topic := range tt.topics { - binary[j] = []byte(topic) - } - filter = NewFilterTopicsFlat(binary...) - if len(filter) != len(tt.topics) { - t.Errorf("test %d: condition count mismatch: have %v, want %v", i, len(filter), len(tt.topics)) - continue - } - for j, condition := range filter { - if len(condition) != len(tt.filter[j]) { - t.Errorf("test %d, condition %d: size mismatch: have %v, want %v", i, j, len(condition), len(tt.filter[j])) - continue - } - for k := 0; k < len(condition); k++ { - if !bytes.Equal(condition[k][:], tt.filter[j][k][:]) { - t.Errorf("test %d, condition %d, segment %d: filter mismatch: have 0x%x, want 0x%x", i, j, k, condition[k], tt.filter[j][k]) - } - } - } - } -} - -var filterCompareTests = []struct { - matcher filterer - message filterer - match bool -}{ - { // Wild-card filter matching anything - matcher: filterer{to: "", from: "", matcher: newTopicMatcher()}, - message: filterer{to: "to", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: true, - }, - { // Filter matching the to field - matcher: filterer{to: "to", from: "", matcher: newTopicMatcher()}, - message: filterer{to: "to", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: true, - }, - { // Filter rejecting the to field - matcher: filterer{to: "to", from: "", matcher: newTopicMatcher()}, - message: filterer{to: "", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: false, - }, - { // Filter matching the from field - matcher: filterer{to: "", from: "from", matcher: newTopicMatcher()}, - message: filterer{to: "to", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: true, - }, - { // Filter rejecting the from field - matcher: filterer{to: "", from: "from", matcher: newTopicMatcher()}, - message: filterer{to: "to", from: "", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: false, - }, - { // Filter matching the topic field - matcher: filterer{to: "", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - message: filterer{to: "to", from: "from", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - match: true, - }, - { // Filter rejecting the topic field - matcher: filterer{to: "", from: "", matcher: newTopicMatcher(NewFilterTopicsFromStringsFlat("topic")...)}, - message: filterer{to: "to", from: "from", matcher: newTopicMatcher()}, - match: false, - }, -} - -func TestFilterCompare(t *testing.T) { - for i, tt := range filterCompareTests { - if match := tt.matcher.Compare(tt.message); match != tt.match { - t.Errorf("test %d: match mismatch: have %v, want %v", i, match, tt.match) - } - } -} diff --git a/whisper/whisperv2/main.go b/whisper/whisperv2/main.go deleted file mode 100644 index be41604890..0000000000 --- a/whisper/whisperv2/main.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -// +build none - -// Contains a simple whisper peer setup and self messaging to allow playing -// around with the protocol and API without a fancy client implementation. - -package main - -import ( - "fmt" - "log" - "os" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/logger" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/nat" - "github.com/ethereum/go-ethereum/whisper" -) - -func main() { - logger.AddLogSystem(logger.NewStdLogSystem(os.Stdout, log.LstdFlags, logger.InfoLevel)) - - // Generate the peer identity - key, err := crypto.GenerateKey() - if err != nil { - fmt.Printf("Failed to generate peer key: %v.\n", err) - os.Exit(-1) - } - name := common.MakeName("whisper-go", "1.0") - shh := whisper.New() - - // Create an Ethereum peer to communicate through - server := p2p.Server{ - PrivateKey: key, - MaxPeers: 10, - Name: name, - Protocols: []p2p.Protocol{shh.Protocol()}, - ListenAddr: ":30300", - NAT: nat.Any(), - } - fmt.Println("Starting Ethereum peer...") - if err := server.Start(); err != nil { - fmt.Printf("Failed to start Ethereum peer: %v.\n", err) - os.Exit(1) - } - - // Send a message to self to check that something works - payload := fmt.Sprintf("Hello world, this is %v. In case you're wondering, the time is %v", name, time.Now()) - if err := selfSend(shh, []byte(payload)); err != nil { - fmt.Printf("Failed to self message: %v.\n", err) - os.Exit(-1) - } -} - -// SendSelf wraps a payload into a Whisper envelope and forwards it to itself. -func selfSend(shh *whisper.Whisper, payload []byte) error { - ok := make(chan struct{}) - - // Start watching for self messages, output any arrivals - id := shh.NewIdentity() - shh.Watch(whisper.Filter{ - To: &id.PublicKey, - Fn: func(msg *whisper.Message) { - fmt.Printf("Message received: %s, signed with 0x%x.\n", string(msg.Payload), msg.Signature) - close(ok) - }, - }) - // Wrap the payload and encrypt it - msg := whisper.NewMessage(payload) - envelope, err := msg.Wrap(whisper.DefaultPoW, whisper.Options{ - From: id, - To: &id.PublicKey, - TTL: whisper.DefaultTTL, - }) - if err != nil { - return fmt.Errorf("failed to seal message: %v", err) - } - // Dump the message into the system and wait for it to pop back out - if err := shh.Send(envelope); err != nil { - return fmt.Errorf("failed to send self-message: %v", err) - } - select { - case <-ok: - case <-time.After(time.Second): - return fmt.Errorf("failed to receive message in time") - } - return nil -} diff --git a/whisper/whisperv2/message.go b/whisper/whisperv2/message.go deleted file mode 100644 index 66648c3be8..0000000000 --- a/whisper/whisperv2/message.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -// Contains the Whisper protocol Message element. For formal details please see -// the specs at https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec#messages. - -package whisperv2 - -import ( - "crypto/ecdsa" - crand "crypto/rand" - "fmt" - "math/rand" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/ecies" - "github.com/ethereum/go-ethereum/log" -) - -// Message represents an end-user data packet to transmit through the Whisper -// protocol. These are wrapped into Envelopes that need not be understood by -// intermediate nodes, just forwarded. -type Message struct { - Flags byte // First bit is signature presence, rest reserved and should be random - Signature []byte - Payload []byte - - Sent time.Time // Time when the message was posted into the network - TTL time.Duration // Maximum time to live allowed for the message - - To *ecdsa.PublicKey // Message recipient (identity used to decode the message) - Hash common.Hash // Message envelope hash to act as a unique id -} - -// Options specifies the exact way a message should be wrapped into an Envelope. -type Options struct { - From *ecdsa.PrivateKey - To *ecdsa.PublicKey - TTL time.Duration - Topics []Topic -} - -// NewMessage creates and initializes a non-signed, non-encrypted Whisper message. -func NewMessage(payload []byte) *Message { - // Construct an initial flag set: no signature, rest random - flags := byte(rand.Intn(256)) - flags &= ^signatureFlag - - // Assemble and return the message - return &Message{ - Flags: flags, - Payload: payload, - Sent: time.Now(), - } -} - -// Wrap bundles the message into an Envelope to transmit over the network. -// -// pow (Proof Of Work) controls how much time to spend on hashing the message, -// inherently controlling its priority through the network (smaller hash, bigger -// priority). -// -// The user can control the amount of identity, privacy and encryption through -// the options parameter as follows: -// - options.From == nil && options.To == nil: anonymous broadcast -// - options.From != nil && options.To == nil: signed broadcast (known sender) -// - options.From == nil && options.To != nil: encrypted anonymous message -// - options.From != nil && options.To != nil: encrypted signed message -func (self *Message) Wrap(pow time.Duration, options Options) (*Envelope, error) { - // Use the default TTL if non was specified - if options.TTL == 0 { - options.TTL = DefaultTTL - } - self.TTL = options.TTL - - // Sign and encrypt the message if requested - if options.From != nil { - if err := self.sign(options.From); err != nil { - return nil, err - } - } - if options.To != nil { - if err := self.encrypt(options.To); err != nil { - return nil, err - } - } - // Wrap the processed message, seal it and return - envelope := NewEnvelope(options.TTL, options.Topics, self) - envelope.Seal(pow) - - return envelope, nil -} - -// sign calculates and sets the cryptographic signature for the message , also -// setting the sign flag. -func (self *Message) sign(key *ecdsa.PrivateKey) (err error) { - self.Flags |= signatureFlag - self.Signature, err = crypto.Sign(self.hash(), key) - return -} - -// Recover retrieves the public key of the message signer. -func (self *Message) Recover() *ecdsa.PublicKey { - defer func() { recover() }() // in case of invalid signature - - // Short circuit if no signature is present - if self.Signature == nil { - return nil - } - // Otherwise try and recover the signature - pub, err := crypto.SigToPub(self.hash(), self.Signature) - if err != nil { - log.Error(fmt.Sprintf("Could not get public key from signature: %v", err)) - return nil - } - return pub -} - -// encrypt encrypts a message payload with a public key. -func (self *Message) encrypt(key *ecdsa.PublicKey) (err error) { - self.Payload, err = ecies.Encrypt(crand.Reader, ecies.ImportECDSAPublic(key), self.Payload, nil, nil) - return -} - -// decrypt decrypts an encrypted payload with a private key. -func (self *Message) decrypt(key *ecdsa.PrivateKey) error { - cleartext, err := ecies.ImportECDSA(key).Decrypt(crand.Reader, self.Payload, nil, nil) - if err == nil { - self.Payload = cleartext - } - return err -} - -// hash calculates the SHA3 checksum of the message flags and payload. -func (self *Message) hash() []byte { - return crypto.Keccak256(append([]byte{self.Flags}, self.Payload...)) -} - -// bytes flattens the message contents (flags, signature and payload) into a -// single binary blob. -func (self *Message) bytes() []byte { - return append([]byte{self.Flags}, append(self.Signature, self.Payload...)...) -} diff --git a/whisper/whisperv2/message_test.go b/whisper/whisperv2/message_test.go deleted file mode 100644 index c760ac54c4..0000000000 --- a/whisper/whisperv2/message_test.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package whisperv2 - -import ( - "bytes" - "crypto/elliptic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/crypto" -) - -// Tests whether a message can be wrapped without any identity or encryption. -func TestMessageSimpleWrap(t *testing.T) { - payload := []byte("hello world") - - msg := NewMessage(payload) - if _, err := msg.Wrap(DefaultPoW, Options{}); err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if msg.Flags&signatureFlag != 0 { - t.Fatalf("signature flag mismatch: have %d, want %d", msg.Flags&signatureFlag, 0) - } - if len(msg.Signature) != 0 { - t.Fatalf("signature found for simple wrapping: 0x%x", msg.Signature) - } - if !bytes.Equal(msg.Payload, payload) { - t.Fatalf("payload mismatch after wrapping: have 0x%x, want 0x%x", msg.Payload, payload) - } - if msg.TTL/time.Second != DefaultTTL/time.Second { - t.Fatalf("message TTL mismatch: have %v, want %v", msg.TTL, DefaultTTL) - } -} - -// Tests whether a message can be signed, and wrapped in plain-text. -func TestMessageCleartextSignRecover(t *testing.T) { - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to create crypto key: %v", err) - } - payload := []byte("hello world") - - msg := NewMessage(payload) - if _, err := msg.Wrap(DefaultPoW, Options{ - From: key, - }); err != nil { - t.Fatalf("failed to sign message: %v", err) - } - if msg.Flags&signatureFlag != signatureFlag { - t.Fatalf("signature flag mismatch: have %d, want %d", msg.Flags&signatureFlag, signatureFlag) - } - if !bytes.Equal(msg.Payload, payload) { - t.Fatalf("payload mismatch after signing: have 0x%x, want 0x%x", msg.Payload, payload) - } - - pubKey := msg.Recover() - if pubKey == nil { - t.Fatalf("failed to recover public key") - } - p1 := elliptic.Marshal(crypto.S256(), key.PublicKey.X, key.PublicKey.Y) - p2 := elliptic.Marshal(crypto.S256(), pubKey.X, pubKey.Y) - if !bytes.Equal(p1, p2) { - t.Fatalf("public key mismatch: have 0x%x, want 0x%x", p2, p1) - } -} - -// Tests whether a message can be encrypted and decrypted using an anonymous -// sender (i.e. no signature). -func TestMessageAnonymousEncryptDecrypt(t *testing.T) { - key, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to create recipient crypto key: %v", err) - } - payload := []byte("hello world") - - msg := NewMessage(payload) - envelope, err := msg.Wrap(DefaultPoW, Options{ - To: &key.PublicKey, - }) - if err != nil { - t.Fatalf("failed to encrypt message: %v", err) - } - if msg.Flags&signatureFlag != 0 { - t.Fatalf("signature flag mismatch: have %d, want %d", msg.Flags&signatureFlag, 0) - } - if len(msg.Signature) != 0 { - t.Fatalf("signature found for anonymous message: 0x%x", msg.Signature) - } - - out, err := envelope.Open(key) - if err != nil { - t.Fatalf("failed to open encrypted message: %v", err) - } - if !bytes.Equal(out.Payload, payload) { - t.Errorf("payload mismatch: have 0x%x, want 0x%x", out.Payload, payload) - } -} - -// Tests whether a message can be properly signed and encrypted. -func TestMessageFullCrypto(t *testing.T) { - fromKey, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to create sender crypto key: %v", err) - } - toKey, err := crypto.GenerateKey() - if err != nil { - t.Fatalf("failed to create recipient crypto key: %v", err) - } - - payload := []byte("hello world") - msg := NewMessage(payload) - envelope, err := msg.Wrap(DefaultPoW, Options{ - From: fromKey, - To: &toKey.PublicKey, - }) - if err != nil { - t.Fatalf("failed to encrypt message: %v", err) - } - if msg.Flags&signatureFlag != signatureFlag { - t.Fatalf("signature flag mismatch: have %d, want %d", msg.Flags&signatureFlag, signatureFlag) - } - if len(msg.Signature) == 0 { - t.Fatalf("no signature found for signed message") - } - - out, err := envelope.Open(toKey) - if err != nil { - t.Fatalf("failed to open encrypted message: %v", err) - } - if !bytes.Equal(out.Payload, payload) { - t.Errorf("payload mismatch: have 0x%x, want 0x%x", out.Payload, payload) - } - - pubKey := out.Recover() - if pubKey == nil { - t.Fatalf("failed to recover public key") - } - p1 := elliptic.Marshal(crypto.S256(), fromKey.PublicKey.X, fromKey.PublicKey.Y) - p2 := elliptic.Marshal(crypto.S256(), pubKey.X, pubKey.Y) - if !bytes.Equal(p1, p2) { - t.Fatalf("public key mismatch: have 0x%x, want 0x%x", p2, p1) - } -} diff --git a/whisper/whisperv2/peer.go b/whisper/whisperv2/peer.go deleted file mode 100644 index 71798408b9..0000000000 --- a/whisper/whisperv2/peer.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package whisperv2 - -import ( - "fmt" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/fatih/set.v0" -) - -// peer represents a whisper protocol peer connection. -type peer struct { - host *Whisper - peer *p2p.Peer - ws p2p.MsgReadWriter - - known *set.Set // Messages already known by the peer to avoid wasting bandwidth - - quit chan struct{} -} - -// newPeer creates a new whisper peer object, but does not run the handshake itself. -func newPeer(host *Whisper, remote *p2p.Peer, rw p2p.MsgReadWriter) *peer { - return &peer{ - host: host, - peer: remote, - ws: rw, - known: set.New(), - quit: make(chan struct{}), - } -} - -// start initiates the peer updater, periodically broadcasting the whisper packets -// into the network. -func (self *peer) start() { - go self.update() - log.Debug(fmt.Sprintf("%v: whisper started", self.peer)) -} - -// stop terminates the peer updater, stopping message forwarding to it. -func (self *peer) stop() { - close(self.quit) - log.Debug(fmt.Sprintf("%v: whisper stopped", self.peer)) -} - -// handshake sends the protocol initiation status message to the remote peer and -// verifies the remote status too. -func (self *peer) handshake() error { - // Send the handshake status message asynchronously - errc := make(chan error, 1) - go func() { - errc <- p2p.SendItems(self.ws, statusCode, protocolVersion) - }() - // Fetch the remote status packet and verify protocol match - packet, err := self.ws.ReadMsg() - if err != nil { - return err - } - if packet.Code != statusCode { - return fmt.Errorf("peer sent %x before status packet", packet.Code) - } - s := rlp.NewStream(packet.Payload, uint64(packet.Size)) - if _, err := s.List(); err != nil { - return fmt.Errorf("bad status message: %v", err) - } - peerVersion, err := s.Uint() - if err != nil { - return fmt.Errorf("bad status message: %v", err) - } - if peerVersion != protocolVersion { - return fmt.Errorf("protocol version mismatch %d != %d", peerVersion, protocolVersion) - } - // Wait until out own status is consumed too - if err := <-errc; err != nil { - return fmt.Errorf("failed to send status packet: %v", err) - } - return nil -} - -// update executes periodic operations on the peer, including message transmission -// and expiration. -func (self *peer) update() { - // Start the tickers for the updates - expire := time.NewTicker(expirationCycle) - transmit := time.NewTicker(transmissionCycle) - - // Loop and transmit until termination is requested - for { - select { - case <-expire.C: - self.expire() - - case <-transmit.C: - if err := self.broadcast(); err != nil { - log.Info(fmt.Sprintf("%v: broadcast failed: %v", self.peer, err)) - return - } - - case <-self.quit: - return - } - } -} - -// mark marks an envelope known to the peer so that it won't be sent back. -func (self *peer) mark(envelope *Envelope) { - self.known.Add(envelope.Hash()) -} - -// marked checks if an envelope is already known to the remote peer. -func (self *peer) marked(envelope *Envelope) bool { - return self.known.Has(envelope.Hash()) -} - -// expire iterates over all the known envelopes in the host and removes all -// expired (unknown) ones from the known list. -func (self *peer) expire() { - // Assemble the list of available envelopes - available := set.NewNonTS() - for _, envelope := range self.host.envelopes() { - available.Add(envelope.Hash()) - } - // Cross reference availability with known status - unmark := make(map[common.Hash]struct{}) - self.known.Each(func(v interface{}) bool { - if !available.Has(v.(common.Hash)) { - unmark[v.(common.Hash)] = struct{}{} - } - return true - }) - // Dump all known but unavailable - for hash := range unmark { - self.known.Remove(hash) - } -} - -// broadcast iterates over the collection of envelopes and transmits yet unknown -// ones over the network. -func (self *peer) broadcast() error { - // Fetch the envelopes and collect the unknown ones - envelopes := self.host.envelopes() - transmit := make([]*Envelope, 0, len(envelopes)) - for _, envelope := range envelopes { - if !self.marked(envelope) { - transmit = append(transmit, envelope) - self.mark(envelope) - } - } - // Transmit the unknown batch (potentially empty) - if err := p2p.Send(self.ws, messagesCode, transmit); err != nil { - return err - } - log.Trace(fmt.Sprint(self.peer, "broadcasted", len(transmit), "message(s)")) - return nil -} diff --git a/whisper/whisperv2/peer_test.go b/whisper/whisperv2/peer_test.go deleted file mode 100644 index 87ca5063df..0000000000 --- a/whisper/whisperv2/peer_test.go +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package whisperv2 - -import ( - "testing" - "time" - - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" -) - -type testPeer struct { - client *Whisper - stream *p2p.MsgPipeRW - termed chan struct{} -} - -func startTestPeer() *testPeer { - // Create a simulated P2P remote peer and data streams to it - remote := p2p.NewPeer(discover.NodeID{}, "", nil) - tester, tested := p2p.MsgPipe() - - // Create a whisper client and connect with it to the tester peer - client := New() - client.Start(nil) - - termed := make(chan struct{}) - go func() { - defer client.Stop() - defer close(termed) - defer tested.Close() - - client.handlePeer(remote, tested) - }() - - return &testPeer{ - client: client, - stream: tester, - termed: termed, - } -} - -func startTestPeerInited() (*testPeer, error) { - peer := startTestPeer() - - if err := p2p.ExpectMsg(peer.stream, statusCode, []uint64{protocolVersion}); err != nil { - peer.stream.Close() - return nil, err - } - if err := p2p.SendItems(peer.stream, statusCode, protocolVersion); err != nil { - peer.stream.Close() - return nil, err - } - return peer, nil -} - -func TestPeerStatusMessage(t *testing.T) { - tester := startTestPeer() - - // Wait for the handshake status message and check it - if err := p2p.ExpectMsg(tester.stream, statusCode, []uint64{protocolVersion}); err != nil { - t.Fatalf("status message mismatch: %v", err) - } - // Terminate the node - tester.stream.Close() - - select { - case <-tester.termed: - case <-time.After(time.Second): - t.Fatalf("local close timed out") - } -} - -func TestPeerHandshakeFail(t *testing.T) { - tester := startTestPeer() - - // Wait for and check the handshake - if err := p2p.ExpectMsg(tester.stream, statusCode, []uint64{protocolVersion}); err != nil { - t.Fatalf("status message mismatch: %v", err) - } - // Send an invalid handshake status and verify disconnect - if err := p2p.SendItems(tester.stream, messagesCode); err != nil { - t.Fatalf("failed to send malformed status: %v", err) - } - select { - case <-tester.termed: - case <-time.After(time.Second): - t.Fatalf("remote close timed out") - } -} - -func TestPeerHandshakeSuccess(t *testing.T) { - tester := startTestPeer() - - // Wait for and check the handshake - if err := p2p.ExpectMsg(tester.stream, statusCode, []uint64{protocolVersion}); err != nil { - t.Fatalf("status message mismatch: %v", err) - } - // Send a valid handshake status and make sure connection stays live - if err := p2p.SendItems(tester.stream, statusCode, protocolVersion); err != nil { - t.Fatalf("failed to send status: %v", err) - } - select { - case <-tester.termed: - t.Fatalf("valid handshake disconnected") - - case <-time.After(100 * time.Millisecond): - } - // Clean up the test - tester.stream.Close() - - select { - case <-tester.termed: - case <-time.After(time.Second): - t.Fatalf("local close timed out") - } -} - -func TestPeerSend(t *testing.T) { - // Start a tester and execute the handshake - tester, err := startTestPeerInited() - if err != nil { - t.Fatalf("failed to start initialized peer: %v", err) - } - defer tester.stream.Close() - - // Construct a message and inject into the tester - message := NewMessage([]byte("peer broadcast test message")) - envelope, err := message.Wrap(DefaultPoW, Options{ - TTL: DefaultTTL, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := tester.client.Send(envelope); err != nil { - t.Fatalf("failed to send message: %v", err) - } - // Check that the message is eventually forwarded - payload := []interface{}{envelope} - if err := p2p.ExpectMsg(tester.stream, messagesCode, payload); err != nil { - t.Fatalf("message mismatch: %v", err) - } - // Make sure that even with a re-insert, an empty batch is received - if err := tester.client.Send(envelope); err != nil { - t.Fatalf("failed to send message: %v", err) - } - if err := p2p.ExpectMsg(tester.stream, messagesCode, []interface{}{}); err != nil { - t.Fatalf("message mismatch: %v", err) - } -} - -func TestPeerDeliver(t *testing.T) { - // Start a tester and execute the handshake - tester, err := startTestPeerInited() - if err != nil { - t.Fatalf("failed to start initialized peer: %v", err) - } - defer tester.stream.Close() - - // Watch for all inbound messages - arrived := make(chan struct{}, 1) - tester.client.Watch(Filter{ - Fn: func(message *Message) { - arrived <- struct{}{} - }, - }) - // Construct a message and deliver it to the tester peer - message := NewMessage([]byte("peer broadcast test message")) - envelope, err := message.Wrap(DefaultPoW, Options{ - TTL: DefaultTTL, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := p2p.Send(tester.stream, messagesCode, []*Envelope{envelope}); err != nil { - t.Fatalf("failed to transfer message: %v", err) - } - // Check that the message is delivered upstream - select { - case <-arrived: - case <-time.After(time.Second): - t.Fatalf("message delivery timeout") - } - // Check that a resend is not delivered - if err := p2p.Send(tester.stream, messagesCode, []*Envelope{envelope}); err != nil { - t.Fatalf("failed to transfer message: %v", err) - } - select { - case <-time.After(2 * transmissionCycle): - case <-arrived: - t.Fatalf("repeating message arrived") - } -} - -func TestPeerMessageExpiration(t *testing.T) { - // Start a tester and execute the handshake - tester, err := startTestPeerInited() - if err != nil { - t.Fatalf("failed to start initialized peer: %v", err) - } - defer tester.stream.Close() - - // Fetch the peer instance for later inspection - tester.client.peerMu.RLock() - if peers := len(tester.client.peers); peers != 1 { - t.Fatalf("peer pool size mismatch: have %v, want %v", peers, 1) - } - var peer *peer - for peer = range tester.client.peers { - break - } - tester.client.peerMu.RUnlock() - - // Construct a message and pass it through the tester - message := NewMessage([]byte("peer test message")) - envelope, err := message.Wrap(DefaultPoW, Options{ - TTL: time.Second, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := tester.client.Send(envelope); err != nil { - t.Fatalf("failed to send message: %v", err) - } - payload := []interface{}{envelope} - if err := p2p.ExpectMsg(tester.stream, messagesCode, payload); err != nil { - // A premature empty message may have been broadcast, check the next too - if err := p2p.ExpectMsg(tester.stream, messagesCode, payload); err != nil { - t.Fatalf("message mismatch: %v", err) - } - } - // Check that the message is inside the cache - if !peer.known.Has(envelope.Hash()) { - t.Fatalf("message not found in cache") - } - // Discard messages until expiration and check cache again - exp := time.Now().Add(time.Second + 2*expirationCycle + 100*time.Millisecond) - for time.Now().Before(exp) { - if err := p2p.ExpectMsg(tester.stream, messagesCode, []interface{}{}); err != nil { - t.Fatalf("message mismatch: %v", err) - } - } - if peer.known.Has(envelope.Hash()) { - t.Fatalf("message not expired from cache") - } -} diff --git a/whisper/whisperv2/topic.go b/whisper/whisperv2/topic.go deleted file mode 100644 index 3e2b47bd3f..0000000000 --- a/whisper/whisperv2/topic.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -// Contains the Whisper protocol Topic element. For formal details please see -// the specs at https://github.com/ethereum/wiki/wiki/Whisper-PoC-1-Protocol-Spec#topics. - -package whisperv2 - -import "github.com/ethereum/go-ethereum/crypto" - -// Topic represents a cryptographically secure, probabilistic partial -// classifications of a message, determined as the first (left) 4 bytes of the -// SHA3 hash of some arbitrary data given by the original author of the message. -type Topic [4]byte - -// NewTopic creates a topic from the 4 byte prefix of the SHA3 hash of the data. -// -// Note, empty topics are considered the wildcard, and cannot be used in messages. -func NewTopic(data []byte) Topic { - prefix := [4]byte{} - copy(prefix[:], crypto.Keccak256(data)[:4]) - return Topic(prefix) -} - -// NewTopics creates a list of topics from a list of binary data elements, by -// iteratively calling NewTopic on each of them. -func NewTopics(data ...[]byte) []Topic { - topics := make([]Topic, len(data)) - for i, element := range data { - topics[i] = NewTopic(element) - } - return topics -} - -// NewTopicFromString creates a topic using the binary data contents of the -// specified string. -func NewTopicFromString(data string) Topic { - return NewTopic([]byte(data)) -} - -// NewTopicsFromStrings creates a list of topics from a list of textual data -// elements, by iteratively calling NewTopicFromString on each of them. -func NewTopicsFromStrings(data ...string) []Topic { - topics := make([]Topic, len(data)) - for i, element := range data { - topics[i] = NewTopicFromString(element) - } - return topics -} - -// String converts a topic byte array to a string representation. -func (self *Topic) String() string { - return string(self[:]) -} - -// topicMatcher is a filter expression to verify if a list of topics contained -// in an arriving message matches some topic conditions. The topic matcher is -// built up of a list of conditions, each of which must be satisfied by the -// corresponding topic in the message. Each condition may require: a) an exact -// topic match; b) a match from a set of topics; or c) a wild-card matching all. -// -// If a message contains more topics than required by the matcher, those beyond -// the condition count are ignored and assumed to match. -// -// Consider the following sample topic matcher: -// sample := { -// {TopicA1, TopicA2, TopicA3}, -// {TopicB}, -// nil, -// {TopicD1, TopicD2} -// } -// In order for a message to pass this filter, it should enumerate at least 4 -// topics, the first any of [TopicA1, TopicA2, TopicA3], the second mandatory -// "TopicB", the third is ignored by the filter and the fourth either "TopicD1" -// or "TopicD2". If the message contains further topics, the filter will match -// them too. -type topicMatcher struct { - conditions []map[Topic]struct{} -} - -// newTopicMatcher create a topic matcher from a list of topic conditions. -func newTopicMatcher(topics ...[]Topic) *topicMatcher { - matcher := make([]map[Topic]struct{}, len(topics)) - for i, condition := range topics { - matcher[i] = make(map[Topic]struct{}) - for _, topic := range condition { - matcher[i][topic] = struct{}{} - } - } - return &topicMatcher{conditions: matcher} -} - -// newTopicMatcherFromBinary create a topic matcher from a list of binary conditions. -func newTopicMatcherFromBinary(data ...[][]byte) *topicMatcher { - topics := make([][]Topic, len(data)) - for i, condition := range data { - topics[i] = NewTopics(condition...) - } - return newTopicMatcher(topics...) -} - -// newTopicMatcherFromStrings creates a topic matcher from a list of textual -// conditions. -func newTopicMatcherFromStrings(data ...[]string) *topicMatcher { - topics := make([][]Topic, len(data)) - for i, condition := range data { - topics[i] = NewTopicsFromStrings(condition...) - } - return newTopicMatcher(topics...) -} - -// Matches checks if a list of topics matches this particular condition set. -func (self *topicMatcher) Matches(topics []Topic) bool { - // Mismatch if there aren't enough topics - if len(self.conditions) > len(topics) { - return false - } - // Check each topic condition for existence (skip wild-cards) - for i := 0; i < len(topics) && i < len(self.conditions); i++ { - if len(self.conditions[i]) > 0 { - if _, ok := self.conditions[i][topics[i]]; !ok { - return false - } - } - } - return true -} diff --git a/whisper/whisperv2/topic_test.go b/whisper/whisperv2/topic_test.go deleted file mode 100644 index bb65689963..0000000000 --- a/whisper/whisperv2/topic_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package whisperv2 - -import ( - "bytes" - "testing" -) - -var topicCreationTests = []struct { - data []byte - hash [4]byte -}{ - {hash: [4]byte{0x8f, 0x9a, 0x2b, 0x7d}, data: []byte("test name")}, - {hash: [4]byte{0xf2, 0x6e, 0x77, 0x79}, data: []byte("some other test")}, -} - -func TestTopicCreation(t *testing.T) { - // Create the topics individually - for i, tt := range topicCreationTests { - topic := NewTopic(tt.data) - if !bytes.Equal(topic[:], tt.hash[:]) { - t.Errorf("binary test %d: hash mismatch: have %v, want %v.", i, topic, tt.hash) - } - } - for i, tt := range topicCreationTests { - topic := NewTopicFromString(string(tt.data)) - if !bytes.Equal(topic[:], tt.hash[:]) { - t.Errorf("textual test %d: hash mismatch: have %v, want %v.", i, topic, tt.hash) - } - } - // Create the topics in batches - binaryData := make([][]byte, len(topicCreationTests)) - for i, tt := range topicCreationTests { - binaryData[i] = tt.data - } - textualData := make([]string, len(topicCreationTests)) - for i, tt := range topicCreationTests { - textualData[i] = string(tt.data) - } - - topics := NewTopics(binaryData...) - for i, tt := range topicCreationTests { - if !bytes.Equal(topics[i][:], tt.hash[:]) { - t.Errorf("binary batch test %d: hash mismatch: have %v, want %v.", i, topics[i], tt.hash) - } - } - topics = NewTopicsFromStrings(textualData...) - for i, tt := range topicCreationTests { - if !bytes.Equal(topics[i][:], tt.hash[:]) { - t.Errorf("textual batch test %d: hash mismatch: have %v, want %v.", i, topics[i], tt.hash) - } - } -} - -var topicMatcherCreationTest = struct { - binary [][][]byte - textual [][]string - matcher []map[[4]byte]struct{} -}{ - binary: [][][]byte{ - {}, - { - []byte("Topic A"), - }, - { - []byte("Topic B1"), - []byte("Topic B2"), - []byte("Topic B3"), - }, - }, - textual: [][]string{ - {}, - {"Topic A"}, - {"Topic B1", "Topic B2", "Topic B3"}, - }, - matcher: []map[[4]byte]struct{}{ - {}, - { - {0x25, 0xfc, 0x95, 0x66}: {}, - }, - { - {0x93, 0x6d, 0xec, 0x09}: {}, - {0x25, 0x23, 0x34, 0xd3}: {}, - {0x6b, 0xc2, 0x73, 0xd1}: {}, - }, - }, -} - -func TestTopicMatcherCreation(t *testing.T) { - test := topicMatcherCreationTest - - matcher := newTopicMatcherFromBinary(test.binary...) - for i, cond := range matcher.conditions { - for topic := range cond { - if _, ok := test.matcher[i][topic]; !ok { - t.Errorf("condition %d; extra topic found: 0x%x", i, topic[:]) - } - } - } - for i, cond := range test.matcher { - for topic := range cond { - if _, ok := matcher.conditions[i][topic]; !ok { - t.Errorf("condition %d; topic not found: 0x%x", i, topic[:]) - } - } - } - - matcher = newTopicMatcherFromStrings(test.textual...) - for i, cond := range matcher.conditions { - for topic := range cond { - if _, ok := test.matcher[i][topic]; !ok { - t.Errorf("condition %d; extra topic found: 0x%x", i, topic[:]) - } - } - } - for i, cond := range test.matcher { - for topic := range cond { - if _, ok := matcher.conditions[i][topic]; !ok { - t.Errorf("condition %d; topic not found: 0x%x", i, topic[:]) - } - } - } -} - -var topicMatcherTests = []struct { - filter [][]string - topics []string - match bool -}{ - // Empty topic matcher should match everything - { - filter: [][]string{}, - topics: []string{}, - match: true, - }, - { - filter: [][]string{}, - topics: []string{"a", "b", "c"}, - match: true, - }, - // Fixed topic matcher should match strictly, but only prefix - { - filter: [][]string{{"a"}, {"b"}}, - topics: []string{"a"}, - match: false, - }, - { - filter: [][]string{{"a"}, {"b"}}, - topics: []string{"a", "b"}, - match: true, - }, - { - filter: [][]string{{"a"}, {"b"}}, - topics: []string{"a", "b", "c"}, - match: true, - }, - // Multi-matcher should match any from a sub-group - { - filter: [][]string{{"a1", "a2"}}, - topics: []string{"a"}, - match: false, - }, - { - filter: [][]string{{"a1", "a2"}}, - topics: []string{"a1"}, - match: true, - }, - { - filter: [][]string{{"a1", "a2"}}, - topics: []string{"a2"}, - match: true, - }, - // Wild-card condition should match anything - { - filter: [][]string{{}, {"b"}}, - topics: []string{"a"}, - match: false, - }, - { - filter: [][]string{{}, {"b"}}, - topics: []string{"a", "b"}, - match: true, - }, - { - filter: [][]string{{}, {"b"}}, - topics: []string{"b", "b"}, - match: true, - }, -} - -func TestTopicMatcher(t *testing.T) { - for i, tt := range topicMatcherTests { - topics := NewTopicsFromStrings(tt.topics...) - - matcher := newTopicMatcherFromStrings(tt.filter...) - if match := matcher.Matches(topics); match != tt.match { - t.Errorf("test %d: match mismatch: have %v, want %v", i, match, tt.match) - } - } -} diff --git a/whisper/whisperv2/whisper.go b/whisper/whisperv2/whisper.go deleted file mode 100644 index e111a34146..0000000000 --- a/whisper/whisperv2/whisper.go +++ /dev/null @@ -1,378 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package whisperv2 - -import ( - "crypto/ecdsa" - "fmt" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/crypto/ecies" - "github.com/ethereum/go-ethereum/event/filter" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rpc" - - "gopkg.in/fatih/set.v0" -) - -const ( - statusCode = 0x00 - messagesCode = 0x01 - - protocolVersion uint64 = 0x02 - protocolName = "shh" - - signatureFlag = byte(1 << 7) - signatureLength = 65 - - expirationCycle = 800 * time.Millisecond - transmissionCycle = 300 * time.Millisecond -) - -const ( - DefaultTTL = 50 * time.Second - DefaultPoW = 50 * time.Millisecond -) - -type MessageEvent struct { - To *ecdsa.PrivateKey - From *ecdsa.PublicKey - Message *Message -} - -// Whisper represents a dark communication interface through the Ethereum -// network, using its very own P2P communication layer. -type Whisper struct { - protocol p2p.Protocol - filters *filter.Filters - - keys map[string]*ecdsa.PrivateKey - - messages map[common.Hash]*Envelope // Pool of messages currently tracked by this node - expirations map[uint32]*set.SetNonTS // Message expiration pool (TODO: something lighter) - poolMu sync.RWMutex // Mutex to sync the message and expiration pools - - peers map[*peer]struct{} // Set of currently active peers - peerMu sync.RWMutex // Mutex to sync the active peer set - - quit chan struct{} -} - -// New creates a Whisper client ready to communicate through the Ethereum P2P -// network. -func New() *Whisper { - whisper := &Whisper{ - filters: filter.New(), - keys: make(map[string]*ecdsa.PrivateKey), - messages: make(map[common.Hash]*Envelope), - expirations: make(map[uint32]*set.SetNonTS), - peers: make(map[*peer]struct{}), - quit: make(chan struct{}), - } - whisper.filters.Start() - - // p2p whisper sub protocol handler - whisper.protocol = p2p.Protocol{ - Name: protocolName, - Version: uint(protocolVersion), - Length: 2, - Run: whisper.handlePeer, - } - - return whisper -} - -// APIs returns the RPC descriptors the Whisper implementation offers -func (s *Whisper) APIs() []rpc.API { - return []rpc.API{ - { - Namespace: "shh", - Version: "1.0", - Service: NewPublicWhisperAPI(s), - Public: true, - }, - } -} - -// Protocols returns the whisper sub-protocols ran by this particular client. -func (self *Whisper) Protocols() []p2p.Protocol { - return []p2p.Protocol{self.protocol} -} - -// Version returns the whisper sub-protocols version number. -func (self *Whisper) Version() uint { - return self.protocol.Version -} - -// NewIdentity generates a new cryptographic identity for the client, and injects -// it into the known identities for message decryption. -func (self *Whisper) NewIdentity() *ecdsa.PrivateKey { - key, err := crypto.GenerateKey() - if err != nil { - panic(err) - } - self.keys[string(crypto.FromECDSAPub(&key.PublicKey))] = key - - return key -} - -// HasIdentity checks if the the whisper node is configured with the private key -// of the specified public pair. -func (self *Whisper) HasIdentity(key *ecdsa.PublicKey) bool { - return self.keys[string(crypto.FromECDSAPub(key))] != nil -} - -// GetIdentity retrieves the private key of the specified public identity. -func (self *Whisper) GetIdentity(key *ecdsa.PublicKey) *ecdsa.PrivateKey { - return self.keys[string(crypto.FromECDSAPub(key))] -} - -// Watch installs a new message handler to run in case a matching packet arrives -// from the whisper network. -func (self *Whisper) Watch(options Filter) int { - filter := filterer{ - to: string(crypto.FromECDSAPub(options.To)), - from: string(crypto.FromECDSAPub(options.From)), - matcher: newTopicMatcher(options.Topics...), - fn: func(data interface{}) { - options.Fn(data.(*Message)) - }, - } - return self.filters.Install(filter) -} - -// Unwatch removes an installed message handler. -func (self *Whisper) Unwatch(id int) { - self.filters.Uninstall(id) -} - -// Send injects a message into the whisper send queue, to be distributed in the -// network in the coming cycles. -func (self *Whisper) Send(envelope *Envelope) error { - return self.add(envelope) -} - -// Start implements node.Service, starting the background data propagation thread -// of the Whisper protocol. -func (self *Whisper) Start(*p2p.Server) error { - log.Info("Whisper started") - go self.update() - return nil -} - -// Stop implements node.Service, stopping the background data propagation thread -// of the Whisper protocol. -func (self *Whisper) Stop() error { - close(self.quit) - log.Info("Whisper stopped") - return nil -} - -// Messages retrieves all the currently pooled messages matching a filter id. -func (self *Whisper) Messages(id int) []*Message { - messages := make([]*Message, 0) - if filter := self.filters.Get(id); filter != nil { - for _, envelope := range self.messages { - if message := self.open(envelope); message != nil { - if self.filters.Match(filter, createFilter(message, envelope.Topics)) { - messages = append(messages, message) - } - } - } - } - return messages -} - -// handlePeer is called by the underlying P2P layer when the whisper sub-protocol -// connection is negotiated. -func (self *Whisper) handlePeer(peer *p2p.Peer, rw p2p.MsgReadWriter) error { - // Create the new peer and start tracking it - whisperPeer := newPeer(self, peer, rw) - - self.peerMu.Lock() - self.peers[whisperPeer] = struct{}{} - self.peerMu.Unlock() - - defer func() { - self.peerMu.Lock() - delete(self.peers, whisperPeer) - self.peerMu.Unlock() - }() - - // Run the peer handshake and state updates - if err := whisperPeer.handshake(); err != nil { - return err - } - whisperPeer.start() - defer whisperPeer.stop() - - // Read and process inbound messages directly to merge into client-global state - for { - // Fetch the next packet and decode the contained envelopes - packet, err := rw.ReadMsg() - if err != nil { - return err - } - var envelopes []*Envelope - if err := packet.Decode(&envelopes); err != nil { - log.Info(fmt.Sprintf("%v: failed to decode envelope: %v", peer, err)) - continue - } - // Inject all envelopes into the internal pool - for _, envelope := range envelopes { - if err := self.add(envelope); err != nil { - // TODO Punish peer here. Invalid envelope. - log.Debug(fmt.Sprintf("%v: failed to pool envelope: %v", peer, err)) - } - whisperPeer.mark(envelope) - } - } -} - -// add inserts a new envelope into the message pool to be distributed within the -// whisper network. It also inserts the envelope into the expiration pool at the -// appropriate time-stamp. -func (self *Whisper) add(envelope *Envelope) error { - self.poolMu.Lock() - defer self.poolMu.Unlock() - - // short circuit when a received envelope has already expired - if envelope.Expiry < uint32(time.Now().Unix()) { - return nil - } - - // Insert the message into the tracked pool - hash := envelope.Hash() - if _, ok := self.messages[hash]; ok { - log.Trace(fmt.Sprintf("whisper envelope already cached: %x\n", hash)) - return nil - } - self.messages[hash] = envelope - - // Insert the message into the expiration pool for later removal - if self.expirations[envelope.Expiry] == nil { - self.expirations[envelope.Expiry] = set.NewNonTS() - } - if !self.expirations[envelope.Expiry].Has(hash) { - self.expirations[envelope.Expiry].Add(hash) - - // Notify the local node of a message arrival - go self.postEvent(envelope) - } - log.Trace(fmt.Sprintf("cached whisper envelope %x\n", hash)) - return nil -} - -// postEvent opens an envelope with the configured identities and delivers the -// message upstream from application processing. -func (self *Whisper) postEvent(envelope *Envelope) { - if message := self.open(envelope); message != nil { - self.filters.Notify(createFilter(message, envelope.Topics), message) - } -} - -// open tries to decrypt a whisper envelope with all the configured identities, -// returning the decrypted message and the key used to achieve it. If not keys -// are configured, open will return the payload as if non encrypted. -func (self *Whisper) open(envelope *Envelope) *Message { - // Short circuit if no identity is set, and assume clear-text - if len(self.keys) == 0 { - if message, err := envelope.Open(nil); err == nil { - return message - } - } - // Iterate over the keys and try to decrypt the message - for _, key := range self.keys { - message, err := envelope.Open(key) - if err == nil { - message.To = &key.PublicKey - return message - } else if err == ecies.ErrInvalidPublicKey { - return message - } - } - // Failed to decrypt, don't return anything - return nil -} - -// createFilter creates a message filter to check against installed handlers. -func createFilter(message *Message, topics []Topic) filter.Filter { - matcher := make([][]Topic, len(topics)) - for i, topic := range topics { - matcher[i] = []Topic{topic} - } - return filterer{ - to: string(crypto.FromECDSAPub(message.To)), - from: string(crypto.FromECDSAPub(message.Recover())), - matcher: newTopicMatcher(matcher...), - } -} - -// update loops until the lifetime of the whisper node, updating its internal -// state by expiring stale messages from the pool. -func (self *Whisper) update() { - // Start a ticker to check for expirations - expire := time.NewTicker(expirationCycle) - - // Repeat updates until termination is requested - for { - select { - case <-expire.C: - self.expire() - - case <-self.quit: - return - } - } -} - -// expire iterates over all the expiration timestamps, removing all stale -// messages from the pools. -func (self *Whisper) expire() { - self.poolMu.Lock() - defer self.poolMu.Unlock() - - now := uint32(time.Now().Unix()) - for then, hashSet := range self.expirations { - // Short circuit if a future time - if then > now { - continue - } - // Dump all expired messages and remove timestamp - hashSet.Each(func(v interface{}) bool { - delete(self.messages, v.(common.Hash)) - return true - }) - self.expirations[then].Clear() - } -} - -// envelopes retrieves all the messages currently pooled by the node. -func (self *Whisper) envelopes() []*Envelope { - self.poolMu.RLock() - defer self.poolMu.RUnlock() - - envelopes := make([]*Envelope, 0, len(self.messages)) - for _, envelope := range self.messages { - envelopes = append(envelopes, envelope) - } - return envelopes -} diff --git a/whisper/whisperv2/whisper_test.go b/whisper/whisperv2/whisper_test.go deleted file mode 100644 index 1e0d3f85d4..0000000000 --- a/whisper/whisperv2/whisper_test.go +++ /dev/null @@ -1,216 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package whisperv2 - -import ( - "testing" - "time" - - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" -) - -func startTestCluster(n int) []*Whisper { - // Create the batch of simulated peers - nodes := make([]*p2p.Peer, n) - for i := 0; i < n; i++ { - nodes[i] = p2p.NewPeer(discover.NodeID{}, "", nil) - } - whispers := make([]*Whisper, n) - for i := 0; i < n; i++ { - whispers[i] = New() - whispers[i].Start(nil) - } - // Wire all the peers to the root one - for i := 1; i < n; i++ { - src, dst := p2p.MsgPipe() - - go whispers[0].handlePeer(nodes[i], src) - go whispers[i].handlePeer(nodes[0], dst) - } - return whispers -} - -func TestSelfMessage(t *testing.T) { - // Start the single node cluster - client := startTestCluster(1)[0] - - // Start watching for self messages, signal any arrivals - self := client.NewIdentity() - done := make(chan struct{}) - - client.Watch(Filter{ - To: &self.PublicKey, - Fn: func(msg *Message) { - close(done) - }, - }) - // Send a dummy message to oneself - msg := NewMessage([]byte("self whisper")) - envelope, err := msg.Wrap(DefaultPoW, Options{ - From: self, - To: &self.PublicKey, - TTL: DefaultTTL, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - // Dump the message into the system and wait for it to pop back out - if err := client.Send(envelope); err != nil { - t.Fatalf("failed to send self-message: %v", err) - } - select { - case <-done: - case <-time.After(time.Second): - t.Fatalf("self-message receive timeout") - } -} - -func TestDirectMessage(t *testing.T) { - // Start the sender-recipient cluster - cluster := startTestCluster(2) - - sender := cluster[0] - senderId := sender.NewIdentity() - - recipient := cluster[1] - recipientId := recipient.NewIdentity() - - // Watch for arriving messages on the recipient - done := make(chan struct{}) - recipient.Watch(Filter{ - To: &recipientId.PublicKey, - Fn: func(msg *Message) { - close(done) - }, - }) - // Send a dummy message from the sender - msg := NewMessage([]byte("direct whisper")) - envelope, err := msg.Wrap(DefaultPoW, Options{ - From: senderId, - To: &recipientId.PublicKey, - TTL: DefaultTTL, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := sender.Send(envelope); err != nil { - t.Fatalf("failed to send direct message: %v", err) - } - // Wait for an arrival or a timeout - select { - case <-done: - case <-time.After(time.Second): - t.Fatalf("direct message receive timeout") - } -} - -func TestAnonymousBroadcast(t *testing.T) { - testBroadcast(true, t) -} - -func TestIdentifiedBroadcast(t *testing.T) { - testBroadcast(false, t) -} - -func testBroadcast(anonymous bool, t *testing.T) { - // Start the single sender multi recipient cluster - cluster := startTestCluster(3) - - sender := cluster[1] - targets := cluster[1:] - for _, target := range targets { - if !anonymous { - target.NewIdentity() - } - } - // Watch for arriving messages on the recipients - dones := make([]chan struct{}, len(targets)) - for i := 0; i < len(targets); i++ { - done := make(chan struct{}) // need for the closure - dones[i] = done - - targets[i].Watch(Filter{ - Topics: NewFilterTopicsFromStringsFlat("broadcast topic"), - Fn: func(msg *Message) { - close(done) - }, - }) - } - // Send a dummy message from the sender - msg := NewMessage([]byte("broadcast whisper")) - envelope, err := msg.Wrap(DefaultPoW, Options{ - Topics: NewTopicsFromStrings("broadcast topic"), - TTL: DefaultTTL, - }) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := sender.Send(envelope); err != nil { - t.Fatalf("failed to send broadcast message: %v", err) - } - // Wait for an arrival on each recipient, or timeouts - timeout := time.After(time.Second) - for _, done := range dones { - select { - case <-done: - case <-timeout: - t.Fatalf("broadcast message receive timeout") - } - } -} - -func TestMessageExpiration(t *testing.T) { - // Start the single node cluster and inject a dummy message - node := startTestCluster(1)[0] - - message := NewMessage([]byte("expiring message")) - envelope, err := message.Wrap(DefaultPoW, Options{TTL: time.Second}) - if err != nil { - t.Fatalf("failed to wrap message: %v", err) - } - if err := node.Send(envelope); err != nil { - t.Fatalf("failed to inject message: %v", err) - } - // Check that the message is inside the cache - node.poolMu.RLock() - _, found := node.messages[envelope.Hash()] - node.poolMu.RUnlock() - - if !found { - t.Fatalf("message not found in cache") - } - // Wait for expiration and check cache again - time.Sleep(time.Second) // wait for expiration - time.Sleep(2 * expirationCycle) // wait for cleanup cycle - - node.poolMu.RLock() - _, found = node.messages[envelope.Hash()] - node.poolMu.RUnlock() - if found { - t.Fatalf("message not expired from cache") - } - - // Check that adding an expired envelope doesn't do anything. - node.add(envelope) - node.poolMu.RLock() - _, found = node.messages[envelope.Hash()] - node.poolMu.RUnlock() - if found { - t.Fatalf("message was added to cache") - } -} From 28b20cff4b76595b695f9d784c98d50d31bb269b Mon Sep 17 00:00:00 2001 From: Felix Lange
Date: Thu, 22 Feb 2018 11:37:57 +0100 Subject: [PATCH 37/94] p2p/protocols: gofmt -w -s --- p2p/protocols/protocol_test.go | 44 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index b041a650a1..053f537a62 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -147,18 +147,18 @@ func protocolTester(t *testing.T, pp *p2ptest.TestPeerPool) *p2ptest.ProtocolTes func protoHandshakeExchange(id discover.NodeID, proto *protoHandshake) []p2ptest.Exchange { return []p2ptest.Exchange{ - p2ptest.Exchange{ + { Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 0, Msg: &protoHandshake{42, "420"}, Peer: id, }, }, }, - p2ptest.Exchange{ + { Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 0, Msg: proto, Peer: id, @@ -200,18 +200,18 @@ func TestProtoHandshakeSuccess(t *testing.T) { func moduleHandshakeExchange(id discover.NodeID, resp uint) []p2ptest.Exchange { return []p2ptest.Exchange{ - p2ptest.Exchange{ + { Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 1, Msg: &hs0{42}, Peer: id, }, }, }, - p2ptest.Exchange{ + { Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 1, Msg: &hs0{resp}, Peer: id, @@ -252,42 +252,42 @@ func TestModuleHandshakeSuccess(t *testing.T) { func testMultiPeerSetup(a, b discover.NodeID) []p2ptest.Exchange { return []p2ptest.Exchange{ - p2ptest.Exchange{ + { Label: "primary handshake", Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 0, Msg: &protoHandshake{42, "420"}, Peer: a, }, - p2ptest.Expect{ + { Code: 0, Msg: &protoHandshake{42, "420"}, Peer: b, }, }, }, - p2ptest.Exchange{ + { Label: "module handshake", Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 0, Msg: &protoHandshake{42, "420"}, Peer: a, }, - p2ptest.Trigger{ + { Code: 0, Msg: &protoHandshake{42, "420"}, Peer: b, }, }, Expects: []p2ptest.Expect{ - p2ptest.Expect{ + { Code: 1, Msg: &hs0{42}, Peer: a, }, - p2ptest.Expect{ + { Code: 1, Msg: &hs0{42}, Peer: b, @@ -295,10 +295,10 @@ func testMultiPeerSetup(a, b discover.NodeID) []p2ptest.Exchange { }, }, - p2ptest.Exchange{Label: "alternative module handshake", Triggers: []p2ptest.Trigger{p2ptest.Trigger{Code: 1, Msg: &hs0{41}, Peer: a}, - p2ptest.Trigger{Code: 1, Msg: &hs0{41}, Peer: b}}}, - p2ptest.Exchange{Label: "repeated module handshake", Triggers: []p2ptest.Trigger{p2ptest.Trigger{Code: 1, Msg: &hs0{1}, Peer: a}}}, - p2ptest.Exchange{Label: "receiving repeated module handshake", Expects: []p2ptest.Expect{p2ptest.Expect{Code: 1, Msg: &hs0{43}, Peer: a}}}} + {Label: "alternative module handshake", Triggers: []p2ptest.Trigger{{Code: 1, Msg: &hs0{41}, Peer: a}, + {Code: 1, Msg: &hs0{41}, Peer: b}}}, + {Label: "repeated module handshake", Triggers: []p2ptest.Trigger{{Code: 1, Msg: &hs0{1}, Peer: a}}}, + {Label: "receiving repeated module handshake", Expects: []p2ptest.Expect{{Code: 1, Msg: &hs0{43}, Peer: a}}}} } func runMultiplePeers(t *testing.T, peer int, errs ...error) { @@ -332,7 +332,7 @@ WAIT: // peer 0 sends kill request for peer with index
err := s.TestExchanges(p2ptest.Exchange{ Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 2, Msg: &kill{s.IDs[peer]}, Peer: s.IDs[0], @@ -347,7 +347,7 @@ WAIT: // the peer not killed sends a drop request err = s.TestExchanges(p2ptest.Exchange{ Triggers: []p2ptest.Trigger{ - p2ptest.Trigger{ + { Code: 3, Msg: &drop{}, Peer: s.IDs[(peer+1)%2], From 1e457b659989478004329c2f3a82b5ac67b32cbf Mon Sep 17 00:00:00 2001 From: Anton Evangelatov
Date: Thu, 22 Feb 2018 11:41:06 +0100 Subject: [PATCH 38/94] p2p: don't send DiscReason when using net.Pipe (#16004) --- p2p/rlpx.go | 10 ++++++++-- p2p/rlpx_test.go | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/p2p/rlpx.go b/p2p/rlpx.go index 24037ecc13..e65a0b6047 100644 --- a/p2p/rlpx.go +++ b/p2p/rlpx.go @@ -108,8 +108,14 @@ func (t *rlpx) close(err error) { // Tell the remote end why we're disconnecting if possible. if t.rw != nil { if r, ok := err.(DiscReason); ok && r != DiscNetworkError { - t.fd.SetWriteDeadline(time.Now().Add(discWriteTimeout)) - SendItems(t.rw, discMsg, r) + // rlpx tries to send DiscReason to disconnected peer + // if the connection is net.Pipe (in-memory simulation) + // it hangs forever, since net.Pipe does not implement + // a write deadline. Because of this only try to send + // the disconnect reason message if there is no error. + if err := t.fd.SetWriteDeadline(time.Now().Add(discWriteTimeout)); err == nil { + SendItems(t.rw, discMsg, r) + } } } t.fd.Close() diff --git a/p2p/rlpx_test.go b/p2p/rlpx_test.go index f4cefa650b..bca4604021 100644 --- a/p2p/rlpx_test.go +++ b/p2p/rlpx_test.go @@ -156,14 +156,18 @@ func TestProtocolHandshake(t *testing.T) { node1 = &discover.Node{ID: discover.PubkeyID(&prv1.PublicKey), IP: net.IP{5, 6, 7, 8}, TCP: 44} hs1 = &protoHandshake{Version: 3, ID: node1.ID, Caps: []Cap{{"c", 1}, {"d", 3}}} - fd0, fd1 = net.Pipe() - wg sync.WaitGroup + wg sync.WaitGroup ) + fd0, fd1, err := tcpPipe() + if err != nil { + t.Fatal(err) + } + wg.Add(2) go func() { defer wg.Done() - defer fd1.Close() + defer fd0.Close() rlpx := newRLPX(fd0) remid, err := rlpx.doEncHandshake(prv0, node1) if err != nil { @@ -597,3 +601,31 @@ func TestHandshakeForwardCompatibility(t *testing.T) { t.Errorf("ingress-mac('foo') mismatch:\ngot %x\nwant %x", fooIngressHash, wantFooIngressHash) } } + +// tcpPipe creates an in process full duplex pipe based on a localhost TCP socket +func tcpPipe() (net.Conn, net.Conn, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, nil, err + } + defer l.Close() + + var aconn net.Conn + aerr := make(chan error, 1) + go func() { + var err error + aconn, err = l.Accept() + aerr <- err + }() + + dconn, err := net.Dial("tcp", l.Addr().String()) + if err != nil { + <-aerr + return nil, nil, err + } + if err := <-aerr; err != nil { + dconn.Close() + return nil, nil, err + } + return aconn, dconn, nil +} From 72c4c50777fbd0e7232c4c0adff030954e3e46ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?=
Date: Thu, 22 Feb 2018 13:20:36 +0200 Subject: [PATCH 39/94] cmd/faucet: resolve twitter user from final redirect --- cmd/faucet/faucet.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 99527f9d1e..095668c868 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -686,8 +686,6 @@ func authTwitter(url string) (string, string, common.Address, error) { if len(parts) < 4 || parts[len(parts)-2] != "status" { return "", "", common.Address{}, errors.New("Invalid Twitter status URL") } - username := parts[len(parts)-3] - // Twitter's API isn't really friendly with direct links. Still, we don't // want to do ask read permissions from users, so just load the public posts and // scrape it for the Ethereum address and profile URL. @@ -697,6 +695,13 @@ func authTwitter(url string) (string, string, common.Address, error) { } defer res.Body.Close() + // Resolve the username from the final redirect, no intermediate junk + parts = strings.Split(res.Request.URL.String(), "/") + if len(parts) < 4 || parts[len(parts)-2] != "status" { + return "", "", common.Address{}, errors.New("Invalid Twitter status URL") + } + username := parts[len(parts)-3] + body, err := ioutil.ReadAll(res.Body) if err != nil { return "", "", common.Address{}, err From 0b3e23f6367eb80de74565e6cf76c18b56774eaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?=
Date: Thu, 22 Feb 2018 15:41:48 +0200 Subject: [PATCH 40/94] core: yeah, funny file, drop it --- core/fees.go | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 core/fees.go diff --git a/core/fees.go b/core/fees.go deleted file mode 100644 index 83275ea36c..0000000000 --- a/core/fees.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -package core - -import ( - "math/big" -) - -var BlockReward = big.NewInt(5e+18) From 45352477933d71ec5055504da74547b0cdf0274b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?=
Date: Thu, 22 Feb 2018 17:39:28 +0100 Subject: [PATCH 41/94] rpc: set rpcRequest.service as methodNotFoundError.service value (#16163) RPC Server readRequest method sets the serverRequest error service value as the rpcRequest.method and this change sets it to the right service value. --- rpc/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/server.go b/rpc/server.go index 30c288349e..11373b504c 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -421,7 +421,7 @@ func (s *Server) readRequest(codec ServerCodec) ([]*serverRequest, bool, Error) } } } else { - requests[i] = &serverRequest{id: r.id, err: &methodNotFoundError{r.method, r.method}} + requests[i] = &serverRequest{id: r.id, err: &methodNotFoundError{r.service, r.method}} } continue } From 5cf1d354704cd2cbc5c64c96d4aaabeeec7dd161 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?=
Date: Thu, 22 Feb 2018 12:48:14 +0200 Subject: [PATCH 42/94] eth, les, light: filter on logs only, derive receipts on demand --- accounts/abi/bind/backends/simulated.go | 13 +++++++ eth/api_backend.go | 12 +++++++ eth/filters/filter.go | 19 +++++++++-- eth/filters/filter_system.go | 22 ++++++++++-- eth/filters/filter_system_test.go | 15 +++++++-- internal/ethapi/api.go | 10 ++++-- les/api_backend.go | 4 +++ light/odr_util.go | 45 +++++++++++++++++++++---- 8 files changed, 123 insertions(+), 17 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index bd342a8cb9..fe7dea4da1 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -428,10 +428,23 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb } return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil } + func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)), nil } +func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { + receipts := core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)) + if receipts == nil { + return nil, nil + } + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs, nil +} + func (fb *filterBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { return event.NewSubscription(func(quit <-chan struct{}) error { <-quit diff --git a/eth/api_backend.go b/eth/api_backend.go index 91f392f94f..ecd5488a24 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -104,6 +104,18 @@ func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)), nil } +func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { + receipts := core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) + if receipts == nil { + return nil, nil + } + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs, nil +} + func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 43d7e2a812..5dfe60e772 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -34,6 +34,7 @@ type Backend interface { EventMux() *event.TypeMux HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription @@ -201,16 +202,28 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e // match the filter criteria. This function is called when the bloom filter signals a potential match. func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { // Get the logs of the block - receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + logsList, err := f.backend.GetLogs(ctx, header.Hash()) if err != nil { return nil, err } var unfiltered []*types.Log - for _, receipt := range receipts { - unfiltered = append(unfiltered, receipt.Logs...) + for _, logs := range logsList { + unfiltered = append(unfiltered, logs...) } logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) if len(logs) > 0 { + // We have matching logs, check if we need to resolve full logs via the light client + if logs[0].TxHash == (common.Hash{}) { + receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + if err != nil { + return nil, err + } + unfiltered = unfiltered[:0] + for _, receipt := range receipts { + unfiltered = append(unfiltered, receipt.Logs...) + } + logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) + } return logs, nil } return nil, nil diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index b09998f9c7..f8097c7b95 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -375,19 +375,35 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // Get the logs of the block ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - receipts, err := es.backend.GetReceipts(ctx, header.Hash()) + logsList, err := es.backend.GetLogs(ctx, header.Hash()) if err != nil { return nil } var unfiltered []*types.Log - for _, receipt := range receipts { - for _, log := range receipt.Logs { + for _, logs := range logsList { + for _, log := range logs { logcopy := *log logcopy.Removed = remove unfiltered = append(unfiltered, &logcopy) } } logs := filterLogs(unfiltered, nil, nil, addresses, topics) + if len(logs) > 0 && logs[0].TxHash == (common.Hash{}) { + // We have matching but non-derived logs + receipts, err := es.backend.GetReceipts(ctx, header.Hash()) + if err != nil { + return nil + } + unfiltered = unfiltered[:0] + for _, receipt := range receipts { + for _, log := range receipt.Logs { + logcopy := *log + logcopy.Removed = remove + unfiltered = append(unfiltered, &logcopy) + } + } + logs = filterLogs(unfiltered, nil, nil, addresses, topics) + } return logs } return nil diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 7ec3b4be7f..61761151a2 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -69,8 +69,19 @@ func (b *testBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumbe } func (b *testBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { - num := core.GetBlockNumber(b.db, blockHash) - return core.GetBlockReceipts(b.db, blockHash, num), nil + number := core.GetBlockNumber(b.db, blockHash) + return core.GetBlockReceipts(b.db, blockHash, number), nil +} + +func (b *testBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { + number := core.GetBlockNumber(b.db, blockHash) + receipts := core.GetBlockReceipts(b.db, blockHash, number) + + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs, nil } func (b *testBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 636d0bfe26..d021b127c5 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1032,15 +1032,19 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, } // GetTransactionReceipt returns the transaction receipt for the given transaction hash. -func (s *PublicTransactionPoolAPI) GetTransactionReceipt(hash common.Hash) (map[string]interface{}, error) { +func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash) if tx == nil { return nil, errors.New("unknown transaction") } - receipt, _, _, _ := core.GetReceipt(s.b.ChainDb(), hash) // Old receipts don't have the lookup data available - if receipt == nil { + receipts, err := s.b.GetReceipts(ctx, blockHash) + if err != nil { + return nil, err + } + if len(receipts) <= int(index) { return nil, errors.New("unknown receipt") } + receipt := receipts[index] var signer types.Signer = types.FrontierSigner{} if tx.Protected() { diff --git a/les/api_backend.go b/les/api_backend.go index 56f617a7db..3fc5c33a44 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -87,6 +87,10 @@ func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) } +func (b *LesApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { + return light.GetBlockLogs(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) +} + func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } diff --git a/light/odr_util.go b/light/odr_util.go index 8f92d64426..97ba440ace 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -126,15 +126,48 @@ func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint // GetBlockReceipts retrieves the receipts generated by the transactions included // in a block given by its hash. func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (types.Receipts, error) { + // Retrieve the potentially incomplete receipts from disk or network receipts := core.GetBlockReceipts(odr.Database(), hash, number) - if receipts != nil { - return receipts, nil + if receipts == nil { + r := &ReceiptsRequest{Hash: hash, Number: number} + if err := odr.Retrieve(ctx, r); err != nil { + return nil, err + } + receipts = r.Receipts } - r := &ReceiptsRequest{Hash: hash, Number: number} - if err := odr.Retrieve(ctx, r); err != nil { - return nil, err + // If the receipts are incomplete, fill the derived fields + if len(receipts) > 0 && receipts[0].TxHash == (common.Hash{}) { + block, err := GetBlock(ctx, odr, hash, number) + if err != nil { + return nil, err + } + genesis := core.GetCanonicalHash(odr.Database(), 0) + config, _ := core.GetChainConfig(odr.Database(), genesis) + + core.SetReceiptsData(config, block, receipts) + core.WriteBlockReceipts(odr.Database(), hash, number, receipts) + } + return receipts, nil +} + +// GetBlockLogs retrieves the logs generated by the transactions included in a +// block given by its hash. +func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) ([][]*types.Log, error) { + // Retrieve the potentially incomplete receipts from disk or network + receipts := core.GetBlockReceipts(odr.Database(), hash, number) + if receipts == nil { + r := &ReceiptsRequest{Hash: hash, Number: number} + if err := odr.Retrieve(ctx, r); err != nil { + return nil, err + } + receipts = r.Receipts + } + // Return the logs without deriving any computed fields on the receipts + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs } - return r.Receipts, nil + return logs, nil } // GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to the given bit index and section indexes From 8522b312210865022618489a46db87df44923d4a Mon Sep 17 00:00:00 2001 From: Ivan Daniluk
Date: Thu, 22 Feb 2018 12:20:28 -0500 Subject: [PATCH 43/94] p2p: remove unused code (#16158) * p2p: remove unused code * p2p: remove unused imports --- p2p/message.go | 26 -------------------------- p2p/server.go | 4 +--- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/p2p/message.go b/p2p/message.go index 5690494bf4..50b419970d 100644 --- a/p2p/message.go +++ b/p2p/message.go @@ -22,8 +22,6 @@ import ( "fmt" "io" "io/ioutil" - "net" - "sync" "sync/atomic" "time" @@ -112,30 +110,6 @@ func SendItems(w MsgWriter, msgcode uint64, elems ...interface{}) error { return Send(w, msgcode, elems) } -// netWrapper wraps a MsgReadWriter with locks around -// ReadMsg/WriteMsg and applies read/write deadlines. -type netWrapper struct { - rmu, wmu sync.Mutex - - rtimeout, wtimeout time.Duration - conn net.Conn - wrapped MsgReadWriter -} - -func (rw *netWrapper) ReadMsg() (Msg, error) { - rw.rmu.Lock() - defer rw.rmu.Unlock() - rw.conn.SetReadDeadline(time.Now().Add(rw.rtimeout)) - return rw.wrapped.ReadMsg() -} - -func (rw *netWrapper) WriteMsg(msg Msg) error { - rw.wmu.Lock() - defer rw.wmu.Unlock() - rw.conn.SetWriteDeadline(time.Now().Add(rw.wtimeout)) - return rw.wrapped.WriteMsg(msg) -} - // eofSignal wraps a reader with eof signaling. the eof channel is // closed when the wrapped reader returns an error or when count bytes // have been read. diff --git a/p2p/server.go b/p2p/server.go index 90e92dc05b..c41d1dc156 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -36,9 +36,7 @@ import ( ) const ( - defaultDialTimeout = 15 * time.Second - refreshPeersInterval = 30 * time.Second - staticPeerCheckInterval = 15 * time.Second + defaultDialTimeout = 15 * time.Second // Connectivity defaults. maxActiveDialTasks = 16 From ae9f97221a96a86e4343a5c3cc4b1db44627a2f3 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov
Date: Fri, 23 Feb 2018 10:56:08 +0100 Subject: [PATCH 44/94] metrics: pull library and introduce ResettingTimer and InfluxDB reporter (#15910) * go-metrics: fork library and introduce ResettingTimer and InfluxDB reporter. * vendor: change nonsense/go-metrics to ethersphere/go-metrics * go-metrics: add tests. move ResettingTimer logic from reporter to type. * all, metrics: pull in metrics package in go-ethereum * metrics/test: make sure metrics are enabled for tests * metrics: apply gosimple rules * metrics/exp, internal/debug: init expvar endpoint when starting pprof server * internal/debug: tiny comment formatting fix --- consensus/ethash/ethash.go | 2 +- core/blockchain.go | 2 +- core/database_util.go | 4 +- core/tx_pool.go | 20 +- dashboard/dashboard.go | 2 +- eth/downloader/downloader.go | 2 +- eth/downloader/metrics.go | 28 +- eth/downloader/queue.go | 2 +- eth/fetcher/metrics.go | 28 +- eth/metrics.go | 64 +- ethdb/database.go | 38 +- internal/debug/flags.go | 6 + les/metrics.go | 8 +- metrics/FORK.md | 1 + .../rcrowley/go-metrics => metrics}/LICENSE | 0 .../rcrowley/go-metrics => metrics}/README.md | 37 +- .../go-metrics => metrics}/counter.go | 2 +- metrics/counter_test.go | 77 + .../rcrowley/go-metrics => metrics}/debug.go | 6 +- metrics/debug_test.go | 48 + .../rcrowley/go-metrics => metrics}/ewma.go | 2 +- metrics/ewma_test.go | 225 ++ .../go-metrics => metrics}/exp/exp.go | 46 +- .../rcrowley/go-metrics => metrics}/gauge.go | 4 +- .../go-metrics => metrics}/gauge_float64.go | 4 +- metrics/gauge_float64_test.go | 59 + metrics/gauge_test.go | 68 + .../go-metrics => metrics}/graphite.go | 2 +- metrics/graphite_test.go | 22 + .../go-metrics => metrics}/healthcheck.go | 2 +- .../go-metrics => metrics}/histogram.go | 2 +- metrics/histogram_test.go | 95 + metrics/influxdb/LICENSE | 19 + metrics/influxdb/README.md | 30 + metrics/influxdb/influxdb.go | 227 ++ metrics/init_test.go | 5 + metrics/json.go | 31 + metrics/json_test.go | 28 + metrics/librato/client.go | 102 + metrics/librato/librato.go | 235 ++ .../rcrowley/go-metrics => metrics}/log.go | 2 +- .../rcrowley/go-metrics => metrics}/memory.md | 0 .../rcrowley/go-metrics => metrics}/meter.go | 50 +- metrics/meter_test.go | 73 + metrics/metrics.go | 77 +- metrics/metrics_test.go | 124 + .../go-metrics => metrics}/opentsdb.go | 2 +- metrics/opentsdb_test.go | 21 + .../go-metrics => metrics}/registry.go | 88 +- metrics/registry_test.go | 305 +++ metrics/resetting_timer.go | 237 ++ metrics/resetting_timer_test.go | 106 + .../go-metrics => metrics}/runtime.go | 2 +- .../go-metrics => metrics}/runtime_cgo.go | 0 .../runtime_gccpufraction.go | 0 .../go-metrics => metrics}/runtime_no_cgo.go | 0 .../runtime_no_gccpufraction.go | 0 metrics/runtime_test.go | 88 + .../rcrowley/go-metrics => metrics}/sample.go | 4 +- metrics/sample_test.go | 363 +++ .../rcrowley/go-metrics => metrics}/syslog.go | 2 +- .../rcrowley/go-metrics => metrics}/timer.go | 22 +- metrics/timer_test.go | 101 + .../go-metrics => metrics}/validate.sh | 2 +- .../rcrowley/go-metrics => metrics}/writer.go | 2 +- metrics/writer_test.go | 22 + node/api.go | 2 +- p2p/metrics.go | 8 +- trie/trie.go | 2 +- vendor/github.com/influxdata/influxdb/LICENSE | 20 + .../influxdb/LICENSE_OF_DEPENDENCIES.md | 62 + .../influxdata/influxdb/client/README.md | 306 +++ .../influxdata/influxdb/client/influxdb.go | 840 ++++++ .../influxdata/influxdb/client/v2/client.go | 635 +++++ .../influxdata/influxdb/client/v2/udp.go | 112 + .../influxdata/influxdb/models/consistency.go | 48 + .../influxdata/influxdb/models/inline_fnv.go | 32 + .../influxdb/models/inline_strconv_parse.go | 44 + .../influxdata/influxdb/models/points.go | 2337 +++++++++++++++++ .../influxdata/influxdb/models/rows.go | 62 + .../influxdata/influxdb/models/statistic.go | 42 + .../influxdata/influxdb/models/time.go | 74 + .../influxdb/models/uint_support.go | 7 + .../influxdata/influxdb/pkg/escape/bytes.go | 115 + .../influxdata/influxdb/pkg/escape/strings.go | 21 + vendor/github.com/rcrowley/go-metrics/json.go | 87 - .../github.com/rcrowley/go-metrics/metrics.go | 13 - vendor/vendor.json | 36 +- 88 files changed, 7837 insertions(+), 346 deletions(-) create mode 100644 metrics/FORK.md rename {vendor/github.com/rcrowley/go-metrics => metrics}/LICENSE (100%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/README.md (81%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/counter.go (99%) create mode 100644 metrics/counter_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/debug.go (94%) create mode 100644 metrics/debug_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/ewma.go (99%) create mode 100644 metrics/ewma_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/exp/exp.go (73%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/gauge.go (98%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/gauge_float64.go (98%) create mode 100644 metrics/gauge_float64_test.go create mode 100644 metrics/gauge_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/graphite.go (99%) create mode 100644 metrics/graphite_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/healthcheck.go (98%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/histogram.go (99%) create mode 100644 metrics/histogram_test.go create mode 100644 metrics/influxdb/LICENSE create mode 100644 metrics/influxdb/README.md create mode 100644 metrics/influxdb/influxdb.go create mode 100644 metrics/init_test.go create mode 100644 metrics/json.go create mode 100644 metrics/json_test.go create mode 100644 metrics/librato/client.go create mode 100644 metrics/librato/librato.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/log.go (98%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/memory.md (100%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/meter.go (81%) create mode 100644 metrics/meter_test.go create mode 100644 metrics/metrics_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/opentsdb.go (99%) create mode 100644 metrics/opentsdb_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/registry.go (76%) create mode 100644 metrics/registry_test.go create mode 100644 metrics/resetting_timer.go create mode 100644 metrics/resetting_timer_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/runtime.go (99%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/runtime_cgo.go (100%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/runtime_gccpufraction.go (100%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/runtime_no_cgo.go (100%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/runtime_no_gccpufraction.go (100%) create mode 100644 metrics/runtime_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/sample.go (99%) create mode 100644 metrics/sample_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/syslog.go (98%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/timer.go (93%) create mode 100644 metrics/timer_test.go rename {vendor/github.com/rcrowley/go-metrics => metrics}/validate.sh (93%) rename {vendor/github.com/rcrowley/go-metrics => metrics}/writer.go (99%) create mode 100644 metrics/writer_test.go create mode 100644 vendor/github.com/influxdata/influxdb/LICENSE create mode 100644 vendor/github.com/influxdata/influxdb/LICENSE_OF_DEPENDENCIES.md create mode 100644 vendor/github.com/influxdata/influxdb/client/README.md create mode 100644 vendor/github.com/influxdata/influxdb/client/influxdb.go create mode 100644 vendor/github.com/influxdata/influxdb/client/v2/client.go create mode 100644 vendor/github.com/influxdata/influxdb/client/v2/udp.go create mode 100644 vendor/github.com/influxdata/influxdb/models/consistency.go create mode 100644 vendor/github.com/influxdata/influxdb/models/inline_fnv.go create mode 100644 vendor/github.com/influxdata/influxdb/models/inline_strconv_parse.go create mode 100644 vendor/github.com/influxdata/influxdb/models/points.go create mode 100644 vendor/github.com/influxdata/influxdb/models/rows.go create mode 100644 vendor/github.com/influxdata/influxdb/models/statistic.go create mode 100644 vendor/github.com/influxdata/influxdb/models/time.go create mode 100644 vendor/github.com/influxdata/influxdb/models/uint_support.go create mode 100644 vendor/github.com/influxdata/influxdb/pkg/escape/bytes.go create mode 100644 vendor/github.com/influxdata/influxdb/pkg/escape/strings.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/json.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/metrics.go diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 91e20112ae..1b3dcee302 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -35,9 +35,9 @@ import ( mmap "github.com/edsrzf/mmap-go" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rpc" "github.com/hashicorp/golang-lru/simplelru" - metrics "github.com/rcrowley/go-metrics" ) var ErrInvalidDumpMagic = errors.New("invalid dump magic") diff --git a/core/blockchain.go b/core/blockchain.go index 4ae0e4f4ec..644df123c5 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -46,7 +46,7 @@ import ( ) var ( - blockInsertTimer = metrics.NewTimer("chain/inserts") + blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) ErrNoGenesis = errors.New("Genesis not found in chain") ) diff --git a/core/database_util.go b/core/database_util.go index c6b125dae9..61ab701349 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -70,8 +70,8 @@ var ( ErrChainConfigNotFound = errors.New("ChainConfig not found") // general config not found error - preimageCounter = metrics.NewCounter("db/preimage/total") - preimageHitCounter = metrics.NewCounter("db/preimage/hits") + preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) + preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) ) // TxLookupEntry is a positional metadata to help looking up the data content of diff --git a/core/tx_pool.go b/core/tx_pool.go index dc3ddc4232..0534fe57a1 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -87,20 +87,20 @@ var ( var ( // Metrics for the pending pool - pendingDiscardCounter = metrics.NewCounter("txpool/pending/discard") - pendingReplaceCounter = metrics.NewCounter("txpool/pending/replace") - pendingRateLimitCounter = metrics.NewCounter("txpool/pending/ratelimit") // Dropped due to rate limiting - pendingNofundsCounter = metrics.NewCounter("txpool/pending/nofunds") // Dropped due to out-of-funds + pendingDiscardCounter = metrics.NewRegisteredCounter("txpool/pending/discard", nil) + pendingReplaceCounter = metrics.NewRegisteredCounter("txpool/pending/replace", nil) + pendingRateLimitCounter = metrics.NewRegisteredCounter("txpool/pending/ratelimit", nil) // Dropped due to rate limiting + pendingNofundsCounter = metrics.NewRegisteredCounter("txpool/pending/nofunds", nil) // Dropped due to out-of-funds // Metrics for the queued pool - queuedDiscardCounter = metrics.NewCounter("txpool/queued/discard") - queuedReplaceCounter = metrics.NewCounter("txpool/queued/replace") - queuedRateLimitCounter = metrics.NewCounter("txpool/queued/ratelimit") // Dropped due to rate limiting - queuedNofundsCounter = metrics.NewCounter("txpool/queued/nofunds") // Dropped due to out-of-funds + queuedDiscardCounter = metrics.NewRegisteredCounter("txpool/queued/discard", nil) + queuedReplaceCounter = metrics.NewRegisteredCounter("txpool/queued/replace", nil) + queuedRateLimitCounter = metrics.NewRegisteredCounter("txpool/queued/ratelimit", nil) // Dropped due to rate limiting + queuedNofundsCounter = metrics.NewRegisteredCounter("txpool/queued/nofunds", nil) // Dropped due to out-of-funds // General tx metrics - invalidTxCounter = metrics.NewCounter("txpool/invalid") - underpricedTxCounter = metrics.NewCounter("txpool/underpriced") + invalidTxCounter = metrics.NewRegisteredCounter("txpool/invalid", nil) + underpricedTxCounter = metrics.NewRegisteredCounter("txpool/underpriced", nil) ) // TxStatus is the current status of a transaction as seen by the pool. diff --git a/dashboard/dashboard.go b/dashboard/dashboard.go index 09038638ed..2ca795187f 100644 --- a/dashboard/dashboard.go +++ b/dashboard/dashboard.go @@ -36,10 +36,10 @@ import ( "github.com/elastic/gosigar" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "github.com/rcrowley/go-metrics" "golang.org/x/net/websocket" ) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 7ede530a94..d13247766a 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -31,8 +31,8 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" - "github.com/rcrowley/go-metrics" ) var ( diff --git a/eth/downloader/metrics.go b/eth/downloader/metrics.go index 58764ccf06..d4eb337946 100644 --- a/eth/downloader/metrics.go +++ b/eth/downloader/metrics.go @@ -23,21 +23,21 @@ import ( ) var ( - headerInMeter = metrics.NewMeter("eth/downloader/headers/in") - headerReqTimer = metrics.NewTimer("eth/downloader/headers/req") - headerDropMeter = metrics.NewMeter("eth/downloader/headers/drop") - headerTimeoutMeter = metrics.NewMeter("eth/downloader/headers/timeout") + headerInMeter = metrics.NewRegisteredMeter("eth/downloader/headers/in", nil) + headerReqTimer = metrics.NewRegisteredTimer("eth/downloader/headers/req", nil) + headerDropMeter = metrics.NewRegisteredMeter("eth/downloader/headers/drop", nil) + headerTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/headers/timeout", nil) - bodyInMeter = metrics.NewMeter("eth/downloader/bodies/in") - bodyReqTimer = metrics.NewTimer("eth/downloader/bodies/req") - bodyDropMeter = metrics.NewMeter("eth/downloader/bodies/drop") - bodyTimeoutMeter = metrics.NewMeter("eth/downloader/bodies/timeout") + bodyInMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/in", nil) + bodyReqTimer = metrics.NewRegisteredTimer("eth/downloader/bodies/req", nil) + bodyDropMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/drop", nil) + bodyTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/timeout", nil) - receiptInMeter = metrics.NewMeter("eth/downloader/receipts/in") - receiptReqTimer = metrics.NewTimer("eth/downloader/receipts/req") - receiptDropMeter = metrics.NewMeter("eth/downloader/receipts/drop") - receiptTimeoutMeter = metrics.NewMeter("eth/downloader/receipts/timeout") + receiptInMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/in", nil) + receiptReqTimer = metrics.NewRegisteredTimer("eth/downloader/receipts/req", nil) + receiptDropMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/drop", nil) + receiptTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/timeout", nil) - stateInMeter = metrics.NewMeter("eth/downloader/states/in") - stateDropMeter = metrics.NewMeter("eth/downloader/states/drop") + stateInMeter = metrics.NewRegisteredMeter("eth/downloader/states/in", nil) + stateDropMeter = metrics.NewRegisteredMeter("eth/downloader/states/drop", nil) ) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index a1a70e46ea..359cce54b5 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -28,7 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" - "github.com/rcrowley/go-metrics" + "github.com/ethereum/go-ethereum/metrics" "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go index 1ed8075bf3..d68d12f000 100644 --- a/eth/fetcher/metrics.go +++ b/eth/fetcher/metrics.go @@ -23,21 +23,21 @@ import ( ) var ( - propAnnounceInMeter = metrics.NewMeter("eth/fetcher/prop/announces/in") - propAnnounceOutTimer = metrics.NewTimer("eth/fetcher/prop/announces/out") - propAnnounceDropMeter = metrics.NewMeter("eth/fetcher/prop/announces/drop") - propAnnounceDOSMeter = metrics.NewMeter("eth/fetcher/prop/announces/dos") + propAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/in", nil) + propAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/announces/out", nil) + propAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/drop", nil) + propAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/dos", nil) - propBroadcastInMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/in") - propBroadcastOutTimer = metrics.NewTimer("eth/fetcher/prop/broadcasts/out") - propBroadcastDropMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/drop") - propBroadcastDOSMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/dos") + propBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/in", nil) + propBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/broadcasts/out", nil) + propBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/drop", nil) + propBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/dos", nil) - headerFetchMeter = metrics.NewMeter("eth/fetcher/fetch/headers") - bodyFetchMeter = metrics.NewMeter("eth/fetcher/fetch/bodies") + headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/headers", nil) + bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/bodies", nil) - headerFilterInMeter = metrics.NewMeter("eth/fetcher/filter/headers/in") - headerFilterOutMeter = metrics.NewMeter("eth/fetcher/filter/headers/out") - bodyFilterInMeter = metrics.NewMeter("eth/fetcher/filter/bodies/in") - bodyFilterOutMeter = metrics.NewMeter("eth/fetcher/filter/bodies/out") + headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/in", nil) + headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/out", nil) + bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/in", nil) + bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/out", nil) ) diff --git a/eth/metrics.go b/eth/metrics.go index 5fa2597d4d..0533a2a875 100644 --- a/eth/metrics.go +++ b/eth/metrics.go @@ -22,38 +22,38 @@ import ( ) var ( - propTxnInPacketsMeter = metrics.NewMeter("eth/prop/txns/in/packets") - propTxnInTrafficMeter = metrics.NewMeter("eth/prop/txns/in/traffic") - propTxnOutPacketsMeter = metrics.NewMeter("eth/prop/txns/out/packets") - propTxnOutTrafficMeter = metrics.NewMeter("eth/prop/txns/out/traffic") - propHashInPacketsMeter = metrics.NewMeter("eth/prop/hashes/in/packets") - propHashInTrafficMeter = metrics.NewMeter("eth/prop/hashes/in/traffic") - propHashOutPacketsMeter = metrics.NewMeter("eth/prop/hashes/out/packets") - propHashOutTrafficMeter = metrics.NewMeter("eth/prop/hashes/out/traffic") - propBlockInPacketsMeter = metrics.NewMeter("eth/prop/blocks/in/packets") - propBlockInTrafficMeter = metrics.NewMeter("eth/prop/blocks/in/traffic") - propBlockOutPacketsMeter = metrics.NewMeter("eth/prop/blocks/out/packets") - propBlockOutTrafficMeter = metrics.NewMeter("eth/prop/blocks/out/traffic") - reqHeaderInPacketsMeter = metrics.NewMeter("eth/req/headers/in/packets") - reqHeaderInTrafficMeter = metrics.NewMeter("eth/req/headers/in/traffic") - reqHeaderOutPacketsMeter = metrics.NewMeter("eth/req/headers/out/packets") - reqHeaderOutTrafficMeter = metrics.NewMeter("eth/req/headers/out/traffic") - reqBodyInPacketsMeter = metrics.NewMeter("eth/req/bodies/in/packets") - reqBodyInTrafficMeter = metrics.NewMeter("eth/req/bodies/in/traffic") - reqBodyOutPacketsMeter = metrics.NewMeter("eth/req/bodies/out/packets") - reqBodyOutTrafficMeter = metrics.NewMeter("eth/req/bodies/out/traffic") - reqStateInPacketsMeter = metrics.NewMeter("eth/req/states/in/packets") - reqStateInTrafficMeter = metrics.NewMeter("eth/req/states/in/traffic") - reqStateOutPacketsMeter = metrics.NewMeter("eth/req/states/out/packets") - reqStateOutTrafficMeter = metrics.NewMeter("eth/req/states/out/traffic") - reqReceiptInPacketsMeter = metrics.NewMeter("eth/req/receipts/in/packets") - reqReceiptInTrafficMeter = metrics.NewMeter("eth/req/receipts/in/traffic") - reqReceiptOutPacketsMeter = metrics.NewMeter("eth/req/receipts/out/packets") - reqReceiptOutTrafficMeter = metrics.NewMeter("eth/req/receipts/out/traffic") - miscInPacketsMeter = metrics.NewMeter("eth/misc/in/packets") - miscInTrafficMeter = metrics.NewMeter("eth/misc/in/traffic") - miscOutPacketsMeter = metrics.NewMeter("eth/misc/out/packets") - miscOutTrafficMeter = metrics.NewMeter("eth/misc/out/traffic") + propTxnInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/txns/in/packets", nil) + propTxnInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/txns/in/traffic", nil) + propTxnOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/txns/out/packets", nil) + propTxnOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/txns/out/traffic", nil) + propHashInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/hashes/in/packets", nil) + propHashInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/hashes/in/traffic", nil) + propHashOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/hashes/out/packets", nil) + propHashOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/hashes/out/traffic", nil) + propBlockInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/blocks/in/packets", nil) + propBlockInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/blocks/in/traffic", nil) + propBlockOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/blocks/out/packets", nil) + propBlockOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/blocks/out/traffic", nil) + reqHeaderInPacketsMeter = metrics.NewRegisteredMeter("eth/req/headers/in/packets", nil) + reqHeaderInTrafficMeter = metrics.NewRegisteredMeter("eth/req/headers/in/traffic", nil) + reqHeaderOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/headers/out/packets", nil) + reqHeaderOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/headers/out/traffic", nil) + reqBodyInPacketsMeter = metrics.NewRegisteredMeter("eth/req/bodies/in/packets", nil) + reqBodyInTrafficMeter = metrics.NewRegisteredMeter("eth/req/bodies/in/traffic", nil) + reqBodyOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/bodies/out/packets", nil) + reqBodyOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/bodies/out/traffic", nil) + reqStateInPacketsMeter = metrics.NewRegisteredMeter("eth/req/states/in/packets", nil) + reqStateInTrafficMeter = metrics.NewRegisteredMeter("eth/req/states/in/traffic", nil) + reqStateOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/states/out/packets", nil) + reqStateOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/states/out/traffic", nil) + reqReceiptInPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/in/packets", nil) + reqReceiptInTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/in/traffic", nil) + reqReceiptOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/packets", nil) + reqReceiptOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/traffic", nil) + miscInPacketsMeter = metrics.NewRegisteredMeter("eth/misc/in/packets", nil) + miscInTrafficMeter = metrics.NewRegisteredMeter("eth/misc/in/traffic", nil) + miscOutPacketsMeter = metrics.NewRegisteredMeter("eth/misc/out/packets", nil) + miscOutTrafficMeter = metrics.NewRegisteredMeter("eth/misc/out/traffic", nil) ) // meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of diff --git a/ethdb/database.go b/ethdb/database.go index d86585f079..57d38f7f5f 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -29,8 +29,6 @@ import ( "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" - - gometrics "github.com/rcrowley/go-metrics" ) var OpenFileLimit = 64 @@ -39,15 +37,15 @@ type LDBDatabase struct { fn string // filename for reporting db *leveldb.DB // LevelDB instance - getTimer gometrics.Timer // Timer for measuring the database get request counts and latencies - putTimer gometrics.Timer // Timer for measuring the database put request counts and latencies - delTimer gometrics.Timer // Timer for measuring the database delete request counts and latencies - missMeter gometrics.Meter // Meter for measuring the missed database get requests - readMeter gometrics.Meter // Meter for measuring the database get request data usage - writeMeter gometrics.Meter // Meter for measuring the database put request data usage - compTimeMeter gometrics.Meter // Meter for measuring the total time spent in database compaction - compReadMeter gometrics.Meter // Meter for measuring the data read during compaction - compWriteMeter gometrics.Meter // Meter for measuring the data written during compaction + getTimer metrics.Timer // Timer for measuring the database get request counts and latencies + putTimer metrics.Timer // Timer for measuring the database put request counts and latencies + delTimer metrics.Timer // Timer for measuring the database delete request counts and latencies + missMeter metrics.Meter // Meter for measuring the missed database get requests + readMeter metrics.Meter // Meter for measuring the database get request data usage + writeMeter metrics.Meter // Meter for measuring the database put request data usage + compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction + compReadMeter metrics.Meter // Meter for measuring the data read during compaction + compWriteMeter metrics.Meter // Meter for measuring the data written during compaction quitLock sync.Mutex // Mutex protecting the quit channel access quitChan chan chan error // Quit channel to stop the metrics collection before closing the database @@ -180,15 +178,15 @@ func (db *LDBDatabase) Meter(prefix string) { return } // Initialize all the metrics collector at the requested prefix - db.getTimer = metrics.NewTimer(prefix + "user/gets") - db.putTimer = metrics.NewTimer(prefix + "user/puts") - db.delTimer = metrics.NewTimer(prefix + "user/dels") - db.missMeter = metrics.NewMeter(prefix + "user/misses") - db.readMeter = metrics.NewMeter(prefix + "user/reads") - db.writeMeter = metrics.NewMeter(prefix + "user/writes") - db.compTimeMeter = metrics.NewMeter(prefix + "compact/time") - db.compReadMeter = metrics.NewMeter(prefix + "compact/input") - db.compWriteMeter = metrics.NewMeter(prefix + "compact/output") + db.getTimer = metrics.NewRegisteredTimer(prefix+"user/gets", nil) + db.putTimer = metrics.NewRegisteredTimer(prefix+"user/puts", nil) + db.delTimer = metrics.NewRegisteredTimer(prefix+"user/dels", nil) + db.missMeter = metrics.NewRegisteredMeter(prefix+"user/misses", nil) + db.readMeter = metrics.NewRegisteredMeter(prefix+"user/reads", nil) + db.writeMeter = metrics.NewRegisteredMeter(prefix+"user/writes", nil) + db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil) + db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil) + db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil) // Create a quit channel for the periodic collector and run it db.quitLock.Lock() diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 6247cc7dc0..1f181bf8b0 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -26,6 +26,8 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log/term" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/metrics/exp" colorable "github.com/mattn/go-colorable" "gopkg.in/urfave/cli.v1" ) @@ -127,6 +129,10 @@ func Setup(ctx *cli.Context) error { // pprof server if ctx.GlobalBool(pprofFlag.Name) { + // Hook go-metrics into expvar on any /debug/metrics request, load all vars + // from the registry into expvar, and execute regular expvar handler. + exp.Exp(metrics.DefaultRegistry) + address := fmt.Sprintf("%s:%d", ctx.GlobalString(pprofAddrFlag.Name), ctx.GlobalInt(pprofPortFlag.Name)) go func() { log.Info("Starting pprof server", "addr", fmt.Sprintf("http://%s/debug/pprof", address)) diff --git a/les/metrics.go b/les/metrics.go index 0162a1d1ad..c282a62a1a 100644 --- a/les/metrics.go +++ b/les/metrics.go @@ -58,10 +58,10 @@ var ( reqReceiptInTrafficMeter = metrics.NewMeter("eth/req/receipts/in/traffic") reqReceiptOutPacketsMeter = metrics.NewMeter("eth/req/receipts/out/packets") reqReceiptOutTrafficMeter = metrics.NewMeter("eth/req/receipts/out/traffic")*/ - miscInPacketsMeter = metrics.NewMeter("les/misc/in/packets") - miscInTrafficMeter = metrics.NewMeter("les/misc/in/traffic") - miscOutPacketsMeter = metrics.NewMeter("les/misc/out/packets") - miscOutTrafficMeter = metrics.NewMeter("les/misc/out/traffic") + miscInPacketsMeter = metrics.NewRegisteredMeter("les/misc/in/packets", nil) + miscInTrafficMeter = metrics.NewRegisteredMeter("les/misc/in/traffic", nil) + miscOutPacketsMeter = metrics.NewRegisteredMeter("les/misc/out/packets", nil) + miscOutTrafficMeter = metrics.NewRegisteredMeter("les/misc/out/traffic", nil) ) // meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of diff --git a/metrics/FORK.md b/metrics/FORK.md new file mode 100644 index 0000000000..b19985bf56 --- /dev/null +++ b/metrics/FORK.md @@ -0,0 +1 @@ +This repo has been forked from https://github.com/rcrowley/go-metrics at commit e181e09 diff --git a/vendor/github.com/rcrowley/go-metrics/LICENSE b/metrics/LICENSE similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/LICENSE rename to metrics/LICENSE diff --git a/vendor/github.com/rcrowley/go-metrics/README.md b/metrics/README.md similarity index 81% rename from vendor/github.com/rcrowley/go-metrics/README.md rename to metrics/README.md index 2d1a6dcfa4..bc2a45a838 100644 --- a/vendor/github.com/rcrowley/go-metrics/README.md +++ b/metrics/README.md @@ -42,12 +42,22 @@ t.Update(47) Register() is not threadsafe. For threadsafe metric registration use GetOrRegister: -``` +```go t := metrics.GetOrRegisterTimer("account.create.latency", nil) t.Time(func() {}) t.Update(47) ``` +**NOTE:** Be sure to unregister short-lived meters and timers otherwise they will +leak memory: + +```go +// Will call Stop() on the Meter to allow for garbage collection +metrics.Unregister("quux") +// Or similarly for a Timer that embeds a Meter +metrics.Unregister("bang") +``` + Periodically log every metric in human-readable form to standard error: ```go @@ -81,12 +91,13 @@ issues [#121](https://github.com/rcrowley/go-metrics/issues/121) and ```go import "github.com/vrischmann/go-metrics-influxdb" -go influxdb.Influxdb(metrics.DefaultRegistry, 10e9, &influxdb.Config{ - Host: "127.0.0.1:8086", - Database: "metrics", - Username: "test", - Password: "test", -}) +go influxdb.InfluxDB(metrics.DefaultRegistry, + 10e9, + "127.0.0.1:8086", + "database-name", + "username", + "password" +) ``` Periodically upload every metric to Librato using the [Librato client](https://github.com/mihasya/go-metrics-librato): @@ -146,8 +157,10 @@ Publishing Metrics Clients are available for the following destinations: -* Librato - [https://github.com/mihasya/go-metrics-librato](https://github.com/mihasya/go-metrics-librato) -* Graphite - [https://github.com/cyberdelia/go-metrics-graphite](https://github.com/cyberdelia/go-metrics-graphite) -* InfluxDB - [https://github.com/vrischmann/go-metrics-influxdb](https://github.com/vrischmann/go-metrics-influxdb) -* Ganglia - [https://github.com/appscode/metlia](https://github.com/appscode/metlia) -* Prometheus - [https://github.com/deathowl/go-metrics-prometheus](https://github.com/deathowl/go-metrics-prometheus) +* Librato - https://github.com/mihasya/go-metrics-librato +* Graphite - https://github.com/cyberdelia/go-metrics-graphite +* InfluxDB - https://github.com/vrischmann/go-metrics-influxdb +* Ganglia - https://github.com/appscode/metlia +* Prometheus - https://github.com/deathowl/go-metrics-prometheus +* DataDog - https://github.com/syntaqx/go-metrics-datadog +* SignalFX - https://github.com/pascallouisperez/go-metrics-signalfx diff --git a/vendor/github.com/rcrowley/go-metrics/counter.go b/metrics/counter.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/counter.go rename to metrics/counter.go index bb7b039cb5..c7f2b4bd3a 100644 --- a/vendor/github.com/rcrowley/go-metrics/counter.go +++ b/metrics/counter.go @@ -22,7 +22,7 @@ func GetOrRegisterCounter(name string, r Registry) Counter { // NewCounter constructs a new StandardCounter. func NewCounter() Counter { - if UseNilMetrics { + if !Enabled { return NilCounter{} } return &StandardCounter{0} diff --git a/metrics/counter_test.go b/metrics/counter_test.go new file mode 100644 index 0000000000..dfb03b4e88 --- /dev/null +++ b/metrics/counter_test.go @@ -0,0 +1,77 @@ +package metrics + +import "testing" + +func BenchmarkCounter(b *testing.B) { + c := NewCounter() + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Inc(1) + } +} + +func TestCounterClear(t *testing.T) { + c := NewCounter() + c.Inc(1) + c.Clear() + if count := c.Count(); 0 != count { + t.Errorf("c.Count(): 0 != %v\n", count) + } +} + +func TestCounterDec1(t *testing.T) { + c := NewCounter() + c.Dec(1) + if count := c.Count(); -1 != count { + t.Errorf("c.Count(): -1 != %v\n", count) + } +} + +func TestCounterDec2(t *testing.T) { + c := NewCounter() + c.Dec(2) + if count := c.Count(); -2 != count { + t.Errorf("c.Count(): -2 != %v\n", count) + } +} + +func TestCounterInc1(t *testing.T) { + c := NewCounter() + c.Inc(1) + if count := c.Count(); 1 != count { + t.Errorf("c.Count(): 1 != %v\n", count) + } +} + +func TestCounterInc2(t *testing.T) { + c := NewCounter() + c.Inc(2) + if count := c.Count(); 2 != count { + t.Errorf("c.Count(): 2 != %v\n", count) + } +} + +func TestCounterSnapshot(t *testing.T) { + c := NewCounter() + c.Inc(1) + snapshot := c.Snapshot() + c.Inc(1) + if count := snapshot.Count(); 1 != count { + t.Errorf("c.Count(): 1 != %v\n", count) + } +} + +func TestCounterZero(t *testing.T) { + c := NewCounter() + if count := c.Count(); 0 != count { + t.Errorf("c.Count(): 0 != %v\n", count) + } +} + +func TestGetOrRegisterCounter(t *testing.T) { + r := NewRegistry() + NewRegisteredCounter("foo", r).Inc(47) + if c := GetOrRegisterCounter("foo", r); 47 != c.Count() { + t.Fatal(c) + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/debug.go b/metrics/debug.go similarity index 94% rename from vendor/github.com/rcrowley/go-metrics/debug.go rename to metrics/debug.go index 043ccefab6..de4a2739fe 100644 --- a/vendor/github.com/rcrowley/go-metrics/debug.go +++ b/metrics/debug.go @@ -22,7 +22,7 @@ var ( // Capture new values for the Go garbage collector statistics exported in // debug.GCStats. This is designed to be called as a goroutine. func CaptureDebugGCStats(r Registry, d time.Duration) { - for _ = range time.Tick(d) { + for range time.Tick(d) { CaptureDebugGCStatsOnce(r) } } @@ -41,8 +41,8 @@ func CaptureDebugGCStatsOnce(r Registry) { debug.ReadGCStats(&gcStats) debugMetrics.ReadGCStats.UpdateSince(t) - debugMetrics.GCStats.LastGC.Update(int64(gcStats.LastGC.UnixNano())) - debugMetrics.GCStats.NumGC.Update(int64(gcStats.NumGC)) + debugMetrics.GCStats.LastGC.Update(gcStats.LastGC.UnixNano()) + debugMetrics.GCStats.NumGC.Update(gcStats.NumGC) if lastGC != gcStats.LastGC && 0 < len(gcStats.Pause) { debugMetrics.GCStats.Pause.Update(int64(gcStats.Pause[0])) } diff --git a/metrics/debug_test.go b/metrics/debug_test.go new file mode 100644 index 0000000000..07eb867841 --- /dev/null +++ b/metrics/debug_test.go @@ -0,0 +1,48 @@ +package metrics + +import ( + "runtime" + "runtime/debug" + "testing" + "time" +) + +func BenchmarkDebugGCStats(b *testing.B) { + r := NewRegistry() + RegisterDebugGCStats(r) + b.ResetTimer() + for i := 0; i < b.N; i++ { + CaptureDebugGCStatsOnce(r) + } +} + +func TestDebugGCStatsBlocking(t *testing.T) { + if g := runtime.GOMAXPROCS(0); g < 2 { + t.Skipf("skipping TestDebugGCMemStatsBlocking with GOMAXPROCS=%d\n", g) + return + } + ch := make(chan int) + go testDebugGCStatsBlocking(ch) + var gcStats debug.GCStats + t0 := time.Now() + debug.ReadGCStats(&gcStats) + t1 := time.Now() + t.Log("i++ during debug.ReadGCStats:", <-ch) + go testDebugGCStatsBlocking(ch) + d := t1.Sub(t0) + t.Log(d) + time.Sleep(d) + t.Log("i++ during time.Sleep:", <-ch) +} + +func testDebugGCStatsBlocking(ch chan int) { + i := 0 + for { + select { + case ch <- i: + return + default: + i++ + } + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/ewma.go b/metrics/ewma.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/ewma.go rename to metrics/ewma.go index 694a1d0330..3aecd4fa35 100644 --- a/vendor/github.com/rcrowley/go-metrics/ewma.go +++ b/metrics/ewma.go @@ -17,7 +17,7 @@ type EWMA interface { // NewEWMA constructs a new EWMA with the given alpha. func NewEWMA(alpha float64) EWMA { - if UseNilMetrics { + if !Enabled { return NilEWMA{} } return &StandardEWMA{alpha: alpha} diff --git a/metrics/ewma_test.go b/metrics/ewma_test.go new file mode 100644 index 0000000000..0430fbd247 --- /dev/null +++ b/metrics/ewma_test.go @@ -0,0 +1,225 @@ +package metrics + +import "testing" + +func BenchmarkEWMA(b *testing.B) { + a := NewEWMA1() + b.ResetTimer() + for i := 0; i < b.N; i++ { + a.Update(1) + a.Tick() + } +} + +func TestEWMA1(t *testing.T) { + a := NewEWMA1() + a.Update(3) + a.Tick() + if rate := a.Rate(); 0.6 != rate { + t.Errorf("initial a.Rate(): 0.6 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.22072766470286553 != rate { + t.Errorf("1 minute a.Rate(): 0.22072766470286553 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.08120116994196772 != rate { + t.Errorf("2 minute a.Rate(): 0.08120116994196772 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.029872241020718428 != rate { + t.Errorf("3 minute a.Rate(): 0.029872241020718428 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.01098938333324054 != rate { + t.Errorf("4 minute a.Rate(): 0.01098938333324054 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.004042768199451294 != rate { + t.Errorf("5 minute a.Rate(): 0.004042768199451294 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.0014872513059998212 != rate { + t.Errorf("6 minute a.Rate(): 0.0014872513059998212 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.0005471291793327122 != rate { + t.Errorf("7 minute a.Rate(): 0.0005471291793327122 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.00020127757674150815 != rate { + t.Errorf("8 minute a.Rate(): 0.00020127757674150815 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 7.404588245200814e-05 != rate { + t.Errorf("9 minute a.Rate(): 7.404588245200814e-05 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 2.7239957857491083e-05 != rate { + t.Errorf("10 minute a.Rate(): 2.7239957857491083e-05 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 1.0021020474147462e-05 != rate { + t.Errorf("11 minute a.Rate(): 1.0021020474147462e-05 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 3.6865274119969525e-06 != rate { + t.Errorf("12 minute a.Rate(): 3.6865274119969525e-06 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 1.3561976441886433e-06 != rate { + t.Errorf("13 minute a.Rate(): 1.3561976441886433e-06 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 4.989172314621449e-07 != rate { + t.Errorf("14 minute a.Rate(): 4.989172314621449e-07 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 1.8354139230109722e-07 != rate { + t.Errorf("15 minute a.Rate(): 1.8354139230109722e-07 != %v\n", rate) + } +} + +func TestEWMA5(t *testing.T) { + a := NewEWMA5() + a.Update(3) + a.Tick() + if rate := a.Rate(); 0.6 != rate { + t.Errorf("initial a.Rate(): 0.6 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.49123845184678905 != rate { + t.Errorf("1 minute a.Rate(): 0.49123845184678905 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.4021920276213837 != rate { + t.Errorf("2 minute a.Rate(): 0.4021920276213837 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.32928698165641596 != rate { + t.Errorf("3 minute a.Rate(): 0.32928698165641596 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.269597378470333 != rate { + t.Errorf("4 minute a.Rate(): 0.269597378470333 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.2207276647028654 != rate { + t.Errorf("5 minute a.Rate(): 0.2207276647028654 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.18071652714732128 != rate { + t.Errorf("6 minute a.Rate(): 0.18071652714732128 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.14795817836496392 != rate { + t.Errorf("7 minute a.Rate(): 0.14795817836496392 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.12113791079679326 != rate { + t.Errorf("8 minute a.Rate(): 0.12113791079679326 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.09917933293295193 != rate { + t.Errorf("9 minute a.Rate(): 0.09917933293295193 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.08120116994196763 != rate { + t.Errorf("10 minute a.Rate(): 0.08120116994196763 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.06648189501740036 != rate { + t.Errorf("11 minute a.Rate(): 0.06648189501740036 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.05443077197364752 != rate { + t.Errorf("12 minute a.Rate(): 0.05443077197364752 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.04456414692860035 != rate { + t.Errorf("13 minute a.Rate(): 0.04456414692860035 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.03648603757513079 != rate { + t.Errorf("14 minute a.Rate(): 0.03648603757513079 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.0298722410207183831020718428 != rate { + t.Errorf("15 minute a.Rate(): 0.0298722410207183831020718428 != %v\n", rate) + } +} + +func TestEWMA15(t *testing.T) { + a := NewEWMA15() + a.Update(3) + a.Tick() + if rate := a.Rate(); 0.6 != rate { + t.Errorf("initial a.Rate(): 0.6 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.5613041910189706 != rate { + t.Errorf("1 minute a.Rate(): 0.5613041910189706 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.5251039914257684 != rate { + t.Errorf("2 minute a.Rate(): 0.5251039914257684 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.4912384518467888184678905 != rate { + t.Errorf("3 minute a.Rate(): 0.4912384518467888184678905 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.459557003018789 != rate { + t.Errorf("4 minute a.Rate(): 0.459557003018789 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.4299187863442732 != rate { + t.Errorf("5 minute a.Rate(): 0.4299187863442732 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.4021920276213831 != rate { + t.Errorf("6 minute a.Rate(): 0.4021920276213831 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.37625345116383313 != rate { + t.Errorf("7 minute a.Rate(): 0.37625345116383313 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.3519877317060185 != rate { + t.Errorf("8 minute a.Rate(): 0.3519877317060185 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.3292869816564153165641596 != rate { + t.Errorf("9 minute a.Rate(): 0.3292869816564153165641596 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.3080502714195546 != rate { + t.Errorf("10 minute a.Rate(): 0.3080502714195546 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.2881831806538789 != rate { + t.Errorf("11 minute a.Rate(): 0.2881831806538789 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.26959737847033216 != rate { + t.Errorf("12 minute a.Rate(): 0.26959737847033216 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.2522102307052083 != rate { + t.Errorf("13 minute a.Rate(): 0.2522102307052083 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.23594443252115815 != rate { + t.Errorf("14 minute a.Rate(): 0.23594443252115815 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.2207276647028646247028654470286553 != rate { + t.Errorf("15 minute a.Rate(): 0.2207276647028646247028654470286553 != %v\n", rate) + } +} + +func elapseMinute(a EWMA) { + for i := 0; i < 12; i++ { + a.Tick() + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/exp/exp.go b/metrics/exp/exp.go similarity index 73% rename from vendor/github.com/rcrowley/go-metrics/exp/exp.go rename to metrics/exp/exp.go index 11dd3f898a..c19d00a94d 100644 --- a/vendor/github.com/rcrowley/go-metrics/exp/exp.go +++ b/metrics/exp/exp.go @@ -8,7 +8,7 @@ import ( "net/http" "sync" - "github.com/rcrowley/go-metrics" + "github.com/ethereum/go-ethereum/metrics" ) type exp struct { @@ -97,22 +97,22 @@ func (exp *exp) publishHistogram(name string, metric metrics.Histogram) { exp.getInt(name + ".count").Set(h.Count()) exp.getFloat(name + ".min").Set(float64(h.Min())) exp.getFloat(name + ".max").Set(float64(h.Max())) - exp.getFloat(name + ".mean").Set(float64(h.Mean())) - exp.getFloat(name + ".std-dev").Set(float64(h.StdDev())) - exp.getFloat(name + ".50-percentile").Set(float64(ps[0])) - exp.getFloat(name + ".75-percentile").Set(float64(ps[1])) - exp.getFloat(name + ".95-percentile").Set(float64(ps[2])) - exp.getFloat(name + ".99-percentile").Set(float64(ps[3])) - exp.getFloat(name + ".999-percentile").Set(float64(ps[4])) + exp.getFloat(name + ".mean").Set(h.Mean()) + exp.getFloat(name + ".std-dev").Set(h.StdDev()) + exp.getFloat(name + ".50-percentile").Set(ps[0]) + exp.getFloat(name + ".75-percentile").Set(ps[1]) + exp.getFloat(name + ".95-percentile").Set(ps[2]) + exp.getFloat(name + ".99-percentile").Set(ps[3]) + exp.getFloat(name + ".999-percentile").Set(ps[4]) } func (exp *exp) publishMeter(name string, metric metrics.Meter) { m := metric.Snapshot() exp.getInt(name + ".count").Set(m.Count()) - exp.getFloat(name + ".one-minute").Set(float64(m.Rate1())) - exp.getFloat(name + ".five-minute").Set(float64(m.Rate5())) - exp.getFloat(name + ".fifteen-minute").Set(float64((m.Rate15()))) - exp.getFloat(name + ".mean").Set(float64(m.RateMean())) + exp.getFloat(name + ".one-minute").Set(m.Rate1()) + exp.getFloat(name + ".five-minute").Set(m.Rate5()) + exp.getFloat(name + ".fifteen-minute").Set((m.Rate15())) + exp.getFloat(name + ".mean").Set(m.RateMean()) } func (exp *exp) publishTimer(name string, metric metrics.Timer) { @@ -121,17 +121,17 @@ func (exp *exp) publishTimer(name string, metric metrics.Timer) { exp.getInt(name + ".count").Set(t.Count()) exp.getFloat(name + ".min").Set(float64(t.Min())) exp.getFloat(name + ".max").Set(float64(t.Max())) - exp.getFloat(name + ".mean").Set(float64(t.Mean())) - exp.getFloat(name + ".std-dev").Set(float64(t.StdDev())) - exp.getFloat(name + ".50-percentile").Set(float64(ps[0])) - exp.getFloat(name + ".75-percentile").Set(float64(ps[1])) - exp.getFloat(name + ".95-percentile").Set(float64(ps[2])) - exp.getFloat(name + ".99-percentile").Set(float64(ps[3])) - exp.getFloat(name + ".999-percentile").Set(float64(ps[4])) - exp.getFloat(name + ".one-minute").Set(float64(t.Rate1())) - exp.getFloat(name + ".five-minute").Set(float64(t.Rate5())) - exp.getFloat(name + ".fifteen-minute").Set(float64((t.Rate15()))) - exp.getFloat(name + ".mean-rate").Set(float64(t.RateMean())) + exp.getFloat(name + ".mean").Set(t.Mean()) + exp.getFloat(name + ".std-dev").Set(t.StdDev()) + exp.getFloat(name + ".50-percentile").Set(ps[0]) + exp.getFloat(name + ".75-percentile").Set(ps[1]) + exp.getFloat(name + ".95-percentile").Set(ps[2]) + exp.getFloat(name + ".99-percentile").Set(ps[3]) + exp.getFloat(name + ".999-percentile").Set(ps[4]) + exp.getFloat(name + ".one-minute").Set(t.Rate1()) + exp.getFloat(name + ".five-minute").Set(t.Rate5()) + exp.getFloat(name + ".fifteen-minute").Set(t.Rate15()) + exp.getFloat(name + ".mean-rate").Set(t.RateMean()) } func (exp *exp) syncToExpvar() { diff --git a/vendor/github.com/rcrowley/go-metrics/gauge.go b/metrics/gauge.go similarity index 98% rename from vendor/github.com/rcrowley/go-metrics/gauge.go rename to metrics/gauge.go index cb57a93889..0fbfdb8603 100644 --- a/vendor/github.com/rcrowley/go-metrics/gauge.go +++ b/metrics/gauge.go @@ -20,7 +20,7 @@ func GetOrRegisterGauge(name string, r Registry) Gauge { // NewGauge constructs a new StandardGauge. func NewGauge() Gauge { - if UseNilMetrics { + if !Enabled { return NilGauge{} } return &StandardGauge{0} @@ -38,7 +38,7 @@ func NewRegisteredGauge(name string, r Registry) Gauge { // NewFunctionalGauge constructs a new FunctionalGauge. func NewFunctionalGauge(f func() int64) Gauge { - if UseNilMetrics { + if !Enabled { return NilGauge{} } return &FunctionalGauge{value: f} diff --git a/vendor/github.com/rcrowley/go-metrics/gauge_float64.go b/metrics/gauge_float64.go similarity index 98% rename from vendor/github.com/rcrowley/go-metrics/gauge_float64.go rename to metrics/gauge_float64.go index 6f93920b2c..66819c9577 100644 --- a/vendor/github.com/rcrowley/go-metrics/gauge_float64.go +++ b/metrics/gauge_float64.go @@ -20,7 +20,7 @@ func GetOrRegisterGaugeFloat64(name string, r Registry) GaugeFloat64 { // NewGaugeFloat64 constructs a new StandardGaugeFloat64. func NewGaugeFloat64() GaugeFloat64 { - if UseNilMetrics { + if !Enabled { return NilGaugeFloat64{} } return &StandardGaugeFloat64{ @@ -40,7 +40,7 @@ func NewRegisteredGaugeFloat64(name string, r Registry) GaugeFloat64 { // NewFunctionalGauge constructs a new FunctionalGauge. func NewFunctionalGaugeFloat64(f func() float64) GaugeFloat64 { - if UseNilMetrics { + if !Enabled { return NilGaugeFloat64{} } return &FunctionalGaugeFloat64{value: f} diff --git a/metrics/gauge_float64_test.go b/metrics/gauge_float64_test.go new file mode 100644 index 0000000000..99e62a4030 --- /dev/null +++ b/metrics/gauge_float64_test.go @@ -0,0 +1,59 @@ +package metrics + +import "testing" + +func BenchmarkGuageFloat64(b *testing.B) { + g := NewGaugeFloat64() + b.ResetTimer() + for i := 0; i < b.N; i++ { + g.Update(float64(i)) + } +} + +func TestGaugeFloat64(t *testing.T) { + g := NewGaugeFloat64() + g.Update(float64(47.0)) + if v := g.Value(); float64(47.0) != v { + t.Errorf("g.Value(): 47.0 != %v\n", v) + } +} + +func TestGaugeFloat64Snapshot(t *testing.T) { + g := NewGaugeFloat64() + g.Update(float64(47.0)) + snapshot := g.Snapshot() + g.Update(float64(0)) + if v := snapshot.Value(); float64(47.0) != v { + t.Errorf("g.Value(): 47.0 != %v\n", v) + } +} + +func TestGetOrRegisterGaugeFloat64(t *testing.T) { + r := NewRegistry() + NewRegisteredGaugeFloat64("foo", r).Update(float64(47.0)) + t.Logf("registry: %v", r) + if g := GetOrRegisterGaugeFloat64("foo", r); float64(47.0) != g.Value() { + t.Fatal(g) + } +} + +func TestFunctionalGaugeFloat64(t *testing.T) { + var counter float64 + fg := NewFunctionalGaugeFloat64(func() float64 { + counter++ + return counter + }) + fg.Value() + fg.Value() + if counter != 2 { + t.Error("counter != 2") + } +} + +func TestGetOrRegisterFunctionalGaugeFloat64(t *testing.T) { + r := NewRegistry() + NewRegisteredFunctionalGaugeFloat64("foo", r, func() float64 { return 47 }) + if g := GetOrRegisterGaugeFloat64("foo", r); 47 != g.Value() { + t.Fatal(g) + } +} diff --git a/metrics/gauge_test.go b/metrics/gauge_test.go new file mode 100644 index 0000000000..1f2603d339 --- /dev/null +++ b/metrics/gauge_test.go @@ -0,0 +1,68 @@ +package metrics + +import ( + "fmt" + "testing" +) + +func BenchmarkGuage(b *testing.B) { + g := NewGauge() + b.ResetTimer() + for i := 0; i < b.N; i++ { + g.Update(int64(i)) + } +} + +func TestGauge(t *testing.T) { + g := NewGauge() + g.Update(int64(47)) + if v := g.Value(); 47 != v { + t.Errorf("g.Value(): 47 != %v\n", v) + } +} + +func TestGaugeSnapshot(t *testing.T) { + g := NewGauge() + g.Update(int64(47)) + snapshot := g.Snapshot() + g.Update(int64(0)) + if v := snapshot.Value(); 47 != v { + t.Errorf("g.Value(): 47 != %v\n", v) + } +} + +func TestGetOrRegisterGauge(t *testing.T) { + r := NewRegistry() + NewRegisteredGauge("foo", r).Update(47) + if g := GetOrRegisterGauge("foo", r); 47 != g.Value() { + t.Fatal(g) + } +} + +func TestFunctionalGauge(t *testing.T) { + var counter int64 + fg := NewFunctionalGauge(func() int64 { + counter++ + return counter + }) + fg.Value() + fg.Value() + if counter != 2 { + t.Error("counter != 2") + } +} + +func TestGetOrRegisterFunctionalGauge(t *testing.T) { + r := NewRegistry() + NewRegisteredFunctionalGauge("foo", r, func() int64 { return 47 }) + if g := GetOrRegisterGauge("foo", r); 47 != g.Value() { + t.Fatal(g) + } +} + +func ExampleGetOrRegisterGauge() { + m := "server.bytes_sent" + g := GetOrRegisterGauge(m, nil) + g.Update(47) + fmt.Println(g.Value()) // Output: 47 +} diff --git a/vendor/github.com/rcrowley/go-metrics/graphite.go b/metrics/graphite.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/graphite.go rename to metrics/graphite.go index abd0a7d291..142eec86be 100644 --- a/vendor/github.com/rcrowley/go-metrics/graphite.go +++ b/metrics/graphite.go @@ -39,7 +39,7 @@ func Graphite(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) { // but it takes a GraphiteConfig instead. func GraphiteWithConfig(c GraphiteConfig) { log.Printf("WARNING: This go-metrics client has been DEPRECATED! It has been moved to https://github.com/cyberdelia/go-metrics-graphite and will be removed from rcrowley/go-metrics on August 12th 2015") - for _ = range time.Tick(c.FlushInterval) { + for range time.Tick(c.FlushInterval) { if err := graphite(&c); nil != err { log.Println(err) } diff --git a/metrics/graphite_test.go b/metrics/graphite_test.go new file mode 100644 index 0000000000..c797c781df --- /dev/null +++ b/metrics/graphite_test.go @@ -0,0 +1,22 @@ +package metrics + +import ( + "net" + "time" +) + +func ExampleGraphite() { + addr, _ := net.ResolveTCPAddr("net", ":2003") + go Graphite(DefaultRegistry, 1*time.Second, "some.prefix", addr) +} + +func ExampleGraphiteWithConfig() { + addr, _ := net.ResolveTCPAddr("net", ":2003") + go GraphiteWithConfig(GraphiteConfig{ + Addr: addr, + Registry: DefaultRegistry, + FlushInterval: 1 * time.Second, + DurationUnit: time.Millisecond, + Percentiles: []float64{0.5, 0.75, 0.99, 0.999}, + }) +} diff --git a/vendor/github.com/rcrowley/go-metrics/healthcheck.go b/metrics/healthcheck.go similarity index 98% rename from vendor/github.com/rcrowley/go-metrics/healthcheck.go rename to metrics/healthcheck.go index 445131caee..f1ae31e34a 100644 --- a/vendor/github.com/rcrowley/go-metrics/healthcheck.go +++ b/metrics/healthcheck.go @@ -11,7 +11,7 @@ type Healthcheck interface { // NewHealthcheck constructs a new Healthcheck which will use the given // function to update its status. func NewHealthcheck(f func(Healthcheck)) Healthcheck { - if UseNilMetrics { + if !Enabled { return NilHealthcheck{} } return &StandardHealthcheck{nil, f} diff --git a/vendor/github.com/rcrowley/go-metrics/histogram.go b/metrics/histogram.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/histogram.go rename to metrics/histogram.go index dbc837fe4d..46f3bbd2f1 100644 --- a/vendor/github.com/rcrowley/go-metrics/histogram.go +++ b/metrics/histogram.go @@ -28,7 +28,7 @@ func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram { // NewHistogram constructs a new StandardHistogram from a Sample. func NewHistogram(s Sample) Histogram { - if UseNilMetrics { + if !Enabled { return NilHistogram{} } return &StandardHistogram{sample: s} diff --git a/metrics/histogram_test.go b/metrics/histogram_test.go new file mode 100644 index 0000000000..d7f4f0171c --- /dev/null +++ b/metrics/histogram_test.go @@ -0,0 +1,95 @@ +package metrics + +import "testing" + +func BenchmarkHistogram(b *testing.B) { + h := NewHistogram(NewUniformSample(100)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + h.Update(int64(i)) + } +} + +func TestGetOrRegisterHistogram(t *testing.T) { + r := NewRegistry() + s := NewUniformSample(100) + NewRegisteredHistogram("foo", r, s).Update(47) + if h := GetOrRegisterHistogram("foo", r, s); 1 != h.Count() { + t.Fatal(h) + } +} + +func TestHistogram10000(t *testing.T) { + h := NewHistogram(NewUniformSample(100000)) + for i := 1; i <= 10000; i++ { + h.Update(int64(i)) + } + testHistogram10000(t, h) +} + +func TestHistogramEmpty(t *testing.T) { + h := NewHistogram(NewUniformSample(100)) + if count := h.Count(); 0 != count { + t.Errorf("h.Count(): 0 != %v\n", count) + } + if min := h.Min(); 0 != min { + t.Errorf("h.Min(): 0 != %v\n", min) + } + if max := h.Max(); 0 != max { + t.Errorf("h.Max(): 0 != %v\n", max) + } + if mean := h.Mean(); 0.0 != mean { + t.Errorf("h.Mean(): 0.0 != %v\n", mean) + } + if stdDev := h.StdDev(); 0.0 != stdDev { + t.Errorf("h.StdDev(): 0.0 != %v\n", stdDev) + } + ps := h.Percentiles([]float64{0.5, 0.75, 0.99}) + if 0.0 != ps[0] { + t.Errorf("median: 0.0 != %v\n", ps[0]) + } + if 0.0 != ps[1] { + t.Errorf("75th percentile: 0.0 != %v\n", ps[1]) + } + if 0.0 != ps[2] { + t.Errorf("99th percentile: 0.0 != %v\n", ps[2]) + } +} + +func TestHistogramSnapshot(t *testing.T) { + h := NewHistogram(NewUniformSample(100000)) + for i := 1; i <= 10000; i++ { + h.Update(int64(i)) + } + snapshot := h.Snapshot() + h.Update(0) + testHistogram10000(t, snapshot) +} + +func testHistogram10000(t *testing.T, h Histogram) { + if count := h.Count(); 10000 != count { + t.Errorf("h.Count(): 10000 != %v\n", count) + } + if min := h.Min(); 1 != min { + t.Errorf("h.Min(): 1 != %v\n", min) + } + if max := h.Max(); 10000 != max { + t.Errorf("h.Max(): 10000 != %v\n", max) + } + if mean := h.Mean(); 5000.5 != mean { + t.Errorf("h.Mean(): 5000.5 != %v\n", mean) + } + if stdDev := h.StdDev(); 2886.751331514372 != stdDev { + t.Errorf("h.StdDev(): 2886.751331514372 != %v\n", stdDev) + } + ps := h.Percentiles([]float64{0.5, 0.75, 0.99}) + if 5000.5 != ps[0] { + t.Errorf("median: 5000.5 != %v\n", ps[0]) + } + if 7500.75 != ps[1] { + t.Errorf("75th percentile: 7500.75 != %v\n", ps[1]) + } + if 9900.99 != ps[2] { + t.Errorf("99th percentile: 9900.99 != %v\n", ps[2]) + } +} diff --git a/metrics/influxdb/LICENSE b/metrics/influxdb/LICENSE new file mode 100644 index 0000000000..e5bf20cdb5 --- /dev/null +++ b/metrics/influxdb/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2015 Vincent Rischmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/metrics/influxdb/README.md b/metrics/influxdb/README.md new file mode 100644 index 0000000000..b76b1a3f99 --- /dev/null +++ b/metrics/influxdb/README.md @@ -0,0 +1,30 @@ +go-metrics-influxdb +=================== + +This is a reporter for the [go-metrics](https://github.com/rcrowley/go-metrics) library which will post the metrics to [InfluxDB](https://influxdb.com/). + +Note +---- + +This is only compatible with InfluxDB 0.9+. + +Usage +----- + +```go +import "github.com/vrischmann/go-metrics-influxdb" + +go influxdb.InfluxDB( + metrics.DefaultRegistry, // metrics registry + time.Second * 10, // interval + "http://localhost:8086", // the InfluxDB url + "mydb", // your InfluxDB database + "myuser", // your InfluxDB user + "mypassword", // your InfluxDB password +) +``` + +License +------- + +go-metrics-influxdb is licensed under the MIT license. See the LICENSE file for details. diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go new file mode 100644 index 0000000000..d5cb4da66f --- /dev/null +++ b/metrics/influxdb/influxdb.go @@ -0,0 +1,227 @@ +package influxdb + +import ( + "fmt" + "log" + uurl "net/url" + "time" + + "github.com/ethereum/go-ethereum/metrics" + "github.com/influxdata/influxdb/client" +) + +type reporter struct { + reg metrics.Registry + interval time.Duration + + url uurl.URL + database string + username string + password string + namespace string + tags map[string]string + + client *client.Client + + cache map[string]int64 +} + +// InfluxDB starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval. +func InfluxDB(r metrics.Registry, d time.Duration, url, database, username, password, namespace string) { + InfluxDBWithTags(r, d, url, database, username, password, namespace, nil) +} + +// InfluxDBWithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags +func InfluxDBWithTags(r metrics.Registry, d time.Duration, url, database, username, password, namespace string, tags map[string]string) { + u, err := uurl.Parse(url) + if err != nil { + log.Printf("unable to parse InfluxDB url %s. err=%v", url, err) + return + } + + rep := &reporter{ + reg: r, + interval: d, + url: *u, + database: database, + username: username, + password: password, + namespace: namespace, + tags: tags, + cache: make(map[string]int64), + } + if err := rep.makeClient(); err != nil { + log.Printf("unable to make InfluxDB client. err=%v", err) + return + } + + rep.run() +} + +func (r *reporter) makeClient() (err error) { + r.client, err = client.NewClient(client.Config{ + URL: r.url, + Username: r.username, + Password: r.password, + }) + + return +} + +func (r *reporter) run() { + intervalTicker := time.Tick(r.interval) + pingTicker := time.Tick(time.Second * 5) + + for { + select { + case <-intervalTicker: + if err := r.send(); err != nil { + log.Printf("unable to send to InfluxDB. err=%v", err) + } + case <-pingTicker: + _, _, err := r.client.Ping() + if err != nil { + log.Printf("got error while sending a ping to InfluxDB, trying to recreate client. err=%v", err) + + if err = r.makeClient(); err != nil { + log.Printf("unable to make InfluxDB client. err=%v", err) + } + } + } + } +} + +func (r *reporter) send() error { + var pts []client.Point + + r.reg.Each(func(name string, i interface{}) { + now := time.Now() + namespace := r.namespace + + switch metric := i.(type) { + case metrics.Counter: + v := metric.Count() + l := r.cache[name] + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.count", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "value": v - l, + }, + Time: now, + }) + r.cache[name] = v + case metrics.Gauge: + ms := metric.Snapshot() + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.gauge", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "value": ms.Value(), + }, + Time: now, + }) + case metrics.GaugeFloat64: + ms := metric.Snapshot() + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.gauge", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "value": ms.Value(), + }, + Time: now, + }) + case metrics.Histogram: + ms := metric.Snapshot() + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.histogram", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "stddev": ms.StdDev(), + "variance": ms.Variance(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], + }, + Time: now, + }) + case metrics.Meter: + ms := metric.Snapshot() + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.meter", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "count": ms.Count(), + "m1": ms.Rate1(), + "m5": ms.Rate5(), + "m15": ms.Rate15(), + "mean": ms.RateMean(), + }, + Time: now, + }) + case metrics.Timer: + ms := metric.Snapshot() + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.timer", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "stddev": ms.StdDev(), + "variance": ms.Variance(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], + "m1": ms.Rate1(), + "m5": ms.Rate5(), + "m15": ms.Rate15(), + "meanrate": ms.RateMean(), + }, + Time: now, + }) + case metrics.ResettingTimer: + t := metric.Snapshot() + + if len(t.Values()) > 0 { + ps := t.Percentiles([]float64{50, 95, 99}) + val := t.Values() + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.span", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "count": len(val), + "max": val[len(val)-1], + "mean": t.Mean(), + "min": val[0], + "p50": ps[0], + "p95": ps[1], + "p99": ps[2], + }, + Time: now, + }) + } + } + }) + + bps := client.BatchPoints{ + Points: pts, + Database: r.database, + } + + _, err := r.client.Write(bps) + return err +} diff --git a/metrics/init_test.go b/metrics/init_test.go new file mode 100644 index 0000000000..43401e833c --- /dev/null +++ b/metrics/init_test.go @@ -0,0 +1,5 @@ +package metrics + +func init() { + Enabled = true +} diff --git a/metrics/json.go b/metrics/json.go new file mode 100644 index 0000000000..2087d8211e --- /dev/null +++ b/metrics/json.go @@ -0,0 +1,31 @@ +package metrics + +import ( + "encoding/json" + "io" + "time" +) + +// MarshalJSON returns a byte slice containing a JSON representation of all +// the metrics in the Registry. +func (r *StandardRegistry) MarshalJSON() ([]byte, error) { + return json.Marshal(r.GetAll()) +} + +// WriteJSON writes metrics from the given registry periodically to the +// specified io.Writer as JSON. +func WriteJSON(r Registry, d time.Duration, w io.Writer) { + for range time.Tick(d) { + WriteJSONOnce(r, w) + } +} + +// WriteJSONOnce writes metrics from the given registry to the specified +// io.Writer as JSON. +func WriteJSONOnce(r Registry, w io.Writer) { + json.NewEncoder(w).Encode(r) +} + +func (p *PrefixedRegistry) MarshalJSON() ([]byte, error) { + return json.Marshal(p.GetAll()) +} diff --git a/metrics/json_test.go b/metrics/json_test.go new file mode 100644 index 0000000000..cf70051f7a --- /dev/null +++ b/metrics/json_test.go @@ -0,0 +1,28 @@ +package metrics + +import ( + "bytes" + "encoding/json" + "testing" +) + +func TestRegistryMarshallJSON(t *testing.T) { + b := &bytes.Buffer{} + enc := json.NewEncoder(b) + r := NewRegistry() + r.Register("counter", NewCounter()) + enc.Encode(r) + if s := b.String(); "{\"counter\":{\"count\":0}}\n" != s { + t.Fatalf(s) + } +} + +func TestRegistryWriteJSONOnce(t *testing.T) { + r := NewRegistry() + r.Register("counter", NewCounter()) + b := &bytes.Buffer{} + WriteJSONOnce(r, b) + if s := b.String(); s != "{\"counter\":{\"count\":0}}\n" { + t.Fail() + } +} diff --git a/metrics/librato/client.go b/metrics/librato/client.go new file mode 100644 index 0000000000..8c0c850e38 --- /dev/null +++ b/metrics/librato/client.go @@ -0,0 +1,102 @@ +package librato + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +const Operations = "operations" +const OperationsShort = "ops" + +type LibratoClient struct { + Email, Token string +} + +// property strings +const ( + // display attributes + Color = "color" + DisplayMax = "display_max" + DisplayMin = "display_min" + DisplayUnitsLong = "display_units_long" + DisplayUnitsShort = "display_units_short" + DisplayStacked = "display_stacked" + DisplayTransform = "display_transform" + // special gauge display attributes + SummarizeFunction = "summarize_function" + Aggregate = "aggregate" + + // metric keys + Name = "name" + Period = "period" + Description = "description" + DisplayName = "display_name" + Attributes = "attributes" + + // measurement keys + MeasureTime = "measure_time" + Source = "source" + Value = "value" + + // special gauge keys + Count = "count" + Sum = "sum" + Max = "max" + Min = "min" + SumSquares = "sum_squares" + + // batch keys + Counters = "counters" + Gauges = "gauges" + + MetricsPostUrl = "https://metrics-api.librato.com/v1/metrics" +) + +type Measurement map[string]interface{} +type Metric map[string]interface{} + +type Batch struct { + Gauges []Measurement `json:"gauges,omitempty"` + Counters []Measurement `json:"counters,omitempty"` + MeasureTime int64 `json:"measure_time"` + Source string `json:"source"` +} + +func (self *LibratoClient) PostMetrics(batch Batch) (err error) { + var ( + js []byte + req *http.Request + resp *http.Response + ) + + if len(batch.Counters) == 0 && len(batch.Gauges) == 0 { + return nil + } + + if js, err = json.Marshal(batch); err != nil { + return + } + + if req, err = http.NewRequest("POST", MetricsPostUrl, bytes.NewBuffer(js)); err != nil { + return + } + + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth(self.Email, self.Token) + + if resp, err = http.DefaultClient.Do(req); err != nil { + return + } + + if resp.StatusCode != http.StatusOK { + var body []byte + if body, err = ioutil.ReadAll(resp.Body); err != nil { + body = []byte(fmt.Sprintf("(could not fetch response body for error: %s)", err)) + } + err = fmt.Errorf("Unable to post to Librato: %d %s %s", resp.StatusCode, resp.Status, string(body)) + } + return +} diff --git a/metrics/librato/librato.go b/metrics/librato/librato.go new file mode 100644 index 0000000000..f8c8c9ecb7 --- /dev/null +++ b/metrics/librato/librato.go @@ -0,0 +1,235 @@ +package librato + +import ( + "fmt" + "log" + "math" + "regexp" + "time" + + "github.com/ethereum/go-ethereum/metrics" +) + +// a regexp for extracting the unit from time.Duration.String +var unitRegexp = regexp.MustCompile(`[^\\d]+$`) + +// a helper that turns a time.Duration into librato display attributes for timer metrics +func translateTimerAttributes(d time.Duration) (attrs map[string]interface{}) { + attrs = make(map[string]interface{}) + attrs[DisplayTransform] = fmt.Sprintf("x/%d", int64(d)) + attrs[DisplayUnitsShort] = string(unitRegexp.Find([]byte(d.String()))) + return +} + +type Reporter struct { + Email, Token string + Namespace string + Source string + Interval time.Duration + Registry metrics.Registry + Percentiles []float64 // percentiles to report on histogram metrics + TimerAttributes map[string]interface{} // units in which timers will be displayed + intervalSec int64 +} + +func NewReporter(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) *Reporter { + return &Reporter{e, t, "", s, d, r, p, translateTimerAttributes(u), int64(d / time.Second)} +} + +func Librato(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) { + NewReporter(r, d, e, t, s, p, u).Run() +} + +func (self *Reporter) Run() { + log.Printf("WARNING: This client has been DEPRECATED! It has been moved to https://github.com/mihasya/go-metrics-librato and will be removed from rcrowley/go-metrics on August 5th 2015") + ticker := time.Tick(self.Interval) + metricsApi := &LibratoClient{self.Email, self.Token} + for now := range ticker { + var metrics Batch + var err error + if metrics, err = self.BuildRequest(now, self.Registry); err != nil { + log.Printf("ERROR constructing librato request body %s", err) + continue + } + if err := metricsApi.PostMetrics(metrics); err != nil { + log.Printf("ERROR sending metrics to librato %s", err) + continue + } + } +} + +// calculate sum of squares from data provided by metrics.Histogram +// see http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods +func sumSquares(s metrics.Sample) float64 { + count := float64(s.Count()) + sumSquared := math.Pow(count*s.Mean(), 2) + sumSquares := math.Pow(count*s.StdDev(), 2) + sumSquared/count + if math.IsNaN(sumSquares) { + return 0.0 + } + return sumSquares +} +func sumSquaresTimer(t metrics.Timer) float64 { + count := float64(t.Count()) + sumSquared := math.Pow(count*t.Mean(), 2) + sumSquares := math.Pow(count*t.StdDev(), 2) + sumSquared/count + if math.IsNaN(sumSquares) { + return 0.0 + } + return sumSquares +} + +func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Batch, err error) { + snapshot = Batch{ + // coerce timestamps to a stepping fn so that they line up in Librato graphs + MeasureTime: (now.Unix() / self.intervalSec) * self.intervalSec, + Source: self.Source, + } + snapshot.Gauges = make([]Measurement, 0) + snapshot.Counters = make([]Measurement, 0) + histogramGaugeCount := 1 + len(self.Percentiles) + r.Each(func(name string, metric interface{}) { + if self.Namespace != "" { + name = fmt.Sprintf("%s.%s", self.Namespace, name) + } + measurement := Measurement{} + measurement[Period] = self.Interval.Seconds() + switch m := metric.(type) { + case metrics.Counter: + if m.Count() > 0 { + measurement[Name] = fmt.Sprintf("%s.%s", name, "count") + measurement[Value] = float64(m.Count()) + measurement[Attributes] = map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + } + snapshot.Counters = append(snapshot.Counters, measurement) + } + case metrics.Gauge: + measurement[Name] = name + measurement[Value] = float64(m.Value()) + snapshot.Gauges = append(snapshot.Gauges, measurement) + case metrics.GaugeFloat64: + measurement[Name] = name + measurement[Value] = m.Value() + snapshot.Gauges = append(snapshot.Gauges, measurement) + case metrics.Histogram: + if m.Count() > 0 { + gauges := make([]Measurement, histogramGaugeCount) + s := m.Sample() + measurement[Name] = fmt.Sprintf("%s.%s", name, "hist") + measurement[Count] = uint64(s.Count()) + measurement[Max] = float64(s.Max()) + measurement[Min] = float64(s.Min()) + measurement[Sum] = float64(s.Sum()) + measurement[SumSquares] = sumSquares(s) + gauges[0] = measurement + for i, p := range self.Percentiles { + gauges[i+1] = Measurement{ + Name: fmt.Sprintf("%s.%.2f", measurement[Name], p), + Value: s.Percentile(p), + Period: measurement[Period], + } + } + snapshot.Gauges = append(snapshot.Gauges, gauges...) + } + case metrics.Meter: + measurement[Name] = name + measurement[Value] = float64(m.Count()) + snapshot.Counters = append(snapshot.Counters, measurement) + snapshot.Gauges = append(snapshot.Gauges, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "1min"), + Value: m.Rate1(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "5min"), + Value: m.Rate5(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "15min"), + Value: m.Rate15(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + ) + case metrics.Timer: + measurement[Name] = name + measurement[Value] = float64(m.Count()) + snapshot.Counters = append(snapshot.Counters, measurement) + if m.Count() > 0 { + libratoName := fmt.Sprintf("%s.%s", name, "timer.mean") + gauges := make([]Measurement, histogramGaugeCount) + gauges[0] = Measurement{ + Name: libratoName, + Count: uint64(m.Count()), + Sum: m.Mean() * float64(m.Count()), + Max: float64(m.Max()), + Min: float64(m.Min()), + SumSquares: sumSquaresTimer(m), + Period: int64(self.Interval.Seconds()), + Attributes: self.TimerAttributes, + } + for i, p := range self.Percentiles { + gauges[i+1] = Measurement{ + Name: fmt.Sprintf("%s.timer.%2.0f", name, p*100), + Value: m.Percentile(p), + Period: int64(self.Interval.Seconds()), + Attributes: self.TimerAttributes, + } + } + snapshot.Gauges = append(snapshot.Gauges, gauges...) + snapshot.Gauges = append(snapshot.Gauges, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "rate.1min"), + Value: m.Rate1(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "rate.5min"), + Value: m.Rate5(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "rate.15min"), + Value: m.Rate15(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + ) + } + } + }) + return +} diff --git a/vendor/github.com/rcrowley/go-metrics/log.go b/metrics/log.go similarity index 98% rename from vendor/github.com/rcrowley/go-metrics/log.go rename to metrics/log.go index f8074c0457..0c8ea7c971 100644 --- a/vendor/github.com/rcrowley/go-metrics/log.go +++ b/metrics/log.go @@ -18,7 +18,7 @@ func LogScaled(r Registry, freq time.Duration, scale time.Duration, l Logger) { du := float64(scale) duSuffix := scale.String()[1:] - for _ = range time.Tick(freq) { + for range time.Tick(freq) { r.Each(func(name string, i interface{}) { switch metric := i.(type) { case Counter: diff --git a/vendor/github.com/rcrowley/go-metrics/memory.md b/metrics/memory.md similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/memory.md rename to metrics/memory.md diff --git a/vendor/github.com/rcrowley/go-metrics/meter.go b/metrics/meter.go similarity index 81% rename from vendor/github.com/rcrowley/go-metrics/meter.go rename to metrics/meter.go index 0389ab0b8f..82b2141a62 100644 --- a/vendor/github.com/rcrowley/go-metrics/meter.go +++ b/metrics/meter.go @@ -15,10 +15,13 @@ type Meter interface { Rate15() float64 RateMean() float64 Snapshot() Meter + Stop() } // GetOrRegisterMeter returns an existing Meter or constructs and registers a // new StandardMeter. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. func GetOrRegisterMeter(name string, r Registry) Meter { if nil == r { r = DefaultRegistry @@ -27,14 +30,15 @@ func GetOrRegisterMeter(name string, r Registry) Meter { } // NewMeter constructs a new StandardMeter and launches a goroutine. +// Be sure to call Stop() once the meter is of no use to allow for garbage collection. func NewMeter() Meter { - if UseNilMetrics { + if !Enabled { return NilMeter{} } m := newStandardMeter() arbiter.Lock() defer arbiter.Unlock() - arbiter.meters = append(arbiter.meters, m) + arbiter.meters[m] = struct{}{} if !arbiter.started { arbiter.started = true go arbiter.tick() @@ -44,6 +48,8 @@ func NewMeter() Meter { // NewMeter constructs and registers a new StandardMeter and launches a // goroutine. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. func NewRegisteredMeter(name string, r Registry) Meter { c := NewMeter() if nil == r { @@ -86,6 +92,9 @@ func (m *MeterSnapshot) RateMean() float64 { return m.rateMean } // Snapshot returns the snapshot. func (m *MeterSnapshot) Snapshot() Meter { return m } +// Stop is a no-op. +func (m *MeterSnapshot) Stop() {} + // NilMeter is a no-op Meter. type NilMeter struct{} @@ -110,12 +119,16 @@ func (NilMeter) RateMean() float64 { return 0.0 } // Snapshot is a no-op. func (NilMeter) Snapshot() Meter { return NilMeter{} } +// Stop is a no-op. +func (NilMeter) Stop() {} + // StandardMeter is the standard implementation of a Meter. type StandardMeter struct { lock sync.RWMutex snapshot *MeterSnapshot a1, a5, a15 EWMA startTime time.Time + stopped bool } func newStandardMeter() *StandardMeter { @@ -128,6 +141,19 @@ func newStandardMeter() *StandardMeter { } } +// Stop stops the meter, Mark() will be a no-op if you use it after being stopped. +func (m *StandardMeter) Stop() { + m.lock.Lock() + stopped := m.stopped + m.stopped = true + m.lock.Unlock() + if !stopped { + arbiter.Lock() + delete(arbiter.meters, m) + arbiter.Unlock() + } +} + // Count returns the number of events recorded. func (m *StandardMeter) Count() int64 { m.lock.RLock() @@ -136,10 +162,13 @@ func (m *StandardMeter) Count() int64 { return count } -// Mark records the occurance of n events. +// Mark records the occurrence of n events. func (m *StandardMeter) Mark(n int64) { m.lock.Lock() defer m.lock.Unlock() + if m.stopped { + return + } m.snapshot.count += n m.a1.Update(n) m.a5.Update(n) @@ -205,29 +234,28 @@ func (m *StandardMeter) tick() { m.updateSnapshot() } +// meterArbiter ticks meters every 5s from a single goroutine. +// meters are references in a set for future stopping. type meterArbiter struct { sync.RWMutex started bool - meters []*StandardMeter + meters map[*StandardMeter]struct{} ticker *time.Ticker } -var arbiter = meterArbiter{ticker: time.NewTicker(5e9)} +var arbiter = meterArbiter{ticker: time.NewTicker(5e9), meters: make(map[*StandardMeter]struct{})} // Ticks meters on the scheduled interval func (ma *meterArbiter) tick() { - for { - select { - case <-ma.ticker.C: - ma.tickMeters() - } + for range ma.ticker.C { + ma.tickMeters() } } func (ma *meterArbiter) tickMeters() { ma.RLock() defer ma.RUnlock() - for _, meter := range ma.meters { + for meter := range ma.meters { meter.tick() } } diff --git a/metrics/meter_test.go b/metrics/meter_test.go new file mode 100644 index 0000000000..e889222601 --- /dev/null +++ b/metrics/meter_test.go @@ -0,0 +1,73 @@ +package metrics + +import ( + "testing" + "time" +) + +func BenchmarkMeter(b *testing.B) { + m := NewMeter() + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.Mark(1) + } +} + +func TestGetOrRegisterMeter(t *testing.T) { + r := NewRegistry() + NewRegisteredMeter("foo", r).Mark(47) + if m := GetOrRegisterMeter("foo", r); 47 != m.Count() { + t.Fatal(m) + } +} + +func TestMeterDecay(t *testing.T) { + ma := meterArbiter{ + ticker: time.NewTicker(time.Millisecond), + meters: make(map[*StandardMeter]struct{}), + } + m := newStandardMeter() + ma.meters[m] = struct{}{} + go ma.tick() + m.Mark(1) + rateMean := m.RateMean() + time.Sleep(100 * time.Millisecond) + if m.RateMean() >= rateMean { + t.Error("m.RateMean() didn't decrease") + } +} + +func TestMeterNonzero(t *testing.T) { + m := NewMeter() + m.Mark(3) + if count := m.Count(); 3 != count { + t.Errorf("m.Count(): 3 != %v\n", count) + } +} + +func TestMeterStop(t *testing.T) { + l := len(arbiter.meters) + m := NewMeter() + if len(arbiter.meters) != l+1 { + t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters)) + } + m.Stop() + if len(arbiter.meters) != l { + t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters)) + } +} + +func TestMeterSnapshot(t *testing.T) { + m := NewMeter() + m.Mark(1) + if snapshot := m.Snapshot(); m.RateMean() != snapshot.RateMean() { + t.Fatal(snapshot) + } +} + +func TestMeterZero(t *testing.T) { + m := NewMeter() + if count := m.Count(); 0 != count { + t.Errorf("m.Count(): 0 != %v\n", count) + } +} diff --git a/metrics/metrics.go b/metrics/metrics.go index c82661d802..4e4e3ecb2b 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,20 +1,8 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. +// Go port of Coda Hale's Metrics library // -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. +//
// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -// Package metrics provides general system and process level metrics collection. +// Coda Hale's original work:
package metrics import ( @@ -24,17 +12,19 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "github.com/rcrowley/go-metrics" - "github.com/rcrowley/go-metrics/exp" ) +// Enabled is checked by the constructor functions for all of the +// standard metrics. If it is true, the metric returned is a stub. +// +// This global kill-switch helps quantify the observer effect and makes +// for less cluttered pprof profiles. +var Enabled bool = false + // MetricsEnabledFlag is the CLI flag name to use to enable metrics collections. const MetricsEnabledFlag = "metrics" const DashboardEnabledFlag = "dashboard" -// Enabled is the flag specifying if metrics are enable or not. -var Enabled = false - // Init enables or disables the metrics system. Since we need this to run before // any other code gets to create meters and timers, we'll actually do an ugly hack // and peek into the command line args for the metrics flag. @@ -45,34 +35,7 @@ func init() { Enabled = true } } - exp.Exp(metrics.DefaultRegistry) -} - -// NewCounter create a new metrics Counter, either a real one of a NOP stub depending -// on the metrics flag. -func NewCounter(name string) metrics.Counter { - if !Enabled { - return new(metrics.NilCounter) - } - return metrics.GetOrRegisterCounter(name, metrics.DefaultRegistry) -} - -// NewMeter create a new metrics Meter, either a real one of a NOP stub depending -// on the metrics flag. -func NewMeter(name string) metrics.Meter { - if !Enabled { - return new(metrics.NilMeter) - } - return metrics.GetOrRegisterMeter(name, metrics.DefaultRegistry) -} - -// NewTimer create a new metrics Timer, either a real one of a NOP stub depending -// on the metrics flag. -func NewTimer(name string) metrics.Timer { - if !Enabled { - return new(metrics.NilTimer) - } - return metrics.GetOrRegisterTimer(name, metrics.DefaultRegistry) + //exp.Exp(DefaultRegistry) } // CollectProcessMetrics periodically collects various metrics about the running @@ -90,17 +53,17 @@ func CollectProcessMetrics(refresh time.Duration) { diskstats[i] = new(DiskStats) } // Define the various metrics to collect - memAllocs := metrics.GetOrRegisterMeter("system/memory/allocs", metrics.DefaultRegistry) - memFrees := metrics.GetOrRegisterMeter("system/memory/frees", metrics.DefaultRegistry) - memInuse := metrics.GetOrRegisterMeter("system/memory/inuse", metrics.DefaultRegistry) - memPauses := metrics.GetOrRegisterMeter("system/memory/pauses", metrics.DefaultRegistry) + memAllocs := GetOrRegisterMeter("system/memory/allocs", DefaultRegistry) + memFrees := GetOrRegisterMeter("system/memory/frees", DefaultRegistry) + memInuse := GetOrRegisterMeter("system/memory/inuse", DefaultRegistry) + memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry) - var diskReads, diskReadBytes, diskWrites, diskWriteBytes metrics.Meter + var diskReads, diskReadBytes, diskWrites, diskWriteBytes Meter if err := ReadDiskStats(diskstats[0]); err == nil { - diskReads = metrics.GetOrRegisterMeter("system/disk/readcount", metrics.DefaultRegistry) - diskReadBytes = metrics.GetOrRegisterMeter("system/disk/readdata", metrics.DefaultRegistry) - diskWrites = metrics.GetOrRegisterMeter("system/disk/writecount", metrics.DefaultRegistry) - diskWriteBytes = metrics.GetOrRegisterMeter("system/disk/writedata", metrics.DefaultRegistry) + diskReads = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry) + diskReadBytes = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry) + diskWrites = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry) + diskWriteBytes = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry) } else { log.Debug("Failed to read disk metrics", "err", err) } diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go new file mode 100644 index 0000000000..726fba3475 --- /dev/null +++ b/metrics/metrics_test.go @@ -0,0 +1,124 @@ +package metrics + +import ( + "fmt" + "io/ioutil" + "log" + "sync" + "testing" +) + +const FANOUT = 128 + +// Stop the compiler from complaining during debugging. +var ( + _ = ioutil.Discard + _ = log.LstdFlags +) + +func BenchmarkMetrics(b *testing.B) { + r := NewRegistry() + c := NewRegisteredCounter("counter", r) + g := NewRegisteredGauge("gauge", r) + gf := NewRegisteredGaugeFloat64("gaugefloat64", r) + h := NewRegisteredHistogram("histogram", r, NewUniformSample(100)) + m := NewRegisteredMeter("meter", r) + t := NewRegisteredTimer("timer", r) + RegisterDebugGCStats(r) + RegisterRuntimeMemStats(r) + b.ResetTimer() + ch := make(chan bool) + + wgD := &sync.WaitGroup{} + /* + wgD.Add(1) + go func() { + defer wgD.Done() + //log.Println("go CaptureDebugGCStats") + for { + select { + case <-ch: + //log.Println("done CaptureDebugGCStats") + return + default: + CaptureDebugGCStatsOnce(r) + } + } + }() + //*/ + + wgR := &sync.WaitGroup{} + //* + wgR.Add(1) + go func() { + defer wgR.Done() + //log.Println("go CaptureRuntimeMemStats") + for { + select { + case <-ch: + //log.Println("done CaptureRuntimeMemStats") + return + default: + CaptureRuntimeMemStatsOnce(r) + } + } + }() + //*/ + + wgW := &sync.WaitGroup{} + /* + wgW.Add(1) + go func() { + defer wgW.Done() + //log.Println("go Write") + for { + select { + case <-ch: + //log.Println("done Write") + return + default: + WriteOnce(r, ioutil.Discard) + } + } + }() + //*/ + + wg := &sync.WaitGroup{} + wg.Add(FANOUT) + for i := 0; i < FANOUT; i++ { + go func(i int) { + defer wg.Done() + //log.Println("go", i) + for i := 0; i < b.N; i++ { + c.Inc(1) + g.Update(int64(i)) + gf.Update(float64(i)) + h.Update(int64(i)) + m.Mark(1) + t.Update(1) + } + //log.Println("done", i) + }(i) + } + wg.Wait() + close(ch) + wgD.Wait() + wgR.Wait() + wgW.Wait() +} + +func Example() { + c := NewCounter() + Register("money", c) + c.Inc(17) + + // Threadsafe registration + t := GetOrRegisterTimer("db.get.latency", nil) + t.Time(func() {}) + t.Update(1) + + fmt.Println(c.Count()) + fmt.Println(t.Min()) + // Output: 17 + // 1 +} diff --git a/vendor/github.com/rcrowley/go-metrics/opentsdb.go b/metrics/opentsdb.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/opentsdb.go rename to metrics/opentsdb.go index 266b6c93d2..df7f152ed2 100644 --- a/vendor/github.com/rcrowley/go-metrics/opentsdb.go +++ b/metrics/opentsdb.go @@ -38,7 +38,7 @@ func OpenTSDB(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) { // OpenTSDBWithConfig is a blocking exporter function just like OpenTSDB, // but it takes a OpenTSDBConfig instead. func OpenTSDBWithConfig(c OpenTSDBConfig) { - for _ = range time.Tick(c.FlushInterval) { + for range time.Tick(c.FlushInterval) { if err := openTSDB(&c); nil != err { log.Println(err) } diff --git a/metrics/opentsdb_test.go b/metrics/opentsdb_test.go new file mode 100644 index 0000000000..c43728960e --- /dev/null +++ b/metrics/opentsdb_test.go @@ -0,0 +1,21 @@ +package metrics + +import ( + "net" + "time" +) + +func ExampleOpenTSDB() { + addr, _ := net.ResolveTCPAddr("net", ":2003") + go OpenTSDB(DefaultRegistry, 1*time.Second, "some.prefix", addr) +} + +func ExampleOpenTSDBWithConfig() { + addr, _ := net.ResolveTCPAddr("net", ":2003") + go OpenTSDBWithConfig(OpenTSDBConfig{ + Addr: addr, + Registry: DefaultRegistry, + FlushInterval: 1 * time.Second, + DurationUnit: time.Millisecond, + }) +} diff --git a/vendor/github.com/rcrowley/go-metrics/registry.go b/metrics/registry.go similarity index 76% rename from vendor/github.com/rcrowley/go-metrics/registry.go rename to metrics/registry.go index 2bb7a1e7d0..cc34c9dfd2 100644 --- a/vendor/github.com/rcrowley/go-metrics/registry.go +++ b/metrics/registry.go @@ -29,6 +29,9 @@ type Registry interface { // Get the metric by the given name or nil if none is registered. Get(string) interface{} + // GetAll metrics in the Registry. + GetAll() map[string]map[string]interface{} + // Gets an existing metric or registers the given one. // The interface can be the metric to register if not found in registry, // or a function returning the metric for lazy instantiation. @@ -109,10 +112,72 @@ func (r *StandardRegistry) RunHealthchecks() { } } +// GetAll metrics in the Registry +func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { + data := make(map[string]map[string]interface{}) + r.Each(func(name string, i interface{}) { + values := make(map[string]interface{}) + switch metric := i.(type) { + case Counter: + values["count"] = metric.Count() + case Gauge: + values["value"] = metric.Value() + case GaugeFloat64: + values["value"] = metric.Value() + case Healthcheck: + values["error"] = nil + metric.Check() + if err := metric.Error(); nil != err { + values["error"] = metric.Error().Error() + } + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = h.Count() + values["min"] = h.Min() + values["max"] = h.Max() + values["mean"] = h.Mean() + values["stddev"] = h.StdDev() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] + case Meter: + m := metric.Snapshot() + values["count"] = m.Count() + values["1m.rate"] = m.Rate1() + values["5m.rate"] = m.Rate5() + values["15m.rate"] = m.Rate15() + values["mean.rate"] = m.RateMean() + case Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = t.Count() + values["min"] = t.Min() + values["max"] = t.Max() + values["mean"] = t.Mean() + values["stddev"] = t.StdDev() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] + values["1m.rate"] = t.Rate1() + values["5m.rate"] = t.Rate5() + values["15m.rate"] = t.Rate15() + values["mean.rate"] = t.RateMean() + } + data[name] = values + }) + return data +} + // Unregister the metric with the given name. func (r *StandardRegistry) Unregister(name string) { r.mutex.Lock() defer r.mutex.Unlock() + r.stop(name) delete(r.metrics, name) } @@ -120,7 +185,8 @@ func (r *StandardRegistry) Unregister(name string) { func (r *StandardRegistry) UnregisterAll() { r.mutex.Lock() defer r.mutex.Unlock() - for name, _ := range r.metrics { + for name := range r.metrics { + r.stop(name) delete(r.metrics, name) } } @@ -130,7 +196,7 @@ func (r *StandardRegistry) register(name string, i interface{}) error { return DuplicateMetric(name) } switch i.(type) { - case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer: + case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer, ResettingTimer: r.metrics[name] = i } return nil @@ -146,6 +212,19 @@ func (r *StandardRegistry) registered() map[string]interface{} { return metrics } +func (r *StandardRegistry) stop(name string) { + if i, ok := r.metrics[name]; ok { + if s, ok := i.(Stoppable); ok { + s.Stop() + } + } +} + +// Stoppable defines the metrics which has to be stopped. +type Stoppable interface { + Stop() +} + type PrefixedRegistry struct { underlying Registry prefix string @@ -216,6 +295,11 @@ func (r *PrefixedRegistry) RunHealthchecks() { r.underlying.RunHealthchecks() } +// GetAll metrics in the Registry +func (r *PrefixedRegistry) GetAll() map[string]map[string]interface{} { + return r.underlying.GetAll() +} + // Unregister the metric with the given name. The name will be prefixed. func (r *PrefixedRegistry) Unregister(name string) { realName := r.prefix + name diff --git a/metrics/registry_test.go b/metrics/registry_test.go new file mode 100644 index 0000000000..a63e485fe9 --- /dev/null +++ b/metrics/registry_test.go @@ -0,0 +1,305 @@ +package metrics + +import ( + "testing" +) + +func BenchmarkRegistry(b *testing.B) { + r := NewRegistry() + r.Register("foo", NewCounter()) + b.ResetTimer() + for i := 0; i < b.N; i++ { + r.Each(func(string, interface{}) {}) + } +} + +func TestRegistry(t *testing.T) { + r := NewRegistry() + r.Register("foo", NewCounter()) + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if "foo" != name { + t.Fatal(name) + } + if _, ok := iface.(Counter); !ok { + t.Fatal(iface) + } + }) + if 1 != i { + t.Fatal(i) + } + r.Unregister("foo") + i = 0 + r.Each(func(string, interface{}) { i++ }) + if 0 != i { + t.Fatal(i) + } +} + +func TestRegistryDuplicate(t *testing.T) { + r := NewRegistry() + if err := r.Register("foo", NewCounter()); nil != err { + t.Fatal(err) + } + if err := r.Register("foo", NewGauge()); nil == err { + t.Fatal(err) + } + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if _, ok := iface.(Counter); !ok { + t.Fatal(iface) + } + }) + if 1 != i { + t.Fatal(i) + } +} + +func TestRegistryGet(t *testing.T) { + r := NewRegistry() + r.Register("foo", NewCounter()) + if count := r.Get("foo").(Counter).Count(); 0 != count { + t.Fatal(count) + } + r.Get("foo").(Counter).Inc(1) + if count := r.Get("foo").(Counter).Count(); 1 != count { + t.Fatal(count) + } +} + +func TestRegistryGetOrRegister(t *testing.T) { + r := NewRegistry() + + // First metric wins with GetOrRegister + _ = r.GetOrRegister("foo", NewCounter()) + m := r.GetOrRegister("foo", NewGauge()) + if _, ok := m.(Counter); !ok { + t.Fatal(m) + } + + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if name != "foo" { + t.Fatal(name) + } + if _, ok := iface.(Counter); !ok { + t.Fatal(iface) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestRegistryGetOrRegisterWithLazyInstantiation(t *testing.T) { + r := NewRegistry() + + // First metric wins with GetOrRegister + _ = r.GetOrRegister("foo", NewCounter) + m := r.GetOrRegister("foo", NewGauge) + if _, ok := m.(Counter); !ok { + t.Fatal(m) + } + + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if name != "foo" { + t.Fatal(name) + } + if _, ok := iface.(Counter); !ok { + t.Fatal(iface) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestRegistryUnregister(t *testing.T) { + l := len(arbiter.meters) + r := NewRegistry() + r.Register("foo", NewCounter()) + r.Register("bar", NewMeter()) + r.Register("baz", NewTimer()) + if len(arbiter.meters) != l+2 { + t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters)) + } + r.Unregister("foo") + r.Unregister("bar") + r.Unregister("baz") + if len(arbiter.meters) != l { + t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters)) + } +} + +func TestPrefixedChildRegistryGetOrRegister(t *testing.T) { + r := NewRegistry() + pr := NewPrefixedChildRegistry(r, "prefix.") + + _ = pr.GetOrRegister("foo", NewCounter()) + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryGetOrRegister(t *testing.T) { + r := NewPrefixedRegistry("prefix.") + + _ = r.GetOrRegister("foo", NewCounter()) + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryRegister(t *testing.T) { + r := NewPrefixedRegistry("prefix.") + err := r.Register("foo", NewCounter()) + c := NewCounter() + Register("bar", c) + if err != nil { + t.Fatal(err.Error()) + } + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryUnregister(t *testing.T) { + r := NewPrefixedRegistry("prefix.") + + _ = r.Register("foo", NewCounter()) + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } + + r.Unregister("foo") + + i = 0 + r.Each(func(name string, m interface{}) { + i++ + }) + + if i != 0 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryGet(t *testing.T) { + pr := NewPrefixedRegistry("prefix.") + name := "foo" + pr.Register(name, NewCounter()) + + fooCounter := pr.Get(name) + if fooCounter == nil { + t.Fatal(name) + } +} + +func TestPrefixedChildRegistryGet(t *testing.T) { + r := NewRegistry() + pr := NewPrefixedChildRegistry(r, "prefix.") + name := "foo" + pr.Register(name, NewCounter()) + fooCounter := pr.Get(name) + if fooCounter == nil { + t.Fatal(name) + } +} + +func TestChildPrefixedRegistryRegister(t *testing.T) { + r := NewPrefixedChildRegistry(DefaultRegistry, "prefix.") + err := r.Register("foo", NewCounter()) + c := NewCounter() + Register("bar", c) + if err != nil { + t.Fatal(err.Error()) + } + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestChildPrefixedRegistryOfChildRegister(t *testing.T) { + r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") + r2 := NewPrefixedChildRegistry(r, "prefix2.") + err := r.Register("foo2", NewCounter()) + if err != nil { + t.Fatal(err.Error()) + } + err = r2.Register("baz", NewCounter()) + c := NewCounter() + Register("bars", c) + + i := 0 + r2.Each(func(name string, m interface{}) { + i++ + if name != "prefix.prefix2.baz" { + //t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestWalkRegistries(t *testing.T) { + r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") + r2 := NewPrefixedChildRegistry(r, "prefix2.") + err := r.Register("foo2", NewCounter()) + if err != nil { + t.Fatal(err.Error()) + } + err = r2.Register("baz", NewCounter()) + c := NewCounter() + Register("bars", c) + + _, prefix := findPrefix(r2, "") + if "prefix.prefix2." != prefix { + t.Fatal(prefix) + } + +} diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go new file mode 100644 index 0000000000..57bcb31343 --- /dev/null +++ b/metrics/resetting_timer.go @@ -0,0 +1,237 @@ +package metrics + +import ( + "math" + "sort" + "sync" + "time" +) + +// Initial slice capacity for the values stored in a ResettingTimer +const InitialResettingTimerSliceCap = 10 + +// ResettingTimer is used for storing aggregated values for timers, which are reset on every flush interval. +type ResettingTimer interface { + Values() []int64 + Snapshot() ResettingTimer + Percentiles([]float64) []int64 + Mean() float64 + Time(func()) + Update(time.Duration) + UpdateSince(time.Time) +} + +// GetOrRegisterResettingTimer returns an existing ResettingTimer or constructs and registers a +// new StandardResettingTimer. +func GetOrRegisterResettingTimer(name string, r Registry) ResettingTimer { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewResettingTimer).(ResettingTimer) +} + +// NewRegisteredResettingTimer constructs and registers a new StandardResettingTimer. +func NewRegisteredResettingTimer(name string, r Registry) ResettingTimer { + c := NewResettingTimer() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// NewResettingTimer constructs a new StandardResettingTimer +func NewResettingTimer() ResettingTimer { + if !Enabled { + return NilResettingTimer{} + } + return &StandardResettingTimer{ + values: make([]int64, 0, InitialResettingTimerSliceCap), + } +} + +// NilResettingTimer is a no-op ResettingTimer. +type NilResettingTimer struct { +} + +// Values is a no-op. +func (NilResettingTimer) Values() []int64 { return nil } + +// Snapshot is a no-op. +func (NilResettingTimer) Snapshot() ResettingTimer { return NilResettingTimer{} } + +// Time is a no-op. +func (NilResettingTimer) Time(func()) {} + +// Update is a no-op. +func (NilResettingTimer) Update(time.Duration) {} + +// Percentiles panics. +func (NilResettingTimer) Percentiles([]float64) []int64 { + panic("Percentiles called on a NilResettingTimer") +} + +// Mean panics. +func (NilResettingTimer) Mean() float64 { + panic("Mean called on a NilResettingTimer") +} + +// UpdateSince is a no-op. +func (NilResettingTimer) UpdateSince(time.Time) {} + +// StandardResettingTimer is the standard implementation of a ResettingTimer. +// and Meter. +type StandardResettingTimer struct { + values []int64 + mutex sync.Mutex +} + +// Values returns a slice with all measurements. +func (t *StandardResettingTimer) Values() []int64 { + return t.values +} + +// Snapshot resets the timer and returns a read-only copy of its contents. +func (t *StandardResettingTimer) Snapshot() ResettingTimer { + t.mutex.Lock() + defer t.mutex.Unlock() + currentValues := t.values + t.values = make([]int64, 0, InitialResettingTimerSliceCap) + + return &ResettingTimerSnapshot{ + values: currentValues, + } +} + +// Percentiles panics. +func (t *StandardResettingTimer) Percentiles([]float64) []int64 { + panic("Percentiles called on a StandardResettingTimer") +} + +// Mean panics. +func (t *StandardResettingTimer) Mean() float64 { + panic("Mean called on a StandardResettingTimer") +} + +// Record the duration of the execution of the given function. +func (t *StandardResettingTimer) Time(f func()) { + ts := time.Now() + f() + t.Update(time.Since(ts)) +} + +// Record the duration of an event. +func (t *StandardResettingTimer) Update(d time.Duration) { + t.mutex.Lock() + defer t.mutex.Unlock() + t.values = append(t.values, int64(d)) +} + +// Record the duration of an event that started at a time and ends now. +func (t *StandardResettingTimer) UpdateSince(ts time.Time) { + t.mutex.Lock() + defer t.mutex.Unlock() + t.values = append(t.values, int64(time.Since(ts))) +} + +// ResettingTimerSnapshot is a point-in-time copy of another ResettingTimer. +type ResettingTimerSnapshot struct { + values []int64 + mean float64 + thresholdBoundaries []int64 + calculated bool +} + +// Snapshot returns the snapshot. +func (t *ResettingTimerSnapshot) Snapshot() ResettingTimer { return t } + +// Time panics. +func (*ResettingTimerSnapshot) Time(func()) { + panic("Time called on a ResettingTimerSnapshot") +} + +// Update panics. +func (*ResettingTimerSnapshot) Update(time.Duration) { + panic("Update called on a ResettingTimerSnapshot") +} + +// UpdateSince panics. +func (*ResettingTimerSnapshot) UpdateSince(time.Time) { + panic("UpdateSince called on a ResettingTimerSnapshot") +} + +// Values returns all values from snapshot. +func (t *ResettingTimerSnapshot) Values() []int64 { + return t.values +} + +// Percentiles returns the boundaries for the input percentiles. +func (t *ResettingTimerSnapshot) Percentiles(percentiles []float64) []int64 { + t.calc(percentiles) + + return t.thresholdBoundaries +} + +// Mean returns the mean of the snapshotted values +func (t *ResettingTimerSnapshot) Mean() float64 { + if !t.calculated { + t.calc([]float64{}) + } + + return t.mean +} + +func (t *ResettingTimerSnapshot) calc(percentiles []float64) { + sort.Sort(Int64Slice(t.values)) + + count := len(t.values) + if count > 0 { + min := t.values[0] + max := t.values[count-1] + + cumulativeValues := make([]int64, count) + cumulativeValues[0] = min + for i := 1; i < count; i++ { + cumulativeValues[i] = t.values[i] + cumulativeValues[i-1] + } + + t.thresholdBoundaries = make([]int64, len(percentiles)) + + thresholdBoundary := max + + for i, pct := range percentiles { + if count > 1 { + var abs float64 + if pct >= 0 { + abs = pct + } else { + abs = 100 + pct + } + // poor man's math.Round(x): + // math.Floor(x + 0.5) + indexOfPerc := int(math.Floor(((abs / 100.0) * float64(count)) + 0.5)) + if pct >= 0 { + indexOfPerc -= 1 // index offset=0 + } + thresholdBoundary = t.values[indexOfPerc] + } + + t.thresholdBoundaries[i] = thresholdBoundary + } + + sum := cumulativeValues[count-1] + t.mean = float64(sum) / float64(count) + } else { + t.thresholdBoundaries = make([]int64, len(percentiles)) + t.mean = 0 + } + + t.calculated = true +} + +// Int64Slice attaches the methods of sort.Interface to []int64, sorting in increasing order. +type Int64Slice []int64 + +func (s Int64Slice) Len() int { return len(s) } +func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Int64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/metrics/resetting_timer_test.go b/metrics/resetting_timer_test.go new file mode 100644 index 0000000000..58fd47f352 --- /dev/null +++ b/metrics/resetting_timer_test.go @@ -0,0 +1,106 @@ +package metrics + +import ( + "testing" + "time" +) + +func TestResettingTimer(t *testing.T) { + tests := []struct { + values []int64 + start int + end int + wantP50 int64 + wantP95 int64 + wantP99 int64 + wantMean float64 + wantMin int64 + wantMax int64 + }{ + { + values: []int64{}, + start: 1, + end: 11, + wantP50: 5, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + { + values: []int64{}, + start: 1, + end: 101, + wantP50: 50, wantP95: 95, wantP99: 99, + wantMin: 1, wantMax: 100, wantMean: 50.5, + }, + { + values: []int64{1}, + start: 0, + end: 0, + wantP50: 1, wantP95: 1, wantP99: 1, + wantMin: 1, wantMax: 1, wantMean: 1, + }, + { + values: []int64{0}, + start: 0, + end: 0, + wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{}, + start: 0, + end: 0, + wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{1, 10}, + start: 0, + end: 0, + wantP50: 1, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + } + for ind, tt := range tests { + timer := NewResettingTimer() + + for i := tt.start; i < tt.end; i++ { + tt.values = append(tt.values, int64(i)) + } + + for _, v := range tt.values { + timer.Update(time.Duration(v)) + } + + snap := timer.Snapshot() + + ps := snap.Percentiles([]float64{50, 95, 99}) + + val := snap.Values() + + if len(val) > 0 { + if tt.wantMin != val[0] { + t.Fatalf("%d: min: got %d, want %d", ind, val[0], tt.wantMin) + } + + if tt.wantMax != val[len(val)-1] { + t.Fatalf("%d: max: got %d, want %d", ind, val[len(val)-1], tt.wantMax) + } + } + + if tt.wantMean != snap.Mean() { + t.Fatalf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean) + } + + if tt.wantP50 != ps[0] { + t.Fatalf("%d: p50: got %d, want %d", ind, ps[0], tt.wantP50) + } + + if tt.wantP95 != ps[1] { + t.Fatalf("%d: p95: got %d, want %d", ind, ps[1], tt.wantP95) + } + + if tt.wantP99 != ps[2] { + t.Fatalf("%d: p99: got %d, want %d", ind, ps[2], tt.wantP99) + } + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/runtime.go b/metrics/runtime.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/runtime.go rename to metrics/runtime.go index 11c6b785a0..9450c479ba 100644 --- a/vendor/github.com/rcrowley/go-metrics/runtime.go +++ b/metrics/runtime.go @@ -55,7 +55,7 @@ var ( // Capture new values for the Go runtime statistics exported in // runtime.MemStats. This is designed to be called as a goroutine. func CaptureRuntimeMemStats(r Registry, d time.Duration) { - for _ = range time.Tick(d) { + for range time.Tick(d) { CaptureRuntimeMemStatsOnce(r) } } diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_cgo.go b/metrics/runtime_cgo.go similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/runtime_cgo.go rename to metrics/runtime_cgo.go diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_gccpufraction.go b/metrics/runtime_gccpufraction.go similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/runtime_gccpufraction.go rename to metrics/runtime_gccpufraction.go diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_no_cgo.go b/metrics/runtime_no_cgo.go similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/runtime_no_cgo.go rename to metrics/runtime_no_cgo.go diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_no_gccpufraction.go b/metrics/runtime_no_gccpufraction.go similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/runtime_no_gccpufraction.go rename to metrics/runtime_no_gccpufraction.go diff --git a/metrics/runtime_test.go b/metrics/runtime_test.go new file mode 100644 index 0000000000..ebbfd501a2 --- /dev/null +++ b/metrics/runtime_test.go @@ -0,0 +1,88 @@ +package metrics + +import ( + "runtime" + "testing" + "time" +) + +func BenchmarkRuntimeMemStats(b *testing.B) { + r := NewRegistry() + RegisterRuntimeMemStats(r) + b.ResetTimer() + for i := 0; i < b.N; i++ { + CaptureRuntimeMemStatsOnce(r) + } +} + +func TestRuntimeMemStats(t *testing.T) { + r := NewRegistry() + RegisterRuntimeMemStats(r) + CaptureRuntimeMemStatsOnce(r) + zero := runtimeMetrics.MemStats.PauseNs.Count() // Get a "zero" since GC may have run before these tests. + runtime.GC() + CaptureRuntimeMemStatsOnce(r) + if count := runtimeMetrics.MemStats.PauseNs.Count(); 1 != count-zero { + t.Fatal(count - zero) + } + runtime.GC() + runtime.GC() + CaptureRuntimeMemStatsOnce(r) + if count := runtimeMetrics.MemStats.PauseNs.Count(); 3 != count-zero { + t.Fatal(count - zero) + } + for i := 0; i < 256; i++ { + runtime.GC() + } + CaptureRuntimeMemStatsOnce(r) + if count := runtimeMetrics.MemStats.PauseNs.Count(); 259 != count-zero { + t.Fatal(count - zero) + } + for i := 0; i < 257; i++ { + runtime.GC() + } + CaptureRuntimeMemStatsOnce(r) + if count := runtimeMetrics.MemStats.PauseNs.Count(); 515 != count-zero { // We lost one because there were too many GCs between captures. + t.Fatal(count - zero) + } +} + +func TestRuntimeMemStatsNumThread(t *testing.T) { + r := NewRegistry() + RegisterRuntimeMemStats(r) + CaptureRuntimeMemStatsOnce(r) + + if value := runtimeMetrics.NumThread.Value(); value < 1 { + t.Fatalf("got NumThread: %d, wanted at least 1", value) + } +} + +func TestRuntimeMemStatsBlocking(t *testing.T) { + if g := runtime.GOMAXPROCS(0); g < 2 { + t.Skipf("skipping TestRuntimeMemStatsBlocking with GOMAXPROCS=%d\n", g) + } + ch := make(chan int) + go testRuntimeMemStatsBlocking(ch) + var memStats runtime.MemStats + t0 := time.Now() + runtime.ReadMemStats(&memStats) + t1 := time.Now() + t.Log("i++ during runtime.ReadMemStats:", <-ch) + go testRuntimeMemStatsBlocking(ch) + d := t1.Sub(t0) + t.Log(d) + time.Sleep(d) + t.Log("i++ during time.Sleep:", <-ch) +} + +func testRuntimeMemStatsBlocking(ch chan int) { + i := 0 + for { + select { + case ch <- i: + return + default: + i++ + } + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/sample.go b/metrics/sample.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/sample.go rename to metrics/sample.go index fecee5ef68..5c4845a4f8 100644 --- a/vendor/github.com/rcrowley/go-metrics/sample.go +++ b/metrics/sample.go @@ -46,7 +46,7 @@ type ExpDecaySample struct { // NewExpDecaySample constructs a new exponentially-decaying sample with the // given reservoir size and alpha. func NewExpDecaySample(reservoirSize int, alpha float64) Sample { - if UseNilMetrics { + if !Enabled { return NilSample{} } s := &ExpDecaySample{ @@ -407,7 +407,7 @@ type UniformSample struct { // NewUniformSample constructs a new uniform sample with the given reservoir // size. func NewUniformSample(reservoirSize int) Sample { - if UseNilMetrics { + if !Enabled { return NilSample{} } return &UniformSample{ diff --git a/metrics/sample_test.go b/metrics/sample_test.go new file mode 100644 index 0000000000..d60e99c5bb --- /dev/null +++ b/metrics/sample_test.go @@ -0,0 +1,363 @@ +package metrics + +import ( + "math/rand" + "runtime" + "testing" + "time" +) + +// Benchmark{Compute,Copy}{1000,1000000} demonstrate that, even for relatively +// expensive computations like Variance, the cost of copying the Sample, as +// approximated by a make and copy, is much greater than the cost of the +// computation for small samples and only slightly less for large samples. +func BenchmarkCompute1000(b *testing.B) { + s := make([]int64, 1000) + for i := 0; i < len(s); i++ { + s[i] = int64(i) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + SampleVariance(s) + } +} +func BenchmarkCompute1000000(b *testing.B) { + s := make([]int64, 1000000) + for i := 0; i < len(s); i++ { + s[i] = int64(i) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + SampleVariance(s) + } +} +func BenchmarkCopy1000(b *testing.B) { + s := make([]int64, 1000) + for i := 0; i < len(s); i++ { + s[i] = int64(i) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + sCopy := make([]int64, len(s)) + copy(sCopy, s) + } +} +func BenchmarkCopy1000000(b *testing.B) { + s := make([]int64, 1000000) + for i := 0; i < len(s); i++ { + s[i] = int64(i) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + sCopy := make([]int64, len(s)) + copy(sCopy, s) + } +} + +func BenchmarkExpDecaySample257(b *testing.B) { + benchmarkSample(b, NewExpDecaySample(257, 0.015)) +} + +func BenchmarkExpDecaySample514(b *testing.B) { + benchmarkSample(b, NewExpDecaySample(514, 0.015)) +} + +func BenchmarkExpDecaySample1028(b *testing.B) { + benchmarkSample(b, NewExpDecaySample(1028, 0.015)) +} + +func BenchmarkUniformSample257(b *testing.B) { + benchmarkSample(b, NewUniformSample(257)) +} + +func BenchmarkUniformSample514(b *testing.B) { + benchmarkSample(b, NewUniformSample(514)) +} + +func BenchmarkUniformSample1028(b *testing.B) { + benchmarkSample(b, NewUniformSample(1028)) +} + +func TestExpDecaySample10(t *testing.T) { + rand.Seed(1) + s := NewExpDecaySample(100, 0.99) + for i := 0; i < 10; i++ { + s.Update(int64(i)) + } + if size := s.Count(); 10 != size { + t.Errorf("s.Count(): 10 != %v\n", size) + } + if size := s.Size(); 10 != size { + t.Errorf("s.Size(): 10 != %v\n", size) + } + if l := len(s.Values()); 10 != l { + t.Errorf("len(s.Values()): 10 != %v\n", l) + } + for _, v := range s.Values() { + if v > 10 || v < 0 { + t.Errorf("out of range [0, 10): %v\n", v) + } + } +} + +func TestExpDecaySample100(t *testing.T) { + rand.Seed(1) + s := NewExpDecaySample(1000, 0.01) + for i := 0; i < 100; i++ { + s.Update(int64(i)) + } + if size := s.Count(); 100 != size { + t.Errorf("s.Count(): 100 != %v\n", size) + } + if size := s.Size(); 100 != size { + t.Errorf("s.Size(): 100 != %v\n", size) + } + if l := len(s.Values()); 100 != l { + t.Errorf("len(s.Values()): 100 != %v\n", l) + } + for _, v := range s.Values() { + if v > 100 || v < 0 { + t.Errorf("out of range [0, 100): %v\n", v) + } + } +} + +func TestExpDecaySample1000(t *testing.T) { + rand.Seed(1) + s := NewExpDecaySample(100, 0.99) + for i := 0; i < 1000; i++ { + s.Update(int64(i)) + } + if size := s.Count(); 1000 != size { + t.Errorf("s.Count(): 1000 != %v\n", size) + } + if size := s.Size(); 100 != size { + t.Errorf("s.Size(): 100 != %v\n", size) + } + if l := len(s.Values()); 100 != l { + t.Errorf("len(s.Values()): 100 != %v\n", l) + } + for _, v := range s.Values() { + if v > 1000 || v < 0 { + t.Errorf("out of range [0, 1000): %v\n", v) + } + } +} + +// This test makes sure that the sample's priority is not amplified by using +// nanosecond duration since start rather than second duration since start. +// The priority becomes +Inf quickly after starting if this is done, +// effectively freezing the set of samples until a rescale step happens. +func TestExpDecaySampleNanosecondRegression(t *testing.T) { + rand.Seed(1) + s := NewExpDecaySample(100, 0.99) + for i := 0; i < 100; i++ { + s.Update(10) + } + time.Sleep(1 * time.Millisecond) + for i := 0; i < 100; i++ { + s.Update(20) + } + v := s.Values() + avg := float64(0) + for i := 0; i < len(v); i++ { + avg += float64(v[i]) + } + avg /= float64(len(v)) + if avg > 16 || avg < 14 { + t.Errorf("out of range [14, 16]: %v\n", avg) + } +} + +func TestExpDecaySampleRescale(t *testing.T) { + s := NewExpDecaySample(2, 0.001).(*ExpDecaySample) + s.update(time.Now(), 1) + s.update(time.Now().Add(time.Hour+time.Microsecond), 1) + for _, v := range s.values.Values() { + if v.k == 0.0 { + t.Fatal("v.k == 0.0") + } + } +} + +func TestExpDecaySampleSnapshot(t *testing.T) { + now := time.Now() + rand.Seed(1) + s := NewExpDecaySample(100, 0.99) + for i := 1; i <= 10000; i++ { + s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i)) + } + snapshot := s.Snapshot() + s.Update(1) + testExpDecaySampleStatistics(t, snapshot) +} + +func TestExpDecaySampleStatistics(t *testing.T) { + now := time.Now() + rand.Seed(1) + s := NewExpDecaySample(100, 0.99) + for i := 1; i <= 10000; i++ { + s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i)) + } + testExpDecaySampleStatistics(t, s) +} + +func TestUniformSample(t *testing.T) { + rand.Seed(1) + s := NewUniformSample(100) + for i := 0; i < 1000; i++ { + s.Update(int64(i)) + } + if size := s.Count(); 1000 != size { + t.Errorf("s.Count(): 1000 != %v\n", size) + } + if size := s.Size(); 100 != size { + t.Errorf("s.Size(): 100 != %v\n", size) + } + if l := len(s.Values()); 100 != l { + t.Errorf("len(s.Values()): 100 != %v\n", l) + } + for _, v := range s.Values() { + if v > 1000 || v < 0 { + t.Errorf("out of range [0, 100): %v\n", v) + } + } +} + +func TestUniformSampleIncludesTail(t *testing.T) { + rand.Seed(1) + s := NewUniformSample(100) + max := 100 + for i := 0; i < max; i++ { + s.Update(int64(i)) + } + v := s.Values() + sum := 0 + exp := (max - 1) * max / 2 + for i := 0; i < len(v); i++ { + sum += int(v[i]) + } + if exp != sum { + t.Errorf("sum: %v != %v\n", exp, sum) + } +} + +func TestUniformSampleSnapshot(t *testing.T) { + s := NewUniformSample(100) + for i := 1; i <= 10000; i++ { + s.Update(int64(i)) + } + snapshot := s.Snapshot() + s.Update(1) + testUniformSampleStatistics(t, snapshot) +} + +func TestUniformSampleStatistics(t *testing.T) { + rand.Seed(1) + s := NewUniformSample(100) + for i := 1; i <= 10000; i++ { + s.Update(int64(i)) + } + testUniformSampleStatistics(t, s) +} + +func benchmarkSample(b *testing.B, s Sample) { + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + pauseTotalNs := memStats.PauseTotalNs + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Update(1) + } + b.StopTimer() + runtime.GC() + runtime.ReadMemStats(&memStats) + b.Logf("GC cost: %d ns/op", int(memStats.PauseTotalNs-pauseTotalNs)/b.N) +} + +func testExpDecaySampleStatistics(t *testing.T, s Sample) { + if count := s.Count(); 10000 != count { + t.Errorf("s.Count(): 10000 != %v\n", count) + } + if min := s.Min(); 107 != min { + t.Errorf("s.Min(): 107 != %v\n", min) + } + if max := s.Max(); 10000 != max { + t.Errorf("s.Max(): 10000 != %v\n", max) + } + if mean := s.Mean(); 4965.98 != mean { + t.Errorf("s.Mean(): 4965.98 != %v\n", mean) + } + if stdDev := s.StdDev(); 2959.825156930727 != stdDev { + t.Errorf("s.StdDev(): 2959.825156930727 != %v\n", stdDev) + } + ps := s.Percentiles([]float64{0.5, 0.75, 0.99}) + if 4615 != ps[0] { + t.Errorf("median: 4615 != %v\n", ps[0]) + } + if 7672 != ps[1] { + t.Errorf("75th percentile: 7672 != %v\n", ps[1]) + } + if 9998.99 != ps[2] { + t.Errorf("99th percentile: 9998.99 != %v\n", ps[2]) + } +} + +func testUniformSampleStatistics(t *testing.T, s Sample) { + if count := s.Count(); 10000 != count { + t.Errorf("s.Count(): 10000 != %v\n", count) + } + if min := s.Min(); 37 != min { + t.Errorf("s.Min(): 37 != %v\n", min) + } + if max := s.Max(); 9989 != max { + t.Errorf("s.Max(): 9989 != %v\n", max) + } + if mean := s.Mean(); 4748.14 != mean { + t.Errorf("s.Mean(): 4748.14 != %v\n", mean) + } + if stdDev := s.StdDev(); 2826.684117548333 != stdDev { + t.Errorf("s.StdDev(): 2826.684117548333 != %v\n", stdDev) + } + ps := s.Percentiles([]float64{0.5, 0.75, 0.99}) + if 4599 != ps[0] { + t.Errorf("median: 4599 != %v\n", ps[0]) + } + if 7380.5 != ps[1] { + t.Errorf("75th percentile: 7380.5 != %v\n", ps[1]) + } + if 9986.429999999998 != ps[2] { + t.Errorf("99th percentile: 9986.429999999998 != %v\n", ps[2]) + } +} + +// TestUniformSampleConcurrentUpdateCount would expose data race problems with +// concurrent Update and Count calls on Sample when test is called with -race +// argument +func TestUniformSampleConcurrentUpdateCount(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + s := NewUniformSample(100) + for i := 0; i < 100; i++ { + s.Update(int64(i)) + } + quit := make(chan struct{}) + go func() { + t := time.NewTicker(10 * time.Millisecond) + for { + select { + case <-t.C: + s.Update(rand.Int63()) + case <-quit: + t.Stop() + return + } + } + }() + for i := 0; i < 1000; i++ { + s.Count() + time.Sleep(5 * time.Millisecond) + } + quit <- struct{}{} +} diff --git a/vendor/github.com/rcrowley/go-metrics/syslog.go b/metrics/syslog.go similarity index 98% rename from vendor/github.com/rcrowley/go-metrics/syslog.go rename to metrics/syslog.go index 693f190855..a0ed4b1b23 100644 --- a/vendor/github.com/rcrowley/go-metrics/syslog.go +++ b/metrics/syslog.go @@ -11,7 +11,7 @@ import ( // Output each metric in the given registry to syslog periodically using // the given syslogger. func Syslog(r Registry, d time.Duration, w *syslog.Writer) { - for _ = range time.Tick(d) { + for range time.Tick(d) { r.Each(func(name string, i interface{}) { switch metric := i.(type) { case Counter: diff --git a/vendor/github.com/rcrowley/go-metrics/timer.go b/metrics/timer.go similarity index 93% rename from vendor/github.com/rcrowley/go-metrics/timer.go rename to metrics/timer.go index 17db8f8d20..89e22208fd 100644 --- a/vendor/github.com/rcrowley/go-metrics/timer.go +++ b/metrics/timer.go @@ -19,6 +19,7 @@ type Timer interface { RateMean() float64 Snapshot() Timer StdDev() float64 + Stop() Sum() int64 Time(func()) Update(time.Duration) @@ -28,6 +29,8 @@ type Timer interface { // GetOrRegisterTimer returns an existing Timer or constructs and registers a // new StandardTimer. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. func GetOrRegisterTimer(name string, r Registry) Timer { if nil == r { r = DefaultRegistry @@ -36,8 +39,9 @@ func GetOrRegisterTimer(name string, r Registry) Timer { } // NewCustomTimer constructs a new StandardTimer from a Histogram and a Meter. +// Be sure to call Stop() once the timer is of no use to allow for garbage collection. func NewCustomTimer(h Histogram, m Meter) Timer { - if UseNilMetrics { + if !Enabled { return NilTimer{} } return &StandardTimer{ @@ -47,6 +51,8 @@ func NewCustomTimer(h Histogram, m Meter) Timer { } // NewRegisteredTimer constructs and registers a new StandardTimer. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. func NewRegisteredTimer(name string, r Registry) Timer { c := NewTimer() if nil == r { @@ -58,8 +64,9 @@ func NewRegisteredTimer(name string, r Registry) Timer { // NewTimer constructs a new StandardTimer using an exponentially-decaying // sample with the same reservoir size and alpha as UNIX load averages. +// Be sure to call Stop() once the timer is of no use to allow for garbage collection. func NewTimer() Timer { - if UseNilMetrics { + if !Enabled { return NilTimer{} } return &StandardTimer{ @@ -112,6 +119,9 @@ func (NilTimer) Snapshot() Timer { return NilTimer{} } // StdDev is a no-op. func (NilTimer) StdDev() float64 { return 0.0 } +// Stop is a no-op. +func (NilTimer) Stop() {} + // Sum is a no-op. func (NilTimer) Sum() int64 { return 0 } @@ -201,6 +211,11 @@ func (t *StandardTimer) StdDev() float64 { return t.histogram.StdDev() } +// Stop stops the meter. +func (t *StandardTimer) Stop() { + t.meter.Stop() +} + // Sum returns the sum in the sample. func (t *StandardTimer) Sum() int64 { return t.histogram.Sum() @@ -288,6 +303,9 @@ func (t *TimerSnapshot) Snapshot() Timer { return t } // was taken. func (t *TimerSnapshot) StdDev() float64 { return t.histogram.StdDev() } +// Stop is a no-op. +func (t *TimerSnapshot) Stop() {} + // Sum returns the sum at the time the snapshot was taken. func (t *TimerSnapshot) Sum() int64 { return t.histogram.Sum() } diff --git a/metrics/timer_test.go b/metrics/timer_test.go new file mode 100644 index 0000000000..f85c9b803d --- /dev/null +++ b/metrics/timer_test.go @@ -0,0 +1,101 @@ +package metrics + +import ( + "fmt" + "math" + "testing" + "time" +) + +func BenchmarkTimer(b *testing.B) { + tm := NewTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + tm.Update(1) + } +} + +func TestGetOrRegisterTimer(t *testing.T) { + r := NewRegistry() + NewRegisteredTimer("foo", r).Update(47) + if tm := GetOrRegisterTimer("foo", r); 1 != tm.Count() { + t.Fatal(tm) + } +} + +func TestTimerExtremes(t *testing.T) { + tm := NewTimer() + tm.Update(math.MaxInt64) + tm.Update(0) + if stdDev := tm.StdDev(); 4.611686018427388e+18 != stdDev { + t.Errorf("tm.StdDev(): 4.611686018427388e+18 != %v\n", stdDev) + } +} + +func TestTimerStop(t *testing.T) { + l := len(arbiter.meters) + tm := NewTimer() + if len(arbiter.meters) != l+1 { + t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters)) + } + tm.Stop() + if len(arbiter.meters) != l { + t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters)) + } +} + +func TestTimerFunc(t *testing.T) { + tm := NewTimer() + tm.Time(func() { time.Sleep(50e6) }) + if max := tm.Max(); 45e6 > max || max > 55e6 { + t.Errorf("tm.Max(): 45e6 > %v || %v > 55e6\n", max, max) + } +} + +func TestTimerZero(t *testing.T) { + tm := NewTimer() + if count := tm.Count(); 0 != count { + t.Errorf("tm.Count(): 0 != %v\n", count) + } + if min := tm.Min(); 0 != min { + t.Errorf("tm.Min(): 0 != %v\n", min) + } + if max := tm.Max(); 0 != max { + t.Errorf("tm.Max(): 0 != %v\n", max) + } + if mean := tm.Mean(); 0.0 != mean { + t.Errorf("tm.Mean(): 0.0 != %v\n", mean) + } + if stdDev := tm.StdDev(); 0.0 != stdDev { + t.Errorf("tm.StdDev(): 0.0 != %v\n", stdDev) + } + ps := tm.Percentiles([]float64{0.5, 0.75, 0.99}) + if 0.0 != ps[0] { + t.Errorf("median: 0.0 != %v\n", ps[0]) + } + if 0.0 != ps[1] { + t.Errorf("75th percentile: 0.0 != %v\n", ps[1]) + } + if 0.0 != ps[2] { + t.Errorf("99th percentile: 0.0 != %v\n", ps[2]) + } + if rate1 := tm.Rate1(); 0.0 != rate1 { + t.Errorf("tm.Rate1(): 0.0 != %v\n", rate1) + } + if rate5 := tm.Rate5(); 0.0 != rate5 { + t.Errorf("tm.Rate5(): 0.0 != %v\n", rate5) + } + if rate15 := tm.Rate15(); 0.0 != rate15 { + t.Errorf("tm.Rate15(): 0.0 != %v\n", rate15) + } + if rateMean := tm.RateMean(); 0.0 != rateMean { + t.Errorf("tm.RateMean(): 0.0 != %v\n", rateMean) + } +} + +func ExampleGetOrRegisterTimer() { + m := "account.create.latency" + t := GetOrRegisterTimer(m, nil) + t.Update(47) + fmt.Println(t.Max()) // Output: 47 +} diff --git a/vendor/github.com/rcrowley/go-metrics/validate.sh b/metrics/validate.sh similarity index 93% rename from vendor/github.com/rcrowley/go-metrics/validate.sh rename to metrics/validate.sh index f6499982e5..c4ae91e642 100755 --- a/vendor/github.com/rcrowley/go-metrics/validate.sh +++ b/metrics/validate.sh @@ -7,4 +7,4 @@ GOFMT_LINES=`gofmt -l . | wc -l | xargs` test $GOFMT_LINES -eq 0 || echo "gofmt needs to be run, ${GOFMT_LINES} files have issues" # run the tests for the root package -go test . +go test -race . diff --git a/vendor/github.com/rcrowley/go-metrics/writer.go b/metrics/writer.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/writer.go rename to metrics/writer.go index 091e971d2e..88521a80d9 100644 --- a/vendor/github.com/rcrowley/go-metrics/writer.go +++ b/metrics/writer.go @@ -10,7 +10,7 @@ import ( // Write sorts writes each metric in the given registry periodically to the // given io.Writer. func Write(r Registry, d time.Duration, w io.Writer) { - for _ = range time.Tick(d) { + for range time.Tick(d) { WriteOnce(r, w) } } diff --git a/metrics/writer_test.go b/metrics/writer_test.go new file mode 100644 index 0000000000..1aacc28712 --- /dev/null +++ b/metrics/writer_test.go @@ -0,0 +1,22 @@ +package metrics + +import ( + "sort" + "testing" +) + +func TestMetricsSorting(t *testing.T) { + var namedMetrics = namedMetricSlice{ + {name: "zzz"}, + {name: "bbb"}, + {name: "fff"}, + {name: "ggg"}, + } + + sort.Sort(namedMetrics) + for i, name := range []string{"bbb", "fff", "ggg", "zzz"} { + if namedMetrics[i].name != name { + t.Fail() + } + } +} diff --git a/node/api.go b/node/api.go index 4e9b1edc47..992a7c4166 100644 --- a/node/api.go +++ b/node/api.go @@ -24,10 +24,10 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/rpc" - "github.com/rcrowley/go-metrics" ) // PrivateAdminAPI is the collection of administrative API methods exposed only diff --git a/p2p/metrics.go b/p2p/metrics.go index 98b61901d5..4cbff90aca 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -25,10 +25,10 @@ import ( ) var ( - ingressConnectMeter = metrics.NewMeter("p2p/InboundConnects") - ingressTrafficMeter = metrics.NewMeter("p2p/InboundTraffic") - egressConnectMeter = metrics.NewMeter("p2p/OutboundConnects") - egressTrafficMeter = metrics.NewMeter("p2p/OutboundTraffic") + ingressConnectMeter = metrics.NewRegisteredMeter("p2p/InboundConnects", nil) + ingressTrafficMeter = metrics.NewRegisteredMeter("p2p/InboundTraffic", nil) + egressConnectMeter = metrics.NewRegisteredMeter("p2p/OutboundConnects", nil) + egressTrafficMeter = metrics.NewRegisteredMeter("p2p/OutboundTraffic", nil) ) // meteredConn is a wrapper around a network TCP connection that meters both the diff --git a/trie/trie.go b/trie/trie.go index e37a1ae109..31a404e3a0 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/rcrowley/go-metrics" + "github.com/ethereum/go-ethereum/metrics" ) var ( diff --git a/vendor/github.com/influxdata/influxdb/LICENSE b/vendor/github.com/influxdata/influxdb/LICENSE new file mode 100644 index 0000000000..63cef79ba6 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013-2016 Errplane Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/influxdata/influxdb/LICENSE_OF_DEPENDENCIES.md b/vendor/github.com/influxdata/influxdb/LICENSE_OF_DEPENDENCIES.md new file mode 100644 index 0000000000..ea6fc69f30 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/LICENSE_OF_DEPENDENCIES.md @@ -0,0 +1,62 @@ +- # List +- bootstrap 3.3.5 [MIT LICENSE](https://github.com/twbs/bootstrap/blob/master/LICENSE) +- collectd.org [ISC LICENSE](https://github.com/collectd/go-collectd/blob/master/LICENSE) +- github.com/BurntSushi/toml [MIT LICENSE](https://github.com/BurntSushi/toml/blob/master/COPYING) +- github.com/RoaringBitmap/roaring [APACHE LICENSE](https://github.com/RoaringBitmap/roaring/blob/master/LICENSE) +- github.com/beorn7/perks [MIT LICENSE](https://github.com/beorn7/perks/blob/master/LICENSE) +- github.com/bmizerany/pat [MIT LICENSE](https://github.com/bmizerany/pat#license) +- github.com/boltdb/bolt [MIT LICENSE](https://github.com/boltdb/bolt/blob/master/LICENSE) +- github.com/cespare/xxhash [MIT LICENSE](https://github.com/cespare/xxhash/blob/master/LICENSE.txt) +- github.com/clarkduvall/hyperloglog [MIT LICENSE](https://github.com/clarkduvall/hyperloglog/blob/master/LICENSE) +- github.com/davecgh/go-spew/spew [ISC LICENSE](https://github.com/davecgh/go-spew/blob/master/LICENSE) +- github.com/dgrijalva/jwt-go [MIT LICENSE](https://github.com/dgrijalva/jwt-go/blob/master/LICENSE) +- github.com/dgryski/go-bits [MIT LICENSE](https://github.com/dgryski/go-bits/blob/master/LICENSE) +- github.com/dgryski/go-bitstream [MIT LICENSE](https://github.com/dgryski/go-bitstream/blob/master/LICENSE) +- github.com/glycerine/go-unsnap-stream [MIT LICENSE](https://github.com/glycerine/go-unsnap-stream/blob/master/LICENSE) +- github.com/gogo/protobuf/proto [BSD LICENSE](https://github.com/gogo/protobuf/blob/master/LICENSE) +- github.com/golang/protobuf [BSD LICENSE](https://github.com/golang/protobuf/blob/master/LICENSE) +- github.com/golang/snappy [BSD LICENSE](https://github.com/golang/snappy/blob/master/LICENSE) +- github.com/google/go-cmp [BSD LICENSE](https://github.com/google/go-cmp/blob/master/LICENSE) +- github.com/influxdata/influxql [MIT LICENSE](https://github.com/influxdata/influxql/blob/master/LICENSE) +- github.com/influxdata/usage-client [MIT LICENSE](https://github.com/influxdata/usage-client/blob/master/LICENSE.txt) +- github.com/influxdata/yamux [MOZILLA PUBLIC LICENSE](https://github.com/influxdata/yamux/blob/master/LICENSE) +- github.com/influxdata/yarpc [MIT LICENSE](https://github.com/influxdata/yarpc/blob/master/LICENSE) +- github.com/jsternberg/zap-logfmt [MIT LICENSE](https://github.com/jsternberg/zap-logfmt/blob/master/LICENSE) +- github.com/jwilder/encoding [MIT LICENSE](https://github.com/jwilder/encoding/blob/master/LICENSE) +- github.com/mattn/go-isatty [MIT LICENSE](https://github.com/mattn/go-isatty/blob/master/LICENSE) +- github.com/matttproud/golang_protobuf_extensions [APACHE LICENSE](https://github.com/matttproud/golang_protobuf_extensions/blob/master/LICENSE) +- github.com/opentracing/opentracing-go [MIT LICENSE](https://github.com/opentracing/opentracing-go/blob/master/LICENSE) +- github.com/paulbellamy/ratecounter [MIT LICENSE](https://github.com/paulbellamy/ratecounter/blob/master/LICENSE) +- github.com/peterh/liner [MIT LICENSE](https://github.com/peterh/liner/blob/master/COPYING) +- github.com/philhofer/fwd [MIT LICENSE](https://github.com/philhofer/fwd/blob/master/LICENSE.md) +- github.com/prometheus/client_golang [MIT LICENSE](https://github.com/prometheus/client_golang/blob/master/LICENSE) +- github.com/prometheus/client_model [MIT LICENSE](https://github.com/prometheus/client_model/blob/master/LICENSE) +- github.com/prometheus/common [APACHE LICENSE](https://github.com/prometheus/common/blob/master/LICENSE) +- github.com/prometheus/procfs [APACHE LICENSE](https://github.com/prometheus/procfs/blob/master/LICENSE) +- github.com/rakyll/statik [APACHE LICENSE](https://github.com/rakyll/statik/blob/master/LICENSE) +- github.com/retailnext/hllpp [BSD LICENSE](https://github.com/retailnext/hllpp/blob/master/LICENSE) +- github.com/tinylib/msgp [MIT LICENSE](https://github.com/tinylib/msgp/blob/master/LICENSE) +- go.uber.org/atomic [MIT LICENSE](https://github.com/uber-go/atomic/blob/master/LICENSE.txt) +- go.uber.org/multierr [MIT LICENSE](https://github.com/uber-go/multierr/blob/master/LICENSE.txt) +- go.uber.org/zap [MIT LICENSE](https://github.com/uber-go/zap/blob/master/LICENSE.txt) +- golang.org/x/crypto [BSD LICENSE](https://github.com/golang/crypto/blob/master/LICENSE) +- golang.org/x/net [BSD LICENSE](https://github.com/golang/net/blob/master/LICENSE) +- golang.org/x/sys [BSD LICENSE](https://github.com/golang/sys/blob/master/LICENSE) +- golang.org/x/text [BSD LICENSE](https://github.com/golang/text/blob/master/LICENSE) +- golang.org/x/time [BSD LICENSE](https://github.com/golang/time/blob/master/LICENSE) +- jquery 2.1.4 [MIT LICENSE](https://github.com/jquery/jquery/blob/master/LICENSE.txt) +- github.com/xlab/treeprint [MIT LICENSE](https://github.com/xlab/treeprint/blob/master/LICENSE) + + + + + + + + + + + + + + diff --git a/vendor/github.com/influxdata/influxdb/client/README.md b/vendor/github.com/influxdata/influxdb/client/README.md new file mode 100644 index 0000000000..773a111228 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/client/README.md @@ -0,0 +1,306 @@ +# InfluxDB Client + +[](http://godoc.org/github.com/influxdata/influxdb/client/v2) + +## Description + +**NOTE:** The Go client library now has a "v2" version, with the old version +being deprecated. The new version can be imported at +`import "github.com/influxdata/influxdb/client/v2"`. It is not backwards-compatible. + +A Go client library written and maintained by the **InfluxDB** team. +This package provides convenience functions to read and write time series data. +It uses the HTTP protocol to communicate with your **InfluxDB** cluster. + + +## Getting Started + +### Connecting To Your Database + +Connecting to an **InfluxDB** database is straightforward. You will need a host +name, a port and the cluster user credentials if applicable. The default port is +8086. You can customize these settings to your specific installation via the +**InfluxDB** configuration file. + +Though not necessary for experimentation, you may want to create a new user +and authenticate the connection to your database. + +For more information please check out the +[Admin Docs](https://docs.influxdata.com/influxdb/latest/administration/). + +For the impatient, you can create a new admin user _bubba_ by firing off the +[InfluxDB CLI](https://github.com/influxdata/influxdb/blob/master/cmd/influx/main.go). + +```shell +influx +> create user bubba with password 'bumblebeetuna' +> grant all privileges to bubba +``` + +And now for good measure set the credentials in you shell environment. +In the example below we will use $INFLUX_USER and $INFLUX_PWD + +Now with the administrivia out of the way, let's connect to our database. + +NOTE: If you've opted out of creating a user, you can omit Username and Password in +the configuration below. + +```go +package main + +import ( + "log" + "time" + + "github.com/influxdata/influxdb/client/v2" +) + +const ( + MyDB = "square_holes" + username = "bubba" + password = "bumblebeetuna" +) + + +func main() { + // Create a new HTTPClient + c, err := client.NewHTTPClient(client.HTTPConfig{ + Addr: "http://localhost:8086", + Username: username, + Password: password, + }) + if err != nil { + log.Fatal(err) + } + + // Create a new point batch + bp, err := client.NewBatchPoints(client.BatchPointsConfig{ + Database: MyDB, + Precision: "s", + }) + if err != nil { + log.Fatal(err) + } + + // Create a point and add to batch + tags := map[string]string{"cpu": "cpu-total"} + fields := map[string]interface{}{ + "idle": 10.1, + "system": 53.3, + "user": 46.6, + } + + pt, err := client.NewPoint("cpu_usage", tags, fields, time.Now()) + if err != nil { + log.Fatal(err) + } + bp.AddPoint(pt) + + // Write the batch + if err := c.Write(bp); err != nil { + log.Fatal(err) + } +} + +``` + +### Inserting Data + +Time series data aka *points* are written to the database using batch inserts. +The mechanism is to create one or more points and then create a batch aka +*batch points* and write these to a given database and series. A series is a +combination of a measurement (time/values) and a set of tags. + +In this sample we will create a batch of a 1,000 points. Each point has a time and +a single value as well as 2 tags indicating a shape and color. We write these points +to a database called _square_holes_ using a measurement named _shapes_. + +NOTE: You can specify a RetentionPolicy as part of the batch points. If not +provided InfluxDB will use the database _default_ retention policy. + +```go + +func writePoints(clnt client.Client) { + sampleSize := 1000 + + bp, err := client.NewBatchPoints(client.BatchPointsConfig{ + Database: "systemstats", + Precision: "us", + }) + if err != nil { + log.Fatal(err) + } + + rand.Seed(time.Now().UnixNano()) + for i := 0; i < sampleSize; i++ { + regions := []string{"us-west1", "us-west2", "us-west3", "us-east1"} + tags := map[string]string{ + "cpu": "cpu-total", + "host": fmt.Sprintf("host%d", rand.Intn(1000)), + "region": regions[rand.Intn(len(regions))], + } + + idle := rand.Float64() * 100.0 + fields := map[string]interface{}{ + "idle": idle, + "busy": 100.0 - idle, + } + + pt, err := client.NewPoint( + "cpu_usage", + tags, + fields, + time.Now(), + ) + if err != nil { + log.Fatal(err) + } + bp.AddPoint(pt) + } + + if err := clnt.Write(bp); err != nil { + log.Fatal(err) + } +} +``` + +#### Uint64 Support + +The `uint64` data type is supported if your server is version `1.4.0` or +greater. To write a data point as an unsigned integer, you must insert +the point as `uint64`. You cannot use `uint` or any of the other +derivatives because previous versions of the client have supported +writing those types as an integer. + +### Querying Data + +One nice advantage of using **InfluxDB** the ability to query your data using familiar +SQL constructs. In this example we can create a convenience function to query the database +as follows: + +```go +// queryDB convenience function to query the database +func queryDB(clnt client.Client, cmd string) (res []client.Result, err error) { + q := client.Query{ + Command: cmd, + Database: MyDB, + } + if response, err := clnt.Query(q); err == nil { + if response.Error() != nil { + return res, response.Error() + } + res = response.Results + } else { + return res, err + } + return res, nil +} +``` + +#### Creating a Database + +```go +_, err := queryDB(clnt, fmt.Sprintf("CREATE DATABASE %s", MyDB)) +if err != nil { + log.Fatal(err) +} +``` + +#### Count Records + +```go +q := fmt.Sprintf("SELECT count(%s) FROM %s", "value", MyMeasurement) +res, err := queryDB(clnt, q) +if err != nil { + log.Fatal(err) +} +count := res[0].Series[0].Values[0][1] +log.Printf("Found a total of %v records\n", count) +``` + +#### Find the last 10 _shapes_ records + +```go +q := fmt.Sprintf("SELECT * FROM %s LIMIT %d", MyMeasurement, 10) +res, err = queryDB(clnt, q) +if err != nil { + log.Fatal(err) +} + +for i, row := range res[0].Series[0].Values { + t, err := time.Parse(time.RFC3339, row[0].(string)) + if err != nil { + log.Fatal(err) + } + val := row[1].(string) + log.Printf("[%2d] %s: %s\n", i, t.Format(time.Stamp), val) +} +``` + +### Using the UDP Client + +The **InfluxDB** client also supports writing over UDP. + +```go +func WriteUDP() { + // Make client + c, err := client.NewUDPClient("localhost:8089") + if err != nil { + panic(err.Error()) + } + + // Create a new point batch + bp, _ := client.NewBatchPoints(client.BatchPointsConfig{ + Precision: "s", + }) + + // Create a point and add to batch + tags := map[string]string{"cpu": "cpu-total"} + fields := map[string]interface{}{ + "idle": 10.1, + "system": 53.3, + "user": 46.6, + } + pt, err := client.NewPoint("cpu_usage", tags, fields, time.Now()) + if err != nil { + panic(err.Error()) + } + bp.AddPoint(pt) + + // Write the batch + c.Write(bp) +} +``` + +### Point Splitting + +The UDP client now supports splitting single points that exceed the configured +payload size. The logic for processing each point is listed here, starting with +an empty payload. + +1. If adding the point to the current (non-empty) payload would exceed the + configured size, send the current payload. Otherwise, add it to the current + payload. +1. If the point is smaller than the configured size, add it to the payload. +1. If the point has no timestamp, just try to send the entire point as a single + UDP payload, and process the next point. +1. Since the point has a timestamp, re-use the existing measurement name, + tagset, and timestamp and create multiple new points by splitting up the + fields. The per-point length will be kept close to the configured size, + staying under it if possible. This does mean that one large field, maybe a + long string, could be sent as a larger-than-configured payload. + +The above logic attempts to respect configured payload sizes, but not sacrifice +any data integrity. Points without a timestamp can't be split, as that may +cause fields to have differing timestamps when processed by the server. + +## Go Docs + +Please refer to +[http://godoc.org/github.com/influxdata/influxdb/client/v2](http://godoc.org/github.com/influxdata/influxdb/client/v2) +for documentation. + +## See Also + +You can also examine how the client library is used by the +[InfluxDB CLI](https://github.com/influxdata/influxdb/blob/master/cmd/influx/main.go). diff --git a/vendor/github.com/influxdata/influxdb/client/influxdb.go b/vendor/github.com/influxdata/influxdb/client/influxdb.go new file mode 100644 index 0000000000..98d362d50a --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/client/influxdb.go @@ -0,0 +1,840 @@ +// Package client implements a now-deprecated client for InfluxDB; +// use github.com/influxdata/influxdb/client/v2 instead. +package client // import "github.com/influxdata/influxdb/client" + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/influxdata/influxdb/models" +) + +const ( + // DefaultHost is the default host used to connect to an InfluxDB instance + DefaultHost = "localhost" + + // DefaultPort is the default port used to connect to an InfluxDB instance + DefaultPort = 8086 + + // DefaultTimeout is the default connection timeout used to connect to an InfluxDB instance + DefaultTimeout = 0 +) + +// Query is used to send a command to the server. Both Command and Database are required. +type Query struct { + Command string + Database string + + // Chunked tells the server to send back chunked responses. This places + // less load on the server by sending back chunks of the response rather + // than waiting for the entire response all at once. + Chunked bool + + // ChunkSize sets the maximum number of rows that will be returned per + // chunk. Chunks are either divided based on their series or if they hit + // the chunk size limit. + // + // Chunked must be set to true for this option to be used. + ChunkSize int +} + +// ParseConnectionString will parse a string to create a valid connection URL +func ParseConnectionString(path string, ssl bool) (url.URL, error) { + var host string + var port int + + h, p, err := net.SplitHostPort(path) + if err != nil { + if path == "" { + host = DefaultHost + } else { + host = path + } + // If they didn't specify a port, always use the default port + port = DefaultPort + } else { + host = h + port, err = strconv.Atoi(p) + if err != nil { + return url.URL{}, fmt.Errorf("invalid port number %q: %s\n", path, err) + } + } + + u := url.URL{ + Scheme: "http", + } + if ssl { + u.Scheme = "https" + } + + u.Host = net.JoinHostPort(host, strconv.Itoa(port)) + + return u, nil +} + +// Config is used to specify what server to connect to. +// URL: The URL of the server connecting to. +// Username/Password are optional. They will be passed via basic auth if provided. +// UserAgent: If not provided, will default "InfluxDBClient", +// Timeout: If not provided, will default to 0 (no timeout) +type Config struct { + URL url.URL + UnixSocket string + Username string + Password string + UserAgent string + Timeout time.Duration + Precision string + WriteConsistency string + UnsafeSsl bool +} + +// NewConfig will create a config to be used in connecting to the client +func NewConfig() Config { + return Config{ + Timeout: DefaultTimeout, + } +} + +// Client is used to make calls to the server. +type Client struct { + url url.URL + unixSocket string + username string + password string + httpClient *http.Client + userAgent string + precision string +} + +const ( + // ConsistencyOne requires at least one data node acknowledged a write. + ConsistencyOne = "one" + + // ConsistencyAll requires all data nodes to acknowledge a write. + ConsistencyAll = "all" + + // ConsistencyQuorum requires a quorum of data nodes to acknowledge a write. + ConsistencyQuorum = "quorum" + + // ConsistencyAny allows for hinted hand off, potentially no write happened yet. + ConsistencyAny = "any" +) + +// NewClient will instantiate and return a connected client to issue commands to the server. +func NewClient(c Config) (*Client, error) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: c.UnsafeSsl, + } + + tr := &http.Transport{ + TLSClientConfig: tlsConfig, + } + + if c.UnixSocket != "" { + // No need for compression in local communications. + tr.DisableCompression = true + + tr.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", c.UnixSocket) + } + } + + client := Client{ + url: c.URL, + unixSocket: c.UnixSocket, + username: c.Username, + password: c.Password, + httpClient: &http.Client{Timeout: c.Timeout, Transport: tr}, + userAgent: c.UserAgent, + precision: c.Precision, + } + if client.userAgent == "" { + client.userAgent = "InfluxDBClient" + } + return &client, nil +} + +// SetAuth will update the username and passwords +func (c *Client) SetAuth(u, p string) { + c.username = u + c.password = p +} + +// SetPrecision will update the precision +func (c *Client) SetPrecision(precision string) { + c.precision = precision +} + +// Query sends a command to the server and returns the Response +func (c *Client) Query(q Query) (*Response, error) { + return c.QueryContext(context.Background(), q) +} + +// QueryContext sends a command to the server and returns the Response +// It uses a context that can be cancelled by the command line client +func (c *Client) QueryContext(ctx context.Context, q Query) (*Response, error) { + u := c.url + + u.Path = "query" + values := u.Query() + values.Set("q", q.Command) + values.Set("db", q.Database) + if q.Chunked { + values.Set("chunked", "true") + if q.ChunkSize > 0 { + values.Set("chunk_size", strconv.Itoa(q.ChunkSize)) + } + } + if c.precision != "" { + values.Set("epoch", c.precision) + } + u.RawQuery = values.Encode() + + req, err := http.NewRequest("POST", u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", c.userAgent) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + req = req.WithContext(ctx) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var response Response + if q.Chunked { + cr := NewChunkedResponse(resp.Body) + for { + r, err := cr.NextResponse() + if err != nil { + // If we got an error while decoding the response, send that back. + return nil, err + } + + if r == nil { + break + } + + response.Results = append(response.Results, r.Results...) + if r.Err != nil { + response.Err = r.Err + break + } + } + } else { + dec := json.NewDecoder(resp.Body) + dec.UseNumber() + if err := dec.Decode(&response); err != nil { + // Ignore EOF errors if we got an invalid status code. + if !(err == io.EOF && resp.StatusCode != http.StatusOK) { + return nil, err + } + } + } + + // If we don't have an error in our json response, and didn't get StatusOK, + // then send back an error. + if resp.StatusCode != http.StatusOK && response.Error() == nil { + return &response, fmt.Errorf("received status code %d from server", resp.StatusCode) + } + return &response, nil +} + +// Write takes BatchPoints and allows for writing of multiple points with defaults +// If successful, error is nil and Response is nil +// If an error occurs, Response may contain additional information if populated. +func (c *Client) Write(bp BatchPoints) (*Response, error) { + u := c.url + u.Path = "write" + + var b bytes.Buffer + for _, p := range bp.Points { + err := checkPointTypes(p) + if err != nil { + return nil, err + } + if p.Raw != "" { + if _, err := b.WriteString(p.Raw); err != nil { + return nil, err + } + } else { + for k, v := range bp.Tags { + if p.Tags == nil { + p.Tags = make(map[string]string, len(bp.Tags)) + } + p.Tags[k] = v + } + + if _, err := b.WriteString(p.MarshalString()); err != nil { + return nil, err + } + } + + if err := b.WriteByte('\n'); err != nil { + return nil, err + } + } + + req, err := http.NewRequest("POST", u.String(), &b) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "") + req.Header.Set("User-Agent", c.userAgent) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + precision := bp.Precision + if precision == "" { + precision = "ns" + } + + params := req.URL.Query() + params.Set("db", bp.Database) + params.Set("rp", bp.RetentionPolicy) + params.Set("precision", precision) + params.Set("consistency", bp.WriteConsistency) + req.URL.RawQuery = params.Encode() + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var response Response + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + var err = fmt.Errorf(string(body)) + response.Err = err + return &response, err + } + + return nil, nil +} + +// WriteLineProtocol takes a string with line returns to delimit each write +// If successful, error is nil and Response is nil +// If an error occurs, Response may contain additional information if populated. +func (c *Client) WriteLineProtocol(data, database, retentionPolicy, precision, writeConsistency string) (*Response, error) { + u := c.url + u.Path = "write" + + r := strings.NewReader(data) + + req, err := http.NewRequest("POST", u.String(), r) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "") + req.Header.Set("User-Agent", c.userAgent) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + params := req.URL.Query() + params.Set("db", database) + params.Set("rp", retentionPolicy) + params.Set("precision", precision) + params.Set("consistency", writeConsistency) + req.URL.RawQuery = params.Encode() + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var response Response + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + err := fmt.Errorf(string(body)) + response.Err = err + return &response, err + } + + return nil, nil +} + +// Ping will check to see if the server is up +// Ping returns how long the request took, the version of the server it connected to, and an error if one occurred. +func (c *Client) Ping() (time.Duration, string, error) { + now := time.Now() + u := c.url + u.Path = "ping" + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return 0, "", err + } + req.Header.Set("User-Agent", c.userAgent) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + + version := resp.Header.Get("X-Influxdb-Version") + return time.Since(now), version, nil +} + +// Structs + +// Message represents a user message. +type Message struct { + Level string `json:"level,omitempty"` + Text string `json:"text,omitempty"` +} + +// Result represents a resultset returned from a single statement. +type Result struct { + Series []models.Row + Messages []*Message + Err error +} + +// MarshalJSON encodes the result into JSON. +func (r *Result) MarshalJSON() ([]byte, error) { + // Define a struct that outputs "error" as a string. + var o struct { + Series []models.Row `json:"series,omitempty"` + Messages []*Message `json:"messages,omitempty"` + Err string `json:"error,omitempty"` + } + + // Copy fields to output struct. + o.Series = r.Series + o.Messages = r.Messages + if r.Err != nil { + o.Err = r.Err.Error() + } + + return json.Marshal(&o) +} + +// UnmarshalJSON decodes the data into the Result struct +func (r *Result) UnmarshalJSON(b []byte) error { + var o struct { + Series []models.Row `json:"series,omitempty"` + Messages []*Message `json:"messages,omitempty"` + Err string `json:"error,omitempty"` + } + + dec := json.NewDecoder(bytes.NewBuffer(b)) + dec.UseNumber() + err := dec.Decode(&o) + if err != nil { + return err + } + r.Series = o.Series + r.Messages = o.Messages + if o.Err != "" { + r.Err = errors.New(o.Err) + } + return nil +} + +// Response represents a list of statement results. +type Response struct { + Results []Result + Err error +} + +// MarshalJSON encodes the response into JSON. +func (r *Response) MarshalJSON() ([]byte, error) { + // Define a struct that outputs "error" as a string. + var o struct { + Results []Result `json:"results,omitempty"` + Err string `json:"error,omitempty"` + } + + // Copy fields to output struct. + o.Results = r.Results + if r.Err != nil { + o.Err = r.Err.Error() + } + + return json.Marshal(&o) +} + +// UnmarshalJSON decodes the data into the Response struct +func (r *Response) UnmarshalJSON(b []byte) error { + var o struct { + Results []Result `json:"results,omitempty"` + Err string `json:"error,omitempty"` + } + + dec := json.NewDecoder(bytes.NewBuffer(b)) + dec.UseNumber() + err := dec.Decode(&o) + if err != nil { + return err + } + r.Results = o.Results + if o.Err != "" { + r.Err = errors.New(o.Err) + } + return nil +} + +// Error returns the first error from any statement. +// Returns nil if no errors occurred on any statements. +func (r *Response) Error() error { + if r.Err != nil { + return r.Err + } + for _, result := range r.Results { + if result.Err != nil { + return result.Err + } + } + return nil +} + +// duplexReader reads responses and writes it to another writer while +// satisfying the reader interface. +type duplexReader struct { + r io.Reader + w io.Writer +} + +func (r *duplexReader) Read(p []byte) (n int, err error) { + n, err = r.r.Read(p) + if err == nil { + r.w.Write(p[:n]) + } + return n, err +} + +// ChunkedResponse represents a response from the server that +// uses chunking to stream the output. +type ChunkedResponse struct { + dec *json.Decoder + duplex *duplexReader + buf bytes.Buffer +} + +// NewChunkedResponse reads a stream and produces responses from the stream. +func NewChunkedResponse(r io.Reader) *ChunkedResponse { + resp := &ChunkedResponse{} + resp.duplex = &duplexReader{r: r, w: &resp.buf} + resp.dec = json.NewDecoder(resp.duplex) + resp.dec.UseNumber() + return resp +} + +// NextResponse reads the next line of the stream and returns a response. +func (r *ChunkedResponse) NextResponse() (*Response, error) { + var response Response + if err := r.dec.Decode(&response); err != nil { + if err == io.EOF { + return nil, nil + } + // A decoding error happened. This probably means the server crashed + // and sent a last-ditch error message to us. Ensure we have read the + // entirety of the connection to get any remaining error text. + io.Copy(ioutil.Discard, r.duplex) + return nil, errors.New(strings.TrimSpace(r.buf.String())) + } + r.buf.Reset() + return &response, nil +} + +// Point defines the fields that will be written to the database +// Measurement, Time, and Fields are required +// Precision can be specified if the time is in epoch format (integer). +// Valid values for Precision are n, u, ms, s, m, and h +type Point struct { + Measurement string + Tags map[string]string + Time time.Time + Fields map[string]interface{} + Precision string + Raw string +} + +// MarshalJSON will format the time in RFC3339Nano +// Precision is also ignored as it is only used for writing, not reading +// Or another way to say it is we always send back in nanosecond precision +func (p *Point) MarshalJSON() ([]byte, error) { + point := struct { + Measurement string `json:"measurement,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Time string `json:"time,omitempty"` + Fields map[string]interface{} `json:"fields,omitempty"` + Precision string `json:"precision,omitempty"` + }{ + Measurement: p.Measurement, + Tags: p.Tags, + Fields: p.Fields, + Precision: p.Precision, + } + // Let it omit empty if it's really zero + if !p.Time.IsZero() { + point.Time = p.Time.UTC().Format(time.RFC3339Nano) + } + return json.Marshal(&point) +} + +// MarshalString renders string representation of a Point with specified +// precision. The default precision is nanoseconds. +func (p *Point) MarshalString() string { + pt, err := models.NewPoint(p.Measurement, models.NewTags(p.Tags), p.Fields, p.Time) + if err != nil { + return "# ERROR: " + err.Error() + " " + p.Measurement + } + if p.Precision == "" || p.Precision == "ns" || p.Precision == "n" { + return pt.String() + } + return pt.PrecisionString(p.Precision) +} + +// UnmarshalJSON decodes the data into the Point struct +func (p *Point) UnmarshalJSON(b []byte) error { + var normal struct { + Measurement string `json:"measurement"` + Tags map[string]string `json:"tags"` + Time time.Time `json:"time"` + Precision string `json:"precision"` + Fields map[string]interface{} `json:"fields"` + } + var epoch struct { + Measurement string `json:"measurement"` + Tags map[string]string `json:"tags"` + Time *int64 `json:"time"` + Precision string `json:"precision"` + Fields map[string]interface{} `json:"fields"` + } + + if err := func() error { + var err error + dec := json.NewDecoder(bytes.NewBuffer(b)) + dec.UseNumber() + if err = dec.Decode(&epoch); err != nil { + return err + } + // Convert from epoch to time.Time, but only if Time + // was actually set. + var ts time.Time + if epoch.Time != nil { + ts, err = EpochToTime(*epoch.Time, epoch.Precision) + if err != nil { + return err + } + } + p.Measurement = epoch.Measurement + p.Tags = epoch.Tags + p.Time = ts + p.Precision = epoch.Precision + p.Fields = normalizeFields(epoch.Fields) + return nil + }(); err == nil { + return nil + } + + dec := json.NewDecoder(bytes.NewBuffer(b)) + dec.UseNumber() + if err := dec.Decode(&normal); err != nil { + return err + } + normal.Time = SetPrecision(normal.Time, normal.Precision) + p.Measurement = normal.Measurement + p.Tags = normal.Tags + p.Time = normal.Time + p.Precision = normal.Precision + p.Fields = normalizeFields(normal.Fields) + + return nil +} + +// Remove any notion of json.Number +func normalizeFields(fields map[string]interface{}) map[string]interface{} { + newFields := map[string]interface{}{} + + for k, v := range fields { + switch v := v.(type) { + case json.Number: + jv, e := v.Float64() + if e != nil { + panic(fmt.Sprintf("unable to convert json.Number to float64: %s", e)) + } + newFields[k] = jv + default: + newFields[k] = v + } + } + return newFields +} + +// BatchPoints is used to send batched data in a single write. +// Database and Points are required +// If no retention policy is specified, it will use the databases default retention policy. +// If tags are specified, they will be "merged" with all points. If a point already has that tag, it will be ignored. +// If time is specified, it will be applied to any point with an empty time. +// Precision can be specified if the time is in epoch format (integer). +// Valid values for Precision are n, u, ms, s, m, and h +type BatchPoints struct { + Points []Point `json:"points,omitempty"` + Database string `json:"database,omitempty"` + RetentionPolicy string `json:"retentionPolicy,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Time time.Time `json:"time,omitempty"` + Precision string `json:"precision,omitempty"` + WriteConsistency string `json:"-"` +} + +// UnmarshalJSON decodes the data into the BatchPoints struct +func (bp *BatchPoints) UnmarshalJSON(b []byte) error { + var normal struct { + Points []Point `json:"points"` + Database string `json:"database"` + RetentionPolicy string `json:"retentionPolicy"` + Tags map[string]string `json:"tags"` + Time time.Time `json:"time"` + Precision string `json:"precision"` + } + var epoch struct { + Points []Point `json:"points"` + Database string `json:"database"` + RetentionPolicy string `json:"retentionPolicy"` + Tags map[string]string `json:"tags"` + Time *int64 `json:"time"` + Precision string `json:"precision"` + } + + if err := func() error { + var err error + if err = json.Unmarshal(b, &epoch); err != nil { + return err + } + // Convert from epoch to time.Time + var ts time.Time + if epoch.Time != nil { + ts, err = EpochToTime(*epoch.Time, epoch.Precision) + if err != nil { + return err + } + } + bp.Points = epoch.Points + bp.Database = epoch.Database + bp.RetentionPolicy = epoch.RetentionPolicy + bp.Tags = epoch.Tags + bp.Time = ts + bp.Precision = epoch.Precision + return nil + }(); err == nil { + return nil + } + + if err := json.Unmarshal(b, &normal); err != nil { + return err + } + normal.Time = SetPrecision(normal.Time, normal.Precision) + bp.Points = normal.Points + bp.Database = normal.Database + bp.RetentionPolicy = normal.RetentionPolicy + bp.Tags = normal.Tags + bp.Time = normal.Time + bp.Precision = normal.Precision + + return nil +} + +// utility functions + +// Addr provides the current url as a string of the server the client is connected to. +func (c *Client) Addr() string { + if c.unixSocket != "" { + return c.unixSocket + } + return c.url.String() +} + +// checkPointTypes ensures no unsupported types are submitted to influxdb, returning error if they are found. +func checkPointTypes(p Point) error { + for _, v := range p.Fields { + switch v.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, bool, string, nil: + return nil + default: + return fmt.Errorf("unsupported point type: %T", v) + } + } + return nil +} + +// helper functions + +// EpochToTime takes a unix epoch time and uses precision to return back a time.Time +func EpochToTime(epoch int64, precision string) (time.Time, error) { + if precision == "" { + precision = "s" + } + var t time.Time + switch precision { + case "h": + t = time.Unix(0, epoch*int64(time.Hour)) + case "m": + t = time.Unix(0, epoch*int64(time.Minute)) + case "s": + t = time.Unix(0, epoch*int64(time.Second)) + case "ms": + t = time.Unix(0, epoch*int64(time.Millisecond)) + case "u": + t = time.Unix(0, epoch*int64(time.Microsecond)) + case "n": + t = time.Unix(0, epoch) + default: + return time.Time{}, fmt.Errorf("Unknown precision %q", precision) + } + return t, nil +} + +// SetPrecision will round a time to the specified precision +func SetPrecision(t time.Time, precision string) time.Time { + switch precision { + case "n": + case "u": + return t.Round(time.Microsecond) + case "ms": + return t.Round(time.Millisecond) + case "s": + return t.Round(time.Second) + case "m": + return t.Round(time.Minute) + case "h": + return t.Round(time.Hour) + } + return t +} diff --git a/vendor/github.com/influxdata/influxdb/client/v2/client.go b/vendor/github.com/influxdata/influxdb/client/v2/client.go new file mode 100644 index 0000000000..77d44f2b34 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/client/v2/client.go @@ -0,0 +1,635 @@ +// Package client (v2) is the current official Go client for InfluxDB. +package client // import "github.com/influxdata/influxdb/client/v2" + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/influxdata/influxdb/models" +) + +// HTTPConfig is the config data needed to create an HTTP Client. +type HTTPConfig struct { + // Addr should be of the form "http://host:port" + // or "http://[ipv6-host%zone]:port". + Addr string + + // Username is the influxdb username, optional. + Username string + + // Password is the influxdb password, optional. + Password string + + // UserAgent is the http User Agent, defaults to "InfluxDBClient". + UserAgent string + + // Timeout for influxdb writes, defaults to no timeout. + Timeout time.Duration + + // InsecureSkipVerify gets passed to the http client, if true, it will + // skip https certificate verification. Defaults to false. + InsecureSkipVerify bool + + // TLSConfig allows the user to set their own TLS config for the HTTP + // Client. If set, this option overrides InsecureSkipVerify. + TLSConfig *tls.Config +} + +// BatchPointsConfig is the config data needed to create an instance of the BatchPoints struct. +type BatchPointsConfig struct { + // Precision is the write precision of the points, defaults to "ns". + Precision string + + // Database is the database to write points to. + Database string + + // RetentionPolicy is the retention policy of the points. + RetentionPolicy string + + // Write consistency is the number of servers required to confirm write. + WriteConsistency string +} + +// Client is a client interface for writing & querying the database. +type Client interface { + // Ping checks that status of cluster, and will always return 0 time and no + // error for UDP clients. + Ping(timeout time.Duration) (time.Duration, string, error) + + // Write takes a BatchPoints object and writes all Points to InfluxDB. + Write(bp BatchPoints) error + + // Query makes an InfluxDB Query on the database. This will fail if using + // the UDP client. + Query(q Query) (*Response, error) + + // Close releases any resources a Client may be using. + Close() error +} + +// NewHTTPClient returns a new Client from the provided config. +// Client is safe for concurrent use by multiple goroutines. +func NewHTTPClient(conf HTTPConfig) (Client, error) { + if conf.UserAgent == "" { + conf.UserAgent = "InfluxDBClient" + } + + u, err := url.Parse(conf.Addr) + if err != nil { + return nil, err + } else if u.Scheme != "http" && u.Scheme != "https" { + m := fmt.Sprintf("Unsupported protocol scheme: %s, your address"+ + " must start with http:// or https://", u.Scheme) + return nil, errors.New(m) + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: conf.InsecureSkipVerify, + }, + } + if conf.TLSConfig != nil { + tr.TLSClientConfig = conf.TLSConfig + } + return &client{ + url: *u, + username: conf.Username, + password: conf.Password, + useragent: conf.UserAgent, + httpClient: &http.Client{ + Timeout: conf.Timeout, + Transport: tr, + }, + transport: tr, + }, nil +} + +// Ping will check to see if the server is up with an optional timeout on waiting for leader. +// Ping returns how long the request took, the version of the server it connected to, and an error if one occurred. +func (c *client) Ping(timeout time.Duration) (time.Duration, string, error) { + now := time.Now() + u := c.url + u.Path = "ping" + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return 0, "", err + } + + req.Header.Set("User-Agent", c.useragent) + + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + if timeout > 0 { + params := req.URL.Query() + params.Set("wait_for_leader", fmt.Sprintf("%.0fs", timeout.Seconds())) + req.URL.RawQuery = params.Encode() + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return 0, "", err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return 0, "", err + } + + if resp.StatusCode != http.StatusNoContent { + var err = fmt.Errorf(string(body)) + return 0, "", err + } + + version := resp.Header.Get("X-Influxdb-Version") + return time.Since(now), version, nil +} + +// Close releases the client's resources. +func (c *client) Close() error { + c.transport.CloseIdleConnections() + return nil +} + +// client is safe for concurrent use as the fields are all read-only +// once the client is instantiated. +type client struct { + // N.B - if url.UserInfo is accessed in future modifications to the + // methods on client, you will need to syncronise access to url. + url url.URL + username string + password string + useragent string + httpClient *http.Client + transport *http.Transport +} + +// BatchPoints is an interface into a batched grouping of points to write into +// InfluxDB together. BatchPoints is NOT thread-safe, you must create a separate +// batch for each goroutine. +type BatchPoints interface { + // AddPoint adds the given point to the Batch of points. + AddPoint(p *Point) + // AddPoints adds the given points to the Batch of points. + AddPoints(ps []*Point) + // Points lists the points in the Batch. + Points() []*Point + + // Precision returns the currently set precision of this Batch. + Precision() string + // SetPrecision sets the precision of this batch. + SetPrecision(s string) error + + // Database returns the currently set database of this Batch. + Database() string + // SetDatabase sets the database of this Batch. + SetDatabase(s string) + + // WriteConsistency returns the currently set write consistency of this Batch. + WriteConsistency() string + // SetWriteConsistency sets the write consistency of this Batch. + SetWriteConsistency(s string) + + // RetentionPolicy returns the currently set retention policy of this Batch. + RetentionPolicy() string + // SetRetentionPolicy sets the retention policy of this Batch. + SetRetentionPolicy(s string) +} + +// NewBatchPoints returns a BatchPoints interface based on the given config. +func NewBatchPoints(conf BatchPointsConfig) (BatchPoints, error) { + if conf.Precision == "" { + conf.Precision = "ns" + } + if _, err := time.ParseDuration("1" + conf.Precision); err != nil { + return nil, err + } + bp := &batchpoints{ + database: conf.Database, + precision: conf.Precision, + retentionPolicy: conf.RetentionPolicy, + writeConsistency: conf.WriteConsistency, + } + return bp, nil +} + +type batchpoints struct { + points []*Point + database string + precision string + retentionPolicy string + writeConsistency string +} + +func (bp *batchpoints) AddPoint(p *Point) { + bp.points = append(bp.points, p) +} + +func (bp *batchpoints) AddPoints(ps []*Point) { + bp.points = append(bp.points, ps...) +} + +func (bp *batchpoints) Points() []*Point { + return bp.points +} + +func (bp *batchpoints) Precision() string { + return bp.precision +} + +func (bp *batchpoints) Database() string { + return bp.database +} + +func (bp *batchpoints) WriteConsistency() string { + return bp.writeConsistency +} + +func (bp *batchpoints) RetentionPolicy() string { + return bp.retentionPolicy +} + +func (bp *batchpoints) SetPrecision(p string) error { + if _, err := time.ParseDuration("1" + p); err != nil { + return err + } + bp.precision = p + return nil +} + +func (bp *batchpoints) SetDatabase(db string) { + bp.database = db +} + +func (bp *batchpoints) SetWriteConsistency(wc string) { + bp.writeConsistency = wc +} + +func (bp *batchpoints) SetRetentionPolicy(rp string) { + bp.retentionPolicy = rp +} + +// Point represents a single data point. +type Point struct { + pt models.Point +} + +// NewPoint returns a point with the given timestamp. If a timestamp is not +// given, then data is sent to the database without a timestamp, in which case +// the server will assign local time upon reception. NOTE: it is recommended to +// send data with a timestamp. +func NewPoint( + name string, + tags map[string]string, + fields map[string]interface{}, + t ...time.Time, +) (*Point, error) { + var T time.Time + if len(t) > 0 { + T = t[0] + } + + pt, err := models.NewPoint(name, models.NewTags(tags), fields, T) + if err != nil { + return nil, err + } + return &Point{ + pt: pt, + }, nil +} + +// String returns a line-protocol string of the Point. +func (p *Point) String() string { + return p.pt.String() +} + +// PrecisionString returns a line-protocol string of the Point, +// with the timestamp formatted for the given precision. +func (p *Point) PrecisionString(precison string) string { + return p.pt.PrecisionString(precison) +} + +// Name returns the measurement name of the point. +func (p *Point) Name() string { + return string(p.pt.Name()) +} + +// Tags returns the tags associated with the point. +func (p *Point) Tags() map[string]string { + return p.pt.Tags().Map() +} + +// Time return the timestamp for the point. +func (p *Point) Time() time.Time { + return p.pt.Time() +} + +// UnixNano returns timestamp of the point in nanoseconds since Unix epoch. +func (p *Point) UnixNano() int64 { + return p.pt.UnixNano() +} + +// Fields returns the fields for the point. +func (p *Point) Fields() (map[string]interface{}, error) { + return p.pt.Fields() +} + +// NewPointFrom returns a point from the provided models.Point. +func NewPointFrom(pt models.Point) *Point { + return &Point{pt: pt} +} + +func (c *client) Write(bp BatchPoints) error { + var b bytes.Buffer + + for _, p := range bp.Points() { + if _, err := b.WriteString(p.pt.PrecisionString(bp.Precision())); err != nil { + return err + } + + if err := b.WriteByte('\n'); err != nil { + return err + } + } + + u := c.url + u.Path = "write" + req, err := http.NewRequest("POST", u.String(), &b) + if err != nil { + return err + } + req.Header.Set("Content-Type", "") + req.Header.Set("User-Agent", c.useragent) + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + params := req.URL.Query() + params.Set("db", bp.Database()) + params.Set("rp", bp.RetentionPolicy()) + params.Set("precision", bp.Precision()) + params.Set("consistency", bp.WriteConsistency()) + req.URL.RawQuery = params.Encode() + + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + var err = fmt.Errorf(string(body)) + return err + } + + return nil +} + +// Query defines a query to send to the server. +type Query struct { + Command string + Database string + Precision string + Chunked bool + ChunkSize int + Parameters map[string]interface{} +} + +// NewQuery returns a query object. +// The database and precision arguments can be empty strings if they are not needed for the query. +func NewQuery(command, database, precision string) Query { + return Query{ + Command: command, + Database: database, + Precision: precision, + Parameters: make(map[string]interface{}), + } +} + +// NewQueryWithParameters returns a query object. +// The database and precision arguments can be empty strings if they are not needed for the query. +// parameters is a map of the parameter names used in the command to their values. +func NewQueryWithParameters(command, database, precision string, parameters map[string]interface{}) Query { + return Query{ + Command: command, + Database: database, + Precision: precision, + Parameters: parameters, + } +} + +// Response represents a list of statement results. +type Response struct { + Results []Result + Err string `json:"error,omitempty"` +} + +// Error returns the first error from any statement. +// It returns nil if no errors occurred on any statements. +func (r *Response) Error() error { + if r.Err != "" { + return fmt.Errorf(r.Err) + } + for _, result := range r.Results { + if result.Err != "" { + return fmt.Errorf(result.Err) + } + } + return nil +} + +// Message represents a user message. +type Message struct { + Level string + Text string +} + +// Result represents a resultset returned from a single statement. +type Result struct { + Series []models.Row + Messages []*Message + Err string `json:"error,omitempty"` +} + +// Query sends a command to the server and returns the Response. +func (c *client) Query(q Query) (*Response, error) { + u := c.url + u.Path = "query" + + jsonParameters, err := json.Marshal(q.Parameters) + + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", u.String(), nil) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "") + req.Header.Set("User-Agent", c.useragent) + + if c.username != "" { + req.SetBasicAuth(c.username, c.password) + } + + params := req.URL.Query() + params.Set("q", q.Command) + params.Set("db", q.Database) + params.Set("params", string(jsonParameters)) + if q.Chunked { + params.Set("chunked", "true") + if q.ChunkSize > 0 { + params.Set("chunk_size", strconv.Itoa(q.ChunkSize)) + } + } + + if q.Precision != "" { + params.Set("epoch", q.Precision) + } + req.URL.RawQuery = params.Encode() + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // If we lack a X-Influxdb-Version header, then we didn't get a response from influxdb + // but instead some other service. If the error code is also a 500+ code, then some + // downstream loadbalancer/proxy/etc had an issue and we should report that. + if resp.Header.Get("X-Influxdb-Version") == "" && resp.StatusCode >= http.StatusInternalServerError { + body, err := ioutil.ReadAll(resp.Body) + if err != nil || len(body) == 0 { + return nil, fmt.Errorf("received status code %d from downstream server", resp.StatusCode) + } + + return nil, fmt.Errorf("received status code %d from downstream server, with response body: %q", resp.StatusCode, body) + } + + // If we get an unexpected content type, then it is also not from influx direct and therefore + // we want to know what we received and what status code was returned for debugging purposes. + if cType, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type")); cType != "application/json" { + // Read up to 1kb of the body to help identify downstream errors and limit the impact of things + // like downstream serving a large file + body, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1024)) + if err != nil || len(body) == 0 { + return nil, fmt.Errorf("expected json response, got empty body, with status: %v", resp.StatusCode) + } + + return nil, fmt.Errorf("expected json response, got %q, with status: %v and response body: %q", cType, resp.StatusCode, body) + } + + var response Response + if q.Chunked { + cr := NewChunkedResponse(resp.Body) + for { + r, err := cr.NextResponse() + if err != nil { + // If we got an error while decoding the response, send that back. + return nil, err + } + + if r == nil { + break + } + + response.Results = append(response.Results, r.Results...) + if r.Err != "" { + response.Err = r.Err + break + } + } + } else { + dec := json.NewDecoder(resp.Body) + dec.UseNumber() + decErr := dec.Decode(&response) + + // ignore this error if we got an invalid status code + if decErr != nil && decErr.Error() == "EOF" && resp.StatusCode != http.StatusOK { + decErr = nil + } + // If we got a valid decode error, send that back + if decErr != nil { + return nil, fmt.Errorf("unable to decode json: received status code %d err: %s", resp.StatusCode, decErr) + } + } + + // If we don't have an error in our json response, and didn't get statusOK + // then send back an error + if resp.StatusCode != http.StatusOK && response.Error() == nil { + return &response, fmt.Errorf("received status code %d from server", resp.StatusCode) + } + return &response, nil +} + +// duplexReader reads responses and writes it to another writer while +// satisfying the reader interface. +type duplexReader struct { + r io.Reader + w io.Writer +} + +func (r *duplexReader) Read(p []byte) (n int, err error) { + n, err = r.r.Read(p) + if err == nil { + r.w.Write(p[:n]) + } + return n, err +} + +// ChunkedResponse represents a response from the server that +// uses chunking to stream the output. +type ChunkedResponse struct { + dec *json.Decoder + duplex *duplexReader + buf bytes.Buffer +} + +// NewChunkedResponse reads a stream and produces responses from the stream. +func NewChunkedResponse(r io.Reader) *ChunkedResponse { + resp := &ChunkedResponse{} + resp.duplex = &duplexReader{r: r, w: &resp.buf} + resp.dec = json.NewDecoder(resp.duplex) + resp.dec.UseNumber() + return resp +} + +// NextResponse reads the next line of the stream and returns a response. +func (r *ChunkedResponse) NextResponse() (*Response, error) { + var response Response + + if err := r.dec.Decode(&response); err != nil { + if err == io.EOF { + return nil, nil + } + // A decoding error happened. This probably means the server crashed + // and sent a last-ditch error message to us. Ensure we have read the + // entirety of the connection to get any remaining error text. + io.Copy(ioutil.Discard, r.duplex) + return nil, errors.New(strings.TrimSpace(r.buf.String())) + } + + r.buf.Reset() + return &response, nil +} diff --git a/vendor/github.com/influxdata/influxdb/client/v2/udp.go b/vendor/github.com/influxdata/influxdb/client/v2/udp.go new file mode 100644 index 0000000000..779a28b33f --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/client/v2/udp.go @@ -0,0 +1,112 @@ +package client + +import ( + "fmt" + "io" + "net" + "time" +) + +const ( + // UDPPayloadSize is a reasonable default payload size for UDP packets that + // could be travelling over the internet. + UDPPayloadSize = 512 +) + +// UDPConfig is the config data needed to create a UDP Client. +type UDPConfig struct { + // Addr should be of the form "host:port" + // or "[ipv6-host%zone]:port". + Addr string + + // PayloadSize is the maximum size of a UDP client message, optional + // Tune this based on your network. Defaults to UDPPayloadSize. + PayloadSize int +} + +// NewUDPClient returns a client interface for writing to an InfluxDB UDP +// service from the given config. +func NewUDPClient(conf UDPConfig) (Client, error) { + var udpAddr *net.UDPAddr + udpAddr, err := net.ResolveUDPAddr("udp", conf.Addr) + if err != nil { + return nil, err + } + + conn, err := net.DialUDP("udp", nil, udpAddr) + if err != nil { + return nil, err + } + + payloadSize := conf.PayloadSize + if payloadSize == 0 { + payloadSize = UDPPayloadSize + } + + return &udpclient{ + conn: conn, + payloadSize: payloadSize, + }, nil +} + +// Close releases the udpclient's resources. +func (uc *udpclient) Close() error { + return uc.conn.Close() +} + +type udpclient struct { + conn io.WriteCloser + payloadSize int +} + +func (uc *udpclient) Write(bp BatchPoints) error { + var b = make([]byte, 0, uc.payloadSize) // initial buffer size, it will grow as needed + var d, _ = time.ParseDuration("1" + bp.Precision()) + + var delayedError error + + var checkBuffer = func(n int) { + if len(b) > 0 && len(b)+n > uc.payloadSize { + if _, err := uc.conn.Write(b); err != nil { + delayedError = err + } + b = b[:0] + } + } + + for _, p := range bp.Points() { + p.pt.Round(d) + pointSize := p.pt.StringSize() + 1 // include newline in size + //point := p.pt.RoundedString(d) + "\n" + + checkBuffer(pointSize) + + if p.Time().IsZero() || pointSize <= uc.payloadSize { + b = p.pt.AppendString(b) + b = append(b, '\n') + continue + } + + points := p.pt.Split(uc.payloadSize - 1) // account for newline character + for _, sp := range points { + checkBuffer(sp.StringSize() + 1) + b = sp.AppendString(b) + b = append(b, '\n') + } + } + + if len(b) > 0 { + if _, err := uc.conn.Write(b); err != nil { + return err + } + } + return delayedError +} + +func (uc *udpclient) Query(q Query) (*Response, error) { + return nil, fmt.Errorf("Querying via UDP is not supported") +} + +func (uc *udpclient) Ping(timeout time.Duration) (time.Duration, string, error) { + return 0, "", nil +} diff --git a/vendor/github.com/influxdata/influxdb/models/consistency.go b/vendor/github.com/influxdata/influxdb/models/consistency.go new file mode 100644 index 0000000000..2a3269bca1 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/consistency.go @@ -0,0 +1,48 @@ +package models + +import ( + "errors" + "strings" +) + +// ConsistencyLevel represent a required replication criteria before a write can +// be returned as successful. +// +// The consistency level is handled in open-source InfluxDB but only applicable to clusters. +type ConsistencyLevel int + +const ( + // ConsistencyLevelAny allows for hinted handoff, potentially no write happened yet. + ConsistencyLevelAny ConsistencyLevel = iota + + // ConsistencyLevelOne requires at least one data node acknowledged a write. + ConsistencyLevelOne + + // ConsistencyLevelQuorum requires a quorum of data nodes to acknowledge a write. + ConsistencyLevelQuorum + + // ConsistencyLevelAll requires all data nodes to acknowledge a write. + ConsistencyLevelAll +) + +var ( + // ErrInvalidConsistencyLevel is returned when parsing the string version + // of a consistency level. + ErrInvalidConsistencyLevel = errors.New("invalid consistency level") +) + +// ParseConsistencyLevel converts a consistency level string to the corresponding ConsistencyLevel const. +func ParseConsistencyLevel(level string) (ConsistencyLevel, error) { + switch strings.ToLower(level) { + case "any": + return ConsistencyLevelAny, nil + case "one": + return ConsistencyLevelOne, nil + case "quorum": + return ConsistencyLevelQuorum, nil + case "all": + return ConsistencyLevelAll, nil + default: + return 0, ErrInvalidConsistencyLevel + } +} diff --git a/vendor/github.com/influxdata/influxdb/models/inline_fnv.go b/vendor/github.com/influxdata/influxdb/models/inline_fnv.go new file mode 100644 index 0000000000..eec1ae8b01 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/inline_fnv.go @@ -0,0 +1,32 @@ +package models // import "github.com/influxdata/influxdb/models" + +// from stdlib hash/fnv/fnv.go +const ( + prime64 = 1099511628211 + offset64 = 14695981039346656037 +) + +// InlineFNV64a is an alloc-free port of the standard library's fnv64a. +// See https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function. +type InlineFNV64a uint64 + +// NewInlineFNV64a returns a new instance of InlineFNV64a. +func NewInlineFNV64a() InlineFNV64a { + return offset64 +} + +// Write adds data to the running hash. +func (s *InlineFNV64a) Write(data []byte) (int, error) { + hash := uint64(*s) + for _, c := range data { + hash ^= uint64(c) + hash *= prime64 + } + *s = InlineFNV64a(hash) + return len(data), nil +} + +// Sum64 returns the uint64 of the current resulting hash. +func (s *InlineFNV64a) Sum64() uint64 { + return uint64(*s) +} diff --git a/vendor/github.com/influxdata/influxdb/models/inline_strconv_parse.go b/vendor/github.com/influxdata/influxdb/models/inline_strconv_parse.go new file mode 100644 index 0000000000..8db4837384 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/inline_strconv_parse.go @@ -0,0 +1,44 @@ +package models // import "github.com/influxdata/influxdb/models" + +import ( + "reflect" + "strconv" + "unsafe" +) + +// parseIntBytes is a zero-alloc wrapper around strconv.ParseInt. +func parseIntBytes(b []byte, base int, bitSize int) (i int64, err error) { + s := unsafeBytesToString(b) + return strconv.ParseInt(s, base, bitSize) +} + +// parseUintBytes is a zero-alloc wrapper around strconv.ParseUint. +func parseUintBytes(b []byte, base int, bitSize int) (i uint64, err error) { + s := unsafeBytesToString(b) + return strconv.ParseUint(s, base, bitSize) +} + +// parseFloatBytes is a zero-alloc wrapper around strconv.ParseFloat. +func parseFloatBytes(b []byte, bitSize int) (float64, error) { + s := unsafeBytesToString(b) + return strconv.ParseFloat(s, bitSize) +} + +// parseBoolBytes is a zero-alloc wrapper around strconv.ParseBool. +func parseBoolBytes(b []byte) (bool, error) { + return strconv.ParseBool(unsafeBytesToString(b)) +} + +// unsafeBytesToString converts a []byte to a string without a heap allocation. +// +// It is unsafe, and is intended to prepare input to short-lived functions +// that require strings. +func unsafeBytesToString(in []byte) string { + src := *(*reflect.SliceHeader)(unsafe.Pointer(&in)) + dst := reflect.StringHeader{ + Data: src.Data, + Len: src.Len, + } + s := *(*string)(unsafe.Pointer(&dst)) + return s +} diff --git a/vendor/github.com/influxdata/influxdb/models/points.go b/vendor/github.com/influxdata/influxdb/models/points.go new file mode 100644 index 0000000000..ad80a816bf --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/points.go @@ -0,0 +1,2337 @@ +// Package models implements basic objects used throughout the TICK stack. +package models // import "github.com/influxdata/influxdb/models" + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + "time" + + "github.com/influxdata/influxdb/pkg/escape" +) + +var ( + measurementEscapeCodes = map[byte][]byte{ + ',': []byte(`\,`), + ' ': []byte(`\ `), + } + + tagEscapeCodes = map[byte][]byte{ + ',': []byte(`\,`), + ' ': []byte(`\ `), + '=': []byte(`\=`), + } + + // ErrPointMustHaveAField is returned when operating on a point that does not have any fields. + ErrPointMustHaveAField = errors.New("point without fields is unsupported") + + // ErrInvalidNumber is returned when a number is expected but not provided. + ErrInvalidNumber = errors.New("invalid number") + + // ErrInvalidPoint is returned when a point cannot be parsed correctly. + ErrInvalidPoint = errors.New("point is invalid") +) + +const ( + // MaxKeyLength is the largest allowed size of the combined measurement and tag keys. + MaxKeyLength = 65535 +) + +// enableUint64Support will enable uint64 support if set to true. +var enableUint64Support = false + +// EnableUintSupport manually enables uint support for the point parser. +// This function will be removed in the future and only exists for unit tests during the +// transition. +func EnableUintSupport() { + enableUint64Support = true +} + +// Point defines the values that will be written to the database. +type Point interface { + // Name return the measurement name for the point. + Name() []byte + + // SetName updates the measurement name for the point. + SetName(string) + + // Tags returns the tag set for the point. + Tags() Tags + + // AddTag adds or replaces a tag value for a point. + AddTag(key, value string) + + // SetTags replaces the tags for the point. + SetTags(tags Tags) + + // HasTag returns true if the tag exists for the point. + HasTag(tag []byte) bool + + // Fields returns the fields for the point. + Fields() (Fields, error) + + // Time return the timestamp for the point. + Time() time.Time + + // SetTime updates the timestamp for the point. + SetTime(t time.Time) + + // UnixNano returns the timestamp of the point as nanoseconds since Unix epoch. + UnixNano() int64 + + // HashID returns a non-cryptographic checksum of the point's key. + HashID() uint64 + + // Key returns the key (measurement joined with tags) of the point. + Key() []byte + + // String returns a string representation of the point. If there is a + // timestamp associated with the point then it will be specified with the default + // precision of nanoseconds. + String() string + + // MarshalBinary returns a binary representation of the point. + MarshalBinary() ([]byte, error) + + // PrecisionString returns a string representation of the point. If there + // is a timestamp associated with the point then it will be specified in the + // given unit. + PrecisionString(precision string) string + + // RoundedString returns a string representation of the point. If there + // is a timestamp associated with the point, then it will be rounded to the + // given duration. + RoundedString(d time.Duration) string + + // Split will attempt to return multiple points with the same timestamp whose + // string representations are no longer than size. Points with a single field or + // a point without a timestamp may exceed the requested size. + Split(size int) []Point + + // Round will round the timestamp of the point to the given duration. + Round(d time.Duration) + + // StringSize returns the length of the string that would be returned by String(). + StringSize() int + + // AppendString appends the result of String() to the provided buffer and returns + // the result, potentially reducing string allocations. + AppendString(buf []byte) []byte + + // FieldIterator retuns a FieldIterator that can be used to traverse the + // fields of a point without constructing the in-memory map. + FieldIterator() FieldIterator +} + +// FieldType represents the type of a field. +type FieldType int + +const ( + // Integer indicates the field's type is integer. + Integer FieldType = iota + + // Float indicates the field's type is float. + Float + + // Boolean indicates the field's type is boolean. + Boolean + + // String indicates the field's type is string. + String + + // Empty is used to indicate that there is no field. + Empty + + // Unsigned indicates the field's type is an unsigned integer. + Unsigned +) + +// FieldIterator provides a low-allocation interface to iterate through a point's fields. +type FieldIterator interface { + // Next indicates whether there any fields remaining. + Next() bool + + // FieldKey returns the key of the current field. + FieldKey() []byte + + // Type returns the FieldType of the current field. + Type() FieldType + + // StringValue returns the string value of the current field. + StringValue() string + + // IntegerValue returns the integer value of the current field. + IntegerValue() (int64, error) + + // UnsignedValue returns the unsigned value of the current field. + UnsignedValue() (uint64, error) + + // BooleanValue returns the boolean value of the current field. + BooleanValue() (bool, error) + + // FloatValue returns the float value of the current field. + FloatValue() (float64, error) + + // Reset resets the iterator to its initial state. + Reset() +} + +// Points represents a sortable list of points by timestamp. +type Points []Point + +// Len implements sort.Interface. +func (a Points) Len() int { return len(a) } + +// Less implements sort.Interface. +func (a Points) Less(i, j int) bool { return a[i].Time().Before(a[j].Time()) } + +// Swap implements sort.Interface. +func (a Points) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// point is the default implementation of Point. +type point struct { + time time.Time + + // text encoding of measurement and tags + // key must always be stored sorted by tags, if the original line was not sorted, + // we need to resort it + key []byte + + // text encoding of field data + fields []byte + + // text encoding of timestamp + ts []byte + + // cached version of parsed fields from data + cachedFields map[string]interface{} + + // cached version of parsed name from key + cachedName string + + // cached version of parsed tags + cachedTags Tags + + it fieldIterator +} + +// type assertions +var ( + _ Point = (*point)(nil) + _ FieldIterator = (*point)(nil) +) + +const ( + // the number of characters for the largest possible int64 (9223372036854775807) + maxInt64Digits = 19 + + // the number of characters for the smallest possible int64 (-9223372036854775808) + minInt64Digits = 20 + + // the number of characters for the largest possible uint64 (18446744073709551615) + maxUint64Digits = 20 + + // the number of characters required for the largest float64 before a range check + // would occur during parsing + maxFloat64Digits = 25 + + // the number of characters required for smallest float64 before a range check occur + // would occur during parsing + minFloat64Digits = 27 +) + +// ParsePoints returns a slice of Points from a text representation of a point +// with each point separated by newlines. If any points fail to parse, a non-nil error +// will be returned in addition to the points that parsed successfully. +func ParsePoints(buf []byte) ([]Point, error) { + return ParsePointsWithPrecision(buf, time.Now().UTC(), "n") +} + +// ParsePointsString is identical to ParsePoints but accepts a string. +func ParsePointsString(buf string) ([]Point, error) { + return ParsePoints([]byte(buf)) +} + +// ParseKey returns the measurement name and tags from a point. +// +// NOTE: to minimize heap allocations, the returned Tags will refer to subslices of buf. +// This can have the unintended effect preventing buf from being garbage collected. +func ParseKey(buf []byte) (string, Tags) { + meas, tags := ParseKeyBytes(buf) + return string(meas), tags +} + +func ParseKeyBytes(buf []byte) ([]byte, Tags) { + // Ignore the error because scanMeasurement returns "missing fields" which we ignore + // when just parsing a key + state, i, _ := scanMeasurement(buf, 0) + + var tags Tags + if state == tagKeyState { + tags = parseTags(buf) + // scanMeasurement returns the location of the comma if there are tags, strip that off + return buf[:i-1], tags + } + return buf[:i], tags +} + +func ParseTags(buf []byte) Tags { + return parseTags(buf) +} + +func ParseName(buf []byte) ([]byte, error) { + // Ignore the error because scanMeasurement returns "missing fields" which we ignore + // when just parsing a key + state, i, _ := scanMeasurement(buf, 0) + if state == tagKeyState { + return buf[:i-1], nil + } + return buf[:i], nil +} + +// ParsePointsWithPrecision is similar to ParsePoints, but allows the +// caller to provide a precision for time. +// +// NOTE: to minimize heap allocations, the returned Points will refer to subslices of buf. +// This can have the unintended effect preventing buf from being garbage collected. +func ParsePointsWithPrecision(buf []byte, defaultTime time.Time, precision string) ([]Point, error) { + points := make([]Point, 0, bytes.Count(buf, []byte{'\n'})+1) + var ( + pos int + block []byte + failed []string + ) + for pos < len(buf) { + pos, block = scanLine(buf, pos) + pos++ + + if len(block) == 0 { + continue + } + + // lines which start with '#' are comments + start := skipWhitespace(block, 0) + + // If line is all whitespace, just skip it + if start >= len(block) { + continue + } + + if block[start] == '#' { + continue + } + + // strip the newline if one is present + if block[len(block)-1] == '\n' { + block = block[:len(block)-1] + } + + pt, err := parsePoint(block[start:], defaultTime, precision) + if err != nil { + failed = append(failed, fmt.Sprintf("unable to parse '%s': %v", string(block[start:]), err)) + } else { + points = append(points, pt) + } + + } + if len(failed) > 0 { + return points, fmt.Errorf("%s", strings.Join(failed, "\n")) + } + return points, nil + +} + +func parsePoint(buf []byte, defaultTime time.Time, precision string) (Point, error) { + // scan the first block which is measurement[,tag1=value1,tag2=value=2...] + pos, key, err := scanKey(buf, 0) + if err != nil { + return nil, err + } + + // measurement name is required + if len(key) == 0 { + return nil, fmt.Errorf("missing measurement") + } + + if len(key) > MaxKeyLength { + return nil, fmt.Errorf("max key length exceeded: %v > %v", len(key), MaxKeyLength) + } + + // scan the second block is which is field1=value1[,field2=value2,...] + pos, fields, err := scanFields(buf, pos) + if err != nil { + return nil, err + } + + // at least one field is required + if len(fields) == 0 { + return nil, fmt.Errorf("missing fields") + } + + var maxKeyErr error + walkFields(fields, func(k, v []byte) bool { + if sz := seriesKeySize(key, k); sz > MaxKeyLength { + maxKeyErr = fmt.Errorf("max key length exceeded: %v > %v", sz, MaxKeyLength) + return false + } + return true + }) + + if maxKeyErr != nil { + return nil, maxKeyErr + } + + // scan the last block which is an optional integer timestamp + pos, ts, err := scanTime(buf, pos) + if err != nil { + return nil, err + } + + pt := &point{ + key: key, + fields: fields, + ts: ts, + } + + if len(ts) == 0 { + pt.time = defaultTime + pt.SetPrecision(precision) + } else { + ts, err := parseIntBytes(ts, 10, 64) + if err != nil { + return nil, err + } + pt.time, err = SafeCalcTime(ts, precision) + if err != nil { + return nil, err + } + + // Determine if there are illegal non-whitespace characters after the + // timestamp block. + for pos < len(buf) { + if buf[pos] != ' ' { + return nil, ErrInvalidPoint + } + pos++ + } + } + return pt, nil +} + +// GetPrecisionMultiplier will return a multiplier for the precision specified. +func GetPrecisionMultiplier(precision string) int64 { + d := time.Nanosecond + switch precision { + case "u": + d = time.Microsecond + case "ms": + d = time.Millisecond + case "s": + d = time.Second + case "m": + d = time.Minute + case "h": + d = time.Hour + } + return int64(d) +} + +// scanKey scans buf starting at i for the measurement and tag portion of the point. +// It returns the ending position and the byte slice of key within buf. If there +// are tags, they will be sorted if they are not already. +func scanKey(buf []byte, i int) (int, []byte, error) { + start := skipWhitespace(buf, i) + + i = start + + // Determines whether the tags are sort, assume they are + sorted := true + + // indices holds the indexes within buf of the start of each tag. For example, + // a buf of 'cpu,host=a,region=b,zone=c' would have indices slice of [4,11,20] + // which indicates that the first tag starts at buf[4], seconds at buf[11], and + // last at buf[20] + indices := make([]int, 100) + + // tracks how many commas we've seen so we know how many values are indices. + // Since indices is an arbitrarily large slice, + // we need to know how many values in the buffer are in use. + commas := 0 + + // First scan the Point's measurement. + state, i, err := scanMeasurement(buf, i) + if err != nil { + return i, buf[start:i], err + } + + // Optionally scan tags if needed. + if state == tagKeyState { + i, commas, indices, err = scanTags(buf, i, indices) + if err != nil { + return i, buf[start:i], err + } + } + + // Now we know where the key region is within buf, and the location of tags, we + // need to determine if duplicate tags exist and if the tags are sorted. This iterates + // over the list comparing each tag in the sequence with each other. + for j := 0; j < commas-1; j++ { + // get the left and right tags + _, left := scanTo(buf[indices[j]:indices[j+1]-1], 0, '=') + _, right := scanTo(buf[indices[j+1]:indices[j+2]-1], 0, '=') + + // If left is greater than right, the tags are not sorted. We do not have to + // continue because the short path no longer works. + // If the tags are equal, then there are duplicate tags, and we should abort. + // If the tags are not sorted, this pass may not find duplicate tags and we + // need to do a more exhaustive search later. + if cmp := bytes.Compare(left, right); cmp > 0 { + sorted = false + break + } else if cmp == 0 { + return i, buf[start:i], fmt.Errorf("duplicate tags") + } + } + + // If the tags are not sorted, then sort them. This sort is inline and + // uses the tag indices we created earlier. The actual buffer is not sorted, the + // indices are using the buffer for value comparison. After the indices are sorted, + // the buffer is reconstructed from the sorted indices. + if !sorted && commas > 0 { + // Get the measurement name for later + measurement := buf[start : indices[0]-1] + + // Sort the indices + indices := indices[:commas] + insertionSort(0, commas, buf, indices) + + // Create a new key using the measurement and sorted indices + b := make([]byte, len(buf[start:i])) + pos := copy(b, measurement) + for _, i := range indices { + b[pos] = ',' + pos++ + _, v := scanToSpaceOr(buf, i, ',') + pos += copy(b[pos:], v) + } + + // Check again for duplicate tags now that the tags are sorted. + for j := 0; j < commas-1; j++ { + // get the left and right tags + _, left := scanTo(buf[indices[j]:], 0, '=') + _, right := scanTo(buf[indices[j+1]:], 0, '=') + + // If the tags are equal, then there are duplicate tags, and we should abort. + // If the tags are not sorted, this pass may not find duplicate tags and we + // need to do a more exhaustive search later. + if bytes.Equal(left, right) { + return i, b, fmt.Errorf("duplicate tags") + } + } + + return i, b, nil + } + + return i, buf[start:i], nil +} + +// The following constants allow us to specify which state to move to +// next, when scanning sections of a Point. +const ( + tagKeyState = iota + tagValueState + fieldsState +) + +// scanMeasurement examines the measurement part of a Point, returning +// the next state to move to, and the current location in the buffer. +func scanMeasurement(buf []byte, i int) (int, int, error) { + // Check first byte of measurement, anything except a comma is fine. + // It can't be a space, since whitespace is stripped prior to this + // function call. + if i >= len(buf) || buf[i] == ',' { + return -1, i, fmt.Errorf("missing measurement") + } + + for { + i++ + if i >= len(buf) { + // cpu + return -1, i, fmt.Errorf("missing fields") + } + + if buf[i-1] == '\\' { + // Skip character (it's escaped). + continue + } + + // Unescaped comma; move onto scanning the tags. + if buf[i] == ',' { + return tagKeyState, i + 1, nil + } + + // Unescaped space; move onto scanning the fields. + if buf[i] == ' ' { + // cpu value=1.0 + return fieldsState, i, nil + } + } +} + +// scanTags examines all the tags in a Point, keeping track of and +// returning the updated indices slice, number of commas and location +// in buf where to start examining the Point fields. +func scanTags(buf []byte, i int, indices []int) (int, int, []int, error) { + var ( + err error + commas int + state = tagKeyState + ) + + for { + switch state { + case tagKeyState: + // Grow our indices slice if we have too many tags. + if commas >= len(indices) { + newIndics := make([]int, cap(indices)*2) + copy(newIndics, indices) + indices = newIndics + } + indices[commas] = i + commas++ + + i, err = scanTagsKey(buf, i) + state = tagValueState // tag value always follows a tag key + case tagValueState: + state, i, err = scanTagsValue(buf, i) + case fieldsState: + indices[commas] = i + 1 + return i, commas, indices, nil + } + + if err != nil { + return i, commas, indices, err + } + } +} + +// scanTagsKey scans each character in a tag key. +func scanTagsKey(buf []byte, i int) (int, error) { + // First character of the key. + if i >= len(buf) || buf[i] == ' ' || buf[i] == ',' || buf[i] == '=' { + // cpu,{'', ' ', ',', '='} + return i, fmt.Errorf("missing tag key") + } + + // Examine each character in the tag key until we hit an unescaped + // equals (the tag value), or we hit an error (i.e., unescaped + // space or comma). + for { + i++ + + // Either we reached the end of the buffer or we hit an + // unescaped comma or space. + if i >= len(buf) || + ((buf[i] == ' ' || buf[i] == ',') && buf[i-1] != '\\') { + // cpu,tag{'', ' ', ','} + return i, fmt.Errorf("missing tag value") + } + + if buf[i] == '=' && buf[i-1] != '\\' { + // cpu,tag= + return i + 1, nil + } + } +} + +// scanTagsValue scans each character in a tag value. +func scanTagsValue(buf []byte, i int) (int, int, error) { + // Tag value cannot be empty. + if i >= len(buf) || buf[i] == ',' || buf[i] == ' ' { + // cpu,tag={',', ' '} + return -1, i, fmt.Errorf("missing tag value") + } + + // Examine each character in the tag value until we hit an unescaped + // comma (move onto next tag key), an unescaped space (move onto + // fields), or we error out. + for { + i++ + if i >= len(buf) { + // cpu,tag=value + return -1, i, fmt.Errorf("missing fields") + } + + // An unescaped equals sign is an invalid tag value. + if buf[i] == '=' && buf[i-1] != '\\' { + // cpu,tag={'=', 'fo=o'} + return -1, i, fmt.Errorf("invalid tag format") + } + + if buf[i] == ',' && buf[i-1] != '\\' { + // cpu,tag=foo, + return tagKeyState, i + 1, nil + } + + // cpu,tag=foo value=1.0 + // cpu, tag=foo\= value=1.0 + if buf[i] == ' ' && buf[i-1] != '\\' { + return fieldsState, i, nil + } + } +} + +func insertionSort(l, r int, buf []byte, indices []int) { + for i := l + 1; i < r; i++ { + for j := i; j > l && less(buf, indices, j, j-1); j-- { + indices[j], indices[j-1] = indices[j-1], indices[j] + } + } +} + +func less(buf []byte, indices []int, i, j int) bool { + // This grabs the tag names for i & j, it ignores the values + _, a := scanTo(buf, indices[i], '=') + _, b := scanTo(buf, indices[j], '=') + return bytes.Compare(a, b) < 0 +} + +// scanFields scans buf, starting at i for the fields section of a point. It returns +// the ending position and the byte slice of the fields within buf. +func scanFields(buf []byte, i int) (int, []byte, error) { + start := skipWhitespace(buf, i) + i = start + quoted := false + + // tracks how many '=' we've seen + equals := 0 + + // tracks how many commas we've seen + commas := 0 + + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // escaped characters? + if buf[i] == '\\' && i+1 < len(buf) { + i += 2 + continue + } + + // If the value is quoted, scan until we get to the end quote + // Only quote values in the field value since quotes are not significant + // in the field key + if buf[i] == '"' && equals > commas { + quoted = !quoted + i++ + continue + } + + // If we see an =, ensure that there is at least on char before and after it + if buf[i] == '=' && !quoted { + equals++ + + // check for "... =123" but allow "a\ =123" + if buf[i-1] == ' ' && buf[i-2] != '\\' { + return i, buf[start:i], fmt.Errorf("missing field key") + } + + // check for "...a=123,=456" but allow "a=123,a\,=456" + if buf[i-1] == ',' && buf[i-2] != '\\' { + return i, buf[start:i], fmt.Errorf("missing field key") + } + + // check for "... value=" + if i+1 >= len(buf) { + return i, buf[start:i], fmt.Errorf("missing field value") + } + + // check for "... value=,value2=..." + if buf[i+1] == ',' || buf[i+1] == ' ' { + return i, buf[start:i], fmt.Errorf("missing field value") + } + + if isNumeric(buf[i+1]) || buf[i+1] == '-' || buf[i+1] == 'N' || buf[i+1] == 'n' { + var err error + i, err = scanNumber(buf, i+1) + if err != nil { + return i, buf[start:i], err + } + continue + } + // If next byte is not a double-quote, the value must be a boolean + if buf[i+1] != '"' { + var err error + i, _, err = scanBoolean(buf, i+1) + if err != nil { + return i, buf[start:i], err + } + continue + } + } + + if buf[i] == ',' && !quoted { + commas++ + } + + // reached end of block? + if buf[i] == ' ' && !quoted { + break + } + i++ + } + + if quoted { + return i, buf[start:i], fmt.Errorf("unbalanced quotes") + } + + // check that all field sections had key and values (e.g. prevent "a=1,b" + if equals == 0 || commas != equals-1 { + return i, buf[start:i], fmt.Errorf("invalid field format") + } + + return i, buf[start:i], nil +} + +// scanTime scans buf, starting at i for the time section of a point. It +// returns the ending position and the byte slice of the timestamp within buf +// and and error if the timestamp is not in the correct numeric format. +func scanTime(buf []byte, i int) (int, []byte, error) { + start := skipWhitespace(buf, i) + i = start + + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // Reached end of block or trailing whitespace? + if buf[i] == '\n' || buf[i] == ' ' { + break + } + + // Handle negative timestamps + if i == start && buf[i] == '-' { + i++ + continue + } + + // Timestamps should be integers, make sure they are so we don't need + // to actually parse the timestamp until needed. + if buf[i] < '0' || buf[i] > '9' { + return i, buf[start:i], fmt.Errorf("bad timestamp") + } + i++ + } + return i, buf[start:i], nil +} + +func isNumeric(b byte) bool { + return (b >= '0' && b <= '9') || b == '.' +} + +// scanNumber returns the end position within buf, start at i after +// scanning over buf for an integer, or float. It returns an +// error if a invalid number is scanned. +func scanNumber(buf []byte, i int) (int, error) { + start := i + var isInt, isUnsigned bool + + // Is negative number? + if i < len(buf) && buf[i] == '-' { + i++ + // There must be more characters now, as just '-' is illegal. + if i == len(buf) { + return i, ErrInvalidNumber + } + } + + // how many decimal points we've see + decimal := false + + // indicates the number is float in scientific notation + scientific := false + + for { + if i >= len(buf) { + break + } + + if buf[i] == ',' || buf[i] == ' ' { + break + } + + if buf[i] == 'i' && i > start && !(isInt || isUnsigned) { + isInt = true + i++ + continue + } else if buf[i] == 'u' && i > start && !(isInt || isUnsigned) { + isUnsigned = true + i++ + continue + } + + if buf[i] == '.' { + // Can't have more than 1 decimal (e.g. 1.1.1 should fail) + if decimal { + return i, ErrInvalidNumber + } + decimal = true + } + + // `e` is valid for floats but not as the first char + if i > start && (buf[i] == 'e' || buf[i] == 'E') { + scientific = true + i++ + continue + } + + // + and - are only valid at this point if they follow an e (scientific notation) + if (buf[i] == '+' || buf[i] == '-') && (buf[i-1] == 'e' || buf[i-1] == 'E') { + i++ + continue + } + + // NaN is an unsupported value + if i+2 < len(buf) && (buf[i] == 'N' || buf[i] == 'n') { + return i, ErrInvalidNumber + } + + if !isNumeric(buf[i]) { + return i, ErrInvalidNumber + } + i++ + } + + if (isInt || isUnsigned) && (decimal || scientific) { + return i, ErrInvalidNumber + } + + numericDigits := i - start + if isInt { + numericDigits-- + } + if decimal { + numericDigits-- + } + if buf[start] == '-' { + numericDigits-- + } + + if numericDigits == 0 { + return i, ErrInvalidNumber + } + + // It's more common that numbers will be within min/max range for their type but we need to prevent + // out or range numbers from being parsed successfully. This uses some simple heuristics to decide + // if we should parse the number to the actual type. It does not do it all the time because it incurs + // extra allocations and we end up converting the type again when writing points to disk. + if isInt { + // Make sure the last char is an 'i' for integers (e.g. 9i10 is not valid) + if buf[i-1] != 'i' { + return i, ErrInvalidNumber + } + // Parse the int to check bounds the number of digits could be larger than the max range + // We subtract 1 from the index to remove the `i` from our tests + if len(buf[start:i-1]) >= maxInt64Digits || len(buf[start:i-1]) >= minInt64Digits { + if _, err := parseIntBytes(buf[start:i-1], 10, 64); err != nil { + return i, fmt.Errorf("unable to parse integer %s: %s", buf[start:i-1], err) + } + } + } else if isUnsigned { + // Return an error if uint64 support has not been enabled. + if !enableUint64Support { + return i, ErrInvalidNumber + } + // Make sure the last char is a 'u' for unsigned + if buf[i-1] != 'u' { + return i, ErrInvalidNumber + } + // Make sure the first char is not a '-' for unsigned + if buf[start] == '-' { + return i, ErrInvalidNumber + } + // Parse the uint to check bounds the number of digits could be larger than the max range + // We subtract 1 from the index to remove the `u` from our tests + if len(buf[start:i-1]) >= maxUint64Digits { + if _, err := parseUintBytes(buf[start:i-1], 10, 64); err != nil { + return i, fmt.Errorf("unable to parse unsigned %s: %s", buf[start:i-1], err) + } + } + } else { + // Parse the float to check bounds if it's scientific or the number of digits could be larger than the max range + if scientific || len(buf[start:i]) >= maxFloat64Digits || len(buf[start:i]) >= minFloat64Digits { + if _, err := parseFloatBytes(buf[start:i], 10); err != nil { + return i, fmt.Errorf("invalid float") + } + } + } + + return i, nil +} + +// scanBoolean returns the end position within buf, start at i after +// scanning over buf for boolean. Valid values for a boolean are +// t, T, true, TRUE, f, F, false, FALSE. It returns an error if a invalid boolean +// is scanned. +func scanBoolean(buf []byte, i int) (int, []byte, error) { + start := i + + if i < len(buf) && (buf[i] != 't' && buf[i] != 'f' && buf[i] != 'T' && buf[i] != 'F') { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + i++ + for { + if i >= len(buf) { + break + } + + if buf[i] == ',' || buf[i] == ' ' { + break + } + i++ + } + + // Single char bool (t, T, f, F) is ok + if i-start == 1 { + return i, buf[start:i], nil + } + + // length must be 4 for true or TRUE + if (buf[start] == 't' || buf[start] == 'T') && i-start != 4 { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + // length must be 5 for false or FALSE + if (buf[start] == 'f' || buf[start] == 'F') && i-start != 5 { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + // Otherwise + valid := false + switch buf[start] { + case 't': + valid = bytes.Equal(buf[start:i], []byte("true")) + case 'f': + valid = bytes.Equal(buf[start:i], []byte("false")) + case 'T': + valid = bytes.Equal(buf[start:i], []byte("TRUE")) || bytes.Equal(buf[start:i], []byte("True")) + case 'F': + valid = bytes.Equal(buf[start:i], []byte("FALSE")) || bytes.Equal(buf[start:i], []byte("False")) + } + + if !valid { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + return i, buf[start:i], nil + +} + +// skipWhitespace returns the end position within buf, starting at i after +// scanning over spaces in tags. +func skipWhitespace(buf []byte, i int) int { + for i < len(buf) { + if buf[i] != ' ' && buf[i] != '\t' && buf[i] != 0 { + break + } + i++ + } + return i +} + +// scanLine returns the end position in buf and the next line found within +// buf. +func scanLine(buf []byte, i int) (int, []byte) { + start := i + quoted := false + fields := false + + // tracks how many '=' and commas we've seen + // this duplicates some of the functionality in scanFields + equals := 0 + commas := 0 + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // skip past escaped characters + if buf[i] == '\\' && i+2 < len(buf) { + i += 2 + continue + } + + if buf[i] == ' ' { + fields = true + } + + // If we see a double quote, makes sure it is not escaped + if fields { + if !quoted && buf[i] == '=' { + i++ + equals++ + continue + } else if !quoted && buf[i] == ',' { + i++ + commas++ + continue + } else if buf[i] == '"' && equals > commas { + i++ + quoted = !quoted + continue + } + } + + if buf[i] == '\n' && !quoted { + break + } + + i++ + } + + return i, buf[start:i] +} + +// scanTo returns the end position in buf and the next consecutive block +// of bytes, starting from i and ending with stop byte, where stop byte +// has not been escaped. +// +// If there are leading spaces, they are skipped. +func scanTo(buf []byte, i int, stop byte) (int, []byte) { + start := i + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // Reached unescaped stop value? + if buf[i] == stop && (i == 0 || buf[i-1] != '\\') { + break + } + i++ + } + + return i, buf[start:i] +} + +// scanTo returns the end position in buf and the next consecutive block +// of bytes, starting from i and ending with stop byte. If there are leading +// spaces, they are skipped. +func scanToSpaceOr(buf []byte, i int, stop byte) (int, []byte) { + start := i + if buf[i] == stop || buf[i] == ' ' { + return i, buf[start:i] + } + + for { + i++ + if buf[i-1] == '\\' { + continue + } + + // reached the end of buf? + if i >= len(buf) { + return i, buf[start:i] + } + + // reached end of block? + if buf[i] == stop || buf[i] == ' ' { + return i, buf[start:i] + } + } +} + +func scanTagValue(buf []byte, i int) (int, []byte) { + start := i + for { + if i >= len(buf) { + break + } + + if buf[i] == ',' && buf[i-1] != '\\' { + break + } + i++ + } + if i > len(buf) { + return i, nil + } + return i, buf[start:i] +} + +func scanFieldValue(buf []byte, i int) (int, []byte) { + start := i + quoted := false + for i < len(buf) { + // Only escape char for a field value is a double-quote and backslash + if buf[i] == '\\' && i+1 < len(buf) && (buf[i+1] == '"' || buf[i+1] == '\\') { + i += 2 + continue + } + + // Quoted value? (e.g. string) + if buf[i] == '"' { + i++ + quoted = !quoted + continue + } + + if buf[i] == ',' && !quoted { + break + } + i++ + } + return i, buf[start:i] +} + +func EscapeMeasurement(in []byte) []byte { + for b, esc := range measurementEscapeCodes { + in = bytes.Replace(in, []byte{b}, esc, -1) + } + return in +} + +func unescapeMeasurement(in []byte) []byte { + for b, esc := range measurementEscapeCodes { + in = bytes.Replace(in, esc, []byte{b}, -1) + } + return in +} + +func escapeTag(in []byte) []byte { + for b, esc := range tagEscapeCodes { + if bytes.IndexByte(in, b) != -1 { + in = bytes.Replace(in, []byte{b}, esc, -1) + } + } + return in +} + +func unescapeTag(in []byte) []byte { + if bytes.IndexByte(in, '\\') == -1 { + return in + } + + for b, esc := range tagEscapeCodes { + if bytes.IndexByte(in, b) != -1 { + in = bytes.Replace(in, esc, []byte{b}, -1) + } + } + return in +} + +// escapeStringFieldReplacer replaces double quotes and backslashes +// with the same character preceded by a backslash. +// As of Go 1.7 this benchmarked better in allocations and CPU time +// compared to iterating through a string byte-by-byte and appending to a new byte slice, +// calling strings.Replace twice, and better than (*Regex).ReplaceAllString. +var escapeStringFieldReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`) + +// EscapeStringField returns a copy of in with any double quotes or +// backslashes with escaped values. +func EscapeStringField(in string) string { + return escapeStringFieldReplacer.Replace(in) +} + +// unescapeStringField returns a copy of in with any escaped double-quotes +// or backslashes unescaped. +func unescapeStringField(in string) string { + if strings.IndexByte(in, '\\') == -1 { + return in + } + + var out []byte + i := 0 + for { + if i >= len(in) { + break + } + // unescape backslashes + if in[i] == '\\' && i+1 < len(in) && in[i+1] == '\\' { + out = append(out, '\\') + i += 2 + continue + } + // unescape double-quotes + if in[i] == '\\' && i+1 < len(in) && in[i+1] == '"' { + out = append(out, '"') + i += 2 + continue + } + out = append(out, in[i]) + i++ + + } + return string(out) +} + +// NewPoint returns a new point with the given measurement name, tags, fields and timestamp. If +// an unsupported field value (NaN) or out of range time is passed, this function returns an error. +func NewPoint(name string, tags Tags, fields Fields, t time.Time) (Point, error) { + key, err := pointKey(name, tags, fields, t) + if err != nil { + return nil, err + } + + return &point{ + key: key, + time: t, + fields: fields.MarshalBinary(), + }, nil +} + +// pointKey checks some basic requirements for valid points, and returns the +// key, along with an possible error. +func pointKey(measurement string, tags Tags, fields Fields, t time.Time) ([]byte, error) { + if len(fields) == 0 { + return nil, ErrPointMustHaveAField + } + + if !t.IsZero() { + if err := CheckTime(t); err != nil { + return nil, err + } + } + + for key, value := range fields { + switch value := value.(type) { + case float64: + // Ensure the caller validates and handles invalid field values + if math.IsNaN(value) { + return nil, fmt.Errorf("NaN is an unsupported value for field %s", key) + } + case float32: + // Ensure the caller validates and handles invalid field values + if math.IsNaN(float64(value)) { + return nil, fmt.Errorf("NaN is an unsupported value for field %s", key) + } + } + if len(key) == 0 { + return nil, fmt.Errorf("all fields must have non-empty names") + } + } + + key := MakeKey([]byte(measurement), tags) + for field := range fields { + sz := seriesKeySize(key, []byte(field)) + if sz > MaxKeyLength { + return nil, fmt.Errorf("max key length exceeded: %v > %v", sz, MaxKeyLength) + } + } + + return key, nil +} + +func seriesKeySize(key, field []byte) int { + // 4 is the length of the tsm1.fieldKeySeparator constant. It's inlined here to avoid a circular + // dependency. + return len(key) + 4 + len(field) +} + +// NewPointFromBytes returns a new Point from a marshalled Point. +func NewPointFromBytes(b []byte) (Point, error) { + p := &point{} + if err := p.UnmarshalBinary(b); err != nil { + return nil, err + } + + // This does some basic validation to ensure there are fields and they + // can be unmarshalled as well. + iter := p.FieldIterator() + var hasField bool + for iter.Next() { + if len(iter.FieldKey()) == 0 { + continue + } + hasField = true + switch iter.Type() { + case Float: + _, err := iter.FloatValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + case Integer: + _, err := iter.IntegerValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + case Unsigned: + _, err := iter.UnsignedValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + case String: + // Skip since this won't return an error + case Boolean: + _, err := iter.BooleanValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + } + } + + if !hasField { + return nil, ErrPointMustHaveAField + } + + return p, nil +} + +// MustNewPoint returns a new point with the given measurement name, tags, fields and timestamp. If +// an unsupported field value (NaN) is passed, this function panics. +func MustNewPoint(name string, tags Tags, fields Fields, time time.Time) Point { + pt, err := NewPoint(name, tags, fields, time) + if err != nil { + panic(err.Error()) + } + return pt +} + +// Key returns the key (measurement joined with tags) of the point. +func (p *point) Key() []byte { + return p.key +} + +func (p *point) name() []byte { + _, name := scanTo(p.key, 0, ',') + return name +} + +func (p *point) Name() []byte { + return escape.Unescape(p.name()) +} + +// SetName updates the measurement name for the point. +func (p *point) SetName(name string) { + p.cachedName = "" + p.key = MakeKey([]byte(name), p.Tags()) +} + +// Time return the timestamp for the point. +func (p *point) Time() time.Time { + return p.time +} + +// SetTime updates the timestamp for the point. +func (p *point) SetTime(t time.Time) { + p.time = t +} + +// Round will round the timestamp of the point to the given duration. +func (p *point) Round(d time.Duration) { + p.time = p.time.Round(d) +} + +// Tags returns the tag set for the point. +func (p *point) Tags() Tags { + if p.cachedTags != nil { + return p.cachedTags + } + p.cachedTags = parseTags(p.key) + return p.cachedTags +} + +func (p *point) HasTag(tag []byte) bool { + if len(p.key) == 0 { + return false + } + + var exists bool + walkTags(p.key, func(key, value []byte) bool { + if bytes.Equal(tag, key) { + exists = true + return false + } + return true + }) + + return exists +} + +func walkTags(buf []byte, fn func(key, value []byte) bool) { + if len(buf) == 0 { + return + } + + pos, name := scanTo(buf, 0, ',') + + // it's an empty key, so there are no tags + if len(name) == 0 { + return + } + + hasEscape := bytes.IndexByte(buf, '\\') != -1 + i := pos + 1 + var key, value []byte + for { + if i >= len(buf) { + break + } + i, key = scanTo(buf, i, '=') + i, value = scanTagValue(buf, i+1) + + if len(value) == 0 { + continue + } + + if hasEscape { + if !fn(unescapeTag(key), unescapeTag(value)) { + return + } + } else { + if !fn(key, value) { + return + } + } + + i++ + } +} + +// walkFields walks each field key and value via fn. If fn returns false, the iteration +// is stopped. The values are the raw byte slices and not the converted types. +func walkFields(buf []byte, fn func(key, value []byte) bool) { + var i int + var key, val []byte + for len(buf) > 0 { + i, key = scanTo(buf, 0, '=') + buf = buf[i+1:] + i, val = scanFieldValue(buf, 0) + buf = buf[i:] + if !fn(key, val) { + break + } + + // slice off comma + if len(buf) > 0 { + buf = buf[1:] + } + } +} + +func parseTags(buf []byte) Tags { + if len(buf) == 0 { + return nil + } + + tags := make(Tags, bytes.Count(buf, []byte(","))) + p := 0 + walkTags(buf, func(key, value []byte) bool { + tags[p].Key = key + tags[p].Value = value + p++ + return true + }) + return tags +} + +// MakeKey creates a key for a set of tags. +func MakeKey(name []byte, tags Tags) []byte { + // unescape the name and then re-escape it to avoid double escaping. + // The key should always be stored in escaped form. + return append(EscapeMeasurement(unescapeMeasurement(name)), tags.HashKey()...) +} + +// SetTags replaces the tags for the point. +func (p *point) SetTags(tags Tags) { + p.key = MakeKey(p.Name(), tags) + p.cachedTags = tags +} + +// AddTag adds or replaces a tag value for a point. +func (p *point) AddTag(key, value string) { + tags := p.Tags() + tags = append(tags, Tag{Key: []byte(key), Value: []byte(value)}) + sort.Sort(tags) + p.cachedTags = tags + p.key = MakeKey(p.Name(), tags) +} + +// Fields returns the fields for the point. +func (p *point) Fields() (Fields, error) { + if p.cachedFields != nil { + return p.cachedFields, nil + } + cf, err := p.unmarshalBinary() + if err != nil { + return nil, err + } + p.cachedFields = cf + return p.cachedFields, nil +} + +// SetPrecision will round a time to the specified precision. +func (p *point) SetPrecision(precision string) { + switch precision { + case "n": + case "u": + p.SetTime(p.Time().Truncate(time.Microsecond)) + case "ms": + p.SetTime(p.Time().Truncate(time.Millisecond)) + case "s": + p.SetTime(p.Time().Truncate(time.Second)) + case "m": + p.SetTime(p.Time().Truncate(time.Minute)) + case "h": + p.SetTime(p.Time().Truncate(time.Hour)) + } +} + +// String returns the string representation of the point. +func (p *point) String() string { + if p.Time().IsZero() { + return string(p.Key()) + " " + string(p.fields) + } + return string(p.Key()) + " " + string(p.fields) + " " + strconv.FormatInt(p.UnixNano(), 10) +} + +// AppendString appends the string representation of the point to buf. +func (p *point) AppendString(buf []byte) []byte { + buf = append(buf, p.key...) + buf = append(buf, ' ') + buf = append(buf, p.fields...) + + if !p.time.IsZero() { + buf = append(buf, ' ') + buf = strconv.AppendInt(buf, p.UnixNano(), 10) + } + + return buf +} + +// StringSize returns the length of the string that would be returned by String(). +func (p *point) StringSize() int { + size := len(p.key) + len(p.fields) + 1 + + if !p.time.IsZero() { + digits := 1 // even "0" has one digit + t := p.UnixNano() + if t < 0 { + // account for negative sign, then negate + digits++ + t = -t + } + for t > 9 { // already accounted for one digit + digits++ + t /= 10 + } + size += digits + 1 // digits and a space + } + + return size +} + +// MarshalBinary returns a binary representation of the point. +func (p *point) MarshalBinary() ([]byte, error) { + if len(p.fields) == 0 { + return nil, ErrPointMustHaveAField + } + + tb, err := p.time.MarshalBinary() + if err != nil { + return nil, err + } + + b := make([]byte, 8+len(p.key)+len(p.fields)+len(tb)) + i := 0 + + binary.BigEndian.PutUint32(b[i:], uint32(len(p.key))) + i += 4 + + i += copy(b[i:], p.key) + + binary.BigEndian.PutUint32(b[i:i+4], uint32(len(p.fields))) + i += 4 + + i += copy(b[i:], p.fields) + + copy(b[i:], tb) + return b, nil +} + +// UnmarshalBinary decodes a binary representation of the point into a point struct. +func (p *point) UnmarshalBinary(b []byte) error { + var n int + + // Read key length. + if len(b) < 4 { + return io.ErrShortBuffer + } + n, b = int(binary.BigEndian.Uint32(b[:4])), b[4:] + + // Read key. + if len(b) < n { + return io.ErrShortBuffer + } + p.key, b = b[:n], b[n:] + + // Read fields length. + if len(b) < 4 { + return io.ErrShortBuffer + } + n, b = int(binary.BigEndian.Uint32(b[:4])), b[4:] + + // Read fields. + if len(b) < n { + return io.ErrShortBuffer + } + p.fields, b = b[:n], b[n:] + + // Read timestamp. + if err := p.time.UnmarshalBinary(b); err != nil { + return err + } + return nil +} + +// PrecisionString returns a string representation of the point. If there +// is a timestamp associated with the point then it will be specified in the +// given unit. +func (p *point) PrecisionString(precision string) string { + if p.Time().IsZero() { + return fmt.Sprintf("%s %s", p.Key(), string(p.fields)) + } + return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), + p.UnixNano()/GetPrecisionMultiplier(precision)) +} + +// RoundedString returns a string representation of the point. If there +// is a timestamp associated with the point, then it will be rounded to the +// given duration. +func (p *point) RoundedString(d time.Duration) string { + if p.Time().IsZero() { + return fmt.Sprintf("%s %s", p.Key(), string(p.fields)) + } + return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), + p.time.Round(d).UnixNano()) +} + +func (p *point) unmarshalBinary() (Fields, error) { + iter := p.FieldIterator() + fields := make(Fields, 8) + for iter.Next() { + if len(iter.FieldKey()) == 0 { + continue + } + switch iter.Type() { + case Float: + v, err := iter.FloatValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + case Integer: + v, err := iter.IntegerValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + case Unsigned: + v, err := iter.UnsignedValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + case String: + fields[string(iter.FieldKey())] = iter.StringValue() + case Boolean: + v, err := iter.BooleanValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + } + } + return fields, nil +} + +// HashID returns a non-cryptographic checksum of the point's key. +func (p *point) HashID() uint64 { + h := NewInlineFNV64a() + h.Write(p.key) + sum := h.Sum64() + return sum +} + +// UnixNano returns the timestamp of the point as nanoseconds since Unix epoch. +func (p *point) UnixNano() int64 { + return p.Time().UnixNano() +} + +// Split will attempt to return multiple points with the same timestamp whose +// string representations are no longer than size. Points with a single field or +// a point without a timestamp may exceed the requested size. +func (p *point) Split(size int) []Point { + if p.time.IsZero() || p.StringSize() <= size { + return []Point{p} + } + + // key string, timestamp string, spaces + size -= len(p.key) + len(strconv.FormatInt(p.time.UnixNano(), 10)) + 2 + + var points []Point + var start, cur int + + for cur < len(p.fields) { + end, _ := scanTo(p.fields, cur, '=') + end, _ = scanFieldValue(p.fields, end+1) + + if cur > start && end-start > size { + points = append(points, &point{ + key: p.key, + time: p.time, + fields: p.fields[start : cur-1], + }) + start = cur + } + + cur = end + 1 + } + + points = append(points, &point{ + key: p.key, + time: p.time, + fields: p.fields[start:], + }) + + return points +} + +// Tag represents a single key/value tag pair. +type Tag struct { + Key []byte + Value []byte +} + +// NewTag returns a new Tag. +func NewTag(key, value []byte) Tag { + return Tag{ + Key: key, + Value: value, + } +} + +// Size returns the size of the key and value. +func (t Tag) Size() int { return len(t.Key) + len(t.Value) } + +// Clone returns a shallow copy of Tag. +// +// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. +// Use Clone to create a Tag with new byte slices that do not refer to the argument to ParsePointsWithPrecision. +func (t Tag) Clone() Tag { + other := Tag{ + Key: make([]byte, len(t.Key)), + Value: make([]byte, len(t.Value)), + } + + copy(other.Key, t.Key) + copy(other.Value, t.Value) + + return other +} + +// String returns the string reprsentation of the tag. +func (t *Tag) String() string { + var buf bytes.Buffer + buf.WriteByte('{') + buf.WriteString(string(t.Key)) + buf.WriteByte(' ') + buf.WriteString(string(t.Value)) + buf.WriteByte('}') + return buf.String() +} + +// Tags represents a sorted list of tags. +type Tags []Tag + +// NewTags returns a new Tags from a map. +func NewTags(m map[string]string) Tags { + if len(m) == 0 { + return nil + } + a := make(Tags, 0, len(m)) + for k, v := range m { + a = append(a, NewTag([]byte(k), []byte(v))) + } + sort.Sort(a) + return a +} + +// Keys returns the list of keys for a tag set. +func (a Tags) Keys() []string { + if len(a) == 0 { + return nil + } + keys := make([]string, len(a)) + for i, tag := range a { + keys[i] = string(tag.Key) + } + return keys +} + +// Values returns the list of values for a tag set. +func (a Tags) Values() []string { + if len(a) == 0 { + return nil + } + values := make([]string, len(a)) + for i, tag := range a { + values[i] = string(tag.Value) + } + return values +} + +// String returns the string representation of the tags. +func (a Tags) String() string { + var buf bytes.Buffer + buf.WriteByte('[') + for i := range a { + buf.WriteString(a[i].String()) + if i < len(a)-1 { + buf.WriteByte(' ') + } + } + buf.WriteByte(']') + return buf.String() +} + +// Size returns the number of bytes needed to store all tags. Note, this is +// the number of bytes needed to store all keys and values and does not account +// for data structures or delimiters for example. +func (a Tags) Size() int { + var total int + for _, t := range a { + total += t.Size() + } + return total +} + +// Clone returns a copy of the slice where the elements are a result of calling `Clone` on the original elements +// +// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. +// Use Clone to create Tags with new byte slices that do not refer to the argument to ParsePointsWithPrecision. +func (a Tags) Clone() Tags { + if len(a) == 0 { + return nil + } + + others := make(Tags, len(a)) + for i := range a { + others[i] = a[i].Clone() + } + + return others +} + +func (a Tags) Len() int { return len(a) } +func (a Tags) Less(i, j int) bool { return bytes.Compare(a[i].Key, a[j].Key) == -1 } +func (a Tags) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// Equal returns true if a equals other. +func (a Tags) Equal(other Tags) bool { + if len(a) != len(other) { + return false + } + for i := range a { + if !bytes.Equal(a[i].Key, other[i].Key) || !bytes.Equal(a[i].Value, other[i].Value) { + return false + } + } + return true +} + +// CompareTags returns -1 if a < b, 1 if a > b, and 0 if a == b. +func CompareTags(a, b Tags) int { + // Compare each key & value until a mismatch. + for i := 0; i < len(a) && i < len(b); i++ { + if cmp := bytes.Compare(a[i].Key, b[i].Key); cmp != 0 { + return cmp + } + if cmp := bytes.Compare(a[i].Value, b[i].Value); cmp != 0 { + return cmp + } + } + + // If all tags are equal up to this point then return shorter tagset. + if len(a) < len(b) { + return -1 + } else if len(a) > len(b) { + return 1 + } + + // All tags are equal. + return 0 +} + +// Get returns the value for a key. +func (a Tags) Get(key []byte) []byte { + // OPTIMIZE: Use sort.Search if tagset is large. + + for _, t := range a { + if bytes.Equal(t.Key, key) { + return t.Value + } + } + return nil +} + +// GetString returns the string value for a string key. +func (a Tags) GetString(key string) string { + return string(a.Get([]byte(key))) +} + +// Set sets the value for a key. +func (a *Tags) Set(key, value []byte) { + for i, t := range *a { + if bytes.Equal(t.Key, key) { + (*a)[i].Value = value + return + } + } + *a = append(*a, Tag{Key: key, Value: value}) + sort.Sort(*a) +} + +// SetString sets the string value for a string key. +func (a *Tags) SetString(key, value string) { + a.Set([]byte(key), []byte(value)) +} + +// Delete removes a tag by key. +func (a *Tags) Delete(key []byte) { + for i, t := range *a { + if bytes.Equal(t.Key, key) { + copy((*a)[i:], (*a)[i+1:]) + (*a)[len(*a)-1] = Tag{} + *a = (*a)[:len(*a)-1] + return + } + } +} + +// Map returns a map representation of the tags. +func (a Tags) Map() map[string]string { + m := make(map[string]string, len(a)) + for _, t := range a { + m[string(t.Key)] = string(t.Value) + } + return m +} + +// Merge merges the tags combining the two. If both define a tag with the +// same key, the merged value overwrites the old value. +// A new map is returned. +func (a Tags) Merge(other map[string]string) Tags { + merged := make(map[string]string, len(a)+len(other)) + for _, t := range a { + merged[string(t.Key)] = string(t.Value) + } + for k, v := range other { + merged[k] = v + } + return NewTags(merged) +} + +// HashKey hashes all of a tag's keys. +func (a Tags) HashKey() []byte { + // Empty maps marshal to empty bytes. + if len(a) == 0 { + return nil + } + + // Type invariant: Tags are sorted + + escaped := make(Tags, 0, len(a)) + sz := 0 + for _, t := range a { + ek := escapeTag(t.Key) + ev := escapeTag(t.Value) + + if len(ev) > 0 { + escaped = append(escaped, Tag{Key: ek, Value: ev}) + sz += len(ek) + len(ev) + } + } + + sz += len(escaped) + (len(escaped) * 2) // separators + + // Generate marshaled bytes. + b := make([]byte, sz) + buf := b + idx := 0 + for _, k := range escaped { + buf[idx] = ',' + idx++ + copy(buf[idx:idx+len(k.Key)], k.Key) + idx += len(k.Key) + buf[idx] = '=' + idx++ + copy(buf[idx:idx+len(k.Value)], k.Value) + idx += len(k.Value) + } + return b[:idx] +} + +// CopyTags returns a shallow copy of tags. +func CopyTags(a Tags) Tags { + other := make(Tags, len(a)) + copy(other, a) + return other +} + +// DeepCopyTags returns a deep copy of tags. +func DeepCopyTags(a Tags) Tags { + // Calculate size of keys/values in bytes. + var n int + for _, t := range a { + n += len(t.Key) + len(t.Value) + } + + // Build single allocation for all key/values. + buf := make([]byte, n) + + // Copy tags to new set. + other := make(Tags, len(a)) + for i, t := range a { + copy(buf, t.Key) + other[i].Key, buf = buf[:len(t.Key)], buf[len(t.Key):] + + copy(buf, t.Value) + other[i].Value, buf = buf[:len(t.Value)], buf[len(t.Value):] + } + + return other +} + +// Fields represents a mapping between a Point's field names and their +// values. +type Fields map[string]interface{} + +// FieldIterator retuns a FieldIterator that can be used to traverse the +// fields of a point without constructing the in-memory map. +func (p *point) FieldIterator() FieldIterator { + p.Reset() + return p +} + +type fieldIterator struct { + start, end int + key, keybuf []byte + valueBuf []byte + fieldType FieldType +} + +// Next indicates whether there any fields remaining. +func (p *point) Next() bool { + p.it.start = p.it.end + if p.it.start >= len(p.fields) { + return false + } + + p.it.end, p.it.key = scanTo(p.fields, p.it.start, '=') + if escape.IsEscaped(p.it.key) { + p.it.keybuf = escape.AppendUnescaped(p.it.keybuf[:0], p.it.key) + p.it.key = p.it.keybuf + } + + p.it.end, p.it.valueBuf = scanFieldValue(p.fields, p.it.end+1) + p.it.end++ + + if len(p.it.valueBuf) == 0 { + p.it.fieldType = Empty + return true + } + + c := p.it.valueBuf[0] + + if c == '"' { + p.it.fieldType = String + return true + } + + if strings.IndexByte(`0123456789-.nNiIu`, c) >= 0 { + if p.it.valueBuf[len(p.it.valueBuf)-1] == 'i' { + p.it.fieldType = Integer + p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] + } else if p.it.valueBuf[len(p.it.valueBuf)-1] == 'u' { + p.it.fieldType = Unsigned + p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] + } else { + p.it.fieldType = Float + } + return true + } + + // to keep the same behavior that currently exists, default to boolean + p.it.fieldType = Boolean + return true +} + +// FieldKey returns the key of the current field. +func (p *point) FieldKey() []byte { + return p.it.key +} + +// Type returns the FieldType of the current field. +func (p *point) Type() FieldType { + return p.it.fieldType +} + +// StringValue returns the string value of the current field. +func (p *point) StringValue() string { + return unescapeStringField(string(p.it.valueBuf[1 : len(p.it.valueBuf)-1])) +} + +// IntegerValue returns the integer value of the current field. +func (p *point) IntegerValue() (int64, error) { + n, err := parseIntBytes(p.it.valueBuf, 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse integer value %q: %v", p.it.valueBuf, err) + } + return n, nil +} + +// UnsignedValue returns the unsigned value of the current field. +func (p *point) UnsignedValue() (uint64, error) { + n, err := parseUintBytes(p.it.valueBuf, 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse unsigned value %q: %v", p.it.valueBuf, err) + } + return n, nil +} + +// BooleanValue returns the boolean value of the current field. +func (p *point) BooleanValue() (bool, error) { + b, err := parseBoolBytes(p.it.valueBuf) + if err != nil { + return false, fmt.Errorf("unable to parse bool value %q: %v", p.it.valueBuf, err) + } + return b, nil +} + +// FloatValue returns the float value of the current field. +func (p *point) FloatValue() (float64, error) { + f, err := parseFloatBytes(p.it.valueBuf, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse floating point value %q: %v", p.it.valueBuf, err) + } + return f, nil +} + +// Reset resets the iterator to its initial state. +func (p *point) Reset() { + p.it.fieldType = Empty + p.it.key = nil + p.it.valueBuf = nil + p.it.start = 0 + p.it.end = 0 +} + +// MarshalBinary encodes all the fields to their proper type and returns the binary +// represenation +// NOTE: uint64 is specifically not supported due to potential overflow when we decode +// again later to an int64 +// NOTE2: uint is accepted, and may be 64 bits, and is for some reason accepted... +func (p Fields) MarshalBinary() []byte { + var b []byte + keys := make([]string, 0, len(p)) + + for k := range p { + keys = append(keys, k) + } + + // Not really necessary, can probably be removed. + sort.Strings(keys) + + for i, k := range keys { + if i > 0 { + b = append(b, ',') + } + b = appendField(b, k, p[k]) + } + + return b +} + +func appendField(b []byte, k string, v interface{}) []byte { + b = append(b, []byte(escape.String(k))...) + b = append(b, '=') + + // check popular types first + switch v := v.(type) { + case float64: + b = strconv.AppendFloat(b, v, 'f', -1, 64) + case int64: + b = strconv.AppendInt(b, v, 10) + b = append(b, 'i') + case string: + b = append(b, '"') + b = append(b, []byte(EscapeStringField(v))...) + b = append(b, '"') + case bool: + b = strconv.AppendBool(b, v) + case int32: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case int16: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case int8: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case int: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint64: + b = strconv.AppendUint(b, v, 10) + b = append(b, 'u') + case uint32: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint16: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint8: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint: + // TODO: 'uint' should be converted to writing as an unsigned integer, + // but we cannot since that would break backwards compatibility. + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case float32: + b = strconv.AppendFloat(b, float64(v), 'f', -1, 32) + case []byte: + b = append(b, v...) + case nil: + // skip + default: + // Can't determine the type, so convert to string + b = append(b, '"') + b = append(b, []byte(EscapeStringField(fmt.Sprintf("%v", v)))...) + b = append(b, '"') + + } + + return b +} + +type byteSlices [][]byte + +func (a byteSlices) Len() int { return len(a) } +func (a byteSlices) Less(i, j int) bool { return bytes.Compare(a[i], a[j]) == -1 } +func (a byteSlices) Swap(i, j int) { a[i], a[j] = a[j], a[i] } diff --git a/vendor/github.com/influxdata/influxdb/models/rows.go b/vendor/github.com/influxdata/influxdb/models/rows.go new file mode 100644 index 0000000000..c087a4882d --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/rows.go @@ -0,0 +1,62 @@ +package models + +import ( + "sort" +) + +// Row represents a single row returned from the execution of a statement. +type Row struct { + Name string `json:"name,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Columns []string `json:"columns,omitempty"` + Values [][]interface{} `json:"values,omitempty"` + Partial bool `json:"partial,omitempty"` +} + +// SameSeries returns true if r contains values for the same series as o. +func (r *Row) SameSeries(o *Row) bool { + return r.tagsHash() == o.tagsHash() && r.Name == o.Name +} + +// tagsHash returns a hash of tag key/value pairs. +func (r *Row) tagsHash() uint64 { + h := NewInlineFNV64a() + keys := r.tagsKeys() + for _, k := range keys { + h.Write([]byte(k)) + h.Write([]byte(r.Tags[k])) + } + return h.Sum64() +} + +// tagKeys returns a sorted list of tag keys. +func (r *Row) tagsKeys() []string { + a := make([]string, 0, len(r.Tags)) + for k := range r.Tags { + a = append(a, k) + } + sort.Strings(a) + return a +} + +// Rows represents a collection of rows. Rows implements sort.Interface. +type Rows []*Row + +// Len implements sort.Interface. +func (p Rows) Len() int { return len(p) } + +// Less implements sort.Interface. +func (p Rows) Less(i, j int) bool { + // Sort by name first. + if p[i].Name != p[j].Name { + return p[i].Name < p[j].Name + } + + // Sort by tag set hash. Tags don't have a meaningful sort order so we + // just compute a hash and sort by that instead. This allows the tests + // to receive rows in a predictable order every time. + return p[i].tagsHash() < p[j].tagsHash() +} + +// Swap implements sort.Interface. +func (p Rows) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/vendor/github.com/influxdata/influxdb/models/statistic.go b/vendor/github.com/influxdata/influxdb/models/statistic.go new file mode 100644 index 0000000000..553e9d09fb --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/statistic.go @@ -0,0 +1,42 @@ +package models + +// Statistic is the representation of a statistic used by the monitoring service. +type Statistic struct { + Name string `json:"name"` + Tags map[string]string `json:"tags"` + Values map[string]interface{} `json:"values"` +} + +// NewStatistic returns an initialized Statistic. +func NewStatistic(name string) Statistic { + return Statistic{ + Name: name, + Tags: make(map[string]string), + Values: make(map[string]interface{}), + } +} + +// StatisticTags is a map that can be merged with others without causing +// mutations to either map. +type StatisticTags map[string]string + +// Merge creates a new map containing the merged contents of tags and t. +// If both tags and the receiver map contain the same key, the value in tags +// is used in the resulting map. +// +// Merge always returns a usable map. +func (t StatisticTags) Merge(tags map[string]string) map[string]string { + // Add everything in tags to the result. + out := make(map[string]string, len(tags)) + for k, v := range tags { + out[k] = v + } + + // Only add values from t that don't appear in tags. + for k, v := range t { + if _, ok := tags[k]; !ok { + out[k] = v + } + } + return out +} diff --git a/vendor/github.com/influxdata/influxdb/models/time.go b/vendor/github.com/influxdata/influxdb/models/time.go new file mode 100644 index 0000000000..e98f2cb336 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/time.go @@ -0,0 +1,74 @@ +package models + +// Helper time methods since parsing time can easily overflow and we only support a +// specific time range. + +import ( + "fmt" + "math" + "time" +) + +const ( + // MinNanoTime is the minumum time that can be represented. + // + // 1677-09-21 00:12:43.145224194 +0000 UTC + // + // The two lowest minimum integers are used as sentinel values. The + // minimum value needs to be used as a value lower than any other value for + // comparisons and another separate value is needed to act as a sentinel + // default value that is unusable by the user, but usable internally. + // Because these two values need to be used for a special purpose, we do + // not allow users to write points at these two times. + MinNanoTime = int64(math.MinInt64) + 2 + + // MaxNanoTime is the maximum time that can be represented. + // + // 2262-04-11 23:47:16.854775806 +0000 UTC + // + // The highest time represented by a nanosecond needs to be used for an + // exclusive range in the shard group, so the maximum time needs to be one + // less than the possible maximum number of nanoseconds representable by an + // int64 so that we don't lose a point at that one time. + MaxNanoTime = int64(math.MaxInt64) - 1 +) + +var ( + minNanoTime = time.Unix(0, MinNanoTime).UTC() + maxNanoTime = time.Unix(0, MaxNanoTime).UTC() + + // ErrTimeOutOfRange gets returned when time is out of the representable range using int64 nanoseconds since the epoch. + ErrTimeOutOfRange = fmt.Errorf("time outside range %d - %d", MinNanoTime, MaxNanoTime) +) + +// SafeCalcTime safely calculates the time given. Will return error if the time is outside the +// supported range. +func SafeCalcTime(timestamp int64, precision string) (time.Time, error) { + mult := GetPrecisionMultiplier(precision) + if t, ok := safeSignedMult(timestamp, mult); ok { + tme := time.Unix(0, t).UTC() + return tme, CheckTime(tme) + } + + return time.Time{}, ErrTimeOutOfRange +} + +// CheckTime checks that a time is within the safe range. +func CheckTime(t time.Time) error { + if t.Before(minNanoTime) || t.After(maxNanoTime) { + return ErrTimeOutOfRange + } + return nil +} + +// Perform the multiplication and check to make sure it didn't overflow. +func safeSignedMult(a, b int64) (int64, bool) { + if a == 0 || b == 0 || a == 1 || b == 1 { + return a * b, true + } + if a == MinNanoTime || b == MaxNanoTime { + return 0, false + } + c := a * b + return c, c/b == a +} diff --git a/vendor/github.com/influxdata/influxdb/models/uint_support.go b/vendor/github.com/influxdata/influxdb/models/uint_support.go new file mode 100644 index 0000000000..18d1ca06e2 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/models/uint_support.go @@ -0,0 +1,7 @@ +// +build uint uint64 + +package models + +func init() { + EnableUintSupport() +} diff --git a/vendor/github.com/influxdata/influxdb/pkg/escape/bytes.go b/vendor/github.com/influxdata/influxdb/pkg/escape/bytes.go new file mode 100644 index 0000000000..f3b31f42d3 --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/pkg/escape/bytes.go @@ -0,0 +1,115 @@ +// Package escape contains utilities for escaping parts of InfluxQL +// and InfluxDB line protocol. +package escape // import "github.com/influxdata/influxdb/pkg/escape" + +import ( + "bytes" + "strings" +) + +// Codes is a map of bytes to be escaped. +var Codes = map[byte][]byte{ + ',': []byte(`\,`), + '"': []byte(`\"`), + ' ': []byte(`\ `), + '=': []byte(`\=`), +} + +// Bytes escapes characters on the input slice, as defined by Codes. +func Bytes(in []byte) []byte { + for b, esc := range Codes { + in = bytes.Replace(in, []byte{b}, esc, -1) + } + return in +} + +const escapeChars = `," =` + +// IsEscaped returns whether b has any escaped characters, +// i.e. whether b seems to have been processed by Bytes. +func IsEscaped(b []byte) bool { + for len(b) > 0 { + i := bytes.IndexByte(b, '\\') + if i < 0 { + return false + } + + if i+1 < len(b) && strings.IndexByte(escapeChars, b[i+1]) >= 0 { + return true + } + b = b[i+1:] + } + return false +} + +// AppendUnescaped appends the unescaped version of src to dst +// and returns the resulting slice. +func AppendUnescaped(dst, src []byte) []byte { + var pos int + for len(src) > 0 { + next := bytes.IndexByte(src[pos:], '\\') + if next < 0 || pos+next+1 >= len(src) { + return append(dst, src...) + } + + if pos+next+1 < len(src) && strings.IndexByte(escapeChars, src[pos+next+1]) >= 0 { + if pos+next > 0 { + dst = append(dst, src[:pos+next]...) + } + src = src[pos+next+1:] + pos = 0 + } else { + pos += next + 1 + } + } + + return dst +} + +// Unescape returns a new slice containing the unescaped version of in. +func Unescape(in []byte) []byte { + if len(in) == 0 { + return nil + } + + if bytes.IndexByte(in, '\\') == -1 { + return in + } + + i := 0 + inLen := len(in) + + // The output size will be no more than inLen. Preallocating the + // capacity of the output is faster and uses less memory than + // letting append() do its own (over)allocation. + out := make([]byte, 0, inLen) + + for { + if i >= inLen { + break + } + if in[i] == '\\' && i+1 < inLen { + switch in[i+1] { + case ',': + out = append(out, ',') + i += 2 + continue + case '"': + out = append(out, '"') + i += 2 + continue + case ' ': + out = append(out, ' ') + i += 2 + continue + case '=': + out = append(out, '=') + i += 2 + continue + } + } + out = append(out, in[i]) + i += 1 + } + return out +} diff --git a/vendor/github.com/influxdata/influxdb/pkg/escape/strings.go b/vendor/github.com/influxdata/influxdb/pkg/escape/strings.go new file mode 100644 index 0000000000..db98033b0d --- /dev/null +++ b/vendor/github.com/influxdata/influxdb/pkg/escape/strings.go @@ -0,0 +1,21 @@ +package escape + +import "strings" + +var ( + escaper = strings.NewReplacer(`,`, `\,`, `"`, `\"`, ` `, `\ `, `=`, `\=`) + unescaper = strings.NewReplacer(`\,`, `,`, `\"`, `"`, `\ `, ` `, `\=`, `=`) +) + +// UnescapeString returns unescaped version of in. +func UnescapeString(in string) string { + if strings.IndexByte(in, '\\') == -1 { + return in + } + return unescaper.Replace(in) +} + +// String returns the escaped version of in. +func String(in string) string { + return escaper.Replace(in) +} diff --git a/vendor/github.com/rcrowley/go-metrics/json.go b/vendor/github.com/rcrowley/go-metrics/json.go deleted file mode 100644 index 2fdcbcfbf1..0000000000 --- a/vendor/github.com/rcrowley/go-metrics/json.go +++ /dev/null @@ -1,87 +0,0 @@ -package metrics - -import ( - "encoding/json" - "io" - "time" -) - -// MarshalJSON returns a byte slice containing a JSON representation of all -// the metrics in the Registry. -func (r *StandardRegistry) MarshalJSON() ([]byte, error) { - data := make(map[string]map[string]interface{}) - r.Each(func(name string, i interface{}) { - values := make(map[string]interface{}) - switch metric := i.(type) { - case Counter: - values["count"] = metric.Count() - case Gauge: - values["value"] = metric.Value() - case GaugeFloat64: - values["value"] = metric.Value() - case Healthcheck: - values["error"] = nil - metric.Check() - if err := metric.Error(); nil != err { - values["error"] = metric.Error().Error() - } - case Histogram: - h := metric.Snapshot() - ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - values["count"] = h.Count() - values["min"] = h.Min() - values["max"] = h.Max() - values["mean"] = h.Mean() - values["stddev"] = h.StdDev() - values["median"] = ps[0] - values["75%"] = ps[1] - values["95%"] = ps[2] - values["99%"] = ps[3] - values["99.9%"] = ps[4] - case Meter: - m := metric.Snapshot() - values["count"] = m.Count() - values["1m.rate"] = m.Rate1() - values["5m.rate"] = m.Rate5() - values["15m.rate"] = m.Rate15() - values["mean.rate"] = m.RateMean() - case Timer: - t := metric.Snapshot() - ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) - values["count"] = t.Count() - values["min"] = t.Min() - values["max"] = t.Max() - values["mean"] = t.Mean() - values["stddev"] = t.StdDev() - values["median"] = ps[0] - values["75%"] = ps[1] - values["95%"] = ps[2] - values["99%"] = ps[3] - values["99.9%"] = ps[4] - values["1m.rate"] = t.Rate1() - values["5m.rate"] = t.Rate5() - values["15m.rate"] = t.Rate15() - values["mean.rate"] = t.RateMean() - } - data[name] = values - }) - return json.Marshal(data) -} - -// WriteJSON writes metrics from the given registry periodically to the -// specified io.Writer as JSON. -func WriteJSON(r Registry, d time.Duration, w io.Writer) { - for _ = range time.Tick(d) { - WriteJSONOnce(r, w) - } -} - -// WriteJSONOnce writes metrics from the given registry to the specified -// io.Writer as JSON. -func WriteJSONOnce(r Registry, w io.Writer) { - json.NewEncoder(w).Encode(r) -} - -func (p *PrefixedRegistry) MarshalJSON() ([]byte, error) { - return json.Marshal(p.underlying) -} diff --git a/vendor/github.com/rcrowley/go-metrics/metrics.go b/vendor/github.com/rcrowley/go-metrics/metrics.go deleted file mode 100644 index b97a49ed12..0000000000 --- a/vendor/github.com/rcrowley/go-metrics/metrics.go +++ /dev/null @@ -1,13 +0,0 @@ -// Go port of Coda Hale's Metrics library -// -//
-// -// Coda Hale's original work:
-package metrics - -// UseNilMetrics is checked by the constructor functions for all of the -// standard metrics. If it is true, the metric returned is a stub. -// -// This global kill-switch helps quantify the observer effect and makes -// for less cluttered pprof profiles. -var UseNilMetrics bool = false diff --git a/vendor/vendor.json b/vendor/vendor.json index e938ce5e64..1341589952 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -206,6 +206,30 @@ "revision": "679507af18f3c7ba2bcc7905392ce23e148661c3", "revisionTime": "2016-12-24T10:41:01Z" }, + { + "checksumSHA1": "6tNwbL5tUS0dxYzADKVZtI2d/lE=", + "path": "github.com/influxdata/influxdb/client", + "revision": "a55dd0f50edd14c9c798d3564189eb4f53914309", + "revisionTime": "2017-10-09T17:24:46Z" + }, + { + "checksumSHA1": "O4XpbSNeUhSIMD2FWtQximJiFIs=", + "path": "github.com/influxdata/influxdb/client/v2", + "revision": "b36b9f109f2da91c8941679caf5356e08eee0b2b", + "revisionTime": "2018-01-17T01:42:09Z" + }, + { + "checksumSHA1": "cfumoC9gHEUROd+fA8qK3WLFAZQ=", + "path": "github.com/influxdata/influxdb/models", + "revision": "b36b9f109f2da91c8941679caf5356e08eee0b2b", + "revisionTime": "2018-01-17T01:42:09Z" + }, + { + "checksumSHA1": "Z0Bb5PWa5WL/j5Dm2KJCLGn1l7U=", + "path": "github.com/influxdata/influxdb/pkg/escape", + "revision": "01288bdb0883a01cac999326bd34421b29acaec8", + "revisionTime": "2018-02-21T22:33:40Z" + }, { "checksumSHA1": "vTGKMIfiMwz43y5bsgx9PrL+AVw=", "path": "github.com/jackpal/go-nat-pmp", @@ -309,18 +333,6 @@ "revision": "3101606756c53221ed58ba94ecba6b26adf89dcc", "revisionTime": "2017-08-14T17:01:13Z" }, - { - "checksumSHA1": "KAzbLjI9MzW2tjfcAsK75lVRp6I=", - "path": "github.com/rcrowley/go-metrics", - "revision": "1f30fe9094a513ce4c700b9a54458bbb0c96996c", - "revisionTime": "2016-11-28T21:05:44Z" - }, - { - "checksumSHA1": "q/d9nXRQYKEJ/EWn+5y6jL8rPGs=", - "path": "github.com/rcrowley/go-metrics/exp", - "revision": "1f30fe9094a513ce4c700b9a54458bbb0c96996c", - "revisionTime": "2016-11-28T21:05:44Z" - }, { "checksumSHA1": "28UVHMmHx0iqO0XiJsjx+fwILyI=", "path": "github.com/rjeczalik/notify", From d7b4b40cb69af0166e24f91cac729d7c07ae65c6 Mon Sep 17 00:00:00 2001 From: Vlad
Date: Fri, 23 Feb 2018 11:10:28 +0100 Subject: [PATCH 45/94] whisper: light client mode introduced --- whisper/whisperv6/api.go | 12 ++++++++++++ whisper/whisperv6/whisper.go | 11 +++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/whisper/whisperv6/api.go b/whisper/whisperv6/api.go index a2c75a41c7..f3d9977fad 100644 --- a/whisper/whisperv6/api.go +++ b/whisper/whisperv6/api.go @@ -219,6 +219,18 @@ func (api *PublicWhisperAPI) DeleteSymKey(ctx context.Context, id string) bool { return api.w.DeleteSymKey(id) } +// MakeLightClient turns the node into light client, which does not forward any incoming messages. +func (api *PublicWhisperAPI) MakeLightClient(ctx context.Context) bool { + api.w.lightClient = true + return api.w.lightClient +} + +// CancelLightClient cancels light client mode. +func (api *PublicWhisperAPI) CancelLightClient(ctx context.Context) bool { + api.w.lightClient = false + return !api.w.lightClient +} + //go:generate gencodec -type NewMessage -field-override newMessageOverride -out gen_newmessage_json.go // NewMessage represents a new whisper message that is posted through the RPC. diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index 600f9cb286..e0ed00bb5d 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -82,6 +82,8 @@ type Whisper struct { syncAllowance int // maximum time in seconds allowed to process the whisper-related messages + lightClient bool // indicates is this node is pure light client (does not forward any messages) + statsMu sync.Mutex // guard stats stats Statistics // Statistics of whisper node @@ -587,7 +589,7 @@ func (whisper *Whisper) Unsubscribe(id string) error { // Send injects a message into the whisper send queue, to be distributed in the // network in the coming cycles. func (whisper *Whisper) Send(envelope *Envelope) error { - ok, err := whisper.add(envelope) + ok, err := whisper.add(envelope, false) if err != nil { return err } @@ -673,7 +675,7 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { trouble := false for _, env := range envelopes { - cached, err := whisper.add(env) + cached, err := whisper.add(env, whisper.lightClient) if err != nil { trouble = true log.Error("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err) @@ -746,7 +748,8 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { // add inserts a new envelope into the message pool to be distributed within the // whisper network. It also inserts the envelope into the expiration pool at the // appropriate time-stamp. In case of error, connection should be dropped. -func (whisper *Whisper) add(envelope *Envelope) (bool, error) { +// param isP2P indicates whether the message is peer-to-peer (should not be forwarded). +func (whisper *Whisper) add(envelope *Envelope, isP2P bool) (bool, error) { now := uint32(time.Now().Unix()) sent := envelope.Expiry - envelope.TTL @@ -811,7 +814,7 @@ func (whisper *Whisper) add(envelope *Envelope) (bool, error) { whisper.statsMu.Lock() whisper.stats.memoryUsed += envelope.size() whisper.statsMu.Unlock() - whisper.postEvent(envelope, false) // notify the local node about the new message + whisper.postEvent(envelope, isP2P) // notify the local node about the new message if whisper.mailServer != nil { whisper.mailServer.Archive(envelope) } From 44d40ffce1200ce8875e187b57ac17e4901db32a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende
Date: Fri, 23 Feb 2018 11:32:57 +0100 Subject: [PATCH 46/94] core, vm, common: define constantinople fork + shift (#16045) * core, vm, common: define constantinople fork, start implementation of shift instructions * vm: more testcases * vm: add tests for intpool erroneous intpool handling * core, vm, common: fix constantinople review concerns * vm: add string<->op definitions for new opcodes --- cmd/puppeth/module_dashboard.go | 1 + common/big.go | 2 +- core/vm/instructions.go | 60 +++++++++++++ core/vm/instructions_test.go | 153 ++++++++++++++++++++++++++++++++ core/vm/interpreter.go | 2 + core/vm/jump_table.go | 33 ++++++- core/vm/opcodes.go | 9 ++ params/config.go | 83 +++++++++-------- 8 files changed, 302 insertions(+), 41 deletions(-) diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index 1cb2d45491..3832b247f8 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -631,6 +631,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da "Tangerine": conf.Genesis.Config.EIP150Block, "Spurious": conf.Genesis.Config.EIP155Block, "Byzantium": conf.Genesis.Config.ByzantiumBlock, + "Constantinople": conf.Genesis.Config.ConstantinopleBlock, }) files[filepath.Join(workdir, "index.html")] = indexfile.Bytes() diff --git a/common/big.go b/common/big.go index b552608bc7..65d4377bf7 100644 --- a/common/big.go +++ b/common/big.go @@ -25,6 +25,6 @@ var ( Big3 = big.NewInt(3) Big0 = big.NewInt(0) Big32 = big.NewInt(32) - Big256 = big.NewInt(0xff) + Big256 = big.NewInt(256) Big257 = big.NewInt(257) ) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 7661725010..6daf4e10df 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -302,6 +302,66 @@ func opMulmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *S return nil, nil } +// opSHL implements Shift Left +// The SHL instruction (shift left) pops 2 values from the stack, first arg1 and then arg2, +// and pushes on the stack arg2 shifted to the left by arg1 number of bits. +func opSHL(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + // Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards + shift, value := math.U256(stack.pop()), math.U256(stack.peek()) + defer evm.interpreter.intPool.put(shift) // First operand back into the pool + + if shift.Cmp(common.Big256) >= 0 { + value.SetUint64(0) + return nil, nil + } + n := uint(shift.Uint64()) + math.U256(value.Lsh(value, n)) + + return nil, nil +} + +// opSHR implements Logical Shift Right +// The SHR instruction (logical shift right) pops 2 values from the stack, first arg1 and then arg2, +// and pushes on the stack arg2 shifted to the right by arg1 number of bits with zero fill. +func opSHR(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + // Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards + shift, value := math.U256(stack.pop()), math.U256(stack.peek()) + defer evm.interpreter.intPool.put(shift) // First operand back into the pool + + if shift.Cmp(common.Big256) >= 0 { + value.SetUint64(0) + return nil, nil + } + n := uint(shift.Uint64()) + math.U256(value.Rsh(value, n)) + + return nil, nil +} + +// opSAR implements Arithmetic Shift Right +// The SAR instruction (arithmetic shift right) pops 2 values from the stack, first arg1 and then arg2, +// and pushes on the stack arg2 shifted to the right by arg1 number of bits with sign extension. +func opSAR(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + // Note, S256 returns (potentially) a new bigint, so we're popping, not peeking this one + shift, value := math.U256(stack.pop()), math.S256(stack.pop()) + defer evm.interpreter.intPool.put(shift) // First operand back into the pool + + if shift.Cmp(common.Big256) >= 0 { + if value.Sign() > 0 { + value.SetUint64(0) + } else { + value.SetInt64(-1) + } + stack.push(math.U256(value)) + return nil, nil + } + n := uint(shift.Uint64()) + value.Rsh(value, n) + stack.push(math.U256(value)) + + return nil, nil +} + func opSha3(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { offset, size := stack.pop(), stack.pop() data := memory.Get(offset.Int64(), size.Int64()) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 180433ea88..eef4328bde 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -24,6 +24,48 @@ import ( "github.com/ethereum/go-ethereum/params" ) +type twoOperandTest struct { + x string + y string + expected string +} + +func testTwoOperandOp(t *testing.T, tests []twoOperandTest, opFn func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error)) { + var ( + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) + stack = newstack() + pc = uint64(0) + ) + for i, test := range tests { + x := new(big.Int).SetBytes(common.Hex2Bytes(test.x)) + shift := new(big.Int).SetBytes(common.Hex2Bytes(test.y)) + expected := new(big.Int).SetBytes(common.Hex2Bytes(test.expected)) + stack.push(x) + stack.push(shift) + opFn(&pc, env, nil, nil, stack) + actual := stack.pop() + if actual.Cmp(expected) != 0 { + t.Errorf("Testcase %d, expected %v, got %v", i, expected, actual) + } + // Check pool usage + // 1.pool is not allowed to contain anything on the stack + // 2.pool is not allowed to contain the same pointers twice + if env.interpreter.intPool.pool.len() > 0 { + + poolvals := make(map[*big.Int]struct{}) + poolvals[actual] = struct{}{} + + for env.interpreter.intPool.pool.len() > 0 { + key := env.interpreter.intPool.get() + if _, exist := poolvals[key]; exist { + t.Errorf("Testcase %d, pool contains double-entry", i) + } + poolvals[key] = struct{}{} + } + } + } +} + func TestByteOp(t *testing.T) { var ( env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) @@ -57,6 +99,98 @@ func TestByteOp(t *testing.T) { } } +func TestSHL(t *testing.T) { + // Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#shl-shift-left + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000002"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "ff", "8000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "0101", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "8000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"}, + } + testTwoOperandOp(t, tests, opSHL) +} + +func TestSHR(t *testing.T) { + // Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#shr-logical-shift-right + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "01", "4000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "ff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0101", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + } + testTwoOperandOp(t, tests, opSHR) +} + +func TestSAR(t *testing.T) { + // Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#sar-arithmetic-shift-right + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "01", "c000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "ff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0100", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0101", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"4000000000000000000000000000000000000000000000000000000000000000", "fe", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "f8", "000000000000000000000000000000000000000000000000000000000000007f"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "fe", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + } + + testTwoOperandOp(t, tests, opSAR) +} + +func TestSGT(t *testing.T) { + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + } + testTwoOperandOp(t, tests, opSgt) +} + +func TestSLT(t *testing.T) { + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"}, + } + testTwoOperandOp(t, tests, opSlt) +} + func opBenchmark(bench *testing.B, op func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error), args ...string) { var ( env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) @@ -259,3 +393,22 @@ func BenchmarkOpMulmod(b *testing.B) { opBenchmark(b, opMulmod, x, y, z) } + +func BenchmarkOpSHL(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + y := "ff" + + opBenchmark(b, opSHL, x, y) +} +func BenchmarkOpSHR(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + y := "ff" + + opBenchmark(b, opSHR, x, y) +} +func BenchmarkOpSAR(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + y := "ff" + + opBenchmark(b, opSAR, x, y) +} diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 82a6d3de6d..e12dd7e5a1 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -68,6 +68,8 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter { // we'll set the default jump table. if !cfg.JumpTable[STOP].valid { switch { + case evm.ChainConfig().IsConstantinople(evm.BlockNumber): + cfg.JumpTable = constantinopleInstructionSet case evm.ChainConfig().IsByzantium(evm.BlockNumber): cfg.JumpTable = byzantiumInstructionSet case evm.ChainConfig().IsHomestead(evm.BlockNumber): diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index a1c5ad9c67..3389941353 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -51,11 +51,38 @@ type operation struct { } var ( - frontierInstructionSet = NewFrontierInstructionSet() - homesteadInstructionSet = NewHomesteadInstructionSet() - byzantiumInstructionSet = NewByzantiumInstructionSet() + frontierInstructionSet = NewFrontierInstructionSet() + homesteadInstructionSet = NewHomesteadInstructionSet() + byzantiumInstructionSet = NewByzantiumInstructionSet() + constantinopleInstructionSet = NewConstantinopleInstructionSet() ) +// NewConstantinopleInstructionSet returns the frontier, homestead +// byzantium and contantinople instructions. +func NewConstantinopleInstructionSet() [256]operation { + // instructions that can be executed during the byzantium phase. + instructionSet := NewByzantiumInstructionSet() + instructionSet[SHL] = operation{ + execute: opSHL, + gasCost: constGasFunc(GasFastestStep), + validateStack: makeStackFunc(2, 1), + valid: true, + } + instructionSet[SHR] = operation{ + execute: opSHR, + gasCost: constGasFunc(GasFastestStep), + validateStack: makeStackFunc(2, 1), + valid: true, + } + instructionSet[SAR] = operation{ + execute: opSAR, + gasCost: constGasFunc(GasFastestStep), + validateStack: makeStackFunc(2, 1), + valid: true, + } + return instructionSet +} + // NewByzantiumInstructionSet returns the frontier, homestead and // byzantium instructions. func NewByzantiumInstructionSet() [256]operation { diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 0c65507355..7fe55b72f6 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -63,6 +63,9 @@ const ( XOR NOT BYTE + SHL + SHR + SAR SHA3 = 0x20 ) @@ -234,6 +237,9 @@ var opCodeToString = map[OpCode]string{ OR: "OR", XOR: "XOR", BYTE: "BYTE", + SHL: "SHL", + SHR: "SHR", + SAR: "SAR", ADDMOD: "ADDMOD", MULMOD: "MULMOD", @@ -400,6 +406,9 @@ var stringToOp = map[string]OpCode{ "OR": OR, "XOR": XOR, "BYTE": BYTE, + "SHL": SHL, + "SHR": SHR, + "SAR": SAR, "ADDMOD": ADDMOD, "MULMOD": MULMOD, "SHA3": SHA3, diff --git a/params/config.go b/params/config.go index 345f6394ab..dc02c7ca37 100644 --- a/params/config.go +++ b/params/config.go @@ -31,46 +31,46 @@ var ( var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(1150000), - DAOForkBlock: big.NewInt(1920000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2463000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2675000), - EIP158Block: big.NewInt(2675000), - ByzantiumBlock: big.NewInt(4370000), - - Ethash: new(EthashConfig), + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(1150000), + DAOForkBlock: big.NewInt(1920000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2463000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2675000), + EIP158Block: big.NewInt(2675000), + ByzantiumBlock: big.NewInt(4370000), + ConstantinopleBlock: nil, + Ethash: new(EthashConfig), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. TestnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - ByzantiumBlock: big.NewInt(1700000), - - Ethash: new(EthashConfig), + ChainId: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + ByzantiumBlock: big.NewInt(1700000), + ConstantinopleBlock: nil, + Ethash: new(EthashConfig), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. RinkebyChainConfig = &ChainConfig{ - ChainId: big.NewInt(4), - HomesteadBlock: big.NewInt(1), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(2), - EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), - EIP155Block: big.NewInt(3), - EIP158Block: big.NewInt(3), - ByzantiumBlock: big.NewInt(1035301), - + ChainId: big.NewInt(4), + HomesteadBlock: big.NewInt(1), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(2), + EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), + EIP155Block: big.NewInt(3), + EIP158Block: big.NewInt(3), + ByzantiumBlock: big.NewInt(1035301), + ConstantinopleBlock: nil, Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -82,16 +82,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -115,7 +115,8 @@ type ChainConfig struct { EIP155Block *big.Int `json:"eip155Block,omitempty"` // EIP155 HF block EIP158Block *big.Int `json:"eip158Block,omitempty"` // EIP158 HF block - ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = already on byzantium) + ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = already on byzantium) + ConstantinopleBlock *big.Int `json:"constantinopleBlock,omitempty"` // Constantinople switch block (nil = no fork, 0 = already activated) // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` @@ -152,7 +153,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Engine: %v}", c.ChainId, c.HomesteadBlock, c.DAOForkBlock, @@ -161,6 +162,7 @@ func (c *ChainConfig) String() string { c.EIP155Block, c.EIP158Block, c.ByzantiumBlock, + c.ConstantinopleBlock, engine, ) } @@ -191,6 +193,10 @@ func (c *ChainConfig) IsByzantium(num *big.Int) bool { return isForked(c.ByzantiumBlock, num) } +func (c *ChainConfig) IsConstantinople(num *big.Int) bool { + return isForked(c.ConstantinopleBlock, num) +} + // GasTable returns the gas table corresponding to the current phase (homestead or homestead reprice). // // The returned GasTable's fields shouldn't, under any circumstances, be changed. @@ -251,6 +257,9 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.ByzantiumBlock, newcfg.ByzantiumBlock, head) { return newCompatError("Byzantium fork block", c.ByzantiumBlock, newcfg.ByzantiumBlock) } + if isForkIncompatible(c.ConstantinopleBlock, newcfg.ConstantinopleBlock, head) { + return newCompatError("Constantinople fork block", c.ConstantinopleBlock, newcfg.ConstantinopleBlock) + } return nil } From 89f914c03075012603267d207de0f433740dcd30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?=
Date: Fri, 23 Feb 2018 14:02:33 +0200 Subject: [PATCH 47/94] core: flush out trie cache more meaningfully on stop (#16143) * core: flush out trie cache more meaningfully on stop * core: upgrade legacy tests to chain maker --- core/blockchain.go | 25 +++-- core/blockchain_test.go | 220 +++++++++++++++++++--------------------- core/genesis_test.go | 6 +- 3 files changed, 123 insertions(+), 128 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 644df123c5..53fe7ee2eb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -648,22 +648,21 @@ func (bc *BlockChain) Stop() { bc.wg.Wait() // Ensure the state of a recent block is also stored to disk before exiting. - // It is fine if this state does not exist (fast start/stop cycle), but it is - // advisable to leave an N block gap from the head so 1) a restart loads up - // the last N blocks as sync assistance to remote nodes; 2) a restart during - // a (small) reorg doesn't require deep reprocesses; 3) chain "repair" from - // missing states are constantly tested. - // - // This may be tuned a bit on mainnet if its too annoying to reprocess the last - // N blocks. + // We're writing three different states to catch different restart scenarios: + // - HEAD: So we don't need to reprocess any blocks in the general case + // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle + // - HEAD-127: So we have a hard limit on the number of blocks reexecuted if !bc.cacheConfig.Disabled { triedb := bc.stateCache.TrieDB() - if number := bc.CurrentBlock().NumberU64(); number >= triesInMemory { - recent := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - triesInMemory + 1) - log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) - if err := triedb.Commit(recent.Root(), true); err != nil { - log.Error("Failed to commit recent state trie", "err", err) + for _, offset := range []uint64{0, 1, triesInMemory - 1} { + if number := bc.CurrentBlock().NumberU64(); number > offset { + recent := bc.GetBlockByNumber(number - offset) + + log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) + if err := triedb.Commit(recent.Root(), true); err != nil { + log.Error("Failed to commit recent state trie", "err", err) + } } } for !bc.triegc.Empty() { diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 635379161c..748cdc5c71 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -34,26 +34,6 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// newTestBlockChain creates a blockchain without validation. -func newTestBlockChain(fake bool) *BlockChain { - db, _ := ethdb.NewMemDatabase() - gspec := &Genesis{ - Config: params.TestChainConfig, - Difficulty: big.NewInt(1), - } - gspec.MustCommit(db) - engine := ethash.NewFullFaker() - if !fake { - engine = ethash.NewTester() - } - blockchain, err := NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) - if err != nil { - panic(err) - } - blockchain.SetValidator(bproc{}) - return blockchain -} - // Test fork of length N starting from block i func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int)) { // Copy old chain up to #i into a new db @@ -183,13 +163,18 @@ func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t * } func TestLastBlock(t *testing.T) { - bchain := newTestBlockChain(false) - defer bchain.Stop() + _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() - block := makeBlockChain(bchain.CurrentBlock(), 1, ethash.NewFaker(), bchain.db, 0)[0] - bchain.insert(block) - if block.Hash() != GetHeadBlockHash(bchain.db) { - t.Errorf("Write/Get HeadBlockHash failed") + blocks := makeBlockChain(blockchain.CurrentBlock(), 1, ethash.NewFullFaker(), blockchain.db, 0) + if _, err := blockchain.InsertChain(blocks); err != nil { + t.Fatalf("Failed to insert block: %v", err) + } + if blocks[len(blocks)-1].Hash() != GetHeadBlockHash(blockchain.db) { + t.Fatalf("Write/Get HeadBlockHash failed") } } @@ -337,55 +322,13 @@ func testBrokenChain(t *testing.T, full bool) { } } -type bproc struct{} - -func (bproc) ValidateBody(*types.Block) error { return nil } -func (bproc) ValidateState(block, parent *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64) error { - return nil -} -func (bproc) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (types.Receipts, []*types.Log, uint64, error) { - return nil, nil, 0, nil -} - -func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Header { - blocks := makeBlockChainWithDiff(genesis, d, seed) - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - return headers -} - -func makeBlockChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block { - var chain []*types.Block - for i, difficulty := range d { - header := &types.Header{ - Coinbase: common.Address{seed}, - Number: big.NewInt(int64(i + 1)), - Difficulty: big.NewInt(int64(difficulty)), - UncleHash: types.EmptyUncleHash, - TxHash: types.EmptyRootHash, - ReceiptHash: types.EmptyRootHash, - Time: big.NewInt(int64(i) + 1), - } - if i == 0 { - header.ParentHash = genesis.Hash() - } else { - header.ParentHash = chain[i-1].Hash() - } - block := types.NewBlockWithHeader(header) - chain = append(chain, block) - } - return chain -} - // Tests that reorganising a long difficult chain after a short easy one // overwrites the canonical numbers and links in the database. func TestReorgLongHeaders(t *testing.T) { testReorgLong(t, false) } func TestReorgLongBlocks(t *testing.T) { testReorgLong(t, true) } func testReorgLong(t *testing.T, full bool) { - testReorg(t, []int{1, 2, 4}, []int{1, 2, 3, 4}, 10, full) + testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280, full) } // Tests that reorganising a short difficult chain after a long easy one @@ -394,45 +337,82 @@ func TestReorgShortHeaders(t *testing.T) { testReorgShort(t, false) } func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true) } func testReorgShort(t *testing.T, full bool) { - testReorg(t, []int{1, 2, 3, 4}, []int{1, 10}, 11, full) + // Create a long easy chain vs. a short heavy one. Due to difficulty adjustment + // we need a fairly long chain of blocks with different difficulties for a short + // one to become heavyer than a long one. The 96 is an empirical value. + easy := make([]int64, 96) + for i := 0; i < len(easy); i++ { + easy[i] = 60 + } + diff := make([]int64, len(easy)-1) + for i := 0; i < len(diff); i++ { + diff[i] = -9 + } + testReorg(t, easy, diff, 12615120, full) } -func testReorg(t *testing.T, first, second []int, td int64, full bool) { - bc := newTestBlockChain(true) - defer bc.Stop() +func testReorg(t *testing.T, first, second []int64, td int64, full bool) { + // Create a pristine chain and database + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() // Insert an easy and a difficult chain afterwards + easyBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(first), func(i int, b *BlockGen) { + b.OffsetTime(first[i]) + }) + diffBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(second), func(i int, b *BlockGen) { + b.OffsetTime(second[i]) + }) if full { - bc.InsertChain(makeBlockChainWithDiff(bc.genesisBlock, first, 11)) - bc.InsertChain(makeBlockChainWithDiff(bc.genesisBlock, second, 22)) + if _, err := blockchain.InsertChain(easyBlocks); err != nil { + t.Fatalf("failed to insert easy chain: %v", err) + } + if _, err := blockchain.InsertChain(diffBlocks); err != nil { + t.Fatalf("failed to insert difficult chain: %v", err) + } } else { - bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, first, 11), 1) - bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, second, 22), 1) + easyHeaders := make([]*types.Header, len(easyBlocks)) + for i, block := range easyBlocks { + easyHeaders[i] = block.Header() + } + diffHeaders := make([]*types.Header, len(diffBlocks)) + for i, block := range diffBlocks { + diffHeaders[i] = block.Header() + } + if _, err := blockchain.InsertHeaderChain(easyHeaders, 1); err != nil { + t.Fatalf("failed to insert easy chain: %v", err) + } + if _, err := blockchain.InsertHeaderChain(diffHeaders, 1); err != nil { + t.Fatalf("failed to insert difficult chain: %v", err) + } } // Check that the chain is valid number and link wise if full { - prev := bc.CurrentBlock() - for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) { + prev := blockchain.CurrentBlock() + for block := blockchain.GetBlockByNumber(blockchain.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, blockchain.GetBlockByNumber(block.NumberU64()-1) { if prev.ParentHash() != block.Hash() { t.Errorf("parent block hash mismatch: have %x, want %x", prev.ParentHash(), block.Hash()) } } } else { - prev := bc.CurrentHeader() - for header := bc.GetHeaderByNumber(bc.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, bc.GetHeaderByNumber(header.Number.Uint64()-1) { + prev := blockchain.CurrentHeader() + for header := blockchain.GetHeaderByNumber(blockchain.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, blockchain.GetHeaderByNumber(header.Number.Uint64()-1) { if prev.ParentHash != header.Hash() { t.Errorf("parent header hash mismatch: have %x, want %x", prev.ParentHash, header.Hash()) } } } // Make sure the chain total difficulty is the correct one - want := new(big.Int).Add(bc.genesisBlock.Difficulty(), big.NewInt(td)) + want := new(big.Int).Add(blockchain.genesisBlock.Difficulty(), big.NewInt(td)) if full { - if have := bc.GetTdByHash(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 { + if have := blockchain.GetTdByHash(blockchain.CurrentBlock().Hash()); have.Cmp(want) != 0 { t.Errorf("total difficulty mismatch: have %v, want %v", have, want) } } else { - if have := bc.GetTdByHash(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 { + if have := blockchain.GetTdByHash(blockchain.CurrentHeader().Hash()); have.Cmp(want) != 0 { t.Errorf("total difficulty mismatch: have %v, want %v", have, want) } } @@ -443,19 +423,28 @@ func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false) } func TestBadBlockHashes(t *testing.T) { testBadHashes(t, true) } func testBadHashes(t *testing.T, full bool) { - bc := newTestBlockChain(true) - defer bc.Stop() + // Create a pristine chain and database + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() // Create a chain, ban a hash and try to import - var err error if full { - blocks := makeBlockChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10) + blocks := makeBlockChain(blockchain.CurrentBlock(), 3, ethash.NewFaker(), db, 10) + BadHashes[blocks[2].Header().Hash()] = true - _, err = bc.InsertChain(blocks) + defer func() { delete(BadHashes, blocks[2].Header().Hash()) }() + + _, err = blockchain.InsertChain(blocks) } else { - headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10) + headers := makeHeaderChain(blockchain.CurrentHeader(), 3, ethash.NewFaker(), db, 10) + BadHashes[headers[2].Hash()] = true - _, err = bc.InsertHeaderChain(headers, 1) + defer func() { delete(BadHashes, headers[2].Hash()) }() + + _, err = blockchain.InsertHeaderChain(headers, 1) } if err != ErrBlacklistedHash { t.Errorf("error mismatch: have: %v, want: %v", err, ErrBlacklistedHash) @@ -468,40 +457,41 @@ func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false) } func TestReorgBadBlockHashes(t *testing.T) { testReorgBadHashes(t, true) } func testReorgBadHashes(t *testing.T, full bool) { - bc := newTestBlockChain(true) - defer bc.Stop() - + // Create a pristine chain and database + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } // Create a chain, import and ban afterwards - headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10) - blocks := makeBlockChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10) + headers := makeHeaderChain(blockchain.CurrentHeader(), 4, ethash.NewFaker(), db, 10) + blocks := makeBlockChain(blockchain.CurrentBlock(), 4, ethash.NewFaker(), db, 10) if full { - if _, err := bc.InsertChain(blocks); err != nil { - t.Fatalf("failed to import blocks: %v", err) + if _, err = blockchain.InsertChain(blocks); err != nil { + t.Errorf("failed to import blocks: %v", err) } - if bc.CurrentBlock().Hash() != blocks[3].Hash() { - t.Errorf("last block hash mismatch: have: %x, want %x", bc.CurrentBlock().Hash(), blocks[3].Header().Hash()) + if blockchain.CurrentBlock().Hash() != blocks[3].Hash() { + t.Errorf("last block hash mismatch: have: %x, want %x", blockchain.CurrentBlock().Hash(), blocks[3].Header().Hash()) } BadHashes[blocks[3].Header().Hash()] = true defer func() { delete(BadHashes, blocks[3].Header().Hash()) }() } else { - if _, err := bc.InsertHeaderChain(headers, 1); err != nil { - t.Fatalf("failed to import headers: %v", err) + if _, err = blockchain.InsertHeaderChain(headers, 1); err != nil { + t.Errorf("failed to import headers: %v", err) } - if bc.CurrentHeader().Hash() != headers[3].Hash() { - t.Errorf("last header hash mismatch: have: %x, want %x", bc.CurrentHeader().Hash(), headers[3].Hash()) + if blockchain.CurrentHeader().Hash() != headers[3].Hash() { + t.Errorf("last header hash mismatch: have: %x, want %x", blockchain.CurrentHeader().Hash(), headers[3].Hash()) } BadHashes[headers[3].Hash()] = true defer func() { delete(BadHashes, headers[3].Hash()) }() } + blockchain.Stop() // Create a new BlockChain and check that it rolled back the state. - ncm, err := NewBlockChain(bc.db, nil, bc.chainConfig, ethash.NewFaker(), vm.Config{}) + ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{}) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } - defer ncm.Stop() - if full { if ncm.CurrentBlock().Hash() != blocks[2].Header().Hash() { t.Errorf("last block hash mismatch: have: %x, want %x", ncm.CurrentBlock().Hash(), blocks[2].Header().Hash()) @@ -514,6 +504,7 @@ func testReorgBadHashes(t *testing.T, full bool) { t.Errorf("last header hash mismatch: have: %x, want %x", ncm.CurrentHeader().Hash(), headers[2].Hash()) } } + ncm.Stop() } // Tests chain insertions in the face of one entity containing an invalid nonce. @@ -989,10 +980,13 @@ done: // Tests if the canonical block can be fetched from the database during chain insertion. func TestCanonicalBlockRetrieval(t *testing.T) { - bc := newTestBlockChain(true) - defer bc.Stop() + _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() - chain, _ := GenerateChain(bc.chainConfig, bc.genesisBlock, ethash.NewFaker(), bc.db, 10, func(i int, gen *BlockGen) {}) + chain, _ := GenerateChain(blockchain.chainConfig, blockchain.genesisBlock, ethash.NewFaker(), blockchain.db, 10, func(i int, gen *BlockGen) {}) var pend sync.WaitGroup pend.Add(len(chain)) @@ -1003,14 +997,14 @@ func TestCanonicalBlockRetrieval(t *testing.T) { // try to retrieve a block by its canonical hash and see if the block data can be retrieved. for { - ch := GetCanonicalHash(bc.db, block.NumberU64()) + ch := GetCanonicalHash(blockchain.db, block.NumberU64()) if ch == (common.Hash{}) { continue // busy wait for canonical hash to be written } if ch != block.Hash() { t.Fatalf("unknown canonical hash, want %s, got %s", block.Hash().Hex(), ch.Hex()) } - fb := GetBlock(bc.db, ch, block.NumberU64()) + fb := GetBlock(blockchain.db, ch, block.NumberU64()) if fb == nil { t.Fatalf("unable to retrieve block %d for canonical hash: %s", block.NumberU64(), ch.Hex()) } @@ -1021,7 +1015,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) { } }(chain[i]) - if _, err := bc.InsertChain(types.Blocks{chain[i]}); err != nil { + if _, err := blockchain.InsertChain(types.Blocks{chain[i]}); err != nil { t.Fatalf("failed to insert block %d: %v", i, err) } } diff --git a/core/genesis_test.go b/core/genesis_test.go index cd548d4b1d..052ded6991 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -118,10 +118,12 @@ func TestSetupGenesis(t *testing.T) { // Commit the 'old' genesis block with Homestead transition at #2. // Advance to block #4, past the homestead transition block of customg. genesis := oldcustomg.MustCommit(db) + bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{}) defer bc.Stop() - bc.SetValidator(bproc{}) - bc.InsertChain(makeBlockChainWithDiff(genesis, []int{2, 3, 4, 5}, 0)) + + blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil) + bc.InsertChain(blocks) bc.CurrentBlock() // This should return a compatibility error. return SetupGenesisBlock(db, &customg) From b677a07d36c957c4221bae952189559ac0c70537 Mon Sep 17 00:00:00 2001 From: Lewis Marshall
Date: Fri, 23 Feb 2018 13:09:01 +0000 Subject: [PATCH 48/94] swarm/api/http: Fix using deprecated bzzr scheme (#16152) Without this, deprecated bzzr requests just return an empty response. Signed-off-by: Lewis Marshall
--- swarm/api/http/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 74341899d2..b89487af7e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -349,7 +349,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { } switch { - case r.uri.Raw(): + case r.uri.Raw() || r.uri.DeprecatedRaw(): // allow the request to overwrite the content type using a query // parameter contentType := "application/octet-stream" From dcca613a0b4c6ce56e52f4607cf71f4f1338db8f Mon Sep 17 00:00:00 2001 From: Anton Evangelatov
Date: Fri, 23 Feb 2018 14:19:59 +0100 Subject: [PATCH 49/94] swarm: initial instrumentation (#15969) * swarm: initial instrumentation with go-metrics * swarm: initialise metrics collection and add ResettingTimer to HTTP requests * swarm: update metrics flags names. remove redundant Timer. * swarm: rename method for periodically updating gauges * swarm: finalise metrics after feedback * swarm/network: always init kad metrics containers * swarm/network: off-by-one index in metrics containers * swarm, metrics: resolved conflicts --- cmd/swarm/main.go | 8 ++- metrics/metrics.go | 1 - swarm/api/api.go | 53 +++++++++++++++++++ swarm/api/http/error.go | 9 ++++ swarm/api/http/server.go | 69 +++++++++++++++++++++++++ swarm/fuse/swarmfs_util.go | 7 --- swarm/metrics/flags.go | 82 ++++++++++++++++++++++++++++++ swarm/network/depo.go | 15 ++++++ swarm/network/hive.go | 10 ++++ swarm/network/kademlia/kademlia.go | 28 +++++++++- swarm/network/protocol.go | 24 +++++++++ swarm/storage/chunker.go | 14 +++++ swarm/storage/dbstore.go | 9 ++++ swarm/storage/localstore.go | 16 ++++++ swarm/storage/memstore.go | 14 +++++ swarm/swarm.go | 32 ++++++++++++ 16 files changed, 381 insertions(+), 10 deletions(-) create mode 100644 swarm/metrics/flags.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 4b08237969..360020b77b 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -43,6 +43,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" + swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics" "gopkg.in/urfave/cli.v1" ) @@ -359,9 +360,14 @@ DEPRECATED: use 'swarm db clean'. DeprecatedEnsAddrFlag, } app.Flags = append(app.Flags, debug.Flags...) + app.Flags = append(app.Flags, swarmmetrics.Flags...) app.Before = func(ctx *cli.Context) error { runtime.GOMAXPROCS(runtime.NumCPU()) - return debug.Setup(ctx) + if err := debug.Setup(ctx); err != nil { + return err + } + swarmmetrics.Setup(ctx) + return nil } app.After = func(ctx *cli.Context) error { debug.Exit() diff --git a/metrics/metrics.go b/metrics/metrics.go index 4e4e3ecb2b..e24324814c 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -35,7 +35,6 @@ func init() { Enabled = true } } - //exp.Exp(DefaultRegistry) } // CollectProcessMetrics periodically collects various metrics about the running diff --git a/swarm/api/api.go b/swarm/api/api.go index fdf76d3901..0cf12fdbed 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -32,11 +32,31 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/storage" ) var hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}") +//setup metrics +var ( + apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil) + apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil) + apiPutCount = metrics.NewRegisteredCounter("api.put.count", nil) + apiPutFail = metrics.NewRegisteredCounter("api.put.fail", nil) + apiGetCount = metrics.NewRegisteredCounter("api.get.count", nil) + apiGetNotFound = metrics.NewRegisteredCounter("api.get.notfound", nil) + apiGetHttp300 = metrics.NewRegisteredCounter("api.get.http.300", nil) + apiModifyCount = metrics.NewRegisteredCounter("api.modify.count", nil) + apiModifyFail = metrics.NewRegisteredCounter("api.modify.fail", nil) + apiAddFileCount = metrics.NewRegisteredCounter("api.addfile.count", nil) + apiAddFileFail = metrics.NewRegisteredCounter("api.addfile.fail", nil) + apiRmFileCount = metrics.NewRegisteredCounter("api.removefile.count", nil) + apiRmFileFail = metrics.NewRegisteredCounter("api.removefile.fail", nil) + apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil) + apiAppendFileFail = metrics.NewRegisteredCounter("api.appendfile.fail", nil) +) + type Resolver interface { Resolve(string) (common.Hash, error) } @@ -155,6 +175,7 @@ type ErrResolve error // DNS Resolver func (self *Api) Resolve(uri *URI) (storage.Key, error) { + apiResolveCount.Inc(1) log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr)) // if the URI is immutable, check if the address is a hash @@ -169,6 +190,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { // if DNS is not configured, check if the address is a hash if self.dns == nil { if !isHash { + apiResolveFail.Inc(1) return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr) } return common.Hex2Bytes(uri.Addr), nil @@ -179,6 +201,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { if err == nil { return resolved[:], nil } else if !isHash { + apiResolveFail.Inc(1) return nil, err } return common.Hex2Bytes(uri.Addr), nil @@ -186,16 +209,19 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { // Put provides singleton manifest creation on top of dpa store func (self *Api) Put(content, contentType string) (storage.Key, error) { + apiPutCount.Inc(1) r := strings.NewReader(content) wg := &sync.WaitGroup{} key, err := self.dpa.Store(r, int64(len(content)), wg, nil) if err != nil { + apiPutFail.Inc(1) return nil, err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) r = strings.NewReader(manifest) key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil) if err != nil { + apiPutFail.Inc(1) return nil, err } wg.Wait() @@ -206,8 +232,10 @@ func (self *Api) Put(content, contentType string) (storage.Key, error) { // to resolve basePath to content using dpa retrieve // it returns a section reader, mimeType, status and an error func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) { + apiGetCount.Inc(1) trie, err := loadManifest(self.dpa, key, nil) if err != nil { + apiGetNotFound.Inc(1) status = http.StatusNotFound log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err)) return @@ -221,6 +249,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe key = common.Hex2Bytes(entry.Hash) status = entry.Status if status == http.StatusMultipleChoices { + apiGetHttp300.Inc(1) return } else { mimeType = entry.ContentType @@ -229,6 +258,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe } } else { status = http.StatusNotFound + apiGetNotFound.Inc(1) err = fmt.Errorf("manifest entry for '%s' not found", path) log.Warn(fmt.Sprintf("%v", err)) } @@ -236,9 +266,11 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe } func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) { + apiModifyCount.Inc(1) quitC := make(chan bool) trie, err := loadManifest(self.dpa, key, quitC) if err != nil { + apiModifyFail.Inc(1) return nil, err } if contentHash != "" { @@ -253,19 +285,23 @@ func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) } if err := trie.recalcAndStore(); err != nil { + apiModifyFail.Inc(1) return nil, err } return trie.hash, nil } func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) { + apiAddFileCount.Inc(1) uri, err := Parse("bzz:/" + mhash) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } mkey, err := self.Resolve(uri) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } @@ -284,16 +320,19 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } fkey, err := mw.AddEntry(bytes.NewReader(content), entry) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } newMkey, err := mw.Store() if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } @@ -303,13 +342,16 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver } func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) { + apiRmFileCount.Inc(1) uri, err := Parse("bzz:/" + mhash) if err != nil { + apiRmFileFail.Inc(1) return "", err } mkey, err := self.Resolve(uri) if err != nil { + apiRmFileFail.Inc(1) return "", err } @@ -320,16 +362,19 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiRmFileFail.Inc(1) return "", err } err = mw.RemoveEntry(filepath.Join(path, fname)) if err != nil { + apiRmFileFail.Inc(1) return "", err } newMkey, err := mw.Store() if err != nil { + apiRmFileFail.Inc(1) return "", err } @@ -338,6 +383,7 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin } func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldKey storage.Key, offset int64, addSize int64, nameresolver bool) (storage.Key, string, error) { + apiAppendFileCount.Inc(1) buffSize := offset + addSize if buffSize < existingSize { @@ -366,10 +412,12 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte uri, err := Parse("bzz:/" + mhash) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } mkey, err := self.Resolve(uri) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -380,11 +428,13 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } err = mw.RemoveEntry(filepath.Join(path, fname)) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -398,11 +448,13 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte fkey, err := mw.AddEntry(io.Reader(combinedReader), entry) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } newMkey, err := mw.Store() if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -412,6 +464,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte } func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) { + uri, err := Parse("bzz:/" + mhash) if err != nil { return nil, nil, err diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go index dbd97182fd..831cf23fe2 100644 --- a/swarm/api/http/error.go +++ b/swarm/api/http/error.go @@ -29,12 +29,19 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" ) //templateMap holds a mapping of an HTTP error code to a template var templateMap map[int]*template.Template +//metrics variables +var ( + htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil) + jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil) +) + //parameters needed for formatting the correct HTML page type ErrorParams struct { Msg string @@ -132,6 +139,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ErrorParams) { //return a HTML page func respondHtml(w http.ResponseWriter, params *ErrorParams) { + htmlCounter.Inc(1) err := params.template.Execute(w, params) if err != nil { log.Error(err.Error()) @@ -140,6 +148,7 @@ func respondHtml(w http.ResponseWriter, params *ErrorParams) { //return JSON func respondJson(w http.ResponseWriter, params *ErrorParams) { + jsonCounter.Inc(1) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(params) } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index b89487af7e..6ebfc80590 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -37,11 +37,35 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/rs/cors" ) +//setup metrics +var ( + postRawCount = metrics.NewRegisteredCounter("api.http.post.raw.count", nil) + postRawFail = metrics.NewRegisteredCounter("api.http.post.raw.fail", nil) + postFilesCount = metrics.NewRegisteredCounter("api.http.post.files.count", nil) + postFilesFail = metrics.NewRegisteredCounter("api.http.post.files.fail", nil) + deleteCount = metrics.NewRegisteredCounter("api.http.delete.count", nil) + deleteFail = metrics.NewRegisteredCounter("api.http.delete.fail", nil) + getCount = metrics.NewRegisteredCounter("api.http.get.count", nil) + getFail = metrics.NewRegisteredCounter("api.http.get.fail", nil) + getFileCount = metrics.NewRegisteredCounter("api.http.get.file.count", nil) + getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil) + getFileFail = metrics.NewRegisteredCounter("api.http.get.file.fail", nil) + getFilesCount = metrics.NewRegisteredCounter("api.http.get.files.count", nil) + getFilesFail = metrics.NewRegisteredCounter("api.http.get.files.fail", nil) + getListCount = metrics.NewRegisteredCounter("api.http.get.list.count", nil) + getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil) + requestCount = metrics.NewRegisteredCounter("http.request.count", nil) + htmlRequestCount = metrics.NewRegisteredCounter("http.request.html.count", nil) + jsonRequestCount = metrics.NewRegisteredCounter("http.request.json.count", nil) + requestTimer = metrics.NewRegisteredResettingTimer("http.request.time", nil) +) + // ServerConfig is the basic configuration needed for the HTTP server and also // includes CORS settings. type ServerConfig struct { @@ -89,18 +113,22 @@ type Request struct { // HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request // body in swarm and returns the resulting storage key as a text/plain response func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { + postRawCount.Inc(1) if r.uri.Path != "" { + postRawFail.Inc(1) s.BadRequest(w, r, "raw POST request cannot contain a path") return } if r.Header.Get("Content-Length") == "" { + postRawFail.Inc(1) s.BadRequest(w, r, "missing Content-Length header in request") return } key, err := s.api.Store(r.Body, r.ContentLength, nil) if err != nil { + postRawFail.Inc(1) s.Error(w, r, err) return } @@ -117,8 +145,10 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { // existing manifest or to a new manifest under
and returns the // resulting manifest hash as a text/plain response func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { + postFilesCount.Inc(1) contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { + postFilesFail.Inc(1) s.BadRequest(w, r, err.Error()) return } @@ -127,12 +157,14 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { if r.uri.Addr != "" { key, err = s.api.Resolve(r.uri) if err != nil { + postFilesFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } } else { key, err = s.api.NewManifest() if err != nil { + postFilesFail.Inc(1) s.Error(w, r, err) return } @@ -152,6 +184,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { } }) if err != nil { + postFilesFail.Inc(1) s.Error(w, r, fmt.Errorf("error creating manifest: %s", err)) return } @@ -270,8 +303,10 @@ func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error //
from
and returns the resulting manifest hash as a // text/plain response func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { + deleteCount.Inc(1) key, err := s.api.Resolve(r.uri) if err != nil { + deleteFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -281,6 +316,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { return mw.RemoveEntry(r.uri.Path) }) if err != nil { + deleteFail.Inc(1) s.Error(w, r, fmt.Errorf("error updating manifest: %s", err)) return } @@ -296,8 +332,10 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { // - bzz-hash://
and responds with the hash of the content stored // at the given storage key as a text/plain response func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { + getCount.Inc(1) key, err := s.api.Resolve(r.uri) if err != nil { + getFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -307,6 +345,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { walker, err := s.api.NewManifestWalker(key, nil) if err != nil { + getFail.Inc(1) s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key)) return } @@ -335,6 +374,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { return api.SkipManifest }) if entry == nil { + getFail.Inc(1) s.NotFound(w, r, fmt.Errorf("Manifest entry could not be loaded")) return } @@ -344,6 +384,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size reader := s.api.Retrieve(key) if _, err := reader.Size(nil); err != nil { + getFail.Inc(1) s.NotFound(w, r, fmt.Errorf("Root chunk not found %s: %s", key, err)) return } @@ -370,19 +411,23 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // header of "application/x-tar" and returns a tar stream of all files // contained in the manifest func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { + getFilesCount.Inc(1) if r.uri.Path != "" { + getFilesFail.Inc(1) s.BadRequest(w, r, "files request cannot contain a path") return } key, err := s.api.Resolve(r.uri) if err != nil { + getFilesFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } walker, err := s.api.NewManifestWalker(key, nil) if err != nil { + getFilesFail.Inc(1) s.Error(w, r, err) return } @@ -430,6 +475,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { return nil }) if err != nil { + getFilesFail.Inc(1) s.logError("error generating tar stream: %s", err) } } @@ -438,6 +484,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { // a list of all files contained in
under
grouped into // common prefixes using "/" as a delimiter func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { + getListCount.Inc(1) // ensure the root path has a trailing slash so that relative URLs work if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) @@ -446,6 +493,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { + getListFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -453,6 +501,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { list, err := s.getManifestList(key, r.uri.Path) if err != nil { + getListFail.Inc(1) s.Error(w, r, err) return } @@ -470,6 +519,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { List: &list, }) if err != nil { + getListFail.Inc(1) s.logError("error rendering list HTML: %s", err) } return @@ -538,6 +588,7 @@ func (s *Server) getManifestList(key storage.Key, prefix string) (list api.Manif // HandleGetFile handles a GET request to bzz://
/
and responds // with the content of the file at
from the given
func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { + getFileCount.Inc(1) // ensure the root path has a trailing slash so that relative URLs work if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) @@ -546,6 +597,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { + getFileFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -554,8 +606,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { if err != nil { switch status { case http.StatusNotFound: + getFileNotFound.Inc(1) s.NotFound(w, r, err) default: + getFileFail.Inc(1) s.Error(w, r, err) } return @@ -567,6 +621,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { list, err := s.getManifestList(key, r.uri.Path) if err != nil { + getFileFail.Inc(1) s.Error(w, r, err) return } @@ -579,6 +634,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size if _, err := reader.Size(nil); err != nil { + getFileNotFound.Inc(1) s.NotFound(w, r, fmt.Errorf("File not found %s: %s", r.uri, err)) return } @@ -589,6 +645,19 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { } func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if metrics.Enabled { + //The increment for request count and request timer themselves have a flag check + //for metrics.Enabled. Nevertheless, we introduce the if here because we + //are looking into the header just to see what request type it is (json/html). + //So let's take advantage and add all metrics related stuff here + requestCount.Inc(1) + defer requestTimer.UpdateSince(time.Now()) + if r.Header.Get("Accept") == "application/json" { + jsonRequestCount.Inc(1) + } else { + htmlRequestCount.Inc(1) + } + } s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) diff --git a/swarm/fuse/swarmfs_util.go b/swarm/fuse/swarmfs_util.go index d39966c0e3..169b674876 100644 --- a/swarm/fuse/swarmfs_util.go +++ b/swarm/fuse/swarmfs_util.go @@ -47,7 +47,6 @@ func externalUnmount(mountPoint string) error { } func addFileToSwarm(sf *SwarmFile, content []byte, size int) error { - fkey, mhash, err := sf.mountInfo.swarmApi.AddFile(sf.mountInfo.LatestManifest, sf.path, sf.name, content, true) if err != nil { return err @@ -64,11 +63,9 @@ func addFileToSwarm(sf *SwarmFile, content []byte, size int) error { log.Info("Added new file:", "fname", sf.name, "New Manifest hash", mhash) return nil - } func removeFileFromSwarm(sf *SwarmFile) error { - mkey, err := sf.mountInfo.swarmApi.RemoveFile(sf.mountInfo.LatestManifest, sf.path, sf.name, true) if err != nil { return err @@ -83,7 +80,6 @@ func removeFileFromSwarm(sf *SwarmFile) error { } func removeDirectoryFromSwarm(sd *SwarmDir) error { - if len(sd.directories) == 0 && len(sd.files) == 0 { return nil } @@ -103,11 +99,9 @@ func removeDirectoryFromSwarm(sd *SwarmDir) error { } return nil - } func appendToExistingFileInSwarm(sf *SwarmFile, content []byte, offset int64, length int64) error { - fkey, mhash, err := sf.mountInfo.swarmApi.AppendFile(sf.mountInfo.LatestManifest, sf.path, sf.name, sf.fileSize, content, sf.key, offset, length, true) if err != nil { return err @@ -124,5 +118,4 @@ func appendToExistingFileInSwarm(sf *SwarmFile, content []byte, offset int64, le log.Info("Appended file:", "fname", sf.name, "New Manifest hash", mhash) return nil - } diff --git a/swarm/metrics/flags.go b/swarm/metrics/flags.go new file mode 100644 index 0000000000..60e34e6e42 --- /dev/null +++ b/swarm/metrics/flags.go @@ -0,0 +1,82 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see
. + +package metrics + +import ( + "time" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + gethmetrics "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/metrics/influxdb" + "gopkg.in/urfave/cli.v1" +) + +var ( + metricsInfluxDBEndpointFlag = cli.StringFlag{ + Name: "metrics.influxdb.endpoint", + Usage: "Metrics InfluxDB endpoint", + Value: "http://127.0.0.1:8086", + } + metricsInfluxDBDatabaseFlag = cli.StringFlag{ + Name: "metrics.influxdb.database", + Usage: "metrics InfluxDB database", + Value: "metrics", + } + metricsInfluxDBUsernameFlag = cli.StringFlag{ + Name: "metrics.influxdb.username", + Usage: "metrics InfluxDB username", + Value: "", + } + metricsInfluxDBPasswordFlag = cli.StringFlag{ + Name: "metrics.influxdb.password", + Usage: "metrics InfluxDB password", + Value: "", + } + // The `host` tag is part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. + // It is used so that we can group all nodes and average a measurement across all of them, but also so + // that we can select a specific node and inspect its measurements. + // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key + metricsInfluxDBHostTagFlag = cli.StringFlag{ + Name: "metrics.influxdb.host.tag", + Usage: "metrics InfluxDB `host` tag attached to all measurements", + Value: "localhost", + } +) + +// Flags holds all command-line flags required for metrics collection. +var Flags = []cli.Flag{ + utils.MetricsEnabledFlag, + metricsInfluxDBEndpointFlag, metricsInfluxDBDatabaseFlag, metricsInfluxDBUsernameFlag, metricsInfluxDBPasswordFlag, metricsInfluxDBHostTagFlag, +} + +func Setup(ctx *cli.Context) { + if gethmetrics.Enabled { + var ( + endpoint = ctx.GlobalString(metricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(metricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(metricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(metricsInfluxDBPasswordFlag.Name) + hosttag = ctx.GlobalString(metricsInfluxDBHostTagFlag.Name) + ) + + log.Info("Enabling swarm metrics collection and export") + go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "swarm.", map[string]string{ + "host": hosttag, + }) + } +} diff --git a/swarm/network/depo.go b/swarm/network/depo.go index 17540d2f9f..5ffbf8be11 100644 --- a/swarm/network/depo.go +++ b/swarm/network/depo.go @@ -23,9 +23,19 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/storage" ) +//metrics variables +var ( + syncReceiveCount = metrics.NewRegisteredCounter("network.sync.recv.count", nil) + syncReceiveIgnore = metrics.NewRegisteredCounter("network.sync.recv.ignore", nil) + syncSendCount = metrics.NewRegisteredCounter("network.sync.send.count", nil) + syncSendRefused = metrics.NewRegisteredCounter("network.sync.send.refused", nil) + syncSendNotFound = metrics.NewRegisteredCounter("network.sync.send.notfound", nil) +) + // Handler for storage/retrieval related protocol requests // implements the StorageHandler interface used by the bzz protocol type Depo struct { @@ -107,6 +117,7 @@ func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) { log.Trace(fmt.Sprintf("Depo.handleStoreRequest: %v not found locally. create new chunk/request", req.Key)) // not found in memory cache, ie., a genuine store request // create chunk + syncReceiveCount.Inc(1) chunk = storage.NewChunk(req.Key, nil) case chunk.SData == nil: @@ -116,6 +127,7 @@ func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) { default: // data is found, store request ignored // this should update access count? + syncReceiveIgnore.Inc(1) log.Trace(fmt.Sprintf("Depo.HandleStoreRequest: %v found locally. ignore.", req)) islocal = true //return @@ -172,11 +184,14 @@ func (self *Depo) HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer) SData: chunk.SData, requestTimeout: req.timeout, // } + syncSendCount.Inc(1) p.syncer.addRequest(sreq, DeliverReq) } else { + syncSendRefused.Inc(1) log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content found, not wanted", req.Key.Log())) } } else { + syncSendNotFound.Inc(1) log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content not found locally. asked swarm for help. will get back", req.Key.Log())) } } diff --git a/swarm/network/hive.go b/swarm/network/hive.go index 2504a46100..8404ffcc2a 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/swarm/network/kademlia" @@ -39,6 +40,12 @@ import ( // connections and disconnections are reported and relayed // to keep the nodetable uptodate +var ( + peersNumGauge = metrics.NewRegisteredGauge("network.peers.num", nil) + addPeerCounter = metrics.NewRegisteredCounter("network.addpeer.count", nil) + removePeerCounter = metrics.NewRegisteredCounter("network.removepeer.count", nil) +) + type Hive struct { listenAddr func() string callInterval uint64 @@ -192,6 +199,7 @@ func (self *Hive) Start(id discover.NodeID, listenAddr func() string, connectPee func (self *Hive) keepAlive() { alarm := time.NewTicker(time.Duration(self.callInterval)).C for { + peersNumGauge.Update(int64(self.kad.Count())) select { case <-alarm: if self.kad.DBCount() > 0 { @@ -223,6 +231,7 @@ func (self *Hive) Stop() error { // called at the end of a successful protocol handshake func (self *Hive) addPeer(p *peer) error { + addPeerCounter.Inc(1) defer func() { select { case self.more <- true: @@ -247,6 +256,7 @@ func (self *Hive) addPeer(p *peer) error { // called after peer disconnected func (self *Hive) removePeer(p *peer) { + removePeerCounter.Inc(1) log.Debug(fmt.Sprintf("bee %v removed", p)) self.kad.Off(p, saveSync) select { diff --git a/swarm/network/kademlia/kademlia.go b/swarm/network/kademlia/kademlia.go index 0abc42a19c..b5999b52d4 100644 --- a/swarm/network/kademlia/kademlia.go +++ b/swarm/network/kademlia/kademlia.go @@ -24,6 +24,16 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +//metrics variables +//For metrics, we want to count how many times peers are added/removed +//at a certain index. Thus we do that with an array of counters with +//entry for each index +var ( + bucketAddIndexCount []metrics.Counter + bucketRmIndexCount []metrics.Counter ) const ( @@ -88,12 +98,14 @@ type Node interface { // params is KadParams configuration func New(addr Address, params *KadParams) *Kademlia { buckets := make([][]Node, params.MaxProx+1) - return &Kademlia{ + kad := &Kademlia{ addr: addr, KadParams: params, buckets: buckets, db: newKadDb(addr, params), } + kad.initMetricsVariables() + return kad } // accessor for KAD base address @@ -138,6 +150,7 @@ func (self *Kademlia) On(node Node, cb func(*NodeRecord, Node) error) (err error // TODO: give priority to peers with active traffic if len(bucket) < self.BucketSize { // >= allows us to add peers beyond the bucketsize limitation self.buckets[index] = append(bucket, node) + bucketAddIndexCount[index].Inc(1) log.Debug(fmt.Sprintf("add node %v to table", node)) self.setProxLimit(index, true) record.node = node @@ -178,6 +191,7 @@ func (self *Kademlia) Off(node Node, cb func(*NodeRecord, Node)) (err error) { defer self.lock.Unlock() index := self.proximityBin(node.Addr()) + bucketRmIndexCount[index].Inc(1) bucket := self.buckets[index] for i := 0; i < len(bucket); i++ { if node.Addr() == bucket[i].Addr() { @@ -426,3 +440,15 @@ func (self *Kademlia) String() string { rows = append(rows, "=========================================================================") return strings.Join(rows, "\n") } + +//We have to build up the array of counters for each index +func (self *Kademlia) initMetricsVariables() { + //create the arrays + bucketAddIndexCount = make([]metrics.Counter, self.MaxProx+1) + bucketRmIndexCount = make([]metrics.Counter, self.MaxProx+1) + //at each index create a metrics counter + for i := 0; i < (self.KadParams.MaxProx + 1); i++ { + bucketAddIndexCount[i] = metrics.NewRegisteredCounter(fmt.Sprintf("network.kademlia.bucket.add.%d.index", i), nil) + bucketRmIndexCount[i] = metrics.NewRegisteredCounter(fmt.Sprintf("network.kademlia.bucket.rm.%d.index", i), nil) + } +} diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index a418c1dbbd..1cbe00a973 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -39,12 +39,26 @@ import ( "github.com/ethereum/go-ethereum/contracts/chequebook" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" bzzswap "github.com/ethereum/go-ethereum/swarm/services/swap" "github.com/ethereum/go-ethereum/swarm/services/swap/swap" "github.com/ethereum/go-ethereum/swarm/storage" ) +//metrics variables +var ( + storeRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.storerequest.count", nil) + retrieveRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.retrieverequest.count", nil) + peersMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.peers.count", nil) + syncRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.syncrequest.count", nil) + unsyncedKeysMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.unsyncedkeys.count", nil) + deliverRequestMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.deliverrequest.count", nil) + paymentMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.payment.count", nil) + invalidMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.invalid.count", nil) + handleStatusMsgCounter = metrics.NewRegisteredCounter("network.protocol.msg.handlestatus.count", nil) +) + const ( Version = 0 ProtocolLength = uint64(8) @@ -206,6 +220,7 @@ func (self *bzz) handle() error { case storeRequestMsg: // store requests are dispatched to netStore + storeRequestMsgCounter.Inc(1) var req storeRequestMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<- %v: %v", msg, err) @@ -221,6 +236,7 @@ func (self *bzz) handle() error { case retrieveRequestMsg: // retrieve Requests are dispatched to netStore + retrieveRequestMsgCounter.Inc(1) var req retrieveRequestMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<- %v: %v", msg, err) @@ -241,6 +257,7 @@ func (self *bzz) handle() error { case peersMsg: // response to lookups and immediate response to retrieve requests // dispatches new peer data to the hive that adds them to KADDB + peersMsgCounter.Inc(1) var req peersMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<- %v: %v", msg, err) @@ -250,6 +267,7 @@ func (self *bzz) handle() error { self.hive.HandlePeersMsg(&req, &peer{bzz: self}) case syncRequestMsg: + syncRequestMsgCounter.Inc(1) var req syncRequestMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<- %v: %v", msg, err) @@ -260,6 +278,7 @@ func (self *bzz) handle() error { case unsyncedKeysMsg: // coming from parent node offering + unsyncedKeysMsgCounter.Inc(1) var req unsyncedKeysMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<- %v: %v", msg, err) @@ -274,6 +293,7 @@ func (self *bzz) handle() error { case deliveryRequestMsg: // response to syncKeysMsg hashes filtered not existing in db // also relays the last synced state to the source + deliverRequestMsgCounter.Inc(1) var req deliveryRequestMsgData if err := msg.Decode(&req); err != nil { return fmt.Errorf("<-msg %v: %v", msg, err) @@ -287,6 +307,7 @@ func (self *bzz) handle() error { case paymentMsg: // swap protocol message for payment, Units paid for, Cheque paid with + paymentMsgCounter.Inc(1) if self.swapEnabled { var req paymentMsgData if err := msg.Decode(&req); err != nil { @@ -298,6 +319,7 @@ func (self *bzz) handle() error { default: // no other message is allowed + invalidMsgCounter.Inc(1) return fmt.Errorf("invalid message code: %v", msg.Code) } return nil @@ -332,6 +354,8 @@ func (self *bzz) handleStatus() (err error) { return fmt.Errorf("first msg has code %x (!= %x)", msg.Code, statusMsg) } + handleStatusMsgCounter.Inc(1) + if msg.Size > ProtocolMaxMsgSize { return fmt.Errorf("message too long: %v > %v", msg.Size, ProtocolMaxMsgSize) } diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 98cd6e75ea..2b397f8010 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -23,6 +23,8 @@ import ( "io" "sync" "time" + + "github.com/ethereum/go-ethereum/metrics" ) /* @@ -63,6 +65,11 @@ var ( errOperationTimedOut = errors.New("operation timed out") ) +//metrics variables +var ( + newChunkCounter = metrics.NewRegisteredCounter("storage.chunks.new", nil) +) + type TreeChunker struct { branches int64 hashFunc SwarmHasher @@ -298,6 +305,13 @@ func (self *TreeChunker) hashChunk(hasher SwarmHash, job *hashJob, chunkC chan * job.parentWg.Done() if chunkC != nil { + //NOTE: this increases the chunk count even if the local node already has this chunk; + //on file upload the node will increase this counter even if the same file has already been uploaded + //So it should be evaluated whether it is worth keeping this counter + //and/or actually better track when the chunk is Put to the local database + //(which may question the need for disambiguation when a completely new chunk has been created + //and/or a chunk is being put to the local DB; for chunk tracking it may be worth distinguishing + newChunkCounter.Inc(1) chunkC <- newChunk } } diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 46a5c16ccc..421bb061d5 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -33,11 +33,18 @@ import ( "sync" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/iterator" ) +//metrics variables +var ( + gcCounter = metrics.NewRegisteredCounter("storage.db.dbstore.gc.count", nil) + dbStoreDeleteCounter = metrics.NewRegisteredCounter("storage.db.dbstore.rm.count", nil) +) + const ( defaultDbCapacity = 5000000 defaultRadius = 0 // not yet used @@ -255,6 +262,7 @@ func (s *DbStore) collectGarbage(ratio float32) { // actual gc for i := 0; i < gcnt; i++ { if s.gcArray[i].value <= cutval { + gcCounter.Inc(1) s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey) } } @@ -383,6 +391,7 @@ func (s *DbStore) delete(idx uint64, idxKey []byte) { batch := new(leveldb.Batch) batch.Delete(idxKey) batch.Delete(getDataKey(idx)) + dbStoreDeleteCounter.Inc(1) s.entryCnt-- batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) s.db.Write(batch) diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index b442e6cc54..ece0c86155 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -18,6 +18,13 @@ package storage import ( "encoding/binary" + + "github.com/ethereum/go-ethereum/metrics" +) + +//metrics variables +var ( + dbStorePutCounter = metrics.NewRegisteredCounter("storage.db.dbstore.put.count", nil) ) // LocalStore is a combination of inmemory db over a disk persisted db @@ -39,6 +46,14 @@ func NewLocalStore(hash SwarmHasher, params *StoreParams) (*LocalStore, error) { }, nil } +func (self *LocalStore) CacheCounter() uint64 { + return uint64(self.memStore.(*MemStore).Counter()) +} + +func (self *LocalStore) DbCounter() uint64 { + return self.DbStore.(*DbStore).Counter() +} + // LocalStore is itself a chunk store // unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { @@ -48,6 +63,7 @@ func (self *LocalStore) Put(chunk *Chunk) { chunk.wg.Add(1) } go func() { + dbStorePutCounter.Inc(1) self.DbStore.Put(chunk) if chunk.wg != nil { chunk.wg.Done() diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 3cb25ac625..d6be542203 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -23,6 +23,13 @@ import ( "sync" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +//metrics variables +var ( + memstorePutCounter = metrics.NewRegisteredCounter("storage.db.memstore.put.count", nil) + memstoreRemoveCounter = metrics.NewRegisteredCounter("storage.db.memstore.rm.count", nil) ) const ( @@ -130,6 +137,10 @@ func (s *MemStore) setCapacity(c uint) { s.capacity = c } +func (s *MemStore) Counter() uint { + return s.entryCnt +} + // entry (not its copy) is going to be in MemStore func (s *MemStore) Put(entry *Chunk) { if s.capacity == 0 { @@ -145,6 +156,8 @@ func (s *MemStore) Put(entry *Chunk) { s.accessCnt++ + memstorePutCounter.Inc(1) + node := s.memtree bitpos := uint(0) for node.entry == nil { @@ -289,6 +302,7 @@ func (s *MemStore) removeOldest() { } if node.entry.SData != nil { + memstoreRemoveCounter.Inc(1) node.entry = nil s.entryCnt-- } diff --git a/swarm/swarm.go b/swarm/swarm.go index 3c77d6eab6..0a120db1fa 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" @@ -46,6 +47,16 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) +var ( + startTime time.Time + updateGaugesPeriod = 5 * time.Second + startCounter = metrics.NewRegisteredCounter("stack,start", nil) + stopCounter = metrics.NewRegisteredCounter("stack,stop", nil) + uptimeGauge = metrics.NewRegisteredGauge("stack.uptime", nil) + dbSizeGauge = metrics.NewRegisteredGauge("storage.db.chunks.size", nil) + cacheSizeGauge = metrics.NewRegisteredGauge("storage.db.cache.size", nil) +) + // the swarm stack type Swarm struct { config *api.Config // swarm configuration @@ -262,6 +273,7 @@ Start is called when the stack is started */ // implements the node.Service interface func (self *Swarm) Start(srv *p2p.Server) error { + startTime = time.Now() connectPeer := func(url string) error { node, err := discover.ParseNode(url) if err != nil { @@ -307,9 +319,28 @@ func (self *Swarm) Start(srv *p2p.Server) error { } } + self.periodicallyUpdateGauges() + + startCounter.Inc(1) return nil } +func (self *Swarm) periodicallyUpdateGauges() { + ticker := time.NewTicker(updateGaugesPeriod) + + go func() { + for range ticker.C { + self.updateGauges() + } + }() +} + +func (self *Swarm) updateGauges() { + dbSizeGauge.Update(int64(self.lstore.DbCounter())) + cacheSizeGauge.Update(int64(self.lstore.CacheCounter())) + uptimeGauge.Update(time.Since(startTime).Nanoseconds()) +} + // implements the node.Service interface // stops all component services. func (self *Swarm) Stop() error { @@ -324,6 +355,7 @@ func (self *Swarm) Stop() error { self.lstore.DbStore.Close() } self.sfs.Stop() + stopCounter.Inc(1) return err } From 6919c36432226a9d86dbe4a65ed9563dd42a4e08 Mon Sep 17 00:00:00 2001 From: Vlad
Date: Fri, 23 Feb 2018 14:52:25 +0100 Subject: [PATCH 50/94] whisper: refactoring --- whisper/whisperv5/api.go | 23 ----------------------- whisper/whisperv6/api.go | 26 ++------------------------ whisper/whisperv6/whisper.go | 5 +---- 3 files changed, 3 insertions(+), 51 deletions(-) diff --git a/whisper/whisperv5/api.go b/whisper/whisperv5/api.go index b4494d0d61..ee566625c9 100644 --- a/whisper/whisperv5/api.go +++ b/whisper/whisperv5/api.go @@ -60,32 +60,9 @@ func NewPublicWhisperAPI(w *Whisper) *PublicWhisperAPI { w: w, lastUsed: make(map[string]time.Time), } - - go api.run() return api } -// run the api event loop. -// this loop deletes filter that have not been used within filterTimeout -func (api *PublicWhisperAPI) run() { - timeout := time.NewTicker(2 * time.Minute) - for { - <-timeout.C - - api.mu.Lock() - for id, lastUsed := range api.lastUsed { - if time.Since(lastUsed).Seconds() >= filterTimeout { - delete(api.lastUsed, id) - if err := api.w.Unsubscribe(id); err != nil { - log.Error("could not unsubscribe whisper filter", "error", err) - } - log.Debug("delete whisper filter (timeout)", "id", id) - } - } - api.mu.Unlock() - } -} - // Version returns the Whisper sub-protocol version. func (api *PublicWhisperAPI) Version(ctx context.Context) string { return ProtocolVersionStr diff --git a/whisper/whisperv6/api.go b/whisper/whisperv6/api.go index f3d9977fad..96e2b17e7c 100644 --- a/whisper/whisperv6/api.go +++ b/whisper/whisperv6/api.go @@ -61,32 +61,9 @@ func NewPublicWhisperAPI(w *Whisper) *PublicWhisperAPI { w: w, lastUsed: make(map[string]time.Time), } - - go api.run() return api } -// run the api event loop. -// this loop deletes filter that have not been used within filterTimeout -func (api *PublicWhisperAPI) run() { - timeout := time.NewTicker(2 * time.Minute) - for { - <-timeout.C - - api.mu.Lock() - for id, lastUsed := range api.lastUsed { - if time.Since(lastUsed).Seconds() >= filterTimeout { - delete(api.lastUsed, id) - if err := api.w.Unsubscribe(id); err != nil { - log.Error("could not unsubscribe whisper filter", "error", err) - } - log.Debug("delete whisper filter (timeout)", "id", id) - } - } - api.mu.Unlock() - } -} - // Version returns the Whisper sub-protocol version. func (api *PublicWhisperAPI) Version(ctx context.Context) string { return ProtocolVersionStr @@ -219,7 +196,8 @@ func (api *PublicWhisperAPI) DeleteSymKey(ctx context.Context, id string) bool { return api.w.DeleteSymKey(id) } -// MakeLightClient turns the node into light client, which does not forward any incoming messages. +// MakeLightClient turns the node into light client, which does not forward +// any incoming messages, and sends only messages originated in this node. func (api *PublicWhisperAPI) MakeLightClient(ctx context.Context) bool { api.w.lightClient = true return api.w.lightClient diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index e0ed00bb5d..b0b601f032 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -590,10 +590,7 @@ func (whisper *Whisper) Unsubscribe(id string) error { // network in the coming cycles. func (whisper *Whisper) Send(envelope *Envelope) error { ok, err := whisper.add(envelope, false) - if err != nil { - return err - } - if !ok { + if err == nil && !ok { return fmt.Errorf("failed to add envelope") } return err From 114738982ef2fa15b4f5c0c2d691dca1e22cc2ee Mon Sep 17 00:00:00 2001 From: Anton Evangelatov
Date: Fri, 23 Feb 2018 16:22:16 +0100 Subject: [PATCH 51/94] swarm/metrics: introduce metrics export flag (#16177) --- swarm/metrics/flags.go | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/swarm/metrics/flags.go b/swarm/metrics/flags.go index 60e34e6e42..48b231b219 100644 --- a/swarm/metrics/flags.go +++ b/swarm/metrics/flags.go @@ -27,6 +27,10 @@ import ( ) var ( + metricsEnableInfluxDBExportFlag = cli.BoolFlag{ + Name: "metrics.influxdb.export", + Usage: "Enable metrics export/push to an external InfluxDB database", + } metricsInfluxDBEndpointFlag = cli.StringFlag{ Name: "metrics.influxdb.endpoint", Usage: "Metrics InfluxDB endpoint", @@ -34,17 +38,17 @@ var ( } metricsInfluxDBDatabaseFlag = cli.StringFlag{ Name: "metrics.influxdb.database", - Usage: "metrics InfluxDB database", + Usage: "Metrics InfluxDB database", Value: "metrics", } metricsInfluxDBUsernameFlag = cli.StringFlag{ Name: "metrics.influxdb.username", - Usage: "metrics InfluxDB username", + Usage: "Metrics InfluxDB username", Value: "", } metricsInfluxDBPasswordFlag = cli.StringFlag{ Name: "metrics.influxdb.password", - Usage: "metrics InfluxDB password", + Usage: "Metrics InfluxDB password", Value: "", } // The `host` tag is part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. @@ -53,7 +57,7 @@ var ( // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key metricsInfluxDBHostTagFlag = cli.StringFlag{ Name: "metrics.influxdb.host.tag", - Usage: "metrics InfluxDB `host` tag attached to all measurements", + Usage: "Metrics InfluxDB `host` tag attached to all measurements", Value: "localhost", } ) @@ -61,22 +65,27 @@ var ( // Flags holds all command-line flags required for metrics collection. var Flags = []cli.Flag{ utils.MetricsEnabledFlag, + metricsEnableInfluxDBExportFlag, metricsInfluxDBEndpointFlag, metricsInfluxDBDatabaseFlag, metricsInfluxDBUsernameFlag, metricsInfluxDBPasswordFlag, metricsInfluxDBHostTagFlag, } func Setup(ctx *cli.Context) { if gethmetrics.Enabled { + log.Info("Enabling swarm metrics collection") var ( - endpoint = ctx.GlobalString(metricsInfluxDBEndpointFlag.Name) - database = ctx.GlobalString(metricsInfluxDBDatabaseFlag.Name) - username = ctx.GlobalString(metricsInfluxDBUsernameFlag.Name) - password = ctx.GlobalString(metricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(metricsInfluxDBHostTagFlag.Name) + enableExport = ctx.GlobalBool(metricsEnableInfluxDBExportFlag.Name) + endpoint = ctx.GlobalString(metricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(metricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(metricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(metricsInfluxDBPasswordFlag.Name) + hosttag = ctx.GlobalString(metricsInfluxDBHostTagFlag.Name) ) - log.Info("Enabling swarm metrics collection and export") - go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "swarm.", map[string]string{ - "host": hosttag, - }) + if enableExport { + log.Info("Enabling swarm metrics export to InfluxDB") + go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "swarm.", map[string]string{ + "host": hosttag, + }) + } } } From 22fc6928d720e3cc1f090ce2891f553db1e076ab Mon Sep 17 00:00:00 2001 From: cooganb
Date: Mon, 26 Feb 2018 02:56:40 -0600 Subject: [PATCH 52/94] swarm: creates Swarm landing page for browser 'localhost:xxxx/' GET request when running Swarm (#15926) * swarm: began work on GetHandleFile method re: issue #155 * swarm: now able to serve landing page template * swarm: added landing page template * swarm: landing page has working input * swarm: fixed CSS issue in template * swarm: deleted extra lines * swarm: deleted time header and made redirect a relative path * swarm: removed code mistakenly left --- swarm/api/http/server.go | 9 +++ swarm/api/http/templates.go | 143 ++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 6ebfc80590..df90fd04f9 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -660,6 +660,15 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) + if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "text/html") { + + err := landingPageTemplate.Execute(w, nil) + if err != nil { + s.logError("error rendering landing page: %s", err) + } + return + } + uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) req := &Request{Request: *r, uri: uri} if err != nil { diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go index 189a999122..cd9d212897 100644 --- a/swarm/api/http/templates.go +++ b/swarm/api/http/templates.go @@ -70,3 +70,146 @@ var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.Fu
+
+
+