cmd/swarm/global-store: global store cmd (#19014)

(cherry picked from commit 33d0a0efa6)

cmd/swarm/config.go

@@ -82,6 +82,7 @@ const (
 	SWARM_ENV_BOOTNODE_MODE           = "SWARM_BOOTNODE_MODE"
 	SWARM_ACCESS_PASSWORD             = "SWARM_ACCESS_PASSWORD"
 	SWARM_AUTO_DEFAULTPATH            = "SWARM_AUTO_DEFAULTPATH"
+	SWARM_GLOBALSTORE_API             = "SWARM_GLOBALSTORE_API"
 	GETH_ENV_DATADIR                  = "GETH_DATADIR"
 )
 
@@ -262,6 +263,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name)
 	}
 
+	if ctx.GlobalIsSet(SwarmGlobalStoreAPIFlag.Name) {
+		currentConfig.GlobalStoreAPI = ctx.GlobalString(SwarmGlobalStoreAPIFlag.Name)
+	}
+
 	return currentConfig
 
 }
@@ -375,6 +380,10 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 		currentConfig.BootnodeMode = bootnodeMode
 	}
 
+	if api := os.Getenv(SWARM_GLOBALSTORE_API); api != "" {
+		currentConfig.GlobalStoreAPI = api
+	}
+
 	return currentConfig
 }
 

cmd/swarm/flags.go

@@ -176,4 +176,9 @@ var (
 		Name:  "user",
 		Usage: "Indicates the user who updates the feed",
 	}
+	SwarmGlobalStoreAPIFlag = cli.StringFlag{
+		Name:   "globalstore-api",
+		Usage:  "URL of the Global Store API provider (only for testing)",
+		EnvVar: SWARM_GLOBALSTORE_API,
+	}
 )

cmd/swarm/global-store/global_store.go

@@ -0,0 +1,100 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"net"
+	"net/http"
+	"os"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/db"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+// startHTTP starts a global store with HTTP RPC server.
+// It is used for "http" cli command.
+func startHTTP(ctx *cli.Context) (err error) {
+	server, cleanup, err := newServer(ctx)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	listener, err := net.Listen("tcp", ctx.String("addr"))
+	if err != nil {
+		return err
+	}
+	log.Info("http", "address", listener.Addr().String())
+
+	return http.Serve(listener, server)
+}
+
+// startWS starts a global store with WebSocket RPC server.
+// It is used for "websocket" cli command.
+func startWS(ctx *cli.Context) (err error) {
+	server, cleanup, err := newServer(ctx)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	listener, err := net.Listen("tcp", ctx.String("addr"))
+	if err != nil {
+		return err
+	}
+	origins := ctx.StringSlice("origins")
+	log.Info("websocket", "address", listener.Addr().String(), "origins", origins)
+
+	return http.Serve(listener, server.WebsocketHandler(origins))
+}
+
+// newServer creates a global store and returns its RPC server.
+// Returned cleanup function should be called only if err is nil.
+func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) {
+	log.PrintOrigins(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(false))))
+
+	cleanup = func() {}
+	var globalStore mock.GlobalStorer
+	dir := ctx.String("dir")
+	if dir != "" {
+		dbStore, err := db.NewGlobalStore(dir)
+		if err != nil {
+			return nil, nil, err
+		}
+		cleanup = func() {
+			dbStore.Close()
+		}
+		globalStore = dbStore
+		log.Info("database global store", "dir", dir)
+	} else {
+		globalStore = mem.NewGlobalStore()
+		log.Info("in-memory global store")
+	}
+
+	server = rpc.NewServer()
+	if err := server.RegisterName("mockStore", globalStore); err != nil {
+		cleanup()
+		return nil, nil, err
+	}
+
+	return server, cleanup, nil
+}

cmd/swarm/global-store/global_store_test.go

@@ -0,0 +1,191 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rpc"
+	mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
+)
+
+// TestHTTP_InMemory tests in-memory global store that exposes
+// HTTP server.
+func TestHTTP_InMemory(t *testing.T) {
+	testHTTP(t, true)
+}
+
+// TestHTTP_Database tests global store with persisted database
+// that exposes HTTP server.
+func TestHTTP_Database(t *testing.T) {
+	dir, err := ioutil.TempDir("", "swarm-global-store-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	// create a fresh global store
+	testHTTP(t, true, "--dir", dir)
+
+	// check if data saved by the previous global store instance
+	testHTTP(t, false, "--dir", dir)
+}
+
+// testWebsocket starts global store binary with HTTP server
+// and validates that it can store and retrieve data.
+// If put is false, no data will be stored, only retrieved,
+// giving the possibility to check if data is present in the
+// storage directory.
+func testHTTP(t *testing.T, put bool, args ...string) {
+	addr := findFreeTCPAddress(t)
+	testCmd := runGlobalStore(t, append([]string{"http", "--addr", addr}, args...)...)
+	defer testCmd.Interrupt()
+
+	client, err := rpc.DialHTTP("http://" + addr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// wait until global store process is started as
+	// rpc.DialHTTP is actually not connecting
+	for i := 0; i < 1000; i++ {
+		_, err = http.DefaultClient.Get("http://" + addr)
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	store := mockRPC.NewGlobalStore(client)
+	defer store.Close()
+
+	node := store.NewNodeStore(common.HexToAddress("123abc"))
+
+	wantKey := "key"
+	wantValue := "value"
+
+	if put {
+		err = node.Put([]byte(wantKey), []byte(wantValue))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	gotValue, err := node.Get([]byte(wantKey))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(gotValue) != wantValue {
+		t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue)
+	}
+}
+
+// TestWebsocket_InMemory tests in-memory global store that exposes
+// WebSocket server.
+func TestWebsocket_InMemory(t *testing.T) {
+	testWebsocket(t, true)
+}
+
+// TestWebsocket_Database tests global store with persisted database
+// that exposes HTTP server.
+func TestWebsocket_Database(t *testing.T) {
+	dir, err := ioutil.TempDir("", "swarm-global-store-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	// create a fresh global store
+	testWebsocket(t, true, "--dir", dir)
+
+	// check if data saved by the previous global store instance
+	testWebsocket(t, false, "--dir", dir)
+}
+
+// testWebsocket starts global store binary with WebSocket server
+// and validates that it can store and retrieve data.
+// If put is false, no data will be stored, only retrieved,
+// giving the possibility to check if data is present in the
+// storage directory.
+func testWebsocket(t *testing.T, put bool, args ...string) {
+	addr := findFreeTCPAddress(t)
+	testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...)
+	defer testCmd.Interrupt()
+
+	var client *rpc.Client
+	var err error
+	// wait until global store process is started
+	for i := 0; i < 1000; i++ {
+		client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "")
+		if err == nil {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	store := mockRPC.NewGlobalStore(client)
+	defer store.Close()
+
+	node := store.NewNodeStore(common.HexToAddress("123abc"))
+
+	wantKey := "key"
+	wantValue := "value"
+
+	if put {
+		err = node.Put([]byte(wantKey), []byte(wantValue))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	gotValue, err := node.Get([]byte(wantKey))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(gotValue) != wantValue {
+		t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue)
+	}
+}
+
+// findFreeTCPAddress returns a local address (IP:Port) to which
+// global store can listen on.
+func findFreeTCPAddress(t *testing.T) (addr string) {
+	t.Helper()
+
+	listener, err := net.Listen("tcp", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer listener.Close()
+
+	return listener.Addr().String()
+}

cmd/swarm/global-store/main.go

@@ -0,0 +1,104 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"os"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/log"
+	cli "gopkg.in/urfave/cli.v1"
+)
+
+var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
+
+func main() {
+	err := newApp().Run(os.Args)
+	if err != nil {
+		log.Error(err.Error())
+		os.Exit(1)
+	}
+}
+
+// newApp construct a new instance of Swarm Global Store.
+// Method Run is called on it in the main function and in tests.
+func newApp() (app *cli.App) {
+	app = utils.NewApp(gitCommit, "Swarm Global Store")
+
+	app.Name = "global-store"
+
+	// app flags (for all commands)
+	app.Flags = []cli.Flag{
+		cli.IntFlag{
+			Name:  "verbosity",
+			Value: 3,
+			Usage: "verbosity level",
+		},
+	}
+
+	app.Commands = []cli.Command{
+		{
+			Name:    "http",
+			Aliases: []string{"h"},
+			Usage:   "start swarm global store with http server",
+			Action:  startHTTP,
+			// Flags only for "start" command.
+			// Allow app flags to be specified after the
+			// command argument.
+			Flags: append(app.Flags,
+				cli.StringFlag{
+					Name:  "dir",
+					Value: "",
+					Usage: "data directory",
+				},
+				cli.StringFlag{
+					Name:  "addr",
+					Value: "0.0.0.0:3033",
+					Usage: "address to listen for http connection",
+				},
+			),
+		},
+		{
+			Name:    "websocket",
+			Aliases: []string{"ws"},
+			Usage:   "start swarm global store with websocket server",
+			Action:  startWS,
+			// Flags only for "start" command.
+			// Allow app flags to be specified after the
+			// command argument.
+			Flags: append(app.Flags,
+				cli.StringFlag{
+					Name:  "dir",
+					Value: "",
+					Usage: "data directory",
+				},
+				cli.StringFlag{
+					Name:  "addr",
+					Value: "0.0.0.0:3033",
+					Usage: "address to listen for websocket connection",
+				},
+				cli.StringSliceFlag{
+					Name:  "origins",
+					Value: &cli.StringSlice{"*"},
+					Usage: "websocket origins",
+				},
+			),
+		},
+	}
+
+	return app
+}

cmd/swarm/global-store/run_test.go

@@ -0,0 +1,49 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/ethereum/go-ethereum/internal/cmdtest"
+)
+
+func init() {
+	reexec.Register("swarm-global-store", func() {
+		if err := newApp().Run(os.Args); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	})
+}
+
+func runGlobalStore(t *testing.T, args ...string) *cmdtest.TestCmd {
+	tt := cmdtest.NewTestCmd(t, nil)
+	tt.Run("swarm-global-store", args...)
+	return tt
+}
+
+func TestMain(m *testing.M) {
+	if reexec.Init() {
+		return
+	}
+	os.Exit(m.Run())
+}

cmd/swarm/main.go

@@ -39,13 +39,16 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p/enode"
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
 	swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	mockrpc "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc"
 	"github.com/ethereum/go-ethereum/swarm/tracing"
 	sv "github.com/ethereum/go-ethereum/swarm/version"
 
-	"gopkg.in/urfave/cli.v1"
+	cli "gopkg.in/urfave/cli.v1"
 )
 
 const clientIdentifier = "swarm"
@@ -196,6 +199,7 @@ func init() {
 		SwarmStorePath,
 		SwarmStoreCapacity,
 		SwarmStoreCacheCapacity,
+		SwarmGlobalStoreAPIFlag,
 	}
 	rpcFlags := []cli.Flag{
 		utils.WSEnabledFlag,
@@ -324,8 +328,18 @@ func bzzd(ctx *cli.Context) error {
 func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) {
 	//define the swarm service boot function
 	boot := func(_ *node.ServiceContext) (node.Service, error) {
-		// In production, mockStore must be always nil.
-		return swarm.NewSwarm(bzzconfig, nil)
+		var nodeStore *mock.NodeStore
+		if bzzconfig.GlobalStoreAPI != "" {
+			// connect to global store
+			client, err := rpc.Dial(bzzconfig.GlobalStoreAPI)
+			if err != nil {
+				return nil, fmt.Errorf("global store: %v", err)
+			}
+			globalStore := mockrpc.NewGlobalStore(client)
+			// create a node store for this swarm key on global store
+			nodeStore = globalStore.NewNodeStore(common.HexToAddress(bzzconfig.BzzKey))
+		}
+		return swarm.NewSwarm(bzzconfig, nodeStore)
 	}
 	//register within the ethereum node
 	if err := stack.Register(boot); err != nil {

swarm/api/config.go

@@ -71,6 +71,7 @@ type Config struct {
 	SwapAPI              string
 	Cors                 string
 	BzzAccount           string
+	GlobalStoreAPI       string
 	privateKey           *ecdsa.PrivateKey
 }