ethdb/remotedb, cmd: add support for remote (readonly) databases (#24836)

* ethdb/remotedb, cmd: add support for remote (readonly) databases

* ethdb/remotedb: minor changes

* ethdb/remotedb: close the conn

* cmd, ethdb: add rpc accessor for ancient data

* internal/ethapi: license

* ethdb/remotedb: linter fixes

cmd/utils/flags.go

@@ -49,6 +49,7 @@ import (
 	"github.com/ethereum/go-ethereum/eth/gasprice"
 	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/ethdb/remotedb"
 	"github.com/ethereum/go-ethereum/ethstats"
 	"github.com/ethereum/go-ethereum/graphql"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
@@ -113,6 +114,10 @@ var (
 		Usage: "Data directory for the databases and keystore",
 		Value: DirectoryString(node.DefaultDataDir()),
 	}
+	RemoteDBFlag = cli.StringFlag{
+		Name:  "remotedb",
+		Usage: "URL for remote database",
+	}
 	AncientFlag = DirectoryFlag{
 		Name:  "datadir.ancient",
 		Usage: "Data directory for ancient chain segments (default = inside chaindata)",
@@ -840,6 +845,7 @@ var (
 	DatabasePathFlags = []cli.Flag{
 		DataDirFlag,
 		AncientFlag,
+		RemoteDBFlag,
 	}
 )
 
@@ -1962,12 +1968,14 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
 		err     error
 		chainDb ethdb.Database
 	)
-	if ctx.GlobalString(SyncModeFlag.Name) == "light" {
-		name := "lightchaindata"
-		chainDb, err = stack.OpenDatabase(name, cache, handles, "", readonly)
-	} else {
-		name := "chaindata"
-		chainDb, err = stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly)
+	switch {
+	case ctx.GlobalIsSet(RemoteDBFlag.Name):
+		log.Info("Using remote db", "url", ctx.GlobalString(RemoteDBFlag.Name))
+		chainDb, err = remotedb.New(ctx.GlobalString(RemoteDBFlag.Name))
+	case ctx.GlobalString(SyncModeFlag.Name) == "light":
+		chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly)
+	default:
+		chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly)
 	}
 	if err != nil {
 		Fatalf("Could not open database: %v", err)

ethdb/remotedb/remotedb.go

@@ -0,0 +1,173 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package remotedb implements the key-value database layer based on a remote geth
+// node. Under the hood, it utilises the `debug_dbGet` method to implement a
+// read-only database.
+// There really are no guarantees in this database, since the local geth does not
+// exclusive access, but it can be used for basic diagnostics of a remote node.
+package remotedb
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// Database is a key-value lookup for a remote database via debug_dbGet.
+type Database struct {
+	remote *rpc.Client
+}
+
+func (db *Database) Has(key []byte) (bool, error) {
+	if _, err := db.Get(key); err != nil {
+		return true, nil
+	}
+	return false, nil
+}
+
+func (db *Database) Get(key []byte) ([]byte, error) {
+	var resp hexutil.Bytes
+	err := db.remote.Call(&resp, "debug_dbGet", hexutil.Bytes(key))
+	if err != nil {
+		return nil, err
+	}
+	return resp, nil
+}
+
+func (db *Database) HasAncient(kind string, number uint64) (bool, error) {
+	if _, err := db.Ancient(kind, number); err != nil {
+		return true, nil
+	}
+	return false, nil
+}
+
+func (db *Database) Ancient(kind string, number uint64) ([]byte, error) {
+	var resp hexutil.Bytes
+	err := db.remote.Call(&resp, "debug_dbAncient", kind, number)
+	if err != nil {
+		return nil, err
+	}
+	return resp, nil
+}
+
+func (db *Database) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
+	panic("not supported")
+}
+
+func (db *Database) Ancients() (uint64, error) {
+	var resp uint64
+	err := db.remote.Call(&resp, "debug_dbAncients")
+	return resp, err
+}
+
+func (db *Database) Tail() (uint64, error) {
+	panic("not supported")
+}
+
+func (db *Database) AncientSize(kind string) (uint64, error) {
+	panic("not supported")
+}
+
+func (db *Database) ReadAncients(fn func(op ethdb.AncientReaderOp) error) (err error) {
+	return fn(db)
+}
+
+func (db *Database) Put(key []byte, value []byte) error {
+	panic("not supported")
+}
+
+func (db *Database) Delete(key []byte) error {
+	panic("not supported")
+}
+
+func (db *Database) ModifyAncients(f func(ethdb.AncientWriteOp) error) (int64, error) {
+	panic("not supported")
+}
+
+func (db *Database) TruncateHead(n uint64) error {
+	panic("not supported")
+}
+
+func (db *Database) TruncateTail(n uint64) error {
+	panic("not supported")
+}
+
+func (db *Database) Sync() error {
+	return nil
+}
+
+func (db *Database) MigrateTable(s string, f func([]byte) ([]byte, error)) error {
+	panic("not supported")
+}
+
+func (db *Database) NewBatch() ethdb.Batch {
+	panic("not supported")
+}
+
+func (db *Database) NewBatchWithSize(size int) ethdb.Batch {
+	panic("not supported")
+}
+
+func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
+	panic("not supported")
+}
+
+func (db *Database) Stat(property string) (string, error) {
+	panic("not supported")
+}
+
+func (db *Database) AncientDatadir() (string, error) {
+	panic("not supported")
+}
+
+func (db *Database) Compact(start []byte, limit []byte) error {
+	return nil
+}
+
+func (db *Database) NewSnapshot() (ethdb.Snapshot, error) {
+	panic("not supported")
+}
+
+func (db *Database) Close() error {
+	db.remote.Close()
+	return nil
+}
+
+func dialRPC(endpoint string) (*rpc.Client, error) {
+	if endpoint == "" {
+		return nil, errors.New("endpoint must be specified")
+	}
+	if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
+		// Backwards compatibility with geth < 1.5 which required
+		// these prefixes.
+		endpoint = endpoint[4:]
+	}
+	return rpc.Dial(endpoint)
+}
+
+func New(endpoint string) (ethdb.Database, error) {
+	client, err := dialRPC(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return &Database{
+		remote: client,
+	}, nil
+}

internal/ethapi/api.go

@@ -1959,15 +1959,6 @@ func (api *PrivateDebugAPI) SetHead(number hexutil.Uint64) {
 	api.b.SetHead(uint64(number))
 }
 
-// DbGet returns the raw value of a key stored in the database.
-func (api *PrivateDebugAPI) DbGet(key string) (hexutil.Bytes, error) {
-	blob, err := common.ParseHexOrString(key)
-	if err != nil {
-		return nil, err
-	}
-	return api.b.ChainDb().Get(blob)
-}
-
 // PublicNetAPI offers network related RPC methods
 type PublicNetAPI struct {
 	net            *p2p.Server

internal/ethapi/dbapi.go

@@ -0,0 +1,43 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package ethapi
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+// DbGet returns the raw value of a key stored in the database.
+func (api *PrivateDebugAPI) DbGet(key string) (hexutil.Bytes, error) {
+	blob, err := common.ParseHexOrString(key)
+	if err != nil {
+		return nil, err
+	}
+	return api.b.ChainDb().Get(blob)
+}
+
+// DbAncient retrieves an ancient binary blob from the append-only immutable files.
+// It is a mapping to the `AncientReaderOp.Ancient` method
+func (api *PrivateDebugAPI) DbAncient(kind string, number uint64) (hexutil.Bytes, error) {
+	return api.b.ChainDb().Ancient(kind, number)
+}
+
+// DbAncients returns the ancient item numbers in the ancient store.
+// It is a mapping to the `AncientReaderOp.Ancients` method
+func (api *PrivateDebugAPI) DbAncients() (uint64, error) {
+	return api.b.ChainDb().Ancients()
+}

internal/web3ext/web3ext.go

@@ -475,6 +475,16 @@ web3._extend({
 			call: 'debug_dbGet',
 			params: 1
 		}),
+		new web3._extend.Method({
+			name: 'dbAncient',
+			call: 'debug_dbAncient',
+			params: 2
+		}),
+		new web3._extend.Method({
+			name: 'dbAncients',
+			call: 'debug_dbAncients',
+			params: 0
+		}),
 	],
 	properties: []
 });