From e90958cd29a228b051faeaa25d66e053cf9d2228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 19 Feb 2016 14:29:19 +0200 Subject: [PATCH] cmd, eth, ethdb, node: prioritise chaindata for resources, bump cache --- cmd/utils/fdlimit_test.go | 35 +++++++++++++++++++++++++ cmd/utils/fdlimit_unix.go | 50 ++++++++++++++++++++++++++++++++++++ cmd/utils/fdlimit_windows.go | 41 +++++++++++++++++++++++++++++ cmd/utils/flags.go | 22 ++++++++++++++-- core/bench_test.go | 2 +- core/database_util_test.go | 2 +- eth/backend.go | 9 +++---- eth/filters/filter_test.go | 4 +-- ethdb/database.go | 23 ++++++++++++----- ethdb/database_test.go | 2 +- node/service.go | 4 +-- node/service_test.go | 4 +-- trie/trie_test.go | 2 +- 13 files changed, 176 insertions(+), 24 deletions(-) create mode 100644 cmd/utils/fdlimit_test.go create mode 100644 cmd/utils/fdlimit_unix.go create mode 100644 cmd/utils/fdlimit_windows.go diff --git a/cmd/utils/fdlimit_test.go b/cmd/utils/fdlimit_test.go new file mode 100644 index 0000000000..0a950a6c9d --- /dev/null +++ b/cmd/utils/fdlimit_test.go @@ -0,0 +1,35 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package utils + +import "testing" + +// TestFileDescriptorLimits simply tests whether the file descriptor allowance +// per this process can be retrieved. +func TestFileDescriptorLimits(t *testing.T) { + target := 4096 + + if limit, err := getFdLimit(); err != nil || limit <= 0 { + t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err) + } + if err := raiseFdLimit(uint64(target)); err != nil { + t.Fatalf("failed to raise file allowance") + } + if limit, err := getFdLimit(); err != nil || limit < target { + t.Fatalf("failed to retrieve raised descriptor limit (have %v, want %v): %v", limit, target, err) + } +} diff --git a/cmd/utils/fdlimit_unix.go b/cmd/utils/fdlimit_unix.go new file mode 100644 index 0000000000..2a6dffc8f3 --- /dev/null +++ b/cmd/utils/fdlimit_unix.go @@ -0,0 +1,50 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// +build linux darwin + +package utils + +import "syscall" + +// raiseFdLimit tries to maximize the file descriptor allowance of this process +// to the maximum hard-limit allowed by the OS. +func raiseFdLimit(max uint64) error { + // Get the current limit + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return err + } + // Try to update the limit to the max allowance + limit.Cur = limit.Max + if limit.Cur > max { + limit.Cur = max + } + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return err + } + return nil +} + +// getFdLimit retrieves the number of file descriptors allowed to be opened by this +// process. +func getFdLimit() (int, error) { + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return int(limit.Cur), nil +} diff --git a/cmd/utils/fdlimit_windows.go b/cmd/utils/fdlimit_windows.go new file mode 100644 index 0000000000..53aad3d7a5 --- /dev/null +++ b/cmd/utils/fdlimit_windows.go @@ -0,0 +1,41 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package utils + +import "errors" + +// raiseFdLimit tries to maximize the file descriptor allowance of this process +// to the maximum hard-limit allowed by the OS. +func raiseFdLimit(max uint64) error { + // This method is NOP by design: + // * Linux/Darwin counterparts need to manually increase per process limits + // * On Windows Go uses the CreateFile API, which is limited to 16K files, non + // changeable from within a running process + // This way we can always "request" raising the limits, which will either have + // or not have effect based on the platform we're running on. + if max > 16384 { + return errors.New("file descriptor limit (16384) reached") + } + return nil +} + +// getFdLimit retrieves the number of file descriptors allowed to be opened by this +// process. +func getFdLimit() (int, error) { + // Please see raiseFdLimit for the reason why we use hard coded 16K as the limit + return 16384, nil +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 3efb65e429..69fb0b9db9 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -143,7 +143,7 @@ var ( CacheFlag = cli.IntFlag{ Name: "cache", Usage: "Megabytes of memory allocated to internal caching (min 16MB / database forced)", - Value: 0, + Value: 128, } BlockchainVersionFlag = cli.IntFlag{ Name: "blockchainversion", @@ -527,6 +527,22 @@ func MakeGenesisBlock(ctx *cli.Context) string { return string(data) } +// MakeDatabaseHandles raises out the number of allowed file handles per process +// for Geth and returns half of the allowance to assign to the database. +func MakeDatabaseHandles() int { + if err := raiseFdLimit(2048); err != nil { + Fatalf("Failed to raise file descriptor allowance: %v", err) + } + limit, err := getFdLimit() + if err != nil { + Fatalf("Failed to retrieve file descriptor allowance: %v", err) + } + if limit > 2048 { // cap database file descriptors even if more is available + limit = 2048 + } + return limit / 2 // Leave half for networking and other stuff +} + // MakeAccountManager creates an account manager from set command line flags. func MakeAccountManager(ctx *cli.Context) *accounts.Manager { // Create the keystore crypto primitive, light if requested @@ -649,6 +665,7 @@ func MakeSystemNode(name, version string, extra []byte, ctx *cli.Context) *node. FastSync: ctx.GlobalBool(FastSyncFlag.Name), BlockChainVersion: ctx.GlobalInt(BlockchainVersionFlag.Name), DatabaseCache: ctx.GlobalInt(CacheFlag.Name), + DatabaseHandles: MakeDatabaseHandles(), NetworkId: ctx.GlobalInt(NetworkIdFlag.Name), AccountManager: accman, Etherbase: MakeEtherbase(accman, ctx), @@ -763,9 +780,10 @@ func SetupVM(ctx *cli.Context) { func MakeChain(ctx *cli.Context) (chain *core.BlockChain, chainDb ethdb.Database) { datadir := MustMakeDataDir(ctx) cache := ctx.GlobalInt(CacheFlag.Name) + handles := MakeDatabaseHandles() var err error - if chainDb, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "chaindata"), cache); err != nil { + if chainDb, err = ethdb.NewLDBDatabase(filepath.Join(datadir, "chaindata"), cache, handles); err != nil { Fatalf("Could not open database: %v", err) } if ctx.GlobalBool(OlympicFlag.Name) { diff --git a/core/bench_test.go b/core/bench_test.go index f991e5e7f4..0ff847ed51 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -153,7 +153,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { b.Fatalf("cannot create temporary directory: %v", err) } defer os.RemoveAll(dir) - db, err = ethdb.NewLDBDatabase(dir, 0) + db, err = ethdb.NewLDBDatabase(dir, 128, 128) if err != nil { b.Fatalf("cannot create temporary database: %v", err) } diff --git a/core/database_util_test.go b/core/database_util_test.go index 6b37936358..ce1ffea8ae 100644 --- a/core/database_util_test.go +++ b/core/database_util_test.go @@ -551,7 +551,7 @@ func TestMipmapChain(t *testing.T) { defer os.RemoveAll(dir) var ( - db, _ = ethdb.NewLDBDatabase(dir, 16) + db, _ = ethdb.NewLDBDatabase(dir, 0, 0) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) addr2 = common.BytesToAddress([]byte("jeff")) diff --git a/eth/backend.go b/eth/backend.go index f62ee976d6..d807f8ae87 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -69,6 +69,7 @@ type Config struct { BlockChainVersion int SkipBcVersionCheck bool // e.g. blockchain export DatabaseCache int + DatabaseHandles int NatSpec bool DocRoot string @@ -135,12 +136,8 @@ type Ethereum struct { } func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { - // Let the database take 3/4 of the max open files (TODO figure out a way to get the actual limit of the open files) - const dbCount = 3 - ethdb.OpenFileLimit = 128 / (dbCount + 1) - // Open the chain database and perform any upgrades needed - chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache) + chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles) if err != nil { return nil, err } @@ -154,7 +151,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { return nil, err } - dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache) + dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache, config.DatabaseHandles) if err != nil { return nil, err } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 5772114b3d..b98945bbd0 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -31,7 +31,7 @@ func BenchmarkMipmaps(b *testing.B) { defer os.RemoveAll(dir) var ( - db, _ = ethdb.NewLDBDatabase(dir, 16) + db, _ = ethdb.NewLDBDatabase(dir, 0, 0) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = common.BytesToAddress([]byte("jeff")) @@ -105,7 +105,7 @@ func TestFilters(t *testing.T) { defer os.RemoveAll(dir) var ( - db, _ = ethdb.NewLDBDatabase(dir, 16) + db, _ = ethdb.NewLDBDatabase(dir, 0, 0) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) diff --git a/ethdb/database.go b/ethdb/database.go index 10dc018b0f..dffb42e2b0 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -39,8 +39,15 @@ var OpenFileLimit = 64 // cacheRatio specifies how the total alloted cache is distributed between the // various system databases. var cacheRatio = map[string]float64{ - "dapp": 2.0 / 13.0, - "chaindata": 11.0 / 13.0, + "dapp": 0.0, + "chaindata": 1.0, +} + +// handleRatio specifies how the total alloted file descriptors is distributed +// between the various system databases. +var handleRatio = map[string]float64{ + "dapp": 0.0, + "chaindata": 1.0, } type LDBDatabase struct { @@ -62,17 +69,21 @@ type LDBDatabase struct { } // NewLDBDatabase returns a LevelDB wrapped object. -func NewLDBDatabase(file string, cache int) (*LDBDatabase, error) { - // Calculate the cache allowance for this particular database +func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { + // Calculate the cache and file descriptor allowance for this particular database cache = int(float64(cache) * cacheRatio[filepath.Base(file)]) if cache < 16 { cache = 16 } - glog.V(logger.Info).Infof("Alloted %dMB cache to %s", cache, file) + handles = int(float64(handles) * handleRatio[filepath.Base(file)]) + if handles < 16 { + handles = 16 + } + glog.V(logger.Info).Infof("Alloted %dMB cache and %d file handles to %s", cache, handles, file) // Open the db and recover any potential corruptions db, err := leveldb.OpenFile(file, &opt.Options{ - OpenFilesCacheCapacity: OpenFileLimit, + OpenFilesCacheCapacity: handles, BlockCacheCapacity: cache / 2 * opt.MiB, WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally }) diff --git a/ethdb/database_test.go b/ethdb/database_test.go index ae49061660..0e69a1218c 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -28,7 +28,7 @@ func newDb() *LDBDatabase { if common.FileExist(file) { os.RemoveAll(file) } - db, _ := NewLDBDatabase(file, 0) + db, _ := NewLDBDatabase(file, 0, 0) return db } diff --git a/node/service.go b/node/service.go index 26e9f1624f..77b2ddc928 100644 --- a/node/service.go +++ b/node/service.go @@ -38,11 +38,11 @@ type ServiceContext struct { // OpenDatabase opens an existing database with the given name (or creates one // if no previous can be found) from within the node's data directory. If the // node is an ephemeral one, a memory database is returned. -func (ctx *ServiceContext) OpenDatabase(name string, cache int) (ethdb.Database, error) { +func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int) (ethdb.Database, error) { if ctx.datadir == "" { return ethdb.NewMemDatabase() } - return ethdb.NewLDBDatabase(filepath.Join(ctx.datadir, name), cache) + return ethdb.NewLDBDatabase(filepath.Join(ctx.datadir, name), cache, handles) } // Service retrieves a currently running service registered of a specific type. diff --git a/node/service_test.go b/node/service_test.go index cfe7fe5dc8..7bd94a52e6 100644 --- a/node/service_test.go +++ b/node/service_test.go @@ -39,7 +39,7 @@ func TestContextDatabases(t *testing.T) { } // Request the opening/creation of a database and ensure it persists to disk ctx := &ServiceContext{datadir: dir} - db, err := ctx.OpenDatabase("persistent", 0) + db, err := ctx.OpenDatabase("persistent", 0, 0) if err != nil { t.Fatalf("failed to open persistent database: %v", err) } @@ -50,7 +50,7 @@ func TestContextDatabases(t *testing.T) { } // Request th opening/creation of an ephemeral database and ensure it's not persisted ctx = &ServiceContext{datadir: ""} - db, err = ctx.OpenDatabase("ephemeral", 0) + db, err = ctx.OpenDatabase("ephemeral", 0, 0) if err != nil { t.Fatalf("failed to open ephemeral database: %v", err) } diff --git a/trie/trie_test.go b/trie/trie_test.go index 35d043cdf1..493f99d914 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -461,7 +461,7 @@ func tempDB() (string, Database) { if err != nil { panic(fmt.Sprintf("can't create temporary directory: %v", err)) } - db, err := ethdb.NewLDBDatabase(dir, 300*1024) + db, err := ethdb.NewLDBDatabase(dir, 256, 0) if err != nil { panic(fmt.Sprintf("can't create temporary database: %v", err)) }