mirror of https://github.com/ethereum/go-ethereum
Janoš Guljaš
6 years ago
committed by
Anton Evangelatov
parent
c94d582aa7
commit
996755c4a8
File diff suppressed because it is too large
Load Diff
@ -1,82 +0,0 @@ |
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package storage |
||||
|
||||
// this is a clone of an earlier state of the ethereum ethdb/database
|
||||
// no need for queueing/caching
|
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/metrics" |
||||
"github.com/syndtr/goleveldb/leveldb" |
||||
"github.com/syndtr/goleveldb/leveldb/iterator" |
||||
"github.com/syndtr/goleveldb/leveldb/opt" |
||||
) |
||||
|
||||
const openFileLimit = 128 |
||||
|
||||
type LDBDatabase struct { |
||||
db *leveldb.DB |
||||
} |
||||
|
||||
func NewLDBDatabase(file string) (*LDBDatabase, error) { |
||||
// Open the db
|
||||
db, err := leveldb.OpenFile(file, &opt.Options{OpenFilesCacheCapacity: openFileLimit}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
database := &LDBDatabase{db: db} |
||||
|
||||
return database, nil |
||||
} |
||||
|
||||
func (db *LDBDatabase) Put(key []byte, value []byte) error { |
||||
metrics.GetOrRegisterCounter("ldbdatabase.put", nil).Inc(1) |
||||
|
||||
return db.db.Put(key, value, nil) |
||||
} |
||||
|
||||
func (db *LDBDatabase) Get(key []byte) ([]byte, error) { |
||||
metrics.GetOrRegisterCounter("ldbdatabase.get", nil).Inc(1) |
||||
|
||||
dat, err := db.db.Get(key, nil) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return dat, nil |
||||
} |
||||
|
||||
func (db *LDBDatabase) Delete(key []byte) error { |
||||
return db.db.Delete(key, nil) |
||||
} |
||||
|
||||
func (db *LDBDatabase) NewIterator() iterator.Iterator { |
||||
metrics.GetOrRegisterCounter("ldbdatabase.newiterator", nil).Inc(1) |
||||
|
||||
return db.db.NewIterator(nil, nil) |
||||
} |
||||
|
||||
func (db *LDBDatabase) Write(batch *leveldb.Batch) error { |
||||
metrics.GetOrRegisterCounter("ldbdatabase.write", nil).Inc(1) |
||||
|
||||
return db.db.Write(batch, nil) |
||||
} |
||||
|
||||
func (db *LDBDatabase) Close() { |
||||
// Close the leveldb database
|
||||
db.db.Close() |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -1,788 +0,0 @@ |
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package storage |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"encoding/binary" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"os" |
||||
"strconv" |
||||
"strings" |
||||
"testing" |
||||
|
||||
"github.com/ethereum/go-ethereum/common" |
||||
"github.com/ethereum/go-ethereum/swarm/chunk" |
||||
"github.com/ethereum/go-ethereum/swarm/log" |
||||
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem" |
||||
"github.com/ethereum/go-ethereum/swarm/testutil" |
||||
ldberrors "github.com/syndtr/goleveldb/leveldb/errors" |
||||
) |
||||
|
||||
type testDbStore struct { |
||||
*LDBStore |
||||
dir string |
||||
} |
||||
|
||||
func newTestDbStore(mock bool, trusted bool) (*testDbStore, func(), error) { |
||||
dir, err := ioutil.TempDir("", "bzz-storage-test") |
||||
if err != nil { |
||||
return nil, func() {}, err |
||||
} |
||||
|
||||
var db *LDBStore |
||||
storeparams := NewDefaultStoreParams() |
||||
params := NewLDBStoreParams(storeparams, dir) |
||||
params.Po = testPoFunc |
||||
|
||||
if mock { |
||||
globalStore := mem.NewGlobalStore() |
||||
addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed") |
||||
mockStore := globalStore.NewNodeStore(addr) |
||||
|
||||
db, err = NewMockDbStore(params, mockStore) |
||||
} else { |
||||
db, err = NewLDBStore(params) |
||||
} |
||||
|
||||
cleanup := func() { |
||||
if db != nil { |
||||
db.Close() |
||||
} |
||||
err = os.RemoveAll(dir) |
||||
if err != nil { |
||||
panic(fmt.Sprintf("db cleanup failed: %v", err)) |
||||
} |
||||
} |
||||
|
||||
return &testDbStore{db, dir}, cleanup, err |
||||
} |
||||
|
||||
func testPoFunc(k Address) (ret uint8) { |
||||
basekey := make([]byte, 32) |
||||
return uint8(Proximity(basekey, k[:])) |
||||
} |
||||
|
||||
func testDbStoreRandom(n int, mock bool, t *testing.T) { |
||||
db, cleanup, err := newTestDbStore(mock, true) |
||||
defer cleanup() |
||||
if err != nil { |
||||
t.Fatalf("init dbStore failed: %v", err) |
||||
} |
||||
testStoreRandom(db, n, t) |
||||
} |
||||
|
||||
func testDbStoreCorrect(n int, mock bool, t *testing.T) { |
||||
db, cleanup, err := newTestDbStore(mock, false) |
||||
defer cleanup() |
||||
if err != nil { |
||||
t.Fatalf("init dbStore failed: %v", err) |
||||
} |
||||
testStoreCorrect(db, n, t) |
||||
} |
||||
|
||||
func TestMarkAccessed(t *testing.T) { |
||||
db, cleanup, err := newTestDbStore(false, true) |
||||
defer cleanup() |
||||
if err != nil { |
||||
t.Fatalf("init dbStore failed: %v", err) |
||||
} |
||||
|
||||
h := GenerateRandomChunk(chunk.DefaultSize) |
||||
|
||||
db.Put(context.Background(), h) |
||||
|
||||
var index dpaDBIndex |
||||
addr := h.Address() |
||||
idxk := getIndexKey(addr) |
||||
|
||||
idata, err := db.db.Get(idxk) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
decodeIndex(idata, &index) |
||||
|
||||
if index.Access != 0 { |
||||
t.Fatalf("Expected the access index to be %d, but it is %d", 0, index.Access) |
||||
} |
||||
|
||||
db.MarkAccessed(addr) |
||||
db.writeCurrentBatch() |
||||
|
||||
idata, err = db.db.Get(idxk) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
decodeIndex(idata, &index) |
||||
|
||||
if index.Access != 1 { |
||||
t.Fatalf("Expected the access index to be %d, but it is %d", 1, index.Access) |
||||
} |
||||
|
||||
} |
||||
|
||||
func TestDbStoreRandom_1(t *testing.T) { |
||||
testDbStoreRandom(1, false, t) |
||||
} |
||||
|
||||
func TestDbStoreCorrect_1(t *testing.T) { |
||||
testDbStoreCorrect(1, false, t) |
||||
} |
||||
|
||||
func TestDbStoreRandom_1k(t *testing.T) { |
||||
testDbStoreRandom(1000, false, t) |
||||
} |
||||
|
||||
func TestDbStoreCorrect_1k(t *testing.T) { |
||||
testDbStoreCorrect(1000, false, t) |
||||
} |
||||
|
||||
func TestMockDbStoreRandom_1(t *testing.T) { |
||||
testDbStoreRandom(1, true, t) |
||||
} |
||||
|
||||
func TestMockDbStoreCorrect_1(t *testing.T) { |
||||
testDbStoreCorrect(1, true, t) |
||||
} |
||||
|
||||
func TestMockDbStoreRandom_1k(t *testing.T) { |
||||
testDbStoreRandom(1000, true, t) |
||||
} |
||||
|
||||
func TestMockDbStoreCorrect_1k(t *testing.T) { |
||||
testDbStoreCorrect(1000, true, t) |
||||
} |
||||
|
||||
func testDbStoreNotFound(t *testing.T, mock bool) { |
||||
db, cleanup, err := newTestDbStore(mock, false) |
||||
defer cleanup() |
||||
if err != nil { |
||||
t.Fatalf("init dbStore failed: %v", err) |
||||
} |
||||
|
||||
_, err = db.Get(context.TODO(), ZeroAddr) |
||||
if err != ErrChunkNotFound { |
||||
t.Errorf("Expected ErrChunkNotFound, got %v", err) |
||||
} |
||||
} |
||||
|
||||
func TestDbStoreNotFound(t *testing.T) { |
||||
testDbStoreNotFound(t, false) |
||||
} |
||||
func TestMockDbStoreNotFound(t *testing.T) { |
||||
testDbStoreNotFound(t, true) |
||||
} |
||||
|
||||
func testIterator(t *testing.T, mock bool) { |
||||
var i int |
||||
var poc uint |
||||
chunkcount := 32 |
||||
chunkkeys := NewAddressCollection(chunkcount) |
||||
chunkkeysResults := NewAddressCollection(chunkcount) |
||||
|
||||
db, cleanup, err := newTestDbStore(mock, false) |
||||
defer cleanup() |
||||
if err != nil { |
||||
t.Fatalf("init dbStore failed: %v", err) |
||||
} |
||||
|
||||
chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount) |
||||
|
||||
for i = 0; i < len(chunks); i++ { |
||||
chunkkeys[i] = chunks[i].Address() |
||||
err := db.Put(context.TODO(), chunks[i]) |
||||
if err != nil { |
||||
t.Fatalf("dbStore.Put failed: %v", err) |
||||
} |
||||
} |
||||
|
||||
for i = 0; i < len(chunkkeys); i++ { |
||||
log.Trace(fmt.Sprintf("Chunk array pos %d/%d: '%v'", i, chunkcount, chunkkeys[i])) |
||||
} |
||||
i = 0 |
||||
for poc = 0; poc <= 255; poc++ { |
||||
err := db.SyncIterator(0, uint64(chunkkeys.Len()), uint8(poc), func(k Address, n uint64) bool { |
||||
log.Trace(fmt.Sprintf("Got key %v number %d poc %d", k, n, uint8(poc))) |
||||
chunkkeysResults[n] = k |
||||
i++ |
||||
return true |
||||
}) |
||||
if err != nil { |
||||
t.Fatalf("Iterator call failed: %v", err) |
||||
} |
||||
} |
||||
|
||||
for i = 0; i < chunkcount; i++ { |
||||
if !bytes.Equal(chunkkeys[i], chunkkeysResults[i]) { |
||||
t.Fatalf("Chunk put #%d key '%v' does not match iterator's key '%v'", i, chunkkeys[i], chunkkeysResults[i]) |
||||
} |
||||
} |
||||
|
||||
} |
||||
|
||||
func TestIterator(t *testing.T) { |
||||
testIterator(t, false) |
||||
} |
||||
func TestMockIterator(t *testing.T) { |
||||
testIterator(t, true) |
||||
} |
||||
|
||||
func benchmarkDbStorePut(n int, mock bool, b *testing.B) { |
||||
db, cleanup, err := newTestDbStore(mock, true) |
||||
defer cleanup() |
||||
if err != nil { |
||||
b.Fatalf("init dbStore failed: %v", err) |
||||
} |
||||
benchmarkStorePut(db, n, b) |
||||
} |
||||
|
||||
func benchmarkDbStoreGet(n int, mock bool, b *testing.B) { |
||||
db, cleanup, err := newTestDbStore(mock, true) |
||||
defer cleanup() |
||||
if err != nil { |
||||
b.Fatalf("init dbStore failed: %v", err) |
||||
} |
||||
benchmarkStoreGet(db, n, b) |
||||
} |
||||
|
||||
func BenchmarkDbStorePut_500(b *testing.B) { |
||||
benchmarkDbStorePut(500, false, b) |
||||
} |
||||
|
||||
func BenchmarkDbStoreGet_500(b *testing.B) { |
||||
benchmarkDbStoreGet(500, false, b) |
||||
} |
||||
|
||||
func BenchmarkMockDbStorePut_500(b *testing.B) { |
||||
benchmarkDbStorePut(500, true, b) |
||||
} |
||||
|
||||
func BenchmarkMockDbStoreGet_500(b *testing.B) { |
||||
benchmarkDbStoreGet(500, true, b) |
||||
} |
||||
|
||||
// TestLDBStoreWithoutCollectGarbage tests that we can put a number of random chunks in the LevelDB store, and
|
||||
// retrieve them, provided we don't hit the garbage collection
|
||||
func TestLDBStoreWithoutCollectGarbage(t *testing.T) { |
||||
capacity := 50 |
||||
n := 10 |
||||
|
||||
ldb, cleanup := newLDBStore(t) |
||||
ldb.setCapacity(uint64(capacity)) |
||||
defer cleanup() |
||||
|
||||
chunks, err := mputRandomChunks(ldb, n) |
||||
if err != nil { |
||||
t.Fatal(err.Error()) |
||||
} |
||||
|
||||
log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) |
||||
|
||||
for _, ch := range chunks { |
||||
ret, err := ldb.Get(context.TODO(), ch.Address()) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
if !bytes.Equal(ret.Data(), ch.Data()) { |
||||
t.Fatal("expected to get the same data back, but got smth else") |
||||
} |
||||
} |
||||
|
||||
if ldb.entryCnt != uint64(n) { |
||||
t.Fatalf("expected entryCnt to be equal to %v, but got %v", n, ldb.entryCnt) |
||||
} |
||||
|
||||
if ldb.accessCnt != uint64(2*n) { |
||||
t.Fatalf("expected accessCnt to be equal to %v, but got %v", 2*n, ldb.accessCnt) |
||||
} |
||||
} |
||||
|
||||
// TestLDBStoreCollectGarbage tests that we can put more chunks than LevelDB's capacity, and
|
||||
// retrieve only some of them, because garbage collection must have partially cleared the store
|
||||
// Also tests that we can delete chunks and that we can trigger garbage collection
|
||||
func TestLDBStoreCollectGarbage(t *testing.T) { |
||||
|
||||
// below max ronud
|
||||
initialCap := defaultMaxGCRound / 100 |
||||
cap := initialCap / 2 |
||||
t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) |
||||
|
||||
if testutil.RaceEnabled { |
||||
t.Skip("only the simplest case run as others are flaky with race") |
||||
// Note: some tests fail consistently and even locally with `-race`
|
||||
} |
||||
|
||||
t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) |
||||
|
||||
// at max round
|
||||
cap = initialCap |
||||
t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) |
||||
t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) |
||||
|
||||
// more than max around, not on threshold
|
||||
cap = initialCap + 500 |
||||
t.Run(fmt.Sprintf("A/%d/%d", cap, cap*4), testLDBStoreCollectGarbage) |
||||
t.Run(fmt.Sprintf("B/%d/%d", cap, cap*4), testLDBStoreRemoveThenCollectGarbage) |
||||
|
||||
} |
||||
|
||||
func testLDBStoreCollectGarbage(t *testing.T) { |
||||
params := strings.Split(t.Name(), "/") |
||||
capacity, err := strconv.Atoi(params[2]) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
n, err := strconv.Atoi(params[3]) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
ldb, cleanup := newLDBStore(t) |
||||
ldb.setCapacity(uint64(capacity)) |
||||
defer cleanup() |
||||
|
||||
// retrieve the gc round target count for the db capacity
|
||||
ldb.startGC(capacity) |
||||
roundTarget := ldb.gc.target |
||||
|
||||
// split put counts to gc target count threshold, and wait for gc to finish in between
|
||||
var allChunks []Chunk |
||||
remaining := n |
||||
for remaining > 0 { |
||||
var putCount int |
||||
if remaining < roundTarget { |
||||
putCount = remaining |
||||
} else { |
||||
putCount = roundTarget |
||||
} |
||||
remaining -= putCount |
||||
chunks, err := mputRandomChunks(ldb, putCount) |
||||
if err != nil { |
||||
t.Fatal(err.Error()) |
||||
} |
||||
allChunks = append(allChunks, chunks...) |
||||
ldb.lock.RLock() |
||||
log.Debug("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt, "cap", capacity, "n", n) |
||||
ldb.lock.RUnlock() |
||||
|
||||
waitGc(ldb) |
||||
} |
||||
|
||||
// attempt gets on all put chunks
|
||||
var missing int |
||||
for _, ch := range allChunks { |
||||
ret, err := ldb.Get(context.TODO(), ch.Address()) |
||||
if err == ErrChunkNotFound || err == ldberrors.ErrNotFound { |
||||
missing++ |
||||
continue |
||||
} |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
if !bytes.Equal(ret.Data(), ch.Data()) { |
||||
t.Fatal("expected to get the same data back, but got smth else") |
||||
} |
||||
|
||||
log.Trace("got back chunk", "chunk", ret) |
||||
} |
||||
|
||||
// all surplus chunks should be missing
|
||||
expectMissing := roundTarget + (((n - capacity) / roundTarget) * roundTarget) |
||||
if missing != expectMissing { |
||||
t.Fatalf("gc failure: expected to miss %v chunks, but only %v are actually missing", expectMissing, missing) |
||||
} |
||||
|
||||
log.Info("ldbstore", "total", n, "missing", missing, "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) |
||||
} |
||||
|
||||
// TestLDBStoreAddRemove tests that we can put and then delete a given chunk
|
||||
func TestLDBStoreAddRemove(t *testing.T) { |
||||
ldb, cleanup := newLDBStore(t) |
||||
ldb.setCapacity(200) |
||||
defer cleanup() |
||||
|
||||
n := 100 |
||||
chunks, err := mputRandomChunks(ldb, n) |
||||
if err != nil { |
||||
t.Fatalf(err.Error()) |
||||
} |
||||
|
||||
for i := 0; i < n; i++ { |
||||
// delete all even index chunks
|
||||
if i%2 == 0 { |
||||
ldb.Delete(chunks[i].Address()) |
||||
} |
||||
} |
||||
|
||||
log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) |
||||
|
||||
for i := 0; i < n; i++ { |
||||
ret, err := ldb.Get(context.TODO(), chunks[i].Address()) |
||||
|
||||
if i%2 == 0 { |
||||
// expect even chunks to be missing
|
||||
if err == nil { |
||||
t.Fatal("expected chunk to be missing, but got no error") |
||||
} |
||||
} else { |
||||
// expect odd chunks to be retrieved successfully
|
||||
if err != nil { |
||||
t.Fatalf("expected no error, but got %s", err) |
||||
} |
||||
|
||||
if !bytes.Equal(ret.Data(), chunks[i].Data()) { |
||||
t.Fatal("expected to get the same data back, but got smth else") |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func testLDBStoreRemoveThenCollectGarbage(t *testing.T) { |
||||
t.Skip("flaky with -race flag") |
||||
|
||||
params := strings.Split(t.Name(), "/") |
||||
capacity, err := strconv.Atoi(params[2]) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
n, err := strconv.Atoi(params[3]) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
ldb, cleanup := newLDBStore(t) |
||||
defer cleanup() |
||||
ldb.setCapacity(uint64(capacity)) |
||||
|
||||
// put capacity count number of chunks
|
||||
chunks := make([]Chunk, n) |
||||
for i := 0; i < n; i++ { |
||||
c := GenerateRandomChunk(chunk.DefaultSize) |
||||
chunks[i] = c |
||||
log.Trace("generate random chunk", "idx", i, "chunk", c) |
||||
} |
||||
|
||||
for i := 0; i < n; i++ { |
||||
err := ldb.Put(context.TODO(), chunks[i]) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
} |
||||
|
||||
waitGc(ldb) |
||||
|
||||
// delete all chunks
|
||||
// (only count the ones actually deleted, the rest will have been gc'd)
|
||||
deletes := 0 |
||||
for i := 0; i < n; i++ { |
||||
if ldb.Delete(chunks[i].Address()) == nil { |
||||
deletes++ |
||||
} |
||||
} |
||||
|
||||
log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) |
||||
|
||||
if ldb.entryCnt != 0 { |
||||
t.Fatalf("ldb.entrCnt expected 0 got %v", ldb.entryCnt) |
||||
} |
||||
|
||||
// the manual deletes will have increased accesscnt, so we need to add this when we verify the current count
|
||||
expAccessCnt := uint64(n) |
||||
if ldb.accessCnt != expAccessCnt { |
||||
t.Fatalf("ldb.accessCnt expected %v got %v", expAccessCnt, ldb.accessCnt) |
||||
} |
||||
|
||||
// retrieve the gc round target count for the db capacity
|
||||
ldb.startGC(capacity) |
||||
roundTarget := ldb.gc.target |
||||
|
||||
remaining := n |
||||
var puts int |
||||
for remaining > 0 { |
||||
var putCount int |
||||
if remaining < roundTarget { |
||||
putCount = remaining |
||||
} else { |
||||
putCount = roundTarget |
||||
} |
||||
remaining -= putCount |
||||
for putCount > 0 { |
||||
ldb.Put(context.TODO(), chunks[puts]) |
||||
ldb.lock.RLock() |
||||
log.Debug("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt, "cap", capacity, "n", n, "puts", puts, "remaining", remaining, "roundtarget", roundTarget) |
||||
ldb.lock.RUnlock() |
||||
puts++ |
||||
putCount-- |
||||
} |
||||
|
||||
waitGc(ldb) |
||||
} |
||||
|
||||
// expect first surplus chunks to be missing, because they have the smallest access value
|
||||
expectMissing := roundTarget + (((n - capacity) / roundTarget) * roundTarget) |
||||
for i := 0; i < expectMissing; i++ { |
||||
_, err := ldb.Get(context.TODO(), chunks[i].Address()) |
||||
if err == nil { |
||||
t.Fatalf("expected surplus chunk %d to be missing, but got no error", i) |
||||
} |
||||
} |
||||
|
||||
// expect last chunks to be present, as they have the largest access value
|
||||
for i := expectMissing; i < n; i++ { |
||||
ret, err := ldb.Get(context.TODO(), chunks[i].Address()) |
||||
if err != nil { |
||||
t.Fatalf("chunk %v: expected no error, but got %s", i, err) |
||||
} |
||||
if !bytes.Equal(ret.Data(), chunks[i].Data()) { |
||||
t.Fatal("expected to get the same data back, but got smth else") |
||||
} |
||||
} |
||||
} |
||||
|
||||
// TestLDBStoreCollectGarbageAccessUnlikeIndex tests garbage collection where accesscount differs from indexcount
|
||||
func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { |
||||
|
||||
capacity := defaultMaxGCRound / 100 * 2 |
||||
n := capacity - 1 |
||||
|
||||
ldb, cleanup := newLDBStore(t) |
||||
ldb.setCapacity(uint64(capacity)) |
||||
defer cleanup() |
||||
|
||||
chunks, err := mputRandomChunks(ldb, n) |
||||
if err != nil { |
||||
t.Fatal(err.Error()) |
||||
} |
||||
log.Info("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) |
||||
|
||||
// set first added capacity/2 chunks to highest accesscount
|
||||
for i := 0; i < capacity/2; i++ { |
||||
_, err := ldb.Get(context.TODO(), chunks[i].Address()) |
||||
if err != nil { |
||||
t.Fatalf("fail add chunk #%d - %s: %v", i, chunks[i].Address(), err) |
||||
} |
||||
} |
||||
_, err = mputRandomChunks(ldb, 2) |
||||
if err != nil { |
||||
t.Fatal(err.Error()) |
||||
} |
||||
|
||||
// wait for garbage collection to kick in on the responsible actor
|
||||
waitGc(ldb) |
||||
|
||||
var missing int |
||||
for i, ch := range chunks[2 : capacity/2] { |
||||
ret, err := ldb.Get(context.TODO(), ch.Address()) |
||||
if err == ErrChunkNotFound || err == ldberrors.ErrNotFound { |
||||
t.Fatalf("fail find chunk #%d - %s: %v", i, ch.Address(), err) |
||||
} |
||||
|
||||
if !bytes.Equal(ret.Data(), ch.Data()) { |
||||
t.Fatal("expected to get the same data back, but got smth else") |
||||
} |
||||
log.Trace("got back chunk", "chunk", ret) |
||||
} |
||||
|
||||
log.Info("ldbstore", "total", n, "missing", missing, "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt) |
||||
} |
||||
|
||||
func TestCleanIndex(t *testing.T) { |
||||
if testutil.RaceEnabled { |
||||
t.Skip("disabled because it times out with race detector") |
||||
} |
||||
|
||||
capacity := 5000 |
||||
n := 3 |
||||
|
||||
ldb, cleanup := newLDBStore(t) |
||||
ldb.setCapacity(uint64(capacity)) |
||||
defer cleanup() |
||||
|
||||
chunks, err := mputRandomChunks(ldb, n) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
// remove the data of the first chunk
|
||||
po := ldb.po(chunks[0].Address()[:]) |
||||
dataKey := make([]byte, 10) |
||||
dataKey[0] = keyData |
||||
dataKey[1] = byte(po) |
||||
// dataKey[2:10] = first chunk has storageIdx 0 on [2:10]
|
||||
if _, err := ldb.db.Get(dataKey); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if err := ldb.db.Delete(dataKey); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
// remove the gc index row for the first chunk
|
||||
gcFirstCorrectKey := make([]byte, 9) |
||||
gcFirstCorrectKey[0] = keyGCIdx |
||||
if err := ldb.db.Delete(gcFirstCorrectKey); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
// warp the gc data of the second chunk
|
||||
// this data should be correct again after the clean
|
||||
gcSecondCorrectKey := make([]byte, 9) |
||||
gcSecondCorrectKey[0] = keyGCIdx |
||||
binary.BigEndian.PutUint64(gcSecondCorrectKey[1:], uint64(1)) |
||||
gcSecondCorrectVal, err := ldb.db.Get(gcSecondCorrectKey) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
warpedGCVal := make([]byte, len(gcSecondCorrectVal)+1) |
||||
copy(warpedGCVal[1:], gcSecondCorrectVal) |
||||
if err := ldb.db.Delete(gcSecondCorrectKey); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if err := ldb.db.Put(gcSecondCorrectKey, warpedGCVal); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
if err := ldb.CleanGCIndex(); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
// the index without corresponding data should have been deleted
|
||||
idxKey := make([]byte, 33) |
||||
idxKey[0] = keyIndex |
||||
copy(idxKey[1:], chunks[0].Address()) |
||||
if _, err := ldb.db.Get(idxKey); err == nil { |
||||
t.Fatalf("expected chunk 0 idx to be pruned: %v", idxKey) |
||||
} |
||||
|
||||
// the two other indices should be present
|
||||
copy(idxKey[1:], chunks[1].Address()) |
||||
if _, err := ldb.db.Get(idxKey); err != nil { |
||||
t.Fatalf("expected chunk 1 idx to be present: %v", idxKey) |
||||
} |
||||
|
||||
copy(idxKey[1:], chunks[2].Address()) |
||||
if _, err := ldb.db.Get(idxKey); err != nil { |
||||
t.Fatalf("expected chunk 2 idx to be present: %v", idxKey) |
||||
} |
||||
|
||||
// first gc index should still be gone
|
||||
if _, err := ldb.db.Get(gcFirstCorrectKey); err == nil { |
||||
t.Fatalf("expected gc 0 idx to be pruned: %v", idxKey) |
||||
} |
||||
|
||||
// second gc index should still be fixed
|
||||
if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil { |
||||
t.Fatalf("expected gc 1 idx to be present: %v", idxKey) |
||||
} |
||||
|
||||
// third gc index should be unchanged
|
||||
binary.BigEndian.PutUint64(gcSecondCorrectKey[1:], uint64(2)) |
||||
if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil { |
||||
t.Fatalf("expected gc 2 idx to be present: %v", idxKey) |
||||
} |
||||
|
||||
c, err := ldb.db.Get(keyEntryCnt) |
||||
if err != nil { |
||||
t.Fatalf("expected gc 2 idx to be present: %v", idxKey) |
||||
} |
||||
|
||||
// entrycount should now be one less
|
||||
entryCount := binary.BigEndian.Uint64(c) |
||||
if entryCount != 2 { |
||||
t.Fatalf("expected entrycnt to be 2, was %d", c) |
||||
} |
||||
|
||||
// the chunks might accidentally be in the same bin
|
||||
// if so that bin counter will now be 2 - the highest added index.
|
||||
// if not, the total of them will be 3
|
||||
poBins := []uint8{ldb.po(chunks[1].Address()), ldb.po(chunks[2].Address())} |
||||
if poBins[0] == poBins[1] { |
||||
poBins = poBins[:1] |
||||
} |
||||
|
||||
var binTotal uint64 |
||||
var currentBin [2]byte |
||||
currentBin[0] = keyDistanceCnt |
||||
if len(poBins) == 1 { |
||||
currentBin[1] = poBins[0] |
||||
c, err := ldb.db.Get(currentBin[:]) |
||||
if err != nil { |
||||
t.Fatalf("expected gc 2 idx to be present: %v", idxKey) |
||||
} |
||||
binCount := binary.BigEndian.Uint64(c) |
||||
if binCount != 2 { |
||||
t.Fatalf("expected entrycnt to be 2, was %d", binCount) |
||||
} |
||||
} else { |
||||
for _, bin := range poBins { |
||||
currentBin[1] = bin |
||||
c, err := ldb.db.Get(currentBin[:]) |
||||
if err != nil { |
||||
t.Fatalf("expected gc 2 idx to be present: %v", idxKey) |
||||
} |
||||
binCount := binary.BigEndian.Uint64(c) |
||||
binTotal += binCount |
||||
|
||||
} |
||||
if binTotal != 3 { |
||||
t.Fatalf("expected sum of bin indices to be 3, was %d", binTotal) |
||||
} |
||||
} |
||||
|
||||
// check that the iterator quits properly
|
||||
chunks, err = mputRandomChunks(ldb, 4100) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
po = ldb.po(chunks[4099].Address()[:]) |
||||
dataKey = make([]byte, 10) |
||||
dataKey[0] = keyData |
||||
dataKey[1] = byte(po) |
||||
binary.BigEndian.PutUint64(dataKey[2:], 4099+3) |
||||
if _, err := ldb.db.Get(dataKey); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if err := ldb.db.Delete(dataKey); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
if err := ldb.CleanGCIndex(); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
// entrycount should now be one less of added chunks
|
||||
c, err = ldb.db.Get(keyEntryCnt) |
||||
if err != nil { |
||||
t.Fatalf("expected gc 2 idx to be present: %v", idxKey) |
||||
} |
||||
entryCount = binary.BigEndian.Uint64(c) |
||||
if entryCount != 4099+2 { |
||||
t.Fatalf("expected entrycnt to be 2, was %d", c) |
||||
} |
||||
} |
||||
|
||||
// Note: waitGc does not guarantee that we wait 1 GC round; it only
|
||||
// guarantees that if the GC is running we wait for that run to finish
|
||||
// ticket: https://github.com/ethersphere/go-ethereum/issues/1151
|
||||
func waitGc(ldb *LDBStore) { |
||||
<-ldb.gc.runC |
||||
ldb.gc.runC <- struct{}{} |
||||
} |
||||
@ -1,251 +0,0 @@ |
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package storage |
||||
|
||||
import ( |
||||
"context" |
||||
"path/filepath" |
||||
"sync" |
||||
|
||||
"github.com/ethereum/go-ethereum/metrics" |
||||
"github.com/ethereum/go-ethereum/swarm/log" |
||||
"github.com/ethereum/go-ethereum/swarm/storage/mock" |
||||
) |
||||
|
||||
type LocalStoreParams struct { |
||||
*StoreParams |
||||
ChunkDbPath string |
||||
Validators []ChunkValidator `toml:"-"` |
||||
} |
||||
|
||||
func NewDefaultLocalStoreParams() *LocalStoreParams { |
||||
return &LocalStoreParams{ |
||||
StoreParams: NewDefaultStoreParams(), |
||||
} |
||||
} |
||||
|
||||
//this can only finally be set after all config options (file, cmd line, env vars)
|
||||
//have been evaluated
|
||||
func (p *LocalStoreParams) Init(path string) { |
||||
if p.ChunkDbPath == "" { |
||||
p.ChunkDbPath = filepath.Join(path, "chunks") |
||||
} |
||||
} |
||||
|
||||
// LocalStore is a combination of inmemory db over a disk persisted db
|
||||
// implements a Get/Put with fallback (caching) logic using any 2 ChunkStores
|
||||
type LocalStore struct { |
||||
Validators []ChunkValidator |
||||
memStore *MemStore |
||||
DbStore *LDBStore |
||||
mu sync.Mutex |
||||
} |
||||
|
||||
// This constructor uses MemStore and DbStore as components
|
||||
func NewLocalStore(params *LocalStoreParams, mockStore *mock.NodeStore) (*LocalStore, error) { |
||||
ldbparams := NewLDBStoreParams(params.StoreParams, params.ChunkDbPath) |
||||
dbStore, err := NewMockDbStore(ldbparams, mockStore) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
return &LocalStore{ |
||||
memStore: NewMemStore(params.StoreParams, dbStore), |
||||
DbStore: dbStore, |
||||
Validators: params.Validators, |
||||
}, nil |
||||
} |
||||
|
||||
func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) { |
||||
ldbparams := NewLDBStoreParams(params.StoreParams, params.ChunkDbPath) |
||||
dbStore, err := NewLDBStore(ldbparams) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
localStore := &LocalStore{ |
||||
memStore: NewMemStore(params.StoreParams, dbStore), |
||||
DbStore: dbStore, |
||||
Validators: params.Validators, |
||||
} |
||||
return localStore, nil |
||||
} |
||||
|
||||
// isValid returns true if chunk passes any of the LocalStore Validators.
|
||||
// isValid also returns true if LocalStore has no Validators.
|
||||
func (ls *LocalStore) isValid(chunk Chunk) bool { |
||||
// by default chunks are valid. if we have 0 validators, then all chunks are valid.
|
||||
valid := true |
||||
|
||||
// ls.Validators contains a list of one validator per chunk type.
|
||||
// if one validator succeeds, then the chunk is valid
|
||||
for _, v := range ls.Validators { |
||||
if valid = v.Validate(chunk); valid { |
||||
break |
||||
} |
||||
} |
||||
return valid |
||||
} |
||||
|
||||
// Put is responsible for doing validation and storage of the chunk
|
||||
// by using configured ChunkValidators, MemStore and LDBStore.
|
||||
// If the chunk is not valid, its GetErrored function will
|
||||
// return ErrChunkInvalid.
|
||||
// This method will check if the chunk is already in the MemStore
|
||||
// and it will return it if it is. If there is an error from
|
||||
// the MemStore.Get, it will be returned by calling GetErrored
|
||||
// on the chunk.
|
||||
// This method is responsible for closing Chunk.ReqC channel
|
||||
// when the chunk is stored in memstore.
|
||||
// After the LDBStore.Put, it is ensured that the MemStore
|
||||
// contains the chunk with the same data, but nil ReqC channel.
|
||||
func (ls *LocalStore) Put(ctx context.Context, chunk Chunk) error { |
||||
if !ls.isValid(chunk) { |
||||
return ErrChunkInvalid |
||||
} |
||||
|
||||
log.Trace("localstore.put", "key", chunk.Address()) |
||||
ls.mu.Lock() |
||||
defer ls.mu.Unlock() |
||||
|
||||
_, err := ls.memStore.Get(ctx, chunk.Address()) |
||||
if err == nil { |
||||
return nil |
||||
} |
||||
if err != nil && err != ErrChunkNotFound { |
||||
return err |
||||
} |
||||
ls.memStore.Put(ctx, chunk) |
||||
err = ls.DbStore.Put(ctx, chunk) |
||||
return err |
||||
} |
||||
|
||||
// Has queries the underlying DbStore if a chunk with the given address
|
||||
// is being stored there.
|
||||
// Returns true if it is stored, false if not
|
||||
func (ls *LocalStore) Has(ctx context.Context, addr Address) bool { |
||||
return ls.DbStore.Has(ctx, addr) |
||||
} |
||||
|
||||
// Get(chunk *Chunk) looks up a chunk in the local stores
|
||||
// This method is blocking until the chunk is retrieved
|
||||
// so additional timeout may be needed to wrap this call if
|
||||
// ChunkStores are remote and can have long latency
|
||||
func (ls *LocalStore) Get(ctx context.Context, addr Address) (chunk Chunk, err error) { |
||||
ls.mu.Lock() |
||||
defer ls.mu.Unlock() |
||||
|
||||
return ls.get(ctx, addr) |
||||
} |
||||
|
||||
func (ls *LocalStore) get(ctx context.Context, addr Address) (chunk Chunk, err error) { |
||||
chunk, err = ls.memStore.Get(ctx, addr) |
||||
|
||||
if err != nil && err != ErrChunkNotFound { |
||||
metrics.GetOrRegisterCounter("localstore.get.error", nil).Inc(1) |
||||
return nil, err |
||||
} |
||||
|
||||
if err == nil { |
||||
metrics.GetOrRegisterCounter("localstore.get.cachehit", nil).Inc(1) |
||||
go ls.DbStore.MarkAccessed(addr) |
||||
return chunk, nil |
||||
} |
||||
|
||||
metrics.GetOrRegisterCounter("localstore.get.cachemiss", nil).Inc(1) |
||||
chunk, err = ls.DbStore.Get(ctx, addr) |
||||
if err != nil { |
||||
metrics.GetOrRegisterCounter("localstore.get.error", nil).Inc(1) |
||||
return nil, err |
||||
} |
||||
|
||||
ls.memStore.Put(ctx, chunk) |
||||
return chunk, nil |
||||
} |
||||
|
||||
func (ls *LocalStore) FetchFunc(ctx context.Context, addr Address) func(context.Context) error { |
||||
ls.mu.Lock() |
||||
defer ls.mu.Unlock() |
||||
|
||||
_, err := ls.get(ctx, addr) |
||||
if err == nil { |
||||
return nil |
||||
} |
||||
return func(context.Context) error { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
func (ls *LocalStore) BinIndex(po uint8) uint64 { |
||||
return ls.DbStore.BinIndex(po) |
||||
} |
||||
|
||||
func (ls *LocalStore) Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error { |
||||
return ls.DbStore.SyncIterator(from, to, po, f) |
||||
} |
||||
|
||||
// Close the local store
|
||||
func (ls *LocalStore) Close() { |
||||
ls.DbStore.Close() |
||||
} |
||||
|
||||
// Migrate checks the datastore schema vs the runtime schema and runs
|
||||
// migrations if they don't match
|
||||
func (ls *LocalStore) Migrate() error { |
||||
actualDbSchema, err := ls.DbStore.GetSchema() |
||||
if err != nil { |
||||
log.Error(err.Error()) |
||||
return err |
||||
} |
||||
|
||||
if actualDbSchema == CurrentDbSchema { |
||||
return nil |
||||
} |
||||
|
||||
log.Debug("running migrations for", "schema", actualDbSchema, "runtime-schema", CurrentDbSchema) |
||||
|
||||
if actualDbSchema == DbSchemaNone { |
||||
ls.migrateFromNoneToPurity() |
||||
actualDbSchema = DbSchemaPurity |
||||
} |
||||
|
||||
if err := ls.DbStore.PutSchema(actualDbSchema); err != nil { |
||||
return err |
||||
} |
||||
|
||||
if actualDbSchema == DbSchemaPurity { |
||||
if err := ls.migrateFromPurityToHalloween(); err != nil { |
||||
return err |
||||
} |
||||
actualDbSchema = DbSchemaHalloween |
||||
} |
||||
|
||||
if err := ls.DbStore.PutSchema(actualDbSchema); err != nil { |
||||
return err |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (ls *LocalStore) migrateFromNoneToPurity() { |
||||
// delete chunks that are not valid, i.e. chunks that do not pass
|
||||
// any of the ls.Validators
|
||||
ls.DbStore.Cleanup(func(c Chunk) bool { |
||||
return !ls.isValid(c) |
||||
}) |
||||
} |
||||
|
||||
func (ls *LocalStore) migrateFromPurityToHalloween() error { |
||||
return ls.DbStore.CleanGCIndex() |
||||
} |
||||
@ -0,0 +1,204 @@ |
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package localstore |
||||
|
||||
import ( |
||||
"archive/tar" |
||||
"context" |
||||
"encoding/hex" |
||||
"fmt" |
||||
"io" |
||||
"io/ioutil" |
||||
"sync" |
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk" |
||||
"github.com/ethereum/go-ethereum/swarm/log" |
||||
"github.com/ethereum/go-ethereum/swarm/shed" |
||||
) |
||||
|
||||
const ( |
||||
// filename in tar archive that holds the information
|
||||
// about exported data format version
|
||||
exportVersionFilename = ".swarm-export-version" |
||||
// legacy version for previous LDBStore
|
||||
legacyExportVersion = "1" |
||||
// current export format version
|
||||
currentExportVersion = "2" |
||||
) |
||||
|
||||
// Export writes a tar structured data to the writer of
|
||||
// all chunks in the retrieval data index. It returns the
|
||||
// number of chunks exported.
|
||||
func (db *DB) Export(w io.Writer) (count int64, err error) { |
||||
tw := tar.NewWriter(w) |
||||
defer tw.Close() |
||||
|
||||
if err := tw.WriteHeader(&tar.Header{ |
||||
Name: exportVersionFilename, |
||||
Mode: 0644, |
||||
Size: int64(len(currentExportVersion)), |
||||
}); err != nil { |
||||
return 0, err |
||||
} |
||||
if _, err := tw.Write([]byte(currentExportVersion)); err != nil { |
||||
return 0, err |
||||
} |
||||
|
||||
err = db.retrievalDataIndex.Iterate(func(item shed.Item) (stop bool, err error) { |
||||
hdr := &tar.Header{ |
||||
Name: hex.EncodeToString(item.Address), |
||||
Mode: 0644, |
||||
Size: int64(len(item.Data)), |
||||
} |
||||
if err := tw.WriteHeader(hdr); err != nil { |
||||
return false, err |
||||
} |
||||
if _, err := tw.Write(item.Data); err != nil { |
||||
return false, err |
||||
} |
||||
count++ |
||||
return false, nil |
||||
}, nil) |
||||
|
||||
return count, err |
||||
} |
||||
|
||||
// Import reads a tar structured data from the reader and
|
||||
// stores chunks in the database. It returns the number of
|
||||
// chunks imported.
|
||||
func (db *DB) Import(r io.Reader, legacy bool) (count int64, err error) { |
||||
tr := tar.NewReader(r) |
||||
|
||||
ctx, cancel := context.WithCancel(context.Background()) |
||||
defer cancel() |
||||
|
||||
errC := make(chan error) |
||||
doneC := make(chan struct{}) |
||||
tokenPool := make(chan struct{}, 100) |
||||
var wg sync.WaitGroup |
||||
go func() { |
||||
var ( |
||||
firstFile = true |
||||
// if exportVersionFilename file is not present
|
||||
// assume legacy version
|
||||
version = legacyExportVersion |
||||
) |
||||
for { |
||||
hdr, err := tr.Next() |
||||
if err != nil { |
||||
if err == io.EOF { |
||||
break |
||||
} |
||||
select { |
||||
case errC <- err: |
||||
case <-ctx.Done(): |
||||
} |
||||
} |
||||
if firstFile { |
||||
firstFile = false |
||||
if hdr.Name == exportVersionFilename { |
||||
data, err := ioutil.ReadAll(tr) |
||||
if err != nil { |
||||
select { |
||||
case errC <- err: |
||||
case <-ctx.Done(): |
||||
} |
||||
} |
||||
version = string(data) |
||||
continue |
||||
} |
||||
} |
||||
|
||||
if len(hdr.Name) != 64 { |
||||
log.Warn("ignoring non-chunk file", "name", hdr.Name) |
||||
continue |
||||
} |
||||
|
||||
keybytes, err := hex.DecodeString(hdr.Name) |
||||
if err != nil { |
||||
log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err) |
||||
continue |
||||
} |
||||
|
||||
data, err := ioutil.ReadAll(tr) |
||||
if err != nil { |
||||
select { |
||||
case errC <- err: |
||||
case <-ctx.Done(): |
||||
} |
||||
} |
||||
key := chunk.Address(keybytes) |
||||
|
||||
var ch chunk.Chunk |
||||
switch version { |
||||
case legacyExportVersion: |
||||
// LDBStore Export exported chunk data prefixed with the chunk key.
|
||||
// That is not necessary, as the key is in the chunk filename,
|
||||
// but backward compatibility needs to be preserved.
|
||||
ch = chunk.NewChunk(key, data[32:]) |
||||
case currentExportVersion: |
||||
ch = chunk.NewChunk(key, data) |
||||
default: |
||||
select { |
||||
case errC <- fmt.Errorf("unsupported export data version %q", version): |
||||
case <-ctx.Done(): |
||||
} |
||||
} |
||||
tokenPool <- struct{}{} |
||||
wg.Add(1) |
||||
|
||||
go func() { |
||||
_, err := db.Put(ctx, chunk.ModePutUpload, ch) |
||||
select { |
||||
case errC <- err: |
||||
case <-ctx.Done(): |
||||
wg.Done() |
||||
<-tokenPool |
||||
default: |
||||
err := db.Put(ctx, chunk.ModePutUpload, ch) |
||||
if err != nil { |
||||
errC <- err |
||||
} |
||||
wg.Done() |
||||
<-tokenPool |
||||
} |
||||
}() |
||||
|
||||
count++ |
||||
} |
||||
wg.Wait() |
||||
close(doneC) |
||||
}() |
||||
|
||||
// wait for all chunks to be stored
|
||||
for { |
||||
select { |
||||
case err := <-errC: |
||||
if err != nil { |
||||
return count, err |
||||
} |
||||
case <-ctx.Done(): |
||||
return count, ctx.Err() |
||||
default: |
||||
select { |
||||
case <-doneC: |
||||
return count, nil |
||||
default: |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,80 @@ |
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package localstore |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"testing" |
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk" |
||||
) |
||||
|
||||
// TestExportImport constructs two databases, one to put and export
|
||||
// chunks and another one to import and validate that all chunks are
|
||||
// imported.
|
||||
func TestExportImport(t *testing.T) { |
||||
db1, cleanup1 := newTestDB(t, nil) |
||||
defer cleanup1() |
||||
|
||||
var chunkCount = 100 |
||||
|
||||
chunks := make(map[string][]byte, chunkCount) |
||||
for i := 0; i < chunkCount; i++ { |
||||
ch := generateTestRandomChunk() |
||||
|
||||
_, err := db1.Put(context.Background(), chunk.ModePutUpload, ch) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
chunks[string(ch.Address())] = ch.Data() |
||||
} |
||||
|
||||
var buf bytes.Buffer |
||||
|
||||
c, err := db1.Export(&buf) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
wantChunksCount := int64(len(chunks)) |
||||
if c != wantChunksCount { |
||||
t.Errorf("got export count %v, want %v", c, wantChunksCount) |
||||
} |
||||
|
||||
db2, cleanup2 := newTestDB(t, nil) |
||||
defer cleanup2() |
||||
|
||||
c, err = db2.Import(&buf, false) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
if c != wantChunksCount { |
||||
t.Errorf("got import count %v, want %v", c, wantChunksCount) |
||||
} |
||||
|
||||
for a, want := range chunks { |
||||
addr := chunk.Address([]byte(a)) |
||||
ch, err := db2.Get(context.Background(), chunk.ModeGetRequest, addr) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
got := ch.Data() |
||||
if !bytes.Equal(got, want) { |
||||
t.Fatalf("chunk %s: got data %x, want %x", addr.Hex(), got, want) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,52 @@ |
||||
package localstore |
||||
|
||||
import ( |
||||
"github.com/ethereum/go-ethereum/swarm/log" |
||||
"github.com/syndtr/goleveldb/leveldb" |
||||
"github.com/syndtr/goleveldb/leveldb/opt" |
||||
) |
||||
|
||||
// The DB schema we want to use. The actual/current DB schema might differ
|
||||
// until migrations are run.
|
||||
const CurrentDbSchema = DbSchemaSanctuary |
||||
|
||||
// There was a time when we had no schema at all.
|
||||
const DbSchemaNone = "" |
||||
|
||||
// "purity" is the first formal schema of LevelDB we release together with Swarm 0.3.5
|
||||
const DbSchemaPurity = "purity" |
||||
|
||||
// "halloween" is here because we had a screw in the garbage collector index.
|
||||
// Because of that we had to rebuild the GC index to get rid of erroneous
|
||||
// entries and that takes a long time. This schema is used for bookkeeping,
|
||||
// so rebuild index will run just once.
|
||||
const DbSchemaHalloween = "halloween" |
||||
|
||||
const DbSchemaSanctuary = "sanctuary" |
||||
|
||||
// returns true if legacy database is in the datadir
|
||||
func IsLegacyDatabase(datadir string) bool { |
||||
|
||||
var ( |
||||
legacyDbSchemaKey = []byte{8} |
||||
) |
||||
|
||||
db, err := leveldb.OpenFile(datadir, &opt.Options{OpenFilesCacheCapacity: 128}) |
||||
if err != nil { |
||||
log.Error("got an error while trying to open leveldb path", "path", datadir, "err", err) |
||||
return false |
||||
} |
||||
defer db.Close() |
||||
|
||||
data, err := db.Get(legacyDbSchemaKey, nil) |
||||
if err != nil { |
||||
if err == leveldb.ErrNotFound { |
||||
// if we haven't found anything under the legacy db schema key- we are not on legacy
|
||||
return false |
||||
} |
||||
|
||||
log.Error("got an unexpected error fetching legacy name from the database", "err", err) |
||||
} |
||||
log.Trace("checking if database scheme is legacy", "schema name", string(data)) |
||||
return string(data) == DbSchemaHalloween || string(data) == DbSchemaPurity |
||||
} |
||||
@ -1,244 +0,0 @@ |
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package storage |
||||
|
||||
import ( |
||||
"context" |
||||
"io/ioutil" |
||||
"os" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/chunk" |
||||
) |
||||
|
||||
var ( |
||||
hashfunc = MakeHashFunc(DefaultHash) |
||||
) |
||||
|
||||
// tests that the content address validator correctly checks the data
|
||||
// tests that feed update chunks are passed through content address validator
|
||||
// the test checking the resouce update validator internal correctness is found in storage/feeds/handler_test.go
|
||||
func TestValidator(t *testing.T) { |
||||
// set up localstore
|
||||
datadir, err := ioutil.TempDir("", "storage-testvalidator") |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
defer os.RemoveAll(datadir) |
||||
|
||||
params := NewDefaultLocalStoreParams() |
||||
params.Init(datadir) |
||||
store, err := NewLocalStore(params, nil) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
// check puts with no validators, both succeed
|
||||
chunks := GenerateRandomChunks(259, 2) |
||||
goodChunk := chunks[0] |
||||
badChunk := chunks[1] |
||||
copy(badChunk.Data(), goodChunk.Data()) |
||||
|
||||
errs := putChunks(store, goodChunk, badChunk) |
||||
if errs[0] != nil { |
||||
t.Fatalf("expected no error on good content address chunk in spite of no validation, but got: %s", err) |
||||
} |
||||
if errs[1] != nil { |
||||
t.Fatalf("expected no error on bad content address chunk in spite of no validation, but got: %s", err) |
||||
} |
||||
|
||||
// add content address validator and check puts
|
||||
// bad should fail, good should pass
|
||||
store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc)) |
||||
chunks = GenerateRandomChunks(chunk.DefaultSize, 2) |
||||
goodChunk = chunks[0] |
||||
badChunk = chunks[1] |
||||
copy(badChunk.Data(), goodChunk.Data()) |
||||
|
||||
errs = putChunks(store, goodChunk, badChunk) |
||||
if errs[0] != nil { |
||||
t.Fatalf("expected no error on good content address chunk with content address validator only, but got: %s", err) |
||||
} |
||||
if errs[1] == nil { |
||||
t.Fatal("expected error on bad content address chunk with content address validator only, but got nil") |
||||
} |
||||
|
||||
// append a validator that always denies
|
||||
// bad should fail, good should pass,
|
||||
var negV boolTestValidator |
||||
store.Validators = append(store.Validators, negV) |
||||
|
||||
chunks = GenerateRandomChunks(chunk.DefaultSize, 2) |
||||
goodChunk = chunks[0] |
||||
badChunk = chunks[1] |
||||
copy(badChunk.Data(), goodChunk.Data()) |
||||
|
||||
errs = putChunks(store, goodChunk, badChunk) |
||||
if errs[0] != nil { |
||||
t.Fatalf("expected no error on good content address chunk with content address validator only, but got: %s", err) |
||||
} |
||||
if errs[1] == nil { |
||||
t.Fatal("expected error on bad content address chunk with content address validator only, but got nil") |
||||
} |
||||
|
||||
// append a validator that always approves
|
||||
// all shall pass
|
||||
var posV boolTestValidator = true |
||||
store.Validators = append(store.Validators, posV) |
||||
|
||||
chunks = GenerateRandomChunks(chunk.DefaultSize, 2) |
||||
goodChunk = chunks[0] |
||||
badChunk = chunks[1] |
||||
copy(badChunk.Data(), goodChunk.Data()) |
||||
|
||||
errs = putChunks(store, goodChunk, badChunk) |
||||
if errs[0] != nil { |
||||
t.Fatalf("expected no error on good content address chunk with content address validator only, but got: %s", err) |
||||
} |
||||
if errs[1] != nil { |
||||
t.Fatalf("expected no error on bad content address chunk in spite of no validation, but got: %s", err) |
||||
} |
||||
|
||||
} |
||||
|
||||
type boolTestValidator bool |
||||
|
||||
func (self boolTestValidator) Validate(chunk Chunk) bool { |
||||
return bool(self) |
||||
} |
||||
|
||||
// putChunks adds chunks to localstore
|
||||
// It waits for receive on the stored channel
|
||||
// It logs but does not fail on delivery error
|
||||
func putChunks(store *LocalStore, chunks ...Chunk) []error { |
||||
i := 0 |
||||
f := func(n int64) Chunk { |
||||
chunk := chunks[i] |
||||
i++ |
||||
return chunk |
||||
} |
||||
_, errs := put(store, len(chunks), f) |
||||
return errs |
||||
} |
||||
|
||||
func put(store *LocalStore, n int, f func(i int64) Chunk) (hs []Address, errs []error) { |
||||
for i := int64(0); i < int64(n); i++ { |
||||
chunk := f(chunk.DefaultSize) |
||||
err := store.Put(context.TODO(), chunk) |
||||
errs = append(errs, err) |
||||
hs = append(hs, chunk.Address()) |
||||
} |
||||
return hs, errs |
||||
} |
||||
|
||||
// TestGetFrequentlyAccessedChunkWontGetGarbageCollected tests that the most
|
||||
// frequently accessed chunk is not garbage collected from LDBStore, i.e.,
|
||||
// from disk when we are at the capacity and garbage collector runs. For that
|
||||
// we start putting random chunks into the DB while continuously accessing the
|
||||
// chunk we care about then check if we can still retrieve it from disk.
|
||||
func TestGetFrequentlyAccessedChunkWontGetGarbageCollected(t *testing.T) { |
||||
ldbCap := defaultGCRatio |
||||
store, cleanup := setupLocalStore(t, ldbCap) |
||||
defer cleanup() |
||||
|
||||
var chunks []Chunk |
||||
for i := 0; i < ldbCap; i++ { |
||||
chunks = append(chunks, GenerateRandomChunk(chunk.DefaultSize)) |
||||
} |
||||
|
||||
mostAccessed := chunks[0].Address() |
||||
for _, chunk := range chunks { |
||||
if err := store.Put(context.Background(), chunk); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
if _, err := store.Get(context.Background(), mostAccessed); err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
// Add time for MarkAccessed() to be able to finish in a separate Goroutine
|
||||
time.Sleep(1 * time.Millisecond) |
||||
} |
||||
|
||||
store.DbStore.collectGarbage() |
||||
if _, err := store.DbStore.Get(context.Background(), mostAccessed); err != nil { |
||||
t.Logf("most frequntly accessed chunk not found on disk (key: %v)", mostAccessed) |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
} |
||||
|
||||
func setupLocalStore(t *testing.T, ldbCap int) (ls *LocalStore, cleanup func()) { |
||||
t.Helper() |
||||
|
||||
var err error |
||||
datadir, err := ioutil.TempDir("", "storage") |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
params := &LocalStoreParams{ |
||||
StoreParams: NewStoreParams(uint64(ldbCap), uint(ldbCap), nil, nil), |
||||
} |
||||
params.Init(datadir) |
||||
|
||||
store, err := NewLocalStore(params, nil) |
||||
if err != nil { |
||||
_ = os.RemoveAll(datadir) |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
cleanup = func() { |
||||
store.Close() |
||||
_ = os.RemoveAll(datadir) |
||||
} |
||||
|
||||
return store, cleanup |
||||
} |
||||
|
||||
func TestHas(t *testing.T) { |
||||
ldbCap := defaultGCRatio |
||||
store, cleanup := setupLocalStore(t, ldbCap) |
||||
defer cleanup() |
||||
|
||||
nonStoredAddr := GenerateRandomChunk(128).Address() |
||||
|
||||
has := store.Has(context.Background(), nonStoredAddr) |
||||
if has { |
||||
t.Fatal("Expected Has() to return false, but returned true!") |
||||
} |
||||
|
||||
storeChunks := GenerateRandomChunks(128, 3) |
||||
for _, ch := range storeChunks { |
||||
err := store.Put(context.Background(), ch) |
||||
if err != nil { |
||||
t.Fatalf("Expected store to store chunk, but it failed: %v", err) |
||||
} |
||||
|
||||
has := store.Has(context.Background(), ch.Address()) |
||||
if !has { |
||||
t.Fatal("Expected Has() to return true, but returned false!") |
||||
} |
||||
} |
||||
|
||||
//let's be paranoic and test again that the non-existent chunk returns false
|
||||
has = store.Has(context.Background(), nonStoredAddr) |
||||
if has { |
||||
t.Fatal("Expected Has() to return false, but returned true!") |
||||
} |
||||
|
||||
} |
||||
@ -1,92 +0,0 @@ |
||||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// memory storage layer for the package blockhash
|
||||
|
||||
package storage |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
lru "github.com/hashicorp/golang-lru" |
||||
) |
||||
|
||||
type MemStore struct { |
||||
cache *lru.Cache |
||||
disabled bool |
||||
} |
||||
|
||||
//NewMemStore is instantiating a MemStore cache keeping all frequently requested
|
||||
//chunks in the `cache` LRU cache.
|
||||
func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore) { |
||||
if params.CacheCapacity == 0 { |
||||
return &MemStore{ |
||||
disabled: true, |
||||
} |
||||
} |
||||
|
||||
c, err := lru.New(int(params.CacheCapacity)) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
|
||||
return &MemStore{ |
||||
cache: c, |
||||
} |
||||
} |
||||
|
||||
// Has needed to implement SyncChunkStore
|
||||
func (m *MemStore) Has(_ context.Context, addr Address) bool { |
||||
return m.cache.Contains(addr) |
||||
} |
||||
|
||||
func (m *MemStore) Get(_ context.Context, addr Address) (Chunk, error) { |
||||
if m.disabled { |
||||
return nil, ErrChunkNotFound |
||||
} |
||||
|
||||
c, ok := m.cache.Get(string(addr)) |
||||
if !ok { |
||||
return nil, ErrChunkNotFound |
||||
} |
||||
return c.(Chunk), nil |
||||
} |
||||
|
||||
func (m *MemStore) Put(_ context.Context, c Chunk) error { |
||||
if m.disabled { |
||||
return nil |
||||
} |
||||
|
||||
m.cache.Add(string(c.Address()), c) |
||||
return nil |
||||
} |
||||
|
||||
func (m *MemStore) setCapacity(n int) { |
||||
if n <= 0 { |
||||
m.disabled = true |
||||
} else { |
||||
c, err := lru.New(n) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
|
||||
*m = MemStore{ |
||||
cache: c, |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (s *MemStore) Close() {} |
||||
@ -1,158 +0,0 @@ |
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package storage |
||||
|
||||
import ( |
||||
"context" |
||||
"testing" |
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/log" |
||||
) |
||||
|
||||
func newTestMemStore() *MemStore { |
||||
storeparams := NewDefaultStoreParams() |
||||
return NewMemStore(storeparams, nil) |
||||
} |
||||
|
||||
func testMemStoreRandom(n int, t *testing.T) { |
||||
m := newTestMemStore() |
||||
defer m.Close() |
||||
testStoreRandom(m, n, t) |
||||
} |
||||
|
||||
func testMemStoreCorrect(n int, t *testing.T) { |
||||
m := newTestMemStore() |
||||
defer m.Close() |
||||
testStoreCorrect(m, n, t) |
||||
} |
||||
|
||||
func TestMemStoreRandom_1(t *testing.T) { |
||||
testMemStoreRandom(1, t) |
||||
} |
||||
|
||||
func TestMemStoreCorrect_1(t *testing.T) { |
||||
testMemStoreCorrect(1, t) |
||||
} |
||||
|
||||
func TestMemStoreRandom_1k(t *testing.T) { |
||||
testMemStoreRandom(1000, t) |
||||
} |
||||
|
||||
func TestMemStoreCorrect_1k(t *testing.T) { |
||||
testMemStoreCorrect(100, t) |
||||
} |
||||
|
||||
func TestMemStoreNotFound(t *testing.T) { |
||||
m := newTestMemStore() |
||||
defer m.Close() |
||||
|
||||
_, err := m.Get(context.TODO(), ZeroAddr) |
||||
if err != ErrChunkNotFound { |
||||
t.Errorf("Expected ErrChunkNotFound, got %v", err) |
||||
} |
||||
} |
||||
|
||||
func benchmarkMemStorePut(n int, b *testing.B) { |
||||
m := newTestMemStore() |
||||
defer m.Close() |
||||
benchmarkStorePut(m, n, b) |
||||
} |
||||
|
||||
func benchmarkMemStoreGet(n int, b *testing.B) { |
||||
m := newTestMemStore() |
||||
defer m.Close() |
||||
benchmarkStoreGet(m, n, b) |
||||
} |
||||
|
||||
func BenchmarkMemStorePut_500(b *testing.B) { |
||||
benchmarkMemStorePut(500, b) |
||||
} |
||||
|
||||
func BenchmarkMemStoreGet_500(b *testing.B) { |
||||
benchmarkMemStoreGet(500, b) |
||||
} |
||||
|
||||
func TestMemStoreAndLDBStore(t *testing.T) { |
||||
ldb, cleanup := newLDBStore(t) |
||||
ldb.setCapacity(4000) |
||||
defer cleanup() |
||||
|
||||
cacheCap := 200 |
||||
memStore := NewMemStore(NewStoreParams(4000, 200, nil, nil), nil) |
||||
|
||||
tests := []struct { |
||||
n int // number of chunks to push to memStore
|
||||
chunkSize int64 // size of chunk (by default in Swarm - 4096)
|
||||
}{ |
||||
{ |
||||
n: 1, |
||||
chunkSize: 4096, |
||||
}, |
||||
{ |
||||
n: 101, |
||||
chunkSize: 4096, |
||||
}, |
||||
{ |
||||
n: 501, |
||||
chunkSize: 4096, |
||||
}, |
||||
{ |
||||
n: 1100, |
||||
chunkSize: 4096, |
||||
}, |
||||
} |
||||
|
||||
for i, tt := range tests { |
||||
log.Info("running test", "idx", i, "tt", tt) |
||||
var chunks []Chunk |
||||
|
||||
for i := 0; i < tt.n; i++ { |
||||
c := GenerateRandomChunk(tt.chunkSize) |
||||
chunks = append(chunks, c) |
||||
} |
||||
|
||||
for i := 0; i < tt.n; i++ { |
||||
err := ldb.Put(context.TODO(), chunks[i]) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
err = memStore.Put(context.TODO(), chunks[i]) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
if got := memStore.cache.Len(); got > cacheCap { |
||||
t.Fatalf("expected to get cache capacity less than %v, but got %v", cacheCap, got) |
||||
} |
||||
|
||||
} |
||||
|
||||
for i := 0; i < tt.n; i++ { |
||||
_, err := memStore.Get(context.TODO(), chunks[i].Address()) |
||||
if err != nil { |
||||
if err == ErrChunkNotFound { |
||||
_, err := ldb.Get(context.TODO(), chunks[i].Address()) |
||||
if err != nil { |
||||
t.Fatalf("couldn't get chunk %v from ldb, got error: %v", i, err) |
||||
} |
||||
} else { |
||||
t.Fatalf("got error from memstore: %v", err) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -1,17 +0,0 @@ |
||||
package storage |
||||
|
||||
// The DB schema we want to use. The actual/current DB schema might differ
|
||||
// until migrations are run.
|
||||
const CurrentDbSchema = DbSchemaHalloween |
||||
|
||||
// There was a time when we had no schema at all.
|
||||
const DbSchemaNone = "" |
||||
|
||||
// "purity" is the first formal schema of LevelDB we release together with Swarm 0.3.5
|
||||
const DbSchemaPurity = "purity" |
||||
|
||||
// "halloween" is here because we had a screw in the garbage collector index.
|
||||
// Because of that we had to rebuild the GC index to get rid of erroneous
|
||||
// entries and that takes a long time. This schema is used for bookkeeping,
|
||||
// so rebuild index will run just once.
|
||||
const DbSchemaHalloween = "halloween" |
||||
Loading…
Reference in new issue