swarm/chunk: move chunk related declarations to chunk package (#19170)

ChrisChinchilla-patch-3
Janoš Guljaš 6 years ago committed by Anton Evangelatov
parent b7e0dec6bd
commit f0233948d2
  1. 106
      swarm/chunk/chunk.go
  2. 2
      swarm/chunk/proximity_test.go
  3. 6
      swarm/storage/chunker.go
  4. 4
      swarm/storage/common_test.go
  5. 9
      swarm/storage/error.go
  6. 16
      swarm/storage/hasherstore.go
  7. 14
      swarm/storage/ldbstore.go
  8. 8
      swarm/storage/ldbstore_test.go
  9. 2
      swarm/storage/localstore.go
  10. 47
      swarm/storage/localstore/gc_test.go
  11. 10
      swarm/storage/localstore/index_test.go
  12. 12
      swarm/storage/localstore/localstore.go
  13. 57
      swarm/storage/localstore/localstore_test.go
  14. 12
      swarm/storage/localstore/mode_get.go
  15. 4
      swarm/storage/localstore/mode_get_test.go
  16. 4
      swarm/storage/localstore/mode_put.go
  17. 18
      swarm/storage/localstore/mode_put_test.go
  18. 6
      swarm/storage/localstore/mode_set.go
  19. 6
      swarm/storage/localstore/mode_set_test.go
  20. 10
      swarm/storage/localstore/retrieval_index_test.go
  21. 4
      swarm/storage/localstore/subscription_pull.go
  22. 72
      swarm/storage/localstore/subscription_pull_test.go
  23. 8
      swarm/storage/localstore/subscription_push.go
  24. 10
      swarm/storage/localstore/subscription_push_test.go
  25. 12
      swarm/storage/localstore_test.go
  26. 24
      swarm/storage/netstore_test.go
  27. 6
      swarm/storage/pyramid.go
  28. 120
      swarm/storage/types.go

@ -1,5 +1,109 @@
package chunk
import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/common"
)
const (
DefaultSize = 4096
DefaultSize = 4096
MaxPO = 16
AddressLength = 32
)
var (
ErrChunkNotFound = errors.New("chunk not found")
ErrChunkInvalid = errors.New("invalid chunk")
)
type Chunk interface {
Address() Address
Data() []byte
}
type chunk struct {
addr Address
sdata []byte
}
func NewChunk(addr Address, data []byte) *chunk {
return &chunk{
addr: addr,
sdata: data,
}
}
func (c *chunk) Address() Address {
return c.addr
}
func (c *chunk) Data() []byte {
return c.sdata
}
func (self *chunk) String() string {
return fmt.Sprintf("Address: %v Chunksize: %v", self.addr.Log(), len(self.sdata))
}
type Address []byte
var ZeroAddr = Address(common.Hash{}.Bytes())
func (a Address) Hex() string {
return fmt.Sprintf("%064x", []byte(a[:]))
}
func (a Address) Log() string {
if len(a[:]) < 8 {
return fmt.Sprintf("%x", []byte(a[:]))
}
return fmt.Sprintf("%016x", []byte(a[:8]))
}
func (a Address) String() string {
return fmt.Sprintf("%064x", []byte(a))
}
func (a Address) MarshalJSON() (out []byte, err error) {
return []byte(`"` + a.String() + `"`), nil
}
func (a *Address) UnmarshalJSON(value []byte) error {
s := string(value)
*a = make([]byte, 32)
h := common.Hex2Bytes(s[1 : len(s)-1])
copy(*a, h)
return nil
}
// Proximity returns the proximity order of the MSB distance between x and y
//
// The distance metric MSB(x, y) of two equal length byte sequences x an y is the
// value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed.
// the binary cast is big endian: most significant bit first (=MSB).
//
// Proximity(x, y) is a discrete logarithmic scaling of the MSB distance.
// It is defined as the reverse rank of the integer part of the base 2
// logarithm of the distance.
// It is calculated by counting the number of common leading zeros in the (MSB)
// binary representation of the x^y.
//
// (0 farthest, 255 closest, 256 self)
func Proximity(one, other []byte) (ret int) {
b := (MaxPO-1)/8 + 1
if b > len(one) {
b = len(one)
}
m := 8
for i := 0; i < b; i++ {
oxo := one[i] ^ other[i]
for j := 0; j < m; j++ {
if (oxo>>uint8(7-j))&0x01 != 0 {
return i*8 + j
}
}
}
return MaxPO
}

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package storage
package chunk
import (
"strconv"

@ -25,7 +25,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/metrics"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/spancontext"
opentracing "github.com/opentracing/opentracing-go"
@ -127,7 +127,7 @@ type TreeChunker struct {
func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader {
jp := &JoinerParams{
ChunkerParams: ChunkerParams{
chunkSize: ch.DefaultSize,
chunkSize: chunk.DefaultSize,
hashSize: int64(len(addr)),
},
addr: addr,
@ -147,7 +147,7 @@ func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (
tsp := &TreeSplitterParams{
SplitterParams: SplitterParams{
ChunkerParams: ChunkerParams{
chunkSize: ch.DefaultSize,
chunkSize: chunk.DefaultSize,
hashSize: putter.RefSize(),
},
reader: data,

@ -29,7 +29,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/log"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/mattn/go-colorable"
)
@ -94,7 +94,7 @@ func mput(store ChunkStore, n int, f func(i int64) Chunk) (hs []Chunk, err error
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()
for i := int64(0); i < int64(n); i++ {
chunk := f(ch.DefaultSize)
chunk := f(chunk.DefaultSize)
go func() {
select {
case errc <- store.Put(ctx, chunk):

@ -16,9 +16,7 @@
package storage
import (
"errors"
)
import "github.com/ethereum/go-ethereum/swarm/chunk"
const (
ErrInit = iota
@ -31,7 +29,8 @@ const (
ErrNotSynced
)
// Errors are the same as the ones in chunk package for backward compatibility.
var (
ErrChunkNotFound = errors.New("chunk not found")
ErrChunkInvalid = errors.New("invalid chunk")
ErrChunkNotFound = chunk.ErrChunkNotFound
ErrChunkInvalid = chunk.ErrChunkNotFound
)

@ -21,7 +21,7 @@ import (
"fmt"
"sync/atomic"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/storage/encryption"
"golang.org/x/crypto/sha3"
)
@ -156,7 +156,7 @@ func (h *hasherStore) createHash(chunkData ChunkData) Address {
return hasher.Sum(nil)
}
func (h *hasherStore) createChunk(chunkData ChunkData) *chunk {
func (h *hasherStore) createChunk(chunkData ChunkData) Chunk {
hash := h.createHash(chunkData)
chunk := NewChunk(hash, chunkData)
return chunk
@ -189,9 +189,9 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp
// removing extra bytes which were just added for padding
length := ChunkData(decryptedSpan).Size()
for length > ch.DefaultSize {
length = length + (ch.DefaultSize - 1)
length = length / ch.DefaultSize
for length > chunk.DefaultSize {
length = length + (chunk.DefaultSize - 1)
length = length / chunk.DefaultSize
length *= uint64(h.refSize)
}
@ -232,14 +232,14 @@ func (h *hasherStore) decrypt(chunkData ChunkData, key encryption.Key) ([]byte,
}
func (h *hasherStore) newSpanEncryption(key encryption.Key) encryption.Encryption {
return encryption.New(key, 0, uint32(ch.DefaultSize/h.refSize), sha3.NewLegacyKeccak256)
return encryption.New(key, 0, uint32(chunk.DefaultSize/h.refSize), sha3.NewLegacyKeccak256)
}
func (h *hasherStore) newDataEncryption(key encryption.Key) encryption.Encryption {
return encryption.New(key, int(ch.DefaultSize), 0, sha3.NewLegacyKeccak256)
return encryption.New(key, int(chunk.DefaultSize), 0, sha3.NewLegacyKeccak256)
}
func (h *hasherStore) storeChunk(ctx context.Context, chunk *chunk) {
func (h *hasherStore) storeChunk(ctx context.Context, chunk Chunk) {
atomic.AddUint64(&h.nrChunks, 1)
go func() {
select {

@ -312,7 +312,7 @@ func decodeIndex(data []byte, index *dpaDBIndex) error {
return dec.Decode(index)
}
func decodeData(addr Address, data []byte) (*chunk, error) {
func decodeData(addr Address, data []byte) (Chunk, error) {
return NewChunk(addr, data[32:]), nil
}
@ -502,7 +502,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) {
}
// Cleanup iterates over the database and deletes chunks if they pass the `f` condition
func (s *LDBStore) Cleanup(f func(*chunk) bool) {
func (s *LDBStore) Cleanup(f func(Chunk) bool) {
var errorsFound, removed, total int
it := s.db.NewIterator()
@ -551,12 +551,14 @@ func (s *LDBStore) Cleanup(f func(*chunk) bool) {
continue
}
cs := int64(binary.LittleEndian.Uint64(c.sdata[:8]))
log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs)
sdata := c.Data()
cs := int64(binary.LittleEndian.Uint64(sdata[:8]))
log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(sdata), "size", cs)
// if chunk is to be removed
if f(c) {
log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs)
log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(sdata), "size", cs)
s.deleteNow(&index, getIndexKey(key[1:]), po)
removed++
errorsFound++
@ -980,7 +982,7 @@ func (s *LDBStore) Has(_ context.Context, addr Address) bool {
}
// TODO: To conform with other private methods of this object indices should not be updated
func (s *LDBStore) get(addr Address) (chunk *chunk, err error) {
func (s *LDBStore) get(addr Address) (chunk Chunk, err error) {
if s.closed {
return nil, ErrDBClosed
}

@ -28,7 +28,7 @@ import (
"testing"
"github.com/ethereum/go-ethereum/common"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
ldberrors "github.com/syndtr/goleveldb/leveldb/errors"
@ -103,7 +103,7 @@ func TestMarkAccessed(t *testing.T) {
t.Fatalf("init dbStore failed: %v", err)
}
h := GenerateRandomChunk(ch.DefaultSize)
h := GenerateRandomChunk(chunk.DefaultSize)
db.Put(context.Background(), h)
@ -201,7 +201,7 @@ func testIterator(t *testing.T, mock bool) {
t.Fatalf("init dbStore failed: %v", err)
}
chunks := GenerateRandomChunks(ch.DefaultSize, chunkcount)
chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount)
for i = 0; i < len(chunks); i++ {
chunkkeys[i] = chunks[i].Address()
@ -468,7 +468,7 @@ func testLDBStoreRemoveThenCollectGarbage(t *testing.T) {
// put capacity count number of chunks
chunks := make([]Chunk, n)
for i := 0; i < n; i++ {
c := GenerateRandomChunk(ch.DefaultSize)
c := GenerateRandomChunk(chunk.DefaultSize)
chunks[i] = c
log.Trace("generate random chunk", "idx", i, "chunk", c)
}

@ -241,7 +241,7 @@ func (ls *LocalStore) Migrate() error {
func (ls *LocalStore) migrateFromNoneToPurity() {
// delete chunks that are not valid, i.e. chunks that do not pass
// any of the ls.Validators
ls.DbStore.Cleanup(func(c *chunk) bool {
ls.DbStore.Cleanup(func(c Chunk) bool {
return !ls.isValid(c)
})
}

@ -23,7 +23,7 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// TestDB_collectGarbageWorker tests garbage collection runs
@ -64,11 +64,11 @@ func testDB_collectGarbageWorker(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
syncer := db.NewSetter(ModeSetSync)
addrs := make([]storage.Address, 0)
addrs := make([]chunk.Address, 0)
// upload random chunks
for i := 0; i < chunkCount; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
@ -106,8 +106,8 @@ func testDB_collectGarbageWorker(t *testing.T) {
// the first synced chunk should be removed
t.Run("get the first synced chunk", func(t *testing.T) {
_, err := db.NewGetter(ModeGetRequest).Get(addrs[0])
if err != storage.ErrChunkNotFound {
t.Errorf("got error %v, want %v", err, storage.ErrChunkNotFound)
if err != chunk.ErrChunkNotFound {
t.Errorf("got error %v, want %v", err, chunk.ErrChunkNotFound)
}
})
@ -137,11 +137,11 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) {
testHookCollectGarbageChan <- collectedCount
})()
addrs := make([]storage.Address, 0)
addrs := make([]chunk.Address, 0)
// upload random chunks just up to the capacity
for i := 0; i < int(db.capacity)-1; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
@ -156,6 +156,14 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) {
addrs = append(addrs, chunk.Address())
}
// set update gc test hook to signal when
// update gc goroutine is done by closing
// testHookUpdateGCChan channel
testHookUpdateGCChan := make(chan struct{})
resetTestHookUpdateGC := setTestHookUpdateGC(func() {
close(testHookUpdateGCChan)
})
// request the latest synced chunk
// to prioritize it in the gc index
// not to be collected
@ -164,18 +172,29 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) {
t.Fatal(err)
}
// wait for update gc goroutine to finish for garbage
// collector to be correctly triggered after the last upload
select {
case <-testHookUpdateGCChan:
case <-time.After(10 * time.Second):
t.Fatal("updateGC was not called after getting chunk with ModeGetRequest")
}
// no need to wait for update gc hook anymore
resetTestHookUpdateGC()
// upload and sync another chunk to trigger
// garbage collection
chunk := generateRandomChunk()
err = uploader.Put(chunk)
ch := generateTestRandomChunk()
err = uploader.Put(ch)
if err != nil {
t.Fatal(err)
}
err = syncer.Set(chunk.Address())
err = syncer.Set(ch.Address())
if err != nil {
t.Fatal(err)
}
addrs = append(addrs, chunk.Address())
addrs = append(addrs, ch.Address())
// wait for garbage collection
@ -217,8 +236,8 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) {
// the second synced chunk should be removed
t.Run("get gc-ed chunk", func(t *testing.T) {
_, err := db.NewGetter(ModeGetRequest).Get(addrs[1])
if err != storage.ErrChunkNotFound {
t.Errorf("got error %v, want %v", err, storage.ErrChunkNotFound)
if err != chunk.ErrChunkNotFound {
t.Errorf("got error %v, want %v", err, chunk.ErrChunkNotFound)
}
})
@ -254,7 +273,7 @@ func TestDB_gcSize(t *testing.T) {
count := 100
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {

@ -21,7 +21,7 @@ import (
"math/rand"
"testing"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// TestDB_pullIndex validates the ordering of keys in pull index.
@ -43,7 +43,7 @@ func TestDB_pullIndex(t *testing.T) {
// upload random chunks
for i := 0; i < chunkCount; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
@ -62,8 +62,8 @@ func TestDB_pullIndex(t *testing.T) {
}
testItemsOrder(t, db.pullIndex, chunks, func(i, j int) (less bool) {
poi := storage.Proximity(db.baseKey, chunks[i].Address())
poj := storage.Proximity(db.baseKey, chunks[j].Address())
poi := chunk.Proximity(db.baseKey, chunks[i].Address())
poj := chunk.Proximity(db.baseKey, chunks[j].Address())
if poi < poj {
return true
}
@ -95,7 +95,7 @@ func TestDB_gcIndex(t *testing.T) {
// upload random chunks
for i := 0; i < chunkCount; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {

@ -24,8 +24,8 @@ import (
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
)
@ -392,8 +392,8 @@ func (db *DB) Close() (err error) {
// po computes the proximity order between the address
// and database base key.
func (db *DB) po(addr storage.Address) (bin uint8) {
return uint8(storage.Proximity(db.baseKey, addr))
func (db *DB) po(addr chunk.Address) (bin uint8) {
return uint8(chunk.Proximity(db.baseKey, addr))
}
var (
@ -409,7 +409,7 @@ var (
// If the address is locked this function will check it
// in a for loop for addressLockTimeout time, after which
// it will return ErrAddressLockTimeout error.
func (db *DB) lockAddr(addr storage.Address) (unlock func(), err error) {
func (db *DB) lockAddr(addr chunk.Address) (unlock func(), err error) {
start := time.Now()
lockKey := hex.EncodeToString(addr)
for {
@ -426,7 +426,7 @@ func (db *DB) lockAddr(addr storage.Address) (unlock func(), err error) {
}
// chunkToItem creates new Item with data provided by the Chunk.
func chunkToItem(ch storage.Chunk) shed.Item {
func chunkToItem(ch chunk.Chunk) shed.Item {
return shed.Item{
Address: ch.Address(),
Data: ch.Data(),
@ -434,7 +434,7 @@ func chunkToItem(ch storage.Chunk) shed.Item {
}
// addressToItem creates new Item with a provided address.
func addressToItem(addr storage.Address) shed.Item {
func addressToItem(addr chunk.Address) shed.Item {
return shed.Item{
Address: addr,
}

@ -29,9 +29,8 @@ import (
"testing"
"time"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
@ -61,7 +60,7 @@ func TestDB(t *testing.T) {
db, cleanupFunc := newTestDB(t, nil)
defer cleanupFunc()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {
@ -115,7 +114,7 @@ func TestDB_updateGCSem(t *testing.T) {
db, cleanupFunc := newTestDB(t, nil)
defer cleanupFunc()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {
@ -188,7 +187,7 @@ func BenchmarkNew(b *testing.B) {
uploader := db.NewPutter(ModePutUpload)
syncer := db.NewSetter(ModeSetSync)
for i := 0; i < count; i++ {
chunk := generateFakeRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
b.Fatal(err)
@ -251,53 +250,47 @@ func newTestDB(t testing.TB, o *Options) (db *DB, cleanupFunc func()) {
return db, cleanupFunc
}
// generateRandomChunk generates a valid Chunk with
// data size of default chunk size.
func generateRandomChunk() storage.Chunk {
return storage.GenerateRandomChunk(ch.DefaultSize)
}
func init() {
// needed for generateFakeRandomChunk
// needed for generateTestRandomChunk
rand.Seed(time.Now().UnixNano())
}
// generateFakeRandomChunk generates a Chunk that is not
// generateTestRandomChunk generates a Chunk that is not
// valid, but it contains a random key and a random value.
// This function is faster then storage.GenerateRandomChunk
// This function is faster then storage.generateTestRandomChunk
// which generates a valid chunk.
// Some tests in this package do not need valid chunks, just
// random data, and their execution time can be decreased
// using this function.
func generateFakeRandomChunk() storage.Chunk {
data := make([]byte, ch.DefaultSize)
func generateTestRandomChunk() chunk.Chunk {
data := make([]byte, chunk.DefaultSize)
rand.Read(data)
key := make([]byte, 32)
rand.Read(key)
return storage.NewChunk(key, data)
return chunk.NewChunk(key, data)
}
// TestGenerateFakeRandomChunk validates that
// generateFakeRandomChunk returns random data by comparing
// TestGenerateTestRandomChunk validates that
// generateTestRandomChunk returns random data by comparing
// two generated chunks.
func TestGenerateFakeRandomChunk(t *testing.T) {
c1 := generateFakeRandomChunk()
c2 := generateFakeRandomChunk()
func TestGenerateTestRandomChunk(t *testing.T) {
c1 := generateTestRandomChunk()
c2 := generateTestRandomChunk()
addrLen := len(c1.Address())
if addrLen != 32 {
t.Errorf("first chunk address length %v, want %v", addrLen, 32)
}
dataLen := len(c1.Data())
if dataLen != ch.DefaultSize {
t.Errorf("first chunk data length %v, want %v", dataLen, ch.DefaultSize)
if dataLen != chunk.DefaultSize {
t.Errorf("first chunk data length %v, want %v", dataLen, chunk.DefaultSize)
}
addrLen = len(c2.Address())
if addrLen != 32 {
t.Errorf("second chunk address length %v, want %v", addrLen, 32)
}
dataLen = len(c2.Data())
if dataLen != ch.DefaultSize {
t.Errorf("second chunk data length %v, want %v", dataLen, ch.DefaultSize)
if dataLen != chunk.DefaultSize {
t.Errorf("second chunk data length %v, want %v", dataLen, chunk.DefaultSize)
}
if bytes.Equal(c1.Address(), c2.Address()) {
t.Error("fake chunks addresses do not differ")
@ -309,7 +302,7 @@ func TestGenerateFakeRandomChunk(t *testing.T) {
// newRetrieveIndexesTest returns a test function that validates if the right
// chunk values are in the retrieval indexes.
func newRetrieveIndexesTest(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
func newRetrieveIndexesTest(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address()))
if err != nil {
@ -328,7 +321,7 @@ func newRetrieveIndexesTest(db *DB, chunk storage.Chunk, storeTimestamp, accessT
// newRetrieveIndexesTestWithAccess returns a test function that validates if the right
// chunk values are in the retrieval indexes when access time must be stored.
func newRetrieveIndexesTestWithAccess(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
func newRetrieveIndexesTestWithAccess(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address()))
if err != nil {
@ -348,7 +341,7 @@ func newRetrieveIndexesTestWithAccess(db *DB, chunk storage.Chunk, storeTimestam
// newPullIndexTest returns a test function that validates if the right
// chunk values are in the pull index.
func newPullIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
func newPullIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.pullIndex.Get(shed.Item{
Address: chunk.Address(),
@ -365,7 +358,7 @@ func newPullIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantErr
// newPushIndexTest returns a test function that validates if the right
// chunk values are in the push index.
func newPushIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
func newPushIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.pushIndex.Get(shed.Item{
Address: chunk.Address(),
@ -382,7 +375,7 @@ func newPushIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantErr
// newGCIndexTest returns a test function that validates if the right
// chunk values are in the push index.
func newGCIndexTest(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
func newGCIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.gcIndex.Get(shed.Item{
Address: chunk.Address(),
@ -436,7 +429,7 @@ func newIndexGCSizeTest(db *DB) func(t *testing.T) {
// testIndexChunk embeds storageChunk with additional data that is stored
// in database. It is used for index values validations.
type testIndexChunk struct {
storage.Chunk
chunk.Chunk
storeTimestamp int64
}

@ -18,8 +18,8 @@ package localstore
import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
@ -51,23 +51,23 @@ func (db *DB) NewGetter(mode ModeGet) *Getter {
}
// Get returns a chunk from the database. If the chunk is
// not found storage.ErrChunkNotFound will be returned.
// not found chunk.ErrChunkNotFound will be returned.
// All required indexes will be updated required by the
// Getter Mode.
func (g *Getter) Get(addr storage.Address) (chunk storage.Chunk, err error) {
func (g *Getter) Get(addr chunk.Address) (ch chunk.Chunk, err error) {
out, err := g.db.get(g.mode, addr)
if err != nil {
if err == leveldb.ErrNotFound {
return nil, storage.ErrChunkNotFound
return nil, chunk.ErrChunkNotFound
}
return nil, err
}
return storage.NewChunk(out.Address, out.Data), nil
return chunk.NewChunk(out.Address, out.Data), nil
}
// get returns Item from the retrieval index
// and updates other indexes.
func (db *DB) get(mode ModeGet, addr storage.Address) (out shed.Item, err error) {
func (db *DB) get(mode ModeGet, addr chunk.Address) (out shed.Item, err error) {
item := addressToItem(addr)
out, err = db.retrievalDataIndex.Get(item)

@ -32,7 +32,7 @@ func TestModeGetRequest(t *testing.T) {
return uploadTimestamp
})()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {
@ -146,7 +146,7 @@ func TestModeGetSync(t *testing.T) {
return uploadTimestamp
})()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {

@ -17,8 +17,8 @@
package localstore
import (
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
@ -53,7 +53,7 @@ func (db *DB) NewPutter(mode ModePut) *Putter {
// Put stores the Chunk to database and depending
// on the Putter mode, it updates required indexes.
func (p *Putter) Put(ch storage.Chunk) (err error) {
func (p *Putter) Put(ch chunk.Chunk) (err error) {
return p.db.put(p.mode, chunkToItem(ch))
}

@ -23,7 +23,7 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// TestModePutRequest validates ModePutRequest index values on the provided DB.
@ -33,7 +33,7 @@ func TestModePutRequest(t *testing.T) {
putter := db.NewPutter(ModePutRequest)
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
// keep the record when the chunk is stored
var storeTimestamp int64
@ -87,7 +87,7 @@ func TestModePutSync(t *testing.T) {
return wantTimestamp
})()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutSync).Put(chunk)
if err != nil {
@ -109,7 +109,7 @@ func TestModePutUpload(t *testing.T) {
return wantTimestamp
})()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {
@ -132,7 +132,7 @@ func TestModePutUpload_parallel(t *testing.T) {
chunkCount := 1000
workerCount := 100
chunkChan := make(chan storage.Chunk)
chunkChan := make(chan chunk.Chunk)
errChan := make(chan error)
doneChan := make(chan struct{})
defer close(doneChan)
@ -159,13 +159,13 @@ func TestModePutUpload_parallel(t *testing.T) {
}(i)
}
chunks := make([]storage.Chunk, 0)
chunks := make([]chunk.Chunk, 0)
var chunksMu sync.Mutex
// send chunks to workers
go func() {
for i := 0; i < chunkCount; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
select {
case chunkChan <- chunk:
case <-doneChan:
@ -271,9 +271,9 @@ func benchmarkPutUpload(b *testing.B, o *Options, count, maxParallelUploads int)
defer cleanupFunc()
uploader := db.NewPutter(ModePutUpload)
chunks := make([]storage.Chunk, count)
chunks := make([]chunk.Chunk, count)
for i := 0; i < count; i++ {
chunks[i] = generateFakeRandomChunk()
chunks[i] = generateTestRandomChunk()
}
errs := make(chan error)
b.StartTimer()

@ -17,7 +17,7 @@
package localstore
import (
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/syndtr/goleveldb/leveldb"
)
@ -53,7 +53,7 @@ func (db *DB) NewSetter(mode ModeSet) *Setter {
// Set updates database indexes for a specific
// chunk represented by the address.
func (s *Setter) Set(addr storage.Address) (err error) {
func (s *Setter) Set(addr chunk.Address) (err error) {
return s.db.set(s.mode, addr)
}
@ -61,7 +61,7 @@ func (s *Setter) Set(addr storage.Address) (err error) {
// chunk represented by the address.
// It acquires lockAddr to protect two calls
// of this function for the same address in parallel.
func (db *DB) set(mode ModeSet, addr storage.Address) (err error) {
func (db *DB) set(mode ModeSet, addr chunk.Address) (err error) {
// protect parallel updates
unlock, err := db.lockAddr(addr)
if err != nil {

@ -28,7 +28,7 @@ func TestModeSetAccess(t *testing.T) {
db, cleanupFunc := newTestDB(t, nil)
defer cleanupFunc()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
wantTimestamp := time.Now().UTC().UnixNano()
defer setNow(func() (t int64) {
@ -56,7 +56,7 @@ func TestModeSetSync(t *testing.T) {
db, cleanupFunc := newTestDB(t, nil)
defer cleanupFunc()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
wantTimestamp := time.Now().UTC().UnixNano()
defer setNow(func() (t int64) {
@ -89,7 +89,7 @@ func TestModeSetRemove(t *testing.T) {
db, cleanupFunc := newTestDB(t, nil)
defer cleanupFunc()
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := db.NewPutter(ModePutUpload).Put(chunk)
if err != nil {

@ -20,7 +20,7 @@ import (
"strconv"
"testing"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// BenchmarkRetrievalIndexes uploads a number of chunks in order to measure
@ -64,9 +64,9 @@ func benchmarkRetrievalIndexes(b *testing.B, o *Options, count int) {
uploader := db.NewPutter(ModePutUpload)
syncer := db.NewSetter(ModeSetSync)
requester := db.NewGetter(ModeGetRequest)
addrs := make([]storage.Address, count)
addrs := make([]chunk.Address, count)
for i := 0; i < count; i++ {
chunk := generateFakeRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
b.Fatal(err)
@ -134,9 +134,9 @@ func benchmarkUpload(b *testing.B, o *Options, count int) {
db, cleanupFunc := newTestDB(b, o)
defer cleanupFunc()
uploader := db.NewPutter(ModePutUpload)
chunks := make([]storage.Chunk, count)
chunks := make([]chunk.Chunk, count)
for i := 0; i < count; i++ {
chunk := generateFakeRandomChunk()
chunk := generateTestRandomChunk()
chunks[i] = chunk
}
b.StartTimer()

@ -24,8 +24,8 @@ import (
"sync"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
)
// SubscribePull returns a channel that provides chunk addresses and stored times from pull syncing index.
@ -161,7 +161,7 @@ func (db *DB) SubscribePull(ctx context.Context, bin uint8, since, until *ChunkD
// ChunkDescriptor holds information required for Pull syncing. This struct
// is provided by subscribing to pull index.
type ChunkDescriptor struct {
Address storage.Address
Address chunk.Address
StoreTimestamp int64
}

@ -24,7 +24,7 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// TestDB_SubscribePull uploads some chunks before and after
@ -37,7 +37,7 @@ func TestDB_SubscribePull(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make(map[uint8][]storage.Address)
addrs := make(map[uint8][]chunk.Address)
var addrsMu sync.Mutex
var wantedChunksCount int
@ -53,7 +53,7 @@ func TestDB_SubscribePull(t *testing.T) {
// to validate the number of addresses received by the subscription
errChan := make(chan error)
for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ {
ch, stop := db.SubscribePull(ctx, bin, nil, nil)
defer stop()
@ -84,7 +84,7 @@ func TestDB_SubscribePull_multiple(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make(map[uint8][]storage.Address)
addrs := make(map[uint8][]chunk.Address)
var addrsMu sync.Mutex
var wantedChunksCount int
@ -105,7 +105,7 @@ func TestDB_SubscribePull_multiple(t *testing.T) {
// start a number of subscriptions
// that all of them will write every address error to errChan
for j := 0; j < subsCount; j++ {
for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ {
ch, stop := db.SubscribePull(ctx, bin, nil, nil)
defer stop()
@ -137,7 +137,7 @@ func TestDB_SubscribePull_since(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make(map[uint8][]storage.Address)
addrs := make(map[uint8][]chunk.Address)
var addrsMu sync.Mutex
var wantedChunksCount int
@ -156,20 +156,20 @@ func TestDB_SubscribePull_since(t *testing.T) {
last = make(map[uint8]ChunkDescriptor)
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
ch := generateTestRandomChunk()
err := uploader.Put(chunk)
err := uploader.Put(ch)
if err != nil {
t.Fatal(err)
}
bin := db.po(chunk.Address())
bin := db.po(ch.Address())
if _, ok := addrs[bin]; !ok {
addrs[bin] = make([]storage.Address, 0)
addrs[bin] = make([]chunk.Address, 0)
}
if wanted {
addrs[bin] = append(addrs[bin], chunk.Address())
addrs[bin] = append(addrs[bin], ch.Address())
wantedChunksCount++
}
@ -178,7 +178,7 @@ func TestDB_SubscribePull_since(t *testing.T) {
lastTimestampMu.RUnlock()
last[bin] = ChunkDescriptor{
Address: chunk.Address(),
Address: ch.Address(),
StoreTimestamp: storeTimestamp,
}
}
@ -199,7 +199,7 @@ func TestDB_SubscribePull_since(t *testing.T) {
// to validate the number of addresses received by the subscription
errChan := make(chan error)
for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ {
var since *ChunkDescriptor
if c, ok := last[bin]; ok {
since = &c
@ -228,7 +228,7 @@ func TestDB_SubscribePull_until(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make(map[uint8][]storage.Address)
addrs := make(map[uint8][]chunk.Address)
var addrsMu sync.Mutex
var wantedChunksCount int
@ -247,20 +247,20 @@ func TestDB_SubscribePull_until(t *testing.T) {
last = make(map[uint8]ChunkDescriptor)
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
ch := generateTestRandomChunk()
err := uploader.Put(chunk)
err := uploader.Put(ch)
if err != nil {
t.Fatal(err)
}
bin := db.po(chunk.Address())
bin := db.po(ch.Address())
if _, ok := addrs[bin]; !ok {
addrs[bin] = make([]storage.Address, 0)
addrs[bin] = make([]chunk.Address, 0)
}
if wanted {
addrs[bin] = append(addrs[bin], chunk.Address())
addrs[bin] = append(addrs[bin], ch.Address())
wantedChunksCount++
}
@ -269,7 +269,7 @@ func TestDB_SubscribePull_until(t *testing.T) {
lastTimestampMu.RUnlock()
last[bin] = ChunkDescriptor{
Address: chunk.Address(),
Address: ch.Address(),
StoreTimestamp: storeTimestamp,
}
}
@ -290,7 +290,7 @@ func TestDB_SubscribePull_until(t *testing.T) {
// to validate the number of addresses received by the subscription
errChan := make(chan error)
for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ {
until, ok := last[bin]
if !ok {
continue
@ -318,7 +318,7 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make(map[uint8][]storage.Address)
addrs := make(map[uint8][]chunk.Address)
var addrsMu sync.Mutex
var wantedChunksCount int
@ -337,20 +337,20 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
last = make(map[uint8]ChunkDescriptor)
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
ch := generateTestRandomChunk()
err := uploader.Put(chunk)
err := uploader.Put(ch)
if err != nil {
t.Fatal(err)
}
bin := db.po(chunk.Address())
bin := db.po(ch.Address())
if _, ok := addrs[bin]; !ok {
addrs[bin] = make([]storage.Address, 0)
addrs[bin] = make([]chunk.Address, 0)
}
if wanted {
addrs[bin] = append(addrs[bin], chunk.Address())
addrs[bin] = append(addrs[bin], ch.Address())
wantedChunksCount++
}
@ -359,7 +359,7 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
lastTimestampMu.RUnlock()
last[bin] = ChunkDescriptor{
Address: chunk.Address(),
Address: ch.Address(),
StoreTimestamp: storeTimestamp,
}
}
@ -386,7 +386,7 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
// to validate the number of addresses received by the subscription
errChan := make(chan error)
for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ {
for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ {
var since *ChunkDescriptor
if c, ok := upload1[bin]; ok {
since = &c
@ -412,23 +412,23 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) {
// uploadRandomChunksBin uploads random chunks to database and adds them to
// the map of addresses ber bin.
func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, wantedChunksCount *int, count int) {
func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uint8][]chunk.Address, addrsMu *sync.Mutex, wantedChunksCount *int, count int) {
addrsMu.Lock()
defer addrsMu.Unlock()
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
ch := generateTestRandomChunk()
err := uploader.Put(chunk)
err := uploader.Put(ch)
if err != nil {
t.Fatal(err)
}
bin := db.po(chunk.Address())
bin := db.po(ch.Address())
if _, ok := addrs[bin]; !ok {
addrs[bin] = make([]storage.Address, 0)
addrs[bin] = make([]chunk.Address, 0)
}
addrs[bin] = append(addrs[bin], chunk.Address())
addrs[bin] = append(addrs[bin], ch.Address())
*wantedChunksCount++
}
@ -437,7 +437,7 @@ func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uin
// readPullSubscriptionBin is a helper function that reads all ChunkDescriptors from a channel and
// sends error to errChan, even if it is nil, to count the number of ChunkDescriptors
// returned by the channel.
func readPullSubscriptionBin(ctx context.Context, bin uint8, ch <-chan ChunkDescriptor, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, errChan chan error) {
func readPullSubscriptionBin(ctx context.Context, bin uint8, ch <-chan ChunkDescriptor, addrs map[uint8][]chunk.Address, addrsMu *sync.Mutex, errChan chan error) {
var i int // address index
for {
select {

@ -21,16 +21,16 @@ import (
"sync"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
)
// SubscribePush returns a channel that provides storage chunks with ordering from push syncing index.
// Returned stop function will terminate current and further iterations, and also it will close
// the returned channel without any errors. Make sure that you check the second returned parameter
// from the channel to stop iteration when its value is false.
func (db *DB) SubscribePush(ctx context.Context) (c <-chan storage.Chunk, stop func()) {
chunks := make(chan storage.Chunk)
func (db *DB) SubscribePush(ctx context.Context) (c <-chan chunk.Chunk, stop func()) {
chunks := make(chan chunk.Chunk)
trigger := make(chan struct{}, 1)
db.pushTriggersMu.Lock()
@ -65,7 +65,7 @@ func (db *DB) SubscribePush(ctx context.Context) (c <-chan storage.Chunk, stop f
}
select {
case chunks <- storage.NewChunk(dataItem.Address, dataItem.Data):
case chunks <- chunk.NewChunk(dataItem.Address, dataItem.Data):
// set next iteration start item
// when its chunk is successfully sent to channel
sinceItem = &item

@ -24,7 +24,7 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
// TestDB_SubscribePush uploads some chunks before and after
@ -36,7 +36,7 @@ func TestDB_SubscribePush(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
chunks := make([]storage.Chunk, 0)
chunks := make([]chunk.Chunk, 0)
var chunksMu sync.Mutex
uploadRandomChunks := func(count int) {
@ -44,7 +44,7 @@ func TestDB_SubscribePush(t *testing.T) {
defer chunksMu.Unlock()
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {
@ -124,7 +124,7 @@ func TestDB_SubscribePush_multiple(t *testing.T) {
uploader := db.NewPutter(ModePutUpload)
addrs := make([]storage.Address, 0)
addrs := make([]chunk.Address, 0)
var addrsMu sync.Mutex
uploadRandomChunks := func(count int) {
@ -132,7 +132,7 @@ func TestDB_SubscribePush_multiple(t *testing.T) {
defer addrsMu.Unlock()
for i := 0; i < count; i++ {
chunk := generateRandomChunk()
chunk := generateTestRandomChunk()
err := uploader.Put(chunk)
if err != nil {

@ -23,7 +23,7 @@ import (
"testing"
"time"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
var (
@ -65,7 +65,7 @@ func TestValidator(t *testing.T) {
// add content address validator and check puts
// bad should fail, good should pass
store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc))
chunks = GenerateRandomChunks(ch.DefaultSize, 2)
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
goodChunk = chunks[0]
badChunk = chunks[1]
copy(badChunk.Data(), goodChunk.Data())
@ -83,7 +83,7 @@ func TestValidator(t *testing.T) {
var negV boolTestValidator
store.Validators = append(store.Validators, negV)
chunks = GenerateRandomChunks(ch.DefaultSize, 2)
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
goodChunk = chunks[0]
badChunk = chunks[1]
copy(badChunk.Data(), goodChunk.Data())
@ -101,7 +101,7 @@ func TestValidator(t *testing.T) {
var posV boolTestValidator = true
store.Validators = append(store.Validators, posV)
chunks = GenerateRandomChunks(ch.DefaultSize, 2)
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
goodChunk = chunks[0]
badChunk = chunks[1]
copy(badChunk.Data(), goodChunk.Data())
@ -138,7 +138,7 @@ func putChunks(store *LocalStore, chunks ...Chunk) []error {
func put(store *LocalStore, n int, f func(i int64) Chunk) (hs []Address, errs []error) {
for i := int64(0); i < int64(n); i++ {
chunk := f(ch.DefaultSize)
chunk := f(chunk.DefaultSize)
err := store.Put(context.TODO(), chunk)
errs = append(errs, err)
hs = append(hs, chunk.Address())
@ -158,7 +158,7 @@ func TestGetFrequentlyAccessedChunkWontGetGarbageCollected(t *testing.T) {
var chunks []Chunk
for i := 0; i < ldbCap; i++ {
chunks = append(chunks, GenerateRandomChunk(ch.DefaultSize))
chunks = append(chunks, GenerateRandomChunk(chunk.DefaultSize))
}
mostAccessed := chunks[0].Address()

@ -29,7 +29,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/p2p/enode"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
var sourcePeerID = enode.HexID("99d8594b52298567d2ca3f4c441a5ba0140ee9245e26460d01102a52773c73b9")
@ -114,7 +114,7 @@ func mustNewNetStoreWithFetcher(t *testing.T) (*NetStore, *mockNetFetcher) {
func TestNetStoreGetAndPut(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
@ -174,7 +174,7 @@ func TestNetStoreGetAndPut(t *testing.T) {
func TestNetStoreGetAfterPut(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
@ -209,7 +209,7 @@ func TestNetStoreGetAfterPut(t *testing.T) {
func TestNetStoreGetTimeout(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
@ -261,7 +261,7 @@ func TestNetStoreGetTimeout(t *testing.T) {
func TestNetStoreGetCancel(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
@ -313,7 +313,7 @@ func TestNetStoreGetCancel(t *testing.T) {
func TestNetStoreMultipleGetAndPut(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
@ -387,7 +387,7 @@ func TestNetStoreMultipleGetAndPut(t *testing.T) {
func TestNetStoreFetchFuncTimeout(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
defer cancel()
@ -426,7 +426,7 @@ func TestNetStoreFetchFuncTimeout(t *testing.T) {
func TestNetStoreFetchFuncAfterPut(t *testing.T) {
netStore := mustNewNetStore(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
@ -453,7 +453,7 @@ func TestNetStoreFetchFuncAfterPut(t *testing.T) {
func TestNetStoreGetCallsRequest(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx := context.WithValue(context.Background(), "hopcount", uint8(5))
ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
@ -481,7 +481,7 @@ func TestNetStoreGetCallsRequest(t *testing.T) {
func TestNetStoreGetCallsOffer(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
// If a source peer is added to the context, NetStore will handle it as an offer
ctx := context.WithValue(context.Background(), "source", sourcePeerID.String())
@ -567,7 +567,7 @@ func TestNetStoreFetcherCountPeers(t *testing.T) {
func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
@ -632,7 +632,7 @@ func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) {
func TestNetStoreFetcherLifeCycleWithTimeout(t *testing.T) {
netStore, fetcher := mustNewNetStoreWithFetcher(t)
chunk := GenerateRandomChunk(ch.DefaultSize)
chunk := GenerateRandomChunk(chunk.DefaultSize)
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()

@ -25,7 +25,7 @@ import (
"sync"
"time"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/log"
)
@ -97,11 +97,11 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get
New chunks to store are store using the putter which the caller provides.
*/
func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, ch.DefaultSize)).Split(ctx)
return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx)
}
func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, ch.DefaultSize)).Append(ctx)
return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx)
}
// Entry to create a tree node

@ -22,53 +22,29 @@ import (
"crypto"
"crypto/rand"
"encoding/binary"
"fmt"
"io"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/bmt"
ch "github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/chunk"
"golang.org/x/crypto/sha3"
)
const MaxPO = 16
const AddressLength = 32
// MaxPO is the same as chunk.MaxPO for backward compatibility.
const MaxPO = chunk.MaxPO
// AddressLength is the same as chunk.AddressLength for backward compatibility.
const AddressLength = chunk.AddressLength
type SwarmHasher func() SwarmHash
type Address []byte
// Address is an alias for chunk.Address for backward compatibility.
type Address = chunk.Address
// Proximity(x, y) returns the proximity order of the MSB distance between x and y
//
// The distance metric MSB(x, y) of two equal length byte sequences x an y is the
// value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed.
// the binary cast is big endian: most significant bit first (=MSB).
//
// Proximity(x, y) is a discrete logarithmic scaling of the MSB distance.
// It is defined as the reverse rank of the integer part of the base 2
// logarithm of the distance.
// It is calculated by counting the number of common leading zeros in the (MSB)
// binary representation of the x^y.
//
// (0 farthest, 255 closest, 256 self)
func Proximity(one, other []byte) (ret int) {
b := (MaxPO-1)/8 + 1
if b > len(one) {
b = len(one)
}
m := 8
for i := 0; i < b; i++ {
oxo := one[i] ^ other[i]
for j := 0; j < m; j++ {
if (oxo>>uint8(7-j))&0x01 != 0 {
return i*8 + j
}
}
}
return MaxPO
}
// Proximity is the same as chunk.Proximity for backward compatibility.
var Proximity = chunk.Proximity
var ZeroAddr = Address(common.Hash{}.Bytes())
// ZeroAddr is the same as chunk.ZeroAddr for backward compatibility.
var ZeroAddr = chunk.ZeroAddr
func MakeHashFunc(hash string) SwarmHasher {
switch hash {
@ -80,7 +56,7 @@ func MakeHashFunc(hash string) SwarmHasher {
return func() SwarmHash {
hasher := sha3.NewLegacyKeccak256
hasherSize := hasher().Size()
segmentCount := ch.DefaultSize / hasherSize
segmentCount := chunk.DefaultSize / hasherSize
pool := bmt.NewTreePool(hasher, segmentCount, bmt.PoolSize)
return bmt.New(pool)
}
@ -88,33 +64,6 @@ func MakeHashFunc(hash string) SwarmHasher {
return nil
}
func (a Address) Hex() string {
return fmt.Sprintf("%064x", []byte(a[:]))
}
func (a Address) Log() string {
if len(a[:]) < 8 {
return fmt.Sprintf("%x", []byte(a[:]))
}
return fmt.Sprintf("%016x", []byte(a[:8]))
}
func (a Address) String() string {
return fmt.Sprintf("%064x", []byte(a))
}
func (a Address) MarshalJSON() (out []byte, err error) {
return []byte(`"` + a.String() + `"`), nil
}
func (a *Address) UnmarshalJSON(value []byte) error {
s := string(value)
*a = make([]byte, 32)
h := common.Hex2Bytes(s[1 : len(s)-1])
copy(*a, h)
return nil
}
type AddressCollection []Address
func NewAddressCollection(l int) AddressCollection {
@ -133,38 +82,11 @@ func (c AddressCollection) Swap(i, j int) {
c[i], c[j] = c[j], c[i]
}
// Chunk interface implemented by context.Contexts and data chunks
type Chunk interface {
Address() Address
Data() []byte
}
type chunk struct {
addr Address
sdata []byte
span int64
}
// Chunk is an alias for chunk.Chunk for backward compatibility.
type Chunk = chunk.Chunk
func NewChunk(addr Address, data []byte) *chunk {
return &chunk{
addr: addr,
sdata: data,
span: -1,
}
}
func (c *chunk) Address() Address {
return c.addr
}
func (c *chunk) Data() []byte {
return c.sdata
}
// String() for pretty printing
func (self *chunk) String() string {
return fmt.Sprintf("Address: %v TreeSize: %v Chunksize: %v", self.addr.Log(), self.span, len(self.sdata))
}
// NewChunk is the same as chunk.NewChunk for backward compatibility.
var NewChunk = chunk.NewChunk
func GenerateRandomChunk(dataSize int64) Chunk {
hasher := MakeHashFunc(DefaultHash)()
@ -274,9 +196,9 @@ func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator {
}
// Validate that the given key is a valid content address for the given data
func (v *ContentAddressValidator) Validate(chunk Chunk) bool {
data := chunk.Data()
if l := len(data); l < 9 || l > ch.DefaultSize+8 {
func (v *ContentAddressValidator) Validate(ch Chunk) bool {
data := ch.Data()
if l := len(data); l < 9 || l > chunk.DefaultSize+8 {
// log.Error("invalid chunk size", "chunk", addr.Hex(), "size", l)
return false
}
@ -286,7 +208,7 @@ func (v *ContentAddressValidator) Validate(chunk Chunk) bool {
hasher.Write(data[8:])
hash := hasher.Sum(nil)
return bytes.Equal(hash, chunk.Address())
return bytes.Equal(hash, ch.Address())
}
type ChunkStore interface {

Loading…
Cancel
Save