mirror
/
go-ethereum
mirror of https://github.com/ethereum/go-ethereum
2
0
Fork 1
Code Issues Releases Wiki Activity

cmd/swarm: Smoke test for Swarm Feed (#17892)

Browse Source
pull/17901/merge
lash 6 years ago committed by Anton Evangelatov
parent 4868964bb9
commit 862d6f2fbf
3 changed files with 351 additions and 6 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 323
      cmd/swarm/swarm-smoke/feed_upload_and_sync.go
  2. 16
      cmd/swarm/swarm-smoke/main.go
  3. 18
      cmd/swarm/swarm-smoke/upload_and_sync.go

323
cmd/swarm/swarm-smoke/feed_upload_and_sync.go
Unescape View File

@ -0,0 +1,323 @@
package main
import (
"bytes"
"crypto/md5"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"os/exec"
"strings"
"sync"
"time"
"github.com/pborman/uuid"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/multihash"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
colorable "github.com/mattn/go-colorable"
cli "gopkg.in/urfave/cli.v1"
)
const (
feedRandomDataLength = 8
)
// TODO: retrieve with manifest + extract repeating code
func cliFeedUploadAndSync(c *cli.Context) error {
log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))))
defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now())
generateEndpoints(scheme, cluster, from, to)
log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing")
// create a random private key to sign updates with and derive the address
pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
if err != nil {
return err
}
defer pkFile.Close()
defer os.Remove(pkFile.Name())
privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976"
privKey, err := crypto.HexToECDSA(privkeyHex)
if err != nil {
return err
}
user := crypto.PubkeyToAddress(privKey.PublicKey)
userHex := hexutil.Encode(user.Bytes())
// save the private key to a file
_, err = io.WriteString(pkFile, privkeyHex)
if err != nil {
return err
}
// keep hex strings for topic and subtopic
var topicHex string
var subTopicHex string
// and create combination hex topics for bzz-feed retrieval
// xor'ed with topic (zero-value topic if no topic)
var subTopicOnlyHex string
var mergedSubTopicHex string
// generate random topic and subtopic and put a hex on them
topicBytes, err := generateRandomData(feed.TopicLength)
topicHex = hexutil.Encode(topicBytes)
subTopicBytes, err := generateRandomData(8)
subTopicHex = hexutil.Encode(subTopicBytes)
if err != nil {
return err
}
mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes)
if err != nil {
return err
}
mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:])
subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil)
if err != nil {
return err
}
subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:])
// create feed manifest, topic only
var out bytes.Buffer
cmd := exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--user", userHex)
cmd.Stdout = &out
log.Debug("create feed manifest topic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
return err
}
manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
if len(manifestWithTopic) != 64 {
return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic)
}
log.Debug("create topic feed", "manifest", manifestWithTopic)
out.Reset()
// create feed manifest, subtopic only
cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--name", subTopicHex, "--user", userHex)
cmd.Stdout = &out
log.Debug("create feed manifest subtopic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
return err
}
manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
if len(manifestWithSubTopic) != 64 {
return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic)
}
log.Debug("create subtopic feed", "manifest", manifestWithTopic)
out.Reset()
// create feed manifest, merged topic
cmd = exec.Command("swarm", "--bzzapi", endpoints[0], "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex)
cmd.Stdout = &out
log.Debug("create feed manifest mergetopic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
log.Error(err.Error())
return err
}
manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
if len(manifestWithMergedTopic) != 64 {
return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic)
}
log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic)
out.Reset()
// create test data
data, err := generateRandomData(feedRandomDataLength)
if err != nil {
return err
}
h := md5.New()
h.Write(data)
dataHash := h.Sum(nil)
dataHex := hexutil.Encode(data)
// update with topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, dataHex)
cmd.Stdout = &out
log.Debug("update feed manifest topic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update topic", "out", out)
out.Reset()
// update with subtopic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, dataHex)
cmd.Stdout = &out
log.Debug("update feed manifest subtopic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update subtopic", "out", out)
out.Reset()
// update with merged topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex)
cmd.Stdout = &out
log.Debug("update feed manifest merged topic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update mergedtopic", "out", out)
out.Reset()
time.Sleep(3 * time.Second)
// retrieve the data
wg := sync.WaitGroup{}
for _, endpoint := range endpoints {
// raw retrieve, topic only
for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} {
wg.Add(1)
ruid := uuid.New()[:8]
go func(endpoint string, ruid string) {
for {
err := fetchFeed(hex, userHex, endpoint, dataHash, ruid)
if err != nil {
continue
}
wg.Done()
return
}
}(endpoint, ruid)
}
}
wg.Wait()
log.Info("all endpoints synced random data successfully")
// upload test file
log.Info("uploading to " + endpoints[0] + " and syncing")
f, cleanup := generateRandomFile(filesize * 1000)
defer cleanup()
hash, err := upload(f, endpoints[0])
if err != nil {
return err
}
hashBytes, err := hexutil.Decode("0x" + hash)
if err != nil {
return err
}
multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes))
fileHash, err := digest(f)
if err != nil {
return err
}
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash))
// update file with topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, multihashHex)
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update topic", "out", out)
out.Reset()
// update file with subtopic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--name", subTopicHex, multihashHex)
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update subtopic", "out", out)
out.Reset()
// update file with merged topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", endpoints[0], "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex)
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update mergedtopic", "out", out)
out.Reset()
time.Sleep(3 * time.Second)
for _, endpoint := range endpoints {
// manifest retrieve, topic only
for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} {
wg.Add(1)
ruid := uuid.New()[:8]
go func(endpoint string, ruid string) {
for {
err := fetch(url, endpoint, fileHash, ruid)
if err != nil {
continue
}
wg.Done()
return
}
}(endpoint, ruid)
}
}
wg.Wait()
log.Info("all endpoints synced random file successfully")
return nil
}
func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error {
log.Trace("sleeping", "ruid", ruid)
time.Sleep(3 * time.Second)
log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user)
res, err := http.Get(endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user)
if err != nil {
return err
}
log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength)
if res.StatusCode != 200 {
return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid)
}
defer res.Body.Close()
rdigest, err := digest(res.Body)
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err
}
if !bytes.Equal(rdigest, original) {
err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
log.Warn(err.Error(), "ruid", ruid)
return err
}
log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
return nil
}

16
cmd/swarm/swarm-smoke/main.go
Unescape View File

@ -21,7 +21,6 @@ import (
"sort"
"github.com/ethereum/go-ethereum/log"
colorable "github.com/mattn/go-colorable"
cli "gopkg.in/urfave/cli.v1"
)
@ -34,11 +33,10 @@ var (
filesize int
from int
to int
verbosity int
)
func main() {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
app := cli.NewApp()
app.Name = "smoke-test"
@ -80,6 +78,12 @@ func main() {
Usage: "file size for generated random file in KB",
Destination: &filesize,
},
cli.IntFlag{
Name: "verbosity",
Value: 1,
Usage: "verbosity",
Destination: &verbosity,
},
}
app.Commands = []cli.Command{
@ -89,6 +93,12 @@ func main() {
Usage: "upload and sync",
Action: cliUploadAndSync,
},
{
Name: "feed_sync",
Aliases: []string{"f"},
Usage: "feed update generate, upload and sync",
Action: cliFeedUploadAndSync,
},
}
sort.Sort(cli.FlagsByName(app.Flags))

18
cmd/swarm/swarm-smoke/upload_and_sync.go
Unescape View File

@ -19,7 +19,8 @@ package main
import (
"bytes"
"crypto/md5"
"crypto/rand"
crand "crypto/rand"
"errors"
"fmt"
"io"
"io/ioutil"
@ -85,7 +86,6 @@ func cliUploadAndSync(c *cli.Context) error {
wg := sync.WaitGroup{}
for _, endpoint := range endpoints {
endpoint := endpoint
ruid := uuid.New()[:8]
wg.Add(1)
go func(endpoint string, ruid string) {
@ -166,6 +166,18 @@ func digest(r io.Reader) ([]byte, error) {
return h.Sum(nil), nil
}
// generates random data in heap buffer
func generateRandomData(datasize int) ([]byte, error) {
b := make([]byte, datasize)
c, err := crand.Read(b)
if err != nil {
return nil, err
} else if c != datasize {
return nil, errors.New("short read")
}
return b, nil
}
// generateRandomFile is creating a temporary file with the requested byte size
func generateRandomFile(size int) (f *os.File, teardown func()) {
// create a tmp file
@ -181,7 +193,7 @@ func generateRandomFile(size int) (f *os.File, teardown func()) {
}
buf := make([]byte, size)
_, err = rand.Read(buf)
_, err = crand.Read(buf)
if err != nil {
panic(err)
}

Write Preview
Loading…
Cancel
Save