mirror of https://github.com/ethereum/go-ethereum
cmd/swarm, swarm/api/client: add HTTP API client and 'swarm ls' command (#3742)
This adds a swarm ls command which lists files and directories stored in a manifest. Rather than listing all files, it uses "directory prefixes" in case there are a lot of files in a manifest but you just want to traverse it. This also includes some refactoring to the tests and the introduction of a swarm/api/client package to make things easier to test.pull/13866/head
Lewis Marshall
8 years ago
committed by
Felix Lange
parent
09777952ee
commit
b319f027a0
@ -0,0 +1,58 @@ |
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package main |
||||
|
||||
import ( |
||||
"fmt" |
||||
"os" |
||||
"strings" |
||||
"text/tabwriter" |
||||
|
||||
"github.com/ethereum/go-ethereum/cmd/utils" |
||||
swarm "github.com/ethereum/go-ethereum/swarm/api/client" |
||||
"gopkg.in/urfave/cli.v1" |
||||
) |
||||
|
||||
func list(ctx *cli.Context) { |
||||
args := ctx.Args() |
||||
|
||||
if len(args) < 1 { |
||||
utils.Fatalf("Please supply a manifest reference as the first argument") |
||||
} else if len(args) > 2 { |
||||
utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'") |
||||
} |
||||
manifest := args[0] |
||||
|
||||
var prefix string |
||||
if len(args) == 2 { |
||||
prefix = args[1] |
||||
} |
||||
|
||||
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") |
||||
client := swarm.NewClient(bzzapi) |
||||
entries, err := client.ManifestFileList(manifest, prefix) |
||||
if err != nil { |
||||
utils.Fatalf("Failed to generate file and directory list: %s", err) |
||||
} |
||||
|
||||
w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0) |
||||
defer w.Flush() |
||||
fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH") |
||||
for _, entry := range entries { |
||||
fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path) |
||||
} |
||||
} |
||||
@ -0,0 +1,298 @@ |
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package client |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"fmt" |
||||
"io" |
||||
"io/ioutil" |
||||
"mime" |
||||
"net/http" |
||||
"os" |
||||
"path/filepath" |
||||
"strings" |
||||
|
||||
"github.com/ethereum/go-ethereum/log" |
||||
) |
||||
|
||||
var ( |
||||
DefaultGateway = "http://localhost:8500" |
||||
DefaultClient = NewClient(DefaultGateway) |
||||
) |
||||
|
||||
// Manifest represents a swarm manifest.
|
||||
type Manifest struct { |
||||
Entries []ManifestEntry `json:"entries,omitempty"` |
||||
} |
||||
|
||||
// ManifestEntry represents an entry in a swarm manifest.
|
||||
type ManifestEntry struct { |
||||
Hash string `json:"hash,omitempty"` |
||||
ContentType string `json:"contentType,omitempty"` |
||||
Path string `json:"path,omitempty"` |
||||
} |
||||
|
||||
func NewClient(gateway string) *Client { |
||||
return &Client{ |
||||
Gateway: gateway, |
||||
} |
||||
} |
||||
|
||||
// Client wraps interaction with a swarm HTTP gateway.
|
||||
type Client struct { |
||||
Gateway string |
||||
} |
||||
|
||||
func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) { |
||||
mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}")))) |
||||
if err != nil { |
||||
return "", fmt.Errorf("failed to upload empty manifest") |
||||
} |
||||
if len(defaultPath) > 0 { |
||||
fi, err := os.Stat(defaultPath) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
} |
||||
prefix := filepath.ToSlash(filepath.Clean(dir)) + "/" |
||||
err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { |
||||
if err != nil || fi.IsDir() { |
||||
return err |
||||
} |
||||
if !strings.HasPrefix(path, dir) { |
||||
return fmt.Errorf("path %s outside directory %s", path, dir) |
||||
} |
||||
uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix) |
||||
mhash, err = c.uploadToManifest(mhash, uripath, path, fi) |
||||
return err |
||||
}) |
||||
return mhash, err |
||||
} |
||||
|
||||
func (c *Client) UploadFile(file string, fi os.FileInfo) (ManifestEntry, error) { |
||||
hash, err := c.uploadFileContent(file, fi) |
||||
m := ManifestEntry{ |
||||
Hash: hash, |
||||
ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())), |
||||
} |
||||
return m, err |
||||
} |
||||
|
||||
func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) { |
||||
fd, err := os.Open(file) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
defer fd.Close() |
||||
log.Info("Uploading swarm content", "file", file, "bytes", fi.Size()) |
||||
return c.postRaw("application/octet-stream", fi.Size(), fd) |
||||
} |
||||
|
||||
func (c *Client) UploadManifest(m Manifest) (string, error) { |
||||
jsm, err := json.Marshal(m) |
||||
if err != nil { |
||||
panic(err) |
||||
} |
||||
log.Info("Uploading swarm manifest") |
||||
return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm))) |
||||
} |
||||
|
||||
func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) { |
||||
fd, err := os.Open(fpath) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
defer fd.Close() |
||||
log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path) |
||||
req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name()))) |
||||
req.ContentLength = fi.Size() |
||||
resp, err := http.DefaultClient.Do(req) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
defer resp.Body.Close() |
||||
if resp.StatusCode >= 400 { |
||||
return "", fmt.Errorf("bad status: %s", resp.Status) |
||||
} |
||||
content, err := ioutil.ReadAll(resp.Body) |
||||
return string(content), err |
||||
} |
||||
|
||||
func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) { |
||||
req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
req.Header.Set("content-type", mimetype) |
||||
req.ContentLength = size |
||||
resp, err := http.DefaultClient.Do(req) |
||||
if err != nil { |
||||
return "", err |
||||
} |
||||
defer resp.Body.Close() |
||||
if resp.StatusCode >= 400 { |
||||
return "", fmt.Errorf("bad status: %s", resp.Status) |
||||
} |
||||
content, err := ioutil.ReadAll(resp.Body) |
||||
return string(content), err |
||||
} |
||||
|
||||
func (c *Client) DownloadManifest(mhash string) (Manifest, error) { |
||||
|
||||
mroot := Manifest{} |
||||
req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil) |
||||
if err != nil { |
||||
return mroot, err |
||||
} |
||||
resp, err := http.DefaultClient.Do(req) |
||||
if err != nil { |
||||
return mroot, err |
||||
} |
||||
defer resp.Body.Close() |
||||
|
||||
if resp.StatusCode >= 400 { |
||||
return mroot, fmt.Errorf("bad status: %s", resp.Status) |
||||
|
||||
} |
||||
content, err := ioutil.ReadAll(resp.Body) |
||||
|
||||
err = json.Unmarshal(content, &mroot) |
||||
if err != nil { |
||||
return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err) |
||||
} |
||||
return mroot, err |
||||
} |
||||
|
||||
// ManifestFileList downloads the manifest with the given hash and generates a
|
||||
// list of files and directory prefixes which have the specified prefix.
|
||||
//
|
||||
// For example, if the manifest represents the following directory structure:
|
||||
//
|
||||
// file1.txt
|
||||
// file2.txt
|
||||
// dir1/file3.txt
|
||||
// dir1/dir2/file4.txt
|
||||
//
|
||||
// Then:
|
||||
//
|
||||
// - a prefix of "" would return [dir1/, file1.txt, file2.txt]
|
||||
// - a prefix of "file" would return [file1.txt, file2.txt]
|
||||
// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
|
||||
func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) { |
||||
manifest, err := c.DownloadManifest(hash) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
// handleFile handles a manifest entry which is a direct reference to a
|
||||
// file (i.e. it is not a swarm manifest)
|
||||
handleFile := func(entry ManifestEntry) { |
||||
// ignore the file if it doesn't have the specified prefix
|
||||
if !strings.HasPrefix(entry.Path, prefix) { |
||||
return |
||||
} |
||||
// if the path after the prefix contains a directory separator,
|
||||
// add a directory prefix to the entries, otherwise add the
|
||||
// file
|
||||
suffix := strings.TrimPrefix(entry.Path, prefix) |
||||
if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 { |
||||
entries = append(entries, ManifestEntry{ |
||||
Path: prefix + suffix[:sepIndex+1], |
||||
ContentType: "DIR", |
||||
}) |
||||
} else { |
||||
if entry.Path == "" { |
||||
entry.Path = "/" |
||||
} |
||||
entries = append(entries, entry) |
||||
} |
||||
} |
||||
|
||||
// handleManifest handles a manifest entry which is a reference to
|
||||
// another swarm manifest.
|
||||
handleManifest := func(entry ManifestEntry) error { |
||||
// if the manifest's path is a prefix of the specified prefix
|
||||
// then just recurse into the manifest by stripping its path
|
||||
// from the prefix
|
||||
if strings.HasPrefix(prefix, entry.Path) { |
||||
subPrefix := strings.TrimPrefix(prefix, entry.Path) |
||||
subEntries, err := c.ManifestFileList(entry.Hash, subPrefix) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
// prefix the manifest's path to the sub entries and
|
||||
// add them to the returned entries
|
||||
for i, subEntry := range subEntries { |
||||
subEntry.Path = entry.Path + subEntry.Path |
||||
subEntries[i] = subEntry |
||||
} |
||||
entries = append(entries, subEntries...) |
||||
return nil |
||||
} |
||||
|
||||
// if the manifest's path has the specified prefix, then if the
|
||||
// path after the prefix contains a directory separator, add a
|
||||
// directory prefix to the entries, otherwise recurse into the
|
||||
// manifest
|
||||
if strings.HasPrefix(entry.Path, prefix) { |
||||
suffix := strings.TrimPrefix(entry.Path, prefix) |
||||
sepIndex := strings.Index(suffix, "/") |
||||
if sepIndex > -1 { |
||||
entries = append(entries, ManifestEntry{ |
||||
Path: prefix + suffix[:sepIndex+1], |
||||
ContentType: "DIR", |
||||
}) |
||||
return nil |
||||
} |
||||
subEntries, err := c.ManifestFileList(entry.Hash, "") |
||||
if err != nil { |
||||
return err |
||||
} |
||||
// prefix the manifest's path to the sub entries and
|
||||
// add them to the returned entries
|
||||
for i, subEntry := range subEntries { |
||||
subEntry.Path = entry.Path + subEntry.Path |
||||
subEntries[i] = subEntry |
||||
} |
||||
entries = append(entries, subEntries...) |
||||
return nil |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
for _, entry := range manifest.Entries { |
||||
if entry.ContentType == "application/bzz-manifest+json" { |
||||
if err := handleManifest(entry); err != nil { |
||||
return nil, err |
||||
} |
||||
} else { |
||||
handleFile(entry) |
||||
} |
||||
} |
||||
|
||||
return |
||||
} |
||||
@ -0,0 +1,105 @@ |
||||
// Copyright 2016 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package client |
||||
|
||||
import ( |
||||
"io/ioutil" |
||||
"os" |
||||
"path/filepath" |
||||
"reflect" |
||||
"sort" |
||||
"testing" |
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/testutil" |
||||
) |
||||
|
||||
func TestClientManifestFileList(t *testing.T) { |
||||
srv := testutil.NewTestSwarmServer(t) |
||||
defer srv.Close() |
||||
|
||||
dir, err := ioutil.TempDir("", "swarm-client-test") |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
files := []string{ |
||||
"file1.txt", |
||||
"file2.txt", |
||||
"dir1/file3.txt", |
||||
"dir1/file4.txt", |
||||
"dir2/file5.txt", |
||||
"dir2/dir3/file6.txt", |
||||
"dir2/dir4/file7.txt", |
||||
"dir2/dir4/file8.txt", |
||||
} |
||||
for _, file := range files { |
||||
path := filepath.Join(dir, file) |
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { |
||||
t.Fatalf("error creating dir for %s: %s", path, err) |
||||
} |
||||
if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil { |
||||
t.Fatalf("error writing file %s: %s", path, err) |
||||
} |
||||
} |
||||
|
||||
client := NewClient(srv.URL) |
||||
|
||||
hash, err := client.UploadDirectory(dir, "") |
||||
if err != nil { |
||||
t.Fatalf("error uploading directory: %s", err) |
||||
} |
||||
|
||||
ls := func(prefix string) []string { |
||||
entries, err := client.ManifestFileList(hash, prefix) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
paths := make([]string, len(entries)) |
||||
for i, entry := range entries { |
||||
paths[i] = entry.Path |
||||
} |
||||
sort.Strings(paths) |
||||
return paths |
||||
} |
||||
|
||||
tests := map[string][]string{ |
||||
"": []string{"dir1/", "dir2/", "file1.txt", "file2.txt"}, |
||||
"file": []string{"file1.txt", "file2.txt"}, |
||||
"file1": []string{"file1.txt"}, |
||||
"file2.txt": []string{"file2.txt"}, |
||||
"file12": []string{}, |
||||
"dir": []string{"dir1/", "dir2/"}, |
||||
"dir1": []string{"dir1/"}, |
||||
"dir1/": []string{"dir1/file3.txt", "dir1/file4.txt"}, |
||||
"dir1/file": []string{"dir1/file3.txt", "dir1/file4.txt"}, |
||||
"dir1/file3.txt": []string{"dir1/file3.txt"}, |
||||
"dir1/file34": []string{}, |
||||
"dir2/": []string{"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"}, |
||||
"dir2/file": []string{"dir2/file5.txt"}, |
||||
"dir2/dir": []string{"dir2/dir3/", "dir2/dir4/"}, |
||||
"dir2/dir3/": []string{"dir2/dir3/file6.txt"}, |
||||
"dir2/dir4/": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, |
||||
"dir2/dir4/file": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, |
||||
"dir2/dir4/file7.txt": []string{"dir2/dir4/file7.txt"}, |
||||
"dir2/dir4/file78": []string{}, |
||||
} |
||||
for prefix, expected := range tests { |
||||
actual := ls(prefix) |
||||
if !reflect.DeepEqual(actual, expected) { |
||||
t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,56 @@ |
||||
package testutil |
||||
|
||||
import ( |
||||
"io/ioutil" |
||||
"net/http/httptest" |
||||
"os" |
||||
"testing" |
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/api" |
||||
httpapi "github.com/ethereum/go-ethereum/swarm/api/http" |
||||
"github.com/ethereum/go-ethereum/swarm/storage" |
||||
) |
||||
|
||||
func NewTestSwarmServer(t *testing.T) *TestSwarmServer { |
||||
dir, err := ioutil.TempDir("", "swarm-storage-test") |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
storeparams := &storage.StoreParams{ |
||||
ChunkDbPath: dir, |
||||
DbCapacity: 5000000, |
||||
CacheCapacity: 5000, |
||||
Radius: 0, |
||||
} |
||||
localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams) |
||||
if err != nil { |
||||
os.RemoveAll(dir) |
||||
t.Fatal(err) |
||||
} |
||||
chunker := storage.NewTreeChunker(storage.NewChunkerParams()) |
||||
dpa := &storage.DPA{ |
||||
Chunker: chunker, |
||||
ChunkStore: localStore, |
||||
} |
||||
dpa.Start() |
||||
a := api.NewApi(dpa, nil) |
||||
srv := httptest.NewServer(httpapi.NewServer(a)) |
||||
return &TestSwarmServer{ |
||||
Server: srv, |
||||
Dpa: dpa, |
||||
dir: dir, |
||||
} |
||||
} |
||||
|
||||
type TestSwarmServer struct { |
||||
*httptest.Server |
||||
|
||||
Dpa *storage.DPA |
||||
dir string |
||||
} |
||||
|
||||
func (t *TestSwarmServer) Close() { |
||||
t.Server.Close() |
||||
t.Dpa.Stop() |
||||
os.RemoveAll(t.dir) |
||||
} |
||||
Loading…
Reference in new issue