cmd/swarm, swarm/api/client: add HTTP API client and 'swarm ls' command (#3742)

This adds a swarm ls command which lists files and directories stored in a
manifest. Rather than listing all files, it uses "directory prefixes" in case there are a
lot of files in a manifest but you just want to traverse it.

This also includes some refactoring to the tests and the introduction of a
swarm/api/client package to make things easier to test.
pull/13866/head
Lewis Marshall 8 years ago committed by Felix Lange
parent 09777952ee
commit b319f027a0
  1. 58
      cmd/swarm/list.go
  2. 9
      cmd/swarm/main.go
  3. 51
      cmd/swarm/manifest.go
  4. 163
      cmd/swarm/upload.go
  5. 298
      swarm/api/client/client.go
  6. 105
      swarm/api/client/client_test.go
  7. 42
      swarm/api/http/server.go
  8. 52
      swarm/api/http/server_test.go
  9. 5
      swarm/swarm.go
  10. 56
      swarm/testutil/http.go

@ -0,0 +1,58 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"os"
"strings"
"text/tabwriter"
"github.com/ethereum/go-ethereum/cmd/utils"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
func list(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 1 {
utils.Fatalf("Please supply a manifest reference as the first argument")
} else if len(args) > 2 {
utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
}
manifest := args[0]
var prefix string
if len(args) == 2 {
prefix = args[1]
}
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
entries, err := client.ManifestFileList(manifest, prefix)
if err != nil {
utils.Fatalf("Failed to generate file and directory list: %s", err)
}
w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
defer w.Flush()
fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
for _, entry := range entries {
fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
}
}

@ -145,6 +145,15 @@ The output of this command is supposed to be machine-readable.
ArgsUsage: " <file>", ArgsUsage: " <file>",
Description: ` Description: `
"upload a file or directory to swarm using the HTTP API and prints the root hash", "upload a file or directory to swarm using the HTTP API and prints the root hash",
`,
},
{
Action: list,
Name: "ls",
Usage: "list files and directories contained in a manifest",
ArgsUsage: " <manifest> [<prefix>]",
Description: `
Lists files and directories contained in a manifest.
`, `,
}, },
{ {

@ -25,6 +25,7 @@ import (
"strings" "strings"
"github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/cmd/utils"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1" "gopkg.in/urfave/cli.v1"
) )
@ -41,7 +42,7 @@ func add(ctx *cli.Context) {
ctype string ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot manifest mroot swarm.Manifest
) )
if len(args) > 3 { if len(args) > 3 {
@ -75,7 +76,7 @@ func update(ctx *cli.Context) {
ctype string ctype string
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot manifest mroot swarm.Manifest
) )
if len(args) > 3 { if len(args) > 3 {
ctype = args[3] ctype = args[3]
@ -105,7 +106,7 @@ func remove(ctx *cli.Context) {
path = args[1] path = args[1]
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
mroot manifest mroot swarm.Manifest
) )
newManifest := removeEntryFromManifest(ctx, mhash, path) newManifest := removeEntryFromManifest(ctx, mhash, path)
@ -123,21 +124,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
var ( var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = &client{api: bzzapi} client = swarm.NewClient(bzzapi)
longestPathEntry = manifestEntry{ longestPathEntry = swarm.ManifestEntry{
Path: "", Path: "",
Hash: "", Hash: "",
ContentType: "", ContentType: "",
} }
) )
mroot, err := client.downloadManifest(mhash) mroot, err := client.DownloadManifest(mhash)
if err != nil { if err != nil {
utils.Fatalf("Manifest download failed: %v", err) utils.Fatalf("Manifest download failed: %v", err)
} }
//TODO: check if the "hash" to add is valid and present in swarm //TODO: check if the "hash" to add is valid and present in swarm
_, err = client.downloadManifest(hash) _, err = client.DownloadManifest(hash)
if err != nil { if err != nil {
utils.Fatalf("Hash to add is not present: %v", err) utils.Fatalf("Hash to add is not present: %v", err)
} }
@ -162,7 +163,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype) newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests // Replace the hash for parent Manifests
newMRoot := manifest{} newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries { for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path { if longestPathEntry.Path == entry.Path {
entry.Hash = newHash entry.Hash = newHash
@ -172,7 +173,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot = newMRoot mroot = newMRoot
} else { } else {
// Add the entry in the leaf Manifest // Add the entry in the leaf Manifest
newEntry := manifestEntry{ newEntry := swarm.ManifestEntry{
Path: path, Path: path,
Hash: hash, Hash: hash,
ContentType: ctype, ContentType: ctype,
@ -180,7 +181,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
mroot.Entries = append(mroot.Entries, newEntry) mroot.Entries = append(mroot.Entries, newEntry)
} }
newManifestHash, err := client.uploadManifest(mroot) newManifestHash, err := client.UploadManifest(mroot)
if err != nil { if err != nil {
utils.Fatalf("Manifest upload failed: %v", err) utils.Fatalf("Manifest upload failed: %v", err)
} }
@ -192,20 +193,20 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
var ( var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = &client{api: bzzapi} client = swarm.NewClient(bzzapi)
newEntry = manifestEntry{ newEntry = swarm.ManifestEntry{
Path: "", Path: "",
Hash: "", Hash: "",
ContentType: "", ContentType: "",
} }
longestPathEntry = manifestEntry{ longestPathEntry = swarm.ManifestEntry{
Path: "", Path: "",
Hash: "", Hash: "",
ContentType: "", ContentType: "",
} }
) )
mroot, err := client.downloadManifest(mhash) mroot, err := client.DownloadManifest(mhash)
if err != nil { if err != nil {
utils.Fatalf("Manifest download failed: %v", err) utils.Fatalf("Manifest download failed: %v", err)
} }
@ -236,7 +237,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype) newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
// Replace the hash for parent Manifests // Replace the hash for parent Manifests
newMRoot := manifest{} newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries { for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path { if longestPathEntry.Path == entry.Path {
entry.Hash = newHash entry.Hash = newHash
@ -249,10 +250,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
if newEntry.Path != "" { if newEntry.Path != "" {
// Replace the hash for leaf Manifest // Replace the hash for leaf Manifest
newMRoot := manifest{} newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries { for _, entry := range mroot.Entries {
if newEntry.Path == entry.Path { if newEntry.Path == entry.Path {
myEntry := manifestEntry{ myEntry := swarm.ManifestEntry{
Path: entry.Path, Path: entry.Path,
Hash: hash, Hash: hash,
ContentType: ctype, ContentType: ctype,
@ -265,7 +266,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
mroot = newMRoot mroot = newMRoot
} }
newManifestHash, err := client.uploadManifest(mroot) newManifestHash, err := client.UploadManifest(mroot)
if err != nil { if err != nil {
utils.Fatalf("Manifest upload failed: %v", err) utils.Fatalf("Manifest upload failed: %v", err)
} }
@ -276,20 +277,20 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
var ( var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = &client{api: bzzapi} client = swarm.NewClient(bzzapi)
entryToRemove = manifestEntry{ entryToRemove = swarm.ManifestEntry{
Path: "", Path: "",
Hash: "", Hash: "",
ContentType: "", ContentType: "",
} }
longestPathEntry = manifestEntry{ longestPathEntry = swarm.ManifestEntry{
Path: "", Path: "",
Hash: "", Hash: "",
ContentType: "", ContentType: "",
} }
) )
mroot, err := client.downloadManifest(mhash) mroot, err := client.DownloadManifest(mhash)
if err != nil { if err != nil {
utils.Fatalf("Manifest download failed: %v", err) utils.Fatalf("Manifest download failed: %v", err)
} }
@ -318,7 +319,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath) newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
// Replace the hash for parent Manifests // Replace the hash for parent Manifests
newMRoot := manifest{} newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries { for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path { if longestPathEntry.Path == entry.Path {
entry.Hash = newHash entry.Hash = newHash
@ -330,7 +331,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
if entryToRemove.Path != "" { if entryToRemove.Path != "" {
// remove the entry in this Manifest // remove the entry in this Manifest
newMRoot := manifest{} newMRoot := swarm.Manifest{}
for _, entry := range mroot.Entries { for _, entry := range mroot.Entries {
if entryToRemove.Path != entry.Path { if entryToRemove.Path != entry.Path {
newMRoot.Entries = append(newMRoot.Entries, entry) newMRoot.Entries = append(newMRoot.Entries, entry)
@ -339,7 +340,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
mroot = newMRoot mroot = newMRoot
} }
newManifestHash, err := client.uploadManifest(mroot) newManifestHash, err := client.UploadManifest(mroot)
if err != nil { if err != nil {
utils.Fatalf("Manifest upload failed: %v", err) utils.Fatalf("Manifest upload failed: %v", err)
} }

@ -18,21 +18,15 @@
package main package main
import ( import (
"bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"os" "os"
"os/user" "os/user"
"path" "path"
"path/filepath"
"strings" "strings"
"github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/log" swarm "github.com/ethereum/go-ethereum/swarm/api/client"
"gopkg.in/urfave/cli.v1" "gopkg.in/urfave/cli.v1"
) )
@ -50,7 +44,7 @@ func upload(ctx *cli.Context) {
var ( var (
file = args[0] file = args[0]
client = &client{api: bzzapi} client = swarm.NewClient(bzzapi)
) )
fi, err := os.Stat(expandPath(file)) fi, err := os.Stat(expandPath(file))
if err != nil { if err != nil {
@ -63,25 +57,25 @@ func upload(ctx *cli.Context) {
if !wantManifest { if !wantManifest {
utils.Fatalf("Manifest is required for directory uploads") utils.Fatalf("Manifest is required for directory uploads")
} }
mhash, err := client.uploadDirectory(file, defaultPath) mhash, err := client.UploadDirectory(file, defaultPath)
if err != nil { if err != nil {
utils.Fatalf("Failed to upload directory: %v", err) utils.Fatalf("Failed to upload directory: %v", err)
} }
fmt.Println(mhash) fmt.Println(mhash)
return return
} }
entry, err := client.uploadFile(file, fi) entry, err := client.UploadFile(file, fi)
if err != nil { if err != nil {
utils.Fatalf("Upload failed: %v", err) utils.Fatalf("Upload failed: %v", err)
} }
mroot := manifest{[]manifestEntry{entry}} mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
if !wantManifest { if !wantManifest {
// Print the manifest. This is the only output to stdout. // Print the manifest. This is the only output to stdout.
mrootJSON, _ := json.MarshalIndent(mroot, "", " ") mrootJSON, _ := json.MarshalIndent(mroot, "", " ")
fmt.Println(string(mrootJSON)) fmt.Println(string(mrootJSON))
return return
} }
hash, err := client.uploadManifest(mroot) hash, err := client.UploadManifest(mroot)
if err != nil { if err != nil {
utils.Fatalf("Manifest upload failed: %v", err) utils.Fatalf("Manifest upload failed: %v", err)
} }
@ -111,148 +105,3 @@ func homeDir() string {
} }
return "" return ""
} }
// client wraps interaction with the swarm HTTP gateway.
type client struct {
api string
}
// manifest is the JSON representation of a swarm manifest.
type manifestEntry struct {
Hash string `json:"hash,omitempty"`
ContentType string `json:"contentType,omitempty"`
Path string `json:"path,omitempty"`
}
// manifest is the JSON representation of a swarm manifest.
type manifest struct {
Entries []manifestEntry `json:"entries,omitempty"`
}
func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
if err != nil {
return "", fmt.Errorf("failed to upload empty manifest")
}
if len(defaultPath) > 0 {
fi, err := os.Stat(defaultPath)
if err != nil {
return "", err
}
mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
if err != nil {
return "", err
}
}
prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
if err != nil || fi.IsDir() {
return err
}
if !strings.HasPrefix(path, dir) {
return fmt.Errorf("path %s outside directory %s", path, dir)
}
uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
return err
})
return mhash, err
}
func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
hash, err := c.uploadFileContent(file, fi)
m := manifestEntry{
Hash: hash,
ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
}
return m, err
}
func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
fd, err := os.Open(file)
if err != nil {
return "", err
}
defer fd.Close()
log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
return c.postRaw("application/octet-stream", fi.Size(), fd)
}
func (c *client) uploadManifest(m manifest) (string, error) {
jsm, err := json.Marshal(m)
if err != nil {
panic(err)
}
log.Info("Uploading swarm manifest")
return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
}
func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
fd, err := os.Open(fpath)
if err != nil {
return "", err
}
defer fd.Close()
log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
if err != nil {
return "", err
}
req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
req.ContentLength = fi.Size()
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
return string(content), err
}
func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
if err != nil {
return "", err
}
req.Header.Set("content-type", mimetype)
req.ContentLength = size
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
return string(content), err
}
func (c *client) downloadManifest(mhash string) (manifest, error) {
mroot := manifest{}
req, err := http.NewRequest("GET", c.api+"/bzzr:/"+mhash, nil)
if err != nil {
return mroot, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return mroot, err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return mroot, fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(content, &mroot)
if err != nil {
return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
}
return mroot, err
}

@ -0,0 +1,298 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package client
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/ethereum/go-ethereum/log"
)
var (
DefaultGateway = "http://localhost:8500"
DefaultClient = NewClient(DefaultGateway)
)
// Manifest represents a swarm manifest.
type Manifest struct {
Entries []ManifestEntry `json:"entries,omitempty"`
}
// ManifestEntry represents an entry in a swarm manifest.
type ManifestEntry struct {
Hash string `json:"hash,omitempty"`
ContentType string `json:"contentType,omitempty"`
Path string `json:"path,omitempty"`
}
func NewClient(gateway string) *Client {
return &Client{
Gateway: gateway,
}
}
// Client wraps interaction with a swarm HTTP gateway.
type Client struct {
Gateway string
}
func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) {
mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
if err != nil {
return "", fmt.Errorf("failed to upload empty manifest")
}
if len(defaultPath) > 0 {
fi, err := os.Stat(defaultPath)
if err != nil {
return "", err
}
mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
if err != nil {
return "", err
}
}
prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
if err != nil || fi.IsDir() {
return err
}
if !strings.HasPrefix(path, dir) {
return fmt.Errorf("path %s outside directory %s", path, dir)
}
uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
return err
})
return mhash, err
}
func (c *Client) UploadFile(file string, fi os.FileInfo) (ManifestEntry, error) {
hash, err := c.uploadFileContent(file, fi)
m := ManifestEntry{
Hash: hash,
ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
}
return m, err
}
func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
fd, err := os.Open(file)
if err != nil {
return "", err
}
defer fd.Close()
log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
return c.postRaw("application/octet-stream", fi.Size(), fd)
}
func (c *Client) UploadManifest(m Manifest) (string, error) {
jsm, err := json.Marshal(m)
if err != nil {
panic(err)
}
log.Info("Uploading swarm manifest")
return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
}
func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
fd, err := os.Open(fpath)
if err != nil {
return "", err
}
defer fd.Close()
log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd)
if err != nil {
return "", err
}
req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
req.ContentLength = fi.Size()
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
return string(content), err
}
func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body)
if err != nil {
return "", err
}
req.Header.Set("content-type", mimetype)
req.ContentLength = size
resp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
return string(content), err
}
func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
mroot := Manifest{}
req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil)
if err != nil {
return mroot, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return mroot, err
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
return mroot, fmt.Errorf("bad status: %s", resp.Status)
}
content, err := ioutil.ReadAll(resp.Body)
err = json.Unmarshal(content, &mroot)
if err != nil {
return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
}
return mroot, err
}
// ManifestFileList downloads the manifest with the given hash and generates a
// list of files and directory prefixes which have the specified prefix.
//
// For example, if the manifest represents the following directory structure:
//
// file1.txt
// file2.txt
// dir1/file3.txt
// dir1/dir2/file4.txt
//
// Then:
//
// - a prefix of "" would return [dir1/, file1.txt, file2.txt]
// - a prefix of "file" would return [file1.txt, file2.txt]
// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
manifest, err := c.DownloadManifest(hash)
if err != nil {
return nil, err
}
// handleFile handles a manifest entry which is a direct reference to a
// file (i.e. it is not a swarm manifest)
handleFile := func(entry ManifestEntry) {
// ignore the file if it doesn't have the specified prefix
if !strings.HasPrefix(entry.Path, prefix) {
return
}
// if the path after the prefix contains a directory separator,
// add a directory prefix to the entries, otherwise add the
// file
suffix := strings.TrimPrefix(entry.Path, prefix)
if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
entries = append(entries, ManifestEntry{
Path: prefix + suffix[:sepIndex+1],
ContentType: "DIR",
})
} else {
if entry.Path == "" {
entry.Path = "/"
}
entries = append(entries, entry)
}
}
// handleManifest handles a manifest entry which is a reference to
// another swarm manifest.
handleManifest := func(entry ManifestEntry) error {
// if the manifest's path is a prefix of the specified prefix
// then just recurse into the manifest by stripping its path
// from the prefix
if strings.HasPrefix(prefix, entry.Path) {
subPrefix := strings.TrimPrefix(prefix, entry.Path)
subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
if err != nil {
return err
}
// prefix the manifest's path to the sub entries and
// add them to the returned entries
for i, subEntry := range subEntries {
subEntry.Path = entry.Path + subEntry.Path
subEntries[i] = subEntry
}
entries = append(entries, subEntries...)
return nil
}
// if the manifest's path has the specified prefix, then if the
// path after the prefix contains a directory separator, add a
// directory prefix to the entries, otherwise recurse into the
// manifest
if strings.HasPrefix(entry.Path, prefix) {
suffix := strings.TrimPrefix(entry.Path, prefix)
sepIndex := strings.Index(suffix, "/")
if sepIndex > -1 {
entries = append(entries, ManifestEntry{
Path: prefix + suffix[:sepIndex+1],
ContentType: "DIR",
})
return nil
}
subEntries, err := c.ManifestFileList(entry.Hash, "")
if err != nil {
return err
}
// prefix the manifest's path to the sub entries and
// add them to the returned entries
for i, subEntry := range subEntries {
subEntry.Path = entry.Path + subEntry.Path
subEntries[i] = subEntry
}
entries = append(entries, subEntries...)
return nil
}
return nil
}
for _, entry := range manifest.Entries {
if entry.ContentType == "application/bzz-manifest+json" {
if err := handleManifest(entry); err != nil {
return nil, err
}
} else {
handleFile(entry)
}
}
return
}

@ -0,0 +1,105 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package client
import (
"io/ioutil"
"os"
"path/filepath"
"reflect"
"sort"
"testing"
"github.com/ethereum/go-ethereum/swarm/testutil"
)
func TestClientManifestFileList(t *testing.T) {
srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
dir, err := ioutil.TempDir("", "swarm-client-test")
if err != nil {
t.Fatal(err)
}
files := []string{
"file1.txt",
"file2.txt",
"dir1/file3.txt",
"dir1/file4.txt",
"dir2/file5.txt",
"dir2/dir3/file6.txt",
"dir2/dir4/file7.txt",
"dir2/dir4/file8.txt",
}
for _, file := range files {
path := filepath.Join(dir, file)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
t.Fatalf("error creating dir for %s: %s", path, err)
}
if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
t.Fatalf("error writing file %s: %s", path, err)
}
}
client := NewClient(srv.URL)
hash, err := client.UploadDirectory(dir, "")
if err != nil {
t.Fatalf("error uploading directory: %s", err)
}
ls := func(prefix string) []string {
entries, err := client.ManifestFileList(hash, prefix)
if err != nil {
t.Fatal(err)
}
paths := make([]string, len(entries))
for i, entry := range entries {
paths[i] = entry.Path
}
sort.Strings(paths)
return paths
}
tests := map[string][]string{
"": []string{"dir1/", "dir2/", "file1.txt", "file2.txt"},
"file": []string{"file1.txt", "file2.txt"},
"file1": []string{"file1.txt"},
"file2.txt": []string{"file2.txt"},
"file12": []string{},
"dir": []string{"dir1/", "dir2/"},
"dir1": []string{"dir1/"},
"dir1/": []string{"dir1/file3.txt", "dir1/file4.txt"},
"dir1/file": []string{"dir1/file3.txt", "dir1/file4.txt"},
"dir1/file3.txt": []string{"dir1/file3.txt"},
"dir1/file34": []string{},
"dir2/": []string{"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"},
"dir2/file": []string{"dir2/file5.txt"},
"dir2/dir": []string{"dir2/dir3/", "dir2/dir4/"},
"dir2/dir3/": []string{"dir2/dir3/file6.txt"},
"dir2/dir4/": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
"dir2/dir4/file": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
"dir2/dir4/file7.txt": []string{"dir2/dir4/file7.txt"},
"dir2/dir4/file78": []string{},
}
for prefix, expected := range tests {
actual := ls(prefix)
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
}
}
}

@ -56,9 +56,9 @@ type sequentialReader struct {
lock sync.Mutex lock sync.Mutex
} }
// Server is the basic configuration needs for the HTTP server and also // ServerConfig is the basic configuration needed for the HTTP server and also
// includes CORS settings. // includes CORS settings.
type Server struct { type ServerConfig struct {
Addr string Addr string
CorsString string CorsString string
} }
@ -69,13 +69,9 @@ type Server struct {
// https://github.com/atom/electron/blob/master/docs/api/protocol.md // https://github.com/atom/electron/blob/master/docs/api/protocol.md
// starts up http server // starts up http server
func StartHttpServer(api *api.Api, server *Server) { func StartHttpServer(api *api.Api, config *ServerConfig) {
serveMux := http.NewServeMux()
serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
handler(w, r, api)
})
var allowedOrigins []string var allowedOrigins []string
for _, domain := range strings.Split(server.CorsString, ",") { for _, domain := range strings.Split(config.CorsString, ",") {
allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain)) allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
} }
c := cors.New(cors.Options{ c := cors.New(cors.Options{
@ -84,13 +80,21 @@ func StartHttpServer(api *api.Api, server *Server) {
MaxAge: 600, MaxAge: 600,
AllowedHeaders: []string{"*"}, AllowedHeaders: []string{"*"},
}) })
hdlr := c.Handler(serveMux) hdlr := c.Handler(NewServer(api))
go http.ListenAndServe(config.Addr, hdlr)
log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", config.Addr))
}
go http.ListenAndServe(server.Addr, hdlr) func NewServer(api *api.Api) *Server {
log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", server.Addr)) return &Server{api}
}
type Server struct {
api *api.Api
} }
func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
requestURL := r.URL requestURL := r.URL
// This is wrong // This is wrong
// if requestURL.Host == "" { // if requestURL.Host == "" {
@ -135,7 +139,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest) http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest)
return return
} }
key, err := a.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil) key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
if err == nil { if err == nil {
log.Debug(fmt.Sprintf("Content for %v stored", key.Log())) log.Debug(fmt.Sprintf("Content for %v stored", key.Log()))
} else { } else {
@ -160,7 +164,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
mime := r.Header.Get("Content-Type") mime := r.Header.Get("Content-Type")
// TODO proper root hash separation // TODO proper root hash separation
log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime)) log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime))
newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver) newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
if err == nil { if err == nil {
log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey)) log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
w.Header().Set("Content-Type", "text/plain") w.Header().Set("Content-Type", "text/plain")
@ -178,7 +182,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
} else { } else {
path = api.RegularSlashes(path) path = api.RegularSlashes(path)
log.Debug(fmt.Sprintf("Delete '%s'.", path)) log.Debug(fmt.Sprintf("Delete '%s'.", path))
newKey, err := a.Modify(path, "", "", nameresolver) newKey, err := s.api.Modify(path, "", "", nameresolver)
if err == nil { if err == nil {
log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey)) log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
w.Header().Set("Content-Type", "text/plain") w.Header().Set("Content-Type", "text/plain")
@ -199,16 +203,16 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
parsedurl, _ := api.Parse(path) parsedurl, _ := api.Parse(path)
if parsedurl == path { if parsedurl == path {
key, err := a.Resolve(parsedurl, nameresolver) key, err := s.api.Resolve(parsedurl, nameresolver)
if err != nil { if err != nil {
log.Error(fmt.Sprintf("%v", err)) log.Error(fmt.Sprintf("%v", err))
http.Error(w, err.Error(), http.StatusBadRequest) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
reader = a.Retrieve(key) reader = s.api.Retrieve(key)
} else { } else {
var status int var status int
readertmp, _, status, err := a.Get(path, nameresolver) readertmp, _, status, err := s.api.Get(path, nameresolver)
if err != nil { if err != nil {
http.Error(w, err.Error(), status) http.Error(w, err.Error(), status)
return return
@ -247,7 +251,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
http.Redirect(w, r, path+"/", http.StatusFound) http.Redirect(w, r, path+"/", http.StatusFound)
return return
} }
reader, mimeType, status, err := a.Get(path, nameresolver) reader, mimeType, status, err := s.api.Get(path, nameresolver)
if err != nil { if err != nil {
if _, ok := err.(api.ErrResolve); ok { if _, ok := err.(api.ErrResolve); ok {
log.Debug(fmt.Sprintf("%v", err)) log.Debug(fmt.Sprintf("%v", err))

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License // You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http package http_test
import ( import (
"bytes" "bytes"
@ -22,19 +22,16 @@ import (
"net/http" "net/http"
"sync" "sync"
"testing" "testing"
"time"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/testutil"
) )
func TestBzzrGetPath(t *testing.T) { func TestBzzrGetPath(t *testing.T) {
var err error var err error
maxproxyattempts := 3
testmanifest := []string{ testmanifest := []string{
`{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`, `{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`,
`{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`, `{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`,
@ -54,61 +51,30 @@ func TestBzzrGetPath(t *testing.T) {
key := [3]storage.Key{} key := [3]storage.Key{}
dir, _ := ioutil.TempDir("", "bzz-storage-test") srv := testutil.NewTestSwarmServer(t)
defer srv.Close()
storeparams := &storage.StoreParams{
ChunkDbPath: dir,
DbCapacity: 5000000,
CacheCapacity: 5000,
Radius: 0,
}
localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
if err != nil {
t.Fatal(err)
}
chunker := storage.NewTreeChunker(storage.NewChunkerParams())
dpa := &storage.DPA{
Chunker: chunker,
ChunkStore: localStore,
}
dpa.Start()
defer dpa.Stop()
wg := &sync.WaitGroup{} wg := &sync.WaitGroup{}
for i, mf := range testmanifest { for i, mf := range testmanifest {
reader[i] = bytes.NewReader([]byte(mf)) reader[i] = bytes.NewReader([]byte(mf))
key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil) key[i], err = srv.Dpa.Store(reader[i], int64(len(mf)), wg, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
wg.Wait() wg.Wait()
} }
a := api.NewApi(dpa, nil) _, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
if err != nil {
/// \todo iterate port numbers up if fail t.Fatalf("Failed to connect to proxy: %v", err)
StartHttpServer(a, &Server{Addr: "127.0.0.1:8504", CorsString: ""})
// how to wait for ListenAndServe to have initialized? This is pretty cruuuude
// if we fix it we don't need maxproxyattempts anymore either
time.Sleep(1000 * time.Millisecond)
for i := 0; i <= maxproxyattempts; i++ {
_, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
if i == maxproxyattempts {
t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err)
} else if err != nil {
time.Sleep(100 * time.Millisecond)
continue
}
break
} }
for k, v := range testrequests { for k, v := range testrequests {
var resp *http.Response var resp *http.Response
var respbody []byte var respbody []byte
url := "http://127.0.0.1:8504/bzzr:/" url := srv.URL + "/bzzr:/"
if k[:] != "" { if k[:] != "" {
url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain" url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
} }

@ -195,7 +195,10 @@ func (self *Swarm) Start(net *p2p.Server) error {
// start swarm http proxy server // start swarm http proxy server
if self.config.Port != "" { if self.config.Port != "" {
addr := ":" + self.config.Port addr := ":" + self.config.Port
go httpapi.StartHttpServer(self.api, &httpapi.Server{Addr: addr, CorsString: self.corsString}) go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{
Addr: addr,
CorsString: self.corsString,
})
} }
log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port)) log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port))

@ -0,0 +1,56 @@
package testutil
import (
"io/ioutil"
"net/http/httptest"
"os"
"testing"
"github.com/ethereum/go-ethereum/swarm/api"
httpapi "github.com/ethereum/go-ethereum/swarm/api/http"
"github.com/ethereum/go-ethereum/swarm/storage"
)
func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
dir, err := ioutil.TempDir("", "swarm-storage-test")
if err != nil {
t.Fatal(err)
}
storeparams := &storage.StoreParams{
ChunkDbPath: dir,
DbCapacity: 5000000,
CacheCapacity: 5000,
Radius: 0,
}
localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
if err != nil {
os.RemoveAll(dir)
t.Fatal(err)
}
chunker := storage.NewTreeChunker(storage.NewChunkerParams())
dpa := &storage.DPA{
Chunker: chunker,
ChunkStore: localStore,
}
dpa.Start()
a := api.NewApi(dpa, nil)
srv := httptest.NewServer(httpapi.NewServer(a))
return &TestSwarmServer{
Server: srv,
Dpa: dpa,
dir: dir,
}
}
type TestSwarmServer struct {
*httptest.Server
Dpa *storage.DPA
dir string
}
func (t *TestSwarmServer) Close() {
t.Server.Close()
t.Dpa.Stop()
os.RemoveAll(t.dir)
}
Loading…
Cancel
Save