// Copyright 2017 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . package client import ( "archive/tar" "bytes" "context" "encoding/json" "errors" "fmt" "io" "io/ioutil" "mime/multipart" "net/http" "net/http/httptrace" "net/textproto" "net/url" "os" "path/filepath" "regexp" "strconv" "strings" "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage/feed" "github.com/pborman/uuid" ) var ( ErrUnauthorized = errors.New("unauthorized") ) func NewClient(gateway string) *Client { return &Client{ Gateway: gateway, } } // Client wraps interaction with a swarm HTTP gateway. type Client struct { Gateway string } // UploadRaw uploads raw data to swarm and returns the resulting hash. If toEncrypt is true it // uploads encrypted data func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, error) { if size <= 0 { return "", errors.New("data size must be greater than zero") } addr := "" if toEncrypt { addr = "encrypt" } req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/"+addr, r) if err != nil { return "", err } req.ContentLength = size req.Header.Set(swarmhttp.SwarmTagHeaderName, fmt.Sprintf("raw_upload_%d", time.Now().Unix())) res, err := http.DefaultClient.Do(req) if err != nil { return "", err } defer res.Body.Close() if res.StatusCode != http.StatusOK { return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) } data, err := ioutil.ReadAll(res.Body) if err != nil { return "", err } return string(data), nil } // DownloadRaw downloads raw data from swarm and it returns a ReadCloser and a bool whether the // content was encrypted func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) { uri := c.Gateway + "/bzz-raw:/" + hash res, err := http.DefaultClient.Get(uri) if err != nil { return nil, false, err } if res.StatusCode != http.StatusOK { res.Body.Close() return nil, false, fmt.Errorf("unexpected HTTP status: %s", res.Status) } isEncrypted := (res.Header.Get("X-Decrypted") == "true") return res.Body, isEncrypted, nil } // File represents a file in a swarm manifest and is used for uploading and // downloading content to and from swarm type File struct { io.ReadCloser api.ManifestEntry Tag string } // Open opens a local file which can then be passed to client.Upload to upload // it to swarm func Open(path string) (*File, error) { f, err := os.Open(path) if err != nil { return nil, err } stat, err := f.Stat() if err != nil { f.Close() return nil, err } contentType, err := api.DetectContentType(f.Name(), f) if err != nil { return nil, err } return &File{ ReadCloser: f, ManifestEntry: api.ManifestEntry{ ContentType: contentType, Mode: int64(stat.Mode()), Size: stat.Size(), ModTime: stat.ModTime(), }, Tag: filepath.Base(path), }, nil } // Upload uploads a file to swarm and either adds it to an existing manifest // (if the manifest argument is non-empty) or creates a new manifest containing // the file, returning the resulting manifest hash (the file will then be // available at bzz://) func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, error) { if file.Size <= 0 { return "", errors.New("file size must be greater than zero") } return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt) } // Download downloads a file with the given path from the swarm manifest with // the given hash (i.e. it gets bzz://) func (c *Client) Download(hash, path string) (*File, error) { uri := c.Gateway + "/bzz:/" + hash + "/" + path res, err := http.DefaultClient.Get(uri) if err != nil { return nil, err } if res.StatusCode != http.StatusOK { res.Body.Close() return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) } return &File{ ReadCloser: res.Body, ManifestEntry: api.ManifestEntry{ ContentType: res.Header.Get("Content-Type"), Size: res.ContentLength, }, }, nil } // UploadDirectory uploads a directory tree to swarm and either adds the files // to an existing manifest (if the manifest argument is non-empty) or creates a // new manifest, returning the resulting manifest hash (files from the // directory will then be available at bzz://path/to/file), with // the file specified in defaultPath being uploaded to the root of the manifest // (i.e. bzz://) func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bool) (string, error) { stat, err := os.Stat(dir) if err != nil { return "", err } else if !stat.IsDir() { return "", fmt.Errorf("not a directory: %s", dir) } if defaultPath != "" { if _, err := os.Stat(filepath.Join(dir, defaultPath)); err != nil { if os.IsNotExist(err) { return "", fmt.Errorf("the default path %q was not found in the upload directory %q", defaultPath, dir) } return "", fmt.Errorf("default path: %v", err) } } return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt) } // DownloadDirectory downloads the files contained in a swarm manifest under // the given path into a local directory (existing files will be overwritten) func (c *Client) DownloadDirectory(hash, path, destDir, credentials string) error { stat, err := os.Stat(destDir) if err != nil { return err } else if !stat.IsDir() { return fmt.Errorf("not a directory: %s", destDir) } uri := c.Gateway + "/bzz:/" + hash + "/" + path req, err := http.NewRequest("GET", uri, nil) if err != nil { return err } if credentials != "" { req.SetBasicAuth("", credentials) } req.Header.Set("Accept", "application/x-tar") res, err := http.DefaultClient.Do(req) if err != nil { return err } defer res.Body.Close() switch res.StatusCode { case http.StatusOK: case http.StatusUnauthorized: return ErrUnauthorized default: return fmt.Errorf("unexpected HTTP status: %s", res.Status) } tr := tar.NewReader(res.Body) for { hdr, err := tr.Next() if err == io.EOF { return nil } else if err != nil { return err } // ignore the default path file if hdr.Name == "" { continue } dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path))) if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { return err } var mode os.FileMode = 0644 if hdr.Mode > 0 { mode = os.FileMode(hdr.Mode) } dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return err } n, err := io.Copy(dst, tr) dst.Close() if err != nil { return err } else if n != hdr.Size { return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n) } } } // DownloadFile downloads a single file into the destination directory // if the manifest entry does not specify a file name - it will fallback // to the hash of the file as a filename func (c *Client) DownloadFile(hash, path, dest, credentials string) error { hasDestinationFilename := false if stat, err := os.Stat(dest); err == nil { hasDestinationFilename = !stat.IsDir() } else { if os.IsNotExist(err) { // does not exist - should be created hasDestinationFilename = true } else { return fmt.Errorf("could not stat path: %v", err) } } manifestList, err := c.List(hash, path, credentials) if err != nil { return err } switch len(manifestList.Entries) { case 0: return fmt.Errorf("could not find path requested at manifest address. make sure the path you've specified is correct") case 1: //continue default: return fmt.Errorf("got too many matches for this path") } uri := c.Gateway + "/bzz:/" + hash + "/" + path req, err := http.NewRequest("GET", uri, nil) if err != nil { return err } if credentials != "" { req.SetBasicAuth("", credentials) } res, err := http.DefaultClient.Do(req) if err != nil { return err } defer res.Body.Close() switch res.StatusCode { case http.StatusOK: case http.StatusUnauthorized: return ErrUnauthorized default: return fmt.Errorf("unexpected HTTP status: expected 200 OK, got %d", res.StatusCode) } filename := "" if hasDestinationFilename { filename = dest } else { // try to assert re := regexp.MustCompile("[^/]+$") //everything after last slash if results := re.FindAllString(path, -1); len(results) > 0 { filename = results[len(results)-1] } else { if entry := manifestList.Entries[0]; entry.Path != "" && entry.Path != "/" { filename = entry.Path } else { // assume hash as name if there's nothing from the command line filename = hash } } filename = filepath.Join(dest, filename) } filePath, err := filepath.Abs(filename) if err != nil { return err } if err := os.MkdirAll(filepath.Dir(filePath), 0777); err != nil { return err } dst, err := os.Create(filename) if err != nil { return err } defer dst.Close() _, err = io.Copy(dst, res.Body) return err } // UploadManifest uploads the given manifest to swarm func (c *Client) UploadManifest(m *api.Manifest, toEncrypt bool) (string, error) { data, err := json.Marshal(m) if err != nil { return "", err } return c.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt) } // DownloadManifest downloads a swarm manifest func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) { res, isEncrypted, err := c.DownloadRaw(hash) if err != nil { return nil, isEncrypted, err } defer res.Close() var manifest api.Manifest if err := json.NewDecoder(res).Decode(&manifest); err != nil { return nil, isEncrypted, err } return &manifest, isEncrypted, nil } // List list files in a swarm manifest which have the given prefix, grouping // common prefixes using "/" as a delimiter. // // For example, if the manifest represents the following directory structure: // // file1.txt // file2.txt // dir1/file3.txt // dir1/dir2/file4.txt // // Then: // // - a prefix of "" would return [dir1/, file1.txt, file2.txt] // - a prefix of "file" would return [file1.txt, file2.txt] // - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt] // // where entries ending with "/" are common prefixes. func (c *Client) List(hash, prefix, credentials string) (*api.ManifestList, error) { req, err := http.NewRequest(http.MethodGet, c.Gateway+"/bzz-list:/"+hash+"/"+prefix, nil) if err != nil { return nil, err } if credentials != "" { req.SetBasicAuth("", credentials) } res, err := http.DefaultClient.Do(req) if err != nil { return nil, err } defer res.Body.Close() switch res.StatusCode { case http.StatusOK: case http.StatusUnauthorized: return nil, ErrUnauthorized default: return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) } var list api.ManifestList if err := json.NewDecoder(res.Body).Decode(&list); err != nil { return nil, err } return &list, nil } // Uploader uploads files to swarm using a provided UploadFn type Uploader interface { Upload(UploadFn) error Tag() string } type UploaderFunc func(UploadFn) error func (u UploaderFunc) Upload(upload UploadFn) error { return u(upload) } func (u UploaderFunc) Tag() string { return fmt.Sprintf("multipart_upload_%d", time.Now().Unix()) } // DirectoryUploader implements Uploader var _ Uploader = &DirectoryUploader{} // DirectoryUploader uploads all files in a directory, optionally uploading // a file to the default path type DirectoryUploader struct { Dir string } func (d *DirectoryUploader) Tag() string { return filepath.Base(d.Dir) } // Upload performs the upload of the directory and default path func (d *DirectoryUploader) Upload(upload UploadFn) error { return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error { if err != nil { return err } if f.IsDir() { return nil } file, err := Open(path) if err != nil { return err } relPath, err := filepath.Rel(d.Dir, path) if err != nil { return err } file.Path = filepath.ToSlash(relPath) return upload(file) }) } var _ Uploader = &FileUploader{} // FileUploader uploads a single file type FileUploader struct { File *File } func (f *FileUploader) Tag() string { return f.File.Tag } // Upload performs the upload of the file func (f *FileUploader) Upload(upload UploadFn) error { return upload(f.File) } // UploadFn is the type of function passed to an Uploader to perform the upload // of a single file (for example, a directory uploader would call a provided // UploadFn for each file in the directory tree) type UploadFn func(file *File) error // TarUpload uses the given Uploader to upload files to swarm as a tar stream, // returning the resulting manifest hash func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) { ctx, sp := spancontext.StartSpan(context.Background(), "api.client.tarupload") defer sp.Finish() var tn time.Time reqR, reqW := io.Pipe() defer reqR.Close() addr := hash // If there is a hash already (a manifest), then that manifest will determine if the upload has // to be encrypted or not. If there is no manifest then the toEncrypt parameter decides if // there is encryption or not. if hash == "" && toEncrypt { // This is the built-in address for the encrypted upload endpoint addr = "encrypt" } req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+addr, reqR) if err != nil { return "", err } trace := GetClientTrace("swarm api client - upload tar", "api.client.uploadtar", uuid.New()[:8], &tn) req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) transport := http.DefaultTransport req.Header.Set("Content-Type", "application/x-tar") if defaultPath != "" { q := req.URL.Query() q.Set("defaultpath", defaultPath) req.URL.RawQuery = q.Encode() } tag := uploader.Tag() if tag == "" { tag = "unnamed_tag_" + fmt.Sprintf("%d", time.Now().Unix()) } log.Trace("setting upload tag", "tag", tag) req.Header.Set(swarmhttp.SwarmTagHeaderName, tag) // use 'Expect: 100-continue' so we don't send the request body if // the server refuses the request req.Header.Set("Expect", "100-continue") tw := tar.NewWriter(reqW) // define an UploadFn which adds files to the tar stream uploadFn := func(file *File) error { hdr := &tar.Header{ Name: file.Path, Mode: file.Mode, Size: file.Size, ModTime: file.ModTime, Xattrs: map[string]string{ "user.swarm.content-type": file.ContentType, }, } if err := tw.WriteHeader(hdr); err != nil { return err } _, err = io.Copy(tw, file) return err } // run the upload in a goroutine so we can send the request headers and // wait for a '100 Continue' response before sending the tar stream go func() { err := uploader.Upload(uploadFn) if err == nil { err = tw.Close() } reqW.CloseWithError(err) }() tn = time.Now() res, err := transport.RoundTrip(req) if err != nil { return "", err } defer res.Body.Close() if res.StatusCode != http.StatusOK { return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) } data, err := ioutil.ReadAll(res.Body) if err != nil { return "", err } return string(data), nil } // MultipartUpload uses the given Uploader to upload files to swarm as a // multipart form, returning the resulting manifest hash func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) { reqR, reqW := io.Pipe() defer reqR.Close() req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR) if err != nil { return "", err } // use 'Expect: 100-continue' so we don't send the request body if // the server refuses the request req.Header.Set("Expect", "100-continue") mw := multipart.NewWriter(reqW) req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary())) req.Header.Set(swarmhttp.SwarmTagHeaderName, fmt.Sprintf("multipart_upload_%d", time.Now().Unix())) // define an UploadFn which adds files to the multipart form uploadFn := func(file *File) error { hdr := make(textproto.MIMEHeader) hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path)) hdr.Set("Content-Type", file.ContentType) hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10)) w, err := mw.CreatePart(hdr) if err != nil { return err } _, err = io.Copy(w, file) return err } // run the upload in a goroutine so we can send the request headers and // wait for a '100 Continue' response before sending the multipart form go func() { err := uploader.Upload(uploadFn) if err == nil { err = mw.Close() } reqW.CloseWithError(err) }() res, err := http.DefaultClient.Do(req) if err != nil { return "", err } defer res.Body.Close() if res.StatusCode != http.StatusOK { return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) } data, err := ioutil.ReadAll(res.Body) if err != nil { return "", err } return string(data), nil } // ErrNoFeedUpdatesFound is returned when Swarm cannot find updates of the given feed var ErrNoFeedUpdatesFound = errors.New("No updates found for this feed") // CreateFeedWithManifest creates a feed manifest, initializing it with the provided // data // Returns the resulting feed manifest address that you can use to include in an ENS Resolver (setContent) // or reference future updates (Client.UpdateFeed) func (c *Client) CreateFeedWithManifest(request *feed.Request) (string, error) { responseStream, err := c.updateFeed(request, true) if err != nil { return "", err } defer responseStream.Close() body, err := ioutil.ReadAll(responseStream) if err != nil { return "", err } var manifestAddress string if err = json.Unmarshal(body, &manifestAddress); err != nil { return "", err } return manifestAddress, nil } // UpdateFeed allows you to set a new version of your content func (c *Client) UpdateFeed(request *feed.Request) error { _, err := c.updateFeed(request, false) return err } func (c *Client) updateFeed(request *feed.Request, createManifest bool) (io.ReadCloser, error) { URL, err := url.Parse(c.Gateway) if err != nil { return nil, err } URL.Path = "/bzz-feed:/" values := URL.Query() body := request.AppendValues(values) if createManifest { values.Set("manifest", "1") } URL.RawQuery = values.Encode() req, err := http.NewRequest("POST", URL.String(), bytes.NewBuffer(body)) if err != nil { return nil, err } res, err := http.DefaultClient.Do(req) if err != nil { return nil, err } return res.Body, nil } // QueryFeed returns a byte stream with the raw content of the feed update // manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver // points to that address func (c *Client) QueryFeed(query *feed.Query, manifestAddressOrDomain string) (io.ReadCloser, error) { return c.queryFeed(query, manifestAddressOrDomain, false) } // queryFeed returns a byte stream with the raw content of the feed update // manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver // points to that address // meta set to true will instruct the node return feed metainformation instead func (c *Client) queryFeed(query *feed.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) { URL, err := url.Parse(c.Gateway) if err != nil { return nil, err } URL.Path = "/bzz-feed:/" + manifestAddressOrDomain values := URL.Query() if query != nil { query.AppendValues(values) //adds query parameters } if meta { values.Set("meta", "1") } URL.RawQuery = values.Encode() res, err := http.Get(URL.String()) if err != nil { return nil, err } if res.StatusCode != http.StatusOK { if res.StatusCode == http.StatusNotFound { return nil, ErrNoFeedUpdatesFound } errorMessageBytes, err := ioutil.ReadAll(res.Body) var errorMessage string if err != nil { errorMessage = "cannot retrieve error message: " + err.Error() } else { errorMessage = string(errorMessageBytes) } return nil, fmt.Errorf("Error retrieving feed updates: %s", errorMessage) } return res.Body, nil } // GetFeedRequest returns a structure that describes the referenced feed status // manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver // points to that address func (c *Client) GetFeedRequest(query *feed.Query, manifestAddressOrDomain string) (*feed.Request, error) { responseStream, err := c.queryFeed(query, manifestAddressOrDomain, true) if err != nil { return nil, err } defer responseStream.Close() body, err := ioutil.ReadAll(responseStream) if err != nil { return nil, err } var metadata feed.Request if err := metadata.UnmarshalJSON(body); err != nil { return nil, err } return &metadata, nil } func GetClientTrace(traceMsg, metricPrefix, ruid string, tn *time.Time) *httptrace.ClientTrace { trace := &httptrace.ClientTrace{ GetConn: func(_ string) { log.Trace(traceMsg+" - http get", "event", "GetConn", "ruid", ruid) metrics.GetOrRegisterResettingTimer(metricPrefix+".getconn", nil).Update(time.Since(*tn)) }, GotConn: func(_ httptrace.GotConnInfo) { log.Trace(traceMsg+" - http get", "event", "GotConn", "ruid", ruid) metrics.GetOrRegisterResettingTimer(metricPrefix+".gotconn", nil).Update(time.Since(*tn)) }, PutIdleConn: func(err error) { log.Trace(traceMsg+" - http get", "event", "PutIdleConn", "ruid", ruid, "err", err) metrics.GetOrRegisterResettingTimer(metricPrefix+".putidle", nil).Update(time.Since(*tn)) }, GotFirstResponseByte: func() { log.Trace(traceMsg+" - http get", "event", "GotFirstResponseByte", "ruid", ruid) metrics.GetOrRegisterResettingTimer(metricPrefix+".firstbyte", nil).Update(time.Since(*tn)) }, Got100Continue: func() { log.Trace(traceMsg, "event", "Got100Continue", "ruid", ruid) metrics.GetOrRegisterResettingTimer(metricPrefix+".got100continue", nil).Update(time.Since(*tn)) }, DNSStart: func(_ httptrace.DNSStartInfo) { log.Trace(traceMsg, "event", "DNSStart", "ruid", ruid) metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsstart", nil).Update(time.Since(*tn)) }, DNSDone: func(_ httptrace.DNSDoneInfo) { log.Trace(traceMsg, "event", "DNSDone", "ruid", ruid) metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsdone", nil).Update(time.Since(*tn)) }, ConnectStart: func(network, addr string) { log.Trace(traceMsg, "event", "ConnectStart", "ruid", ruid, "network", network, "addr", addr) metrics.GetOrRegisterResettingTimer(metricPrefix+".connectstart", nil).Update(time.Since(*tn)) }, ConnectDone: func(network, addr string, err error) { log.Trace(traceMsg, "event", "ConnectDone", "ruid", ruid, "network", network, "addr", addr, "err", err) metrics.GetOrRegisterResettingTimer(metricPrefix+".connectdone", nil).Update(time.Since(*tn)) }, WroteHeaders: func() { log.Trace(traceMsg, "event", "WroteHeaders(request)", "ruid", ruid) metrics.GetOrRegisterResettingTimer(metricPrefix+".wroteheaders", nil).Update(time.Since(*tn)) }, Wait100Continue: func() { log.Trace(traceMsg, "event", "Wait100Continue", "ruid", ruid) metrics.GetOrRegisterResettingTimer(metricPrefix+".wait100continue", nil).Update(time.Since(*tn)) }, WroteRequest: func(_ httptrace.WroteRequestInfo) { log.Trace(traceMsg, "event", "WroteRequest", "ruid", ruid) metrics.GetOrRegisterResettingTimer(metricPrefix+".wroterequest", nil).Update(time.Since(*tn)) }, } return trace }