|
|
|
// Copyright 2017 The go-ethereum Authors
|
|
|
|
// This file is part of the go-ethereum library.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
|
|
|
// it under the terms of the GNU Lesser General Public License as published by
|
|
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
|
|
// (at your option) any later version.
|
|
|
|
//
|
|
|
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
// GNU Lesser General Public License for more details.
|
|
|
|
//
|
|
|
|
// You should have received a copy of the GNU Lesser General Public License
|
|
|
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
|
|
|
package client
|
|
|
|
|
|
|
|
import (
|
|
|
|
"archive/tar"
|
|
|
|
"bytes"
|
|
|
|
"context"
|
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io/ioutil"
|
|
|
|
"mime/multipart"
|
|
|
|
"net/http"
|
|
|
|
"net/http/httptrace"
|
|
|
|
"net/textproto"
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
"net/url"
|
|
|
|
"os"
|
|
|
|
"path/filepath"
|
|
|
|
"regexp"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/ethereum/go-ethereum/log"
|
|
|
|
"github.com/ethereum/go-ethereum/metrics"
|
|
|
|
"github.com/ethereum/go-ethereum/swarm/api"
|
|
|
|
"github.com/ethereum/go-ethereum/swarm/spancontext"
|
|
|
|
"github.com/ethereum/go-ethereum/swarm/storage/feed"
|
|
|
|
"github.com/pborman/uuid"
|
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
|
|
|
DefaultGateway = "http://localhost:8500"
|
|
|
|
DefaultClient = NewClient(DefaultGateway)
|
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
|
|
|
ErrUnauthorized = errors.New("unauthorized")
|
|
|
|
)
|
|
|
|
|
|
|
|
func NewClient(gateway string) *Client {
|
|
|
|
return &Client{
|
|
|
|
Gateway: gateway,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Client wraps interaction with a swarm HTTP gateway.
|
|
|
|
type Client struct {
|
|
|
|
Gateway string
|
|
|
|
}
|
|
|
|
|
|
|
|
// UploadRaw uploads raw data to swarm and returns the resulting hash. If toEncrypt is true it
|
|
|
|
// uploads encrypted data
|
|
|
|
func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, error) {
|
|
|
|
if size <= 0 {
|
|
|
|
return "", errors.New("data size must be greater than zero")
|
|
|
|
}
|
|
|
|
addr := ""
|
|
|
|
if toEncrypt {
|
|
|
|
addr = "encrypt"
|
|
|
|
}
|
|
|
|
req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/"+addr, r)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
req.ContentLength = size
|
|
|
|
res, err := http.DefaultClient.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
if res.StatusCode != http.StatusOK {
|
|
|
|
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
|
|
|
|
}
|
|
|
|
data, err := ioutil.ReadAll(res.Body)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return string(data), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// DownloadRaw downloads raw data from swarm and it returns a ReadCloser and a bool whether the
|
|
|
|
// content was encrypted
|
|
|
|
func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) {
|
|
|
|
uri := c.Gateway + "/bzz-raw:/" + hash
|
|
|
|
res, err := http.DefaultClient.Get(uri)
|
|
|
|
if err != nil {
|
|
|
|
return nil, false, err
|
|
|
|
}
|
|
|
|
if res.StatusCode != http.StatusOK {
|
|
|
|
res.Body.Close()
|
|
|
|
return nil, false, fmt.Errorf("unexpected HTTP status: %s", res.Status)
|
|
|
|
}
|
|
|
|
isEncrypted := (res.Header.Get("X-Decrypted") == "true")
|
|
|
|
return res.Body, isEncrypted, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// File represents a file in a swarm manifest and is used for uploading and
|
|
|
|
// downloading content to and from swarm
|
|
|
|
type File struct {
|
|
|
|
io.ReadCloser
|
|
|
|
api.ManifestEntry
|
|
|
|
}
|
|
|
|
|
|
|
|
// Open opens a local file which can then be passed to client.Upload to upload
|
|
|
|
// it to swarm
|
|
|
|
func Open(path string) (*File, error) {
|
|
|
|
f, err := os.Open(path)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
stat, err := f.Stat()
|
|
|
|
if err != nil {
|
|
|
|
f.Close()
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
contentType, err := api.DetectContentType(f.Name(), f)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return &File{
|
|
|
|
ReadCloser: f,
|
|
|
|
ManifestEntry: api.ManifestEntry{
|
|
|
|
ContentType: contentType,
|
|
|
|
Mode: int64(stat.Mode()),
|
|
|
|
Size: stat.Size(),
|
|
|
|
ModTime: stat.ModTime(),
|
|
|
|
},
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Upload uploads a file to swarm and either adds it to an existing manifest
|
|
|
|
// (if the manifest argument is non-empty) or creates a new manifest containing
|
|
|
|
// the file, returning the resulting manifest hash (the file will then be
|
|
|
|
// available at bzz:/<hash>/<path>)
|
|
|
|
func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, error) {
|
|
|
|
if file.Size <= 0 {
|
|
|
|
return "", errors.New("file size must be greater than zero")
|
|
|
|
}
|
|
|
|
return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Download downloads a file with the given path from the swarm manifest with
|
|
|
|
// the given hash (i.e. it gets bzz:/<hash>/<path>)
|
|
|
|
func (c *Client) Download(hash, path string) (*File, error) {
|
|
|
|
uri := c.Gateway + "/bzz:/" + hash + "/" + path
|
|
|
|
res, err := http.DefaultClient.Get(uri)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if res.StatusCode != http.StatusOK {
|
|
|
|
res.Body.Close()
|
|
|
|
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
|
|
|
|
}
|
|
|
|
return &File{
|
|
|
|
ReadCloser: res.Body,
|
|
|
|
ManifestEntry: api.ManifestEntry{
|
|
|
|
ContentType: res.Header.Get("Content-Type"),
|
|
|
|
Size: res.ContentLength,
|
|
|
|
},
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// UploadDirectory uploads a directory tree to swarm and either adds the files
|
|
|
|
// to an existing manifest (if the manifest argument is non-empty) or creates a
|
|
|
|
// new manifest, returning the resulting manifest hash (files from the
|
|
|
|
// directory will then be available at bzz:/<hash>/path/to/file), with
|
|
|
|
// the file specified in defaultPath being uploaded to the root of the manifest
|
|
|
|
// (i.e. bzz:/<hash>/)
|
|
|
|
func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bool) (string, error) {
|
|
|
|
stat, err := os.Stat(dir)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
} else if !stat.IsDir() {
|
|
|
|
return "", fmt.Errorf("not a directory: %s", dir)
|
|
|
|
}
|
|
|
|
if defaultPath != "" {
|
|
|
|
if _, err := os.Stat(filepath.Join(dir, defaultPath)); err != nil {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
return "", fmt.Errorf("the default path %q was not found in the upload directory %q", defaultPath, dir)
|
|
|
|
}
|
|
|
|
return "", fmt.Errorf("default path: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt)
|
|
|
|
}
|
|
|
|
|
|
|
|
// DownloadDirectory downloads the files contained in a swarm manifest under
|
|
|
|
// the given path into a local directory (existing files will be overwritten)
|
|
|
|
func (c *Client) DownloadDirectory(hash, path, destDir, credentials string) error {
|
|
|
|
stat, err := os.Stat(destDir)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
} else if !stat.IsDir() {
|
|
|
|
return fmt.Errorf("not a directory: %s", destDir)
|
|
|
|
}
|
|
|
|
|
|
|
|
uri := c.Gateway + "/bzz:/" + hash + "/" + path
|
|
|
|
req, err := http.NewRequest("GET", uri, nil)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if credentials != "" {
|
|
|
|
req.SetBasicAuth("", credentials)
|
|
|
|
}
|
|
|
|
req.Header.Set("Accept", "application/x-tar")
|
|
|
|
res, err := http.DefaultClient.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
switch res.StatusCode {
|
|
|
|
case http.StatusOK:
|
|
|
|
case http.StatusUnauthorized:
|
|
|
|
return ErrUnauthorized
|
|
|
|
default:
|
|
|
|
return fmt.Errorf("unexpected HTTP status: %s", res.Status)
|
|
|
|
}
|
|
|
|
tr := tar.NewReader(res.Body)
|
|
|
|
for {
|
|
|
|
hdr, err := tr.Next()
|
|
|
|
if err == io.EOF {
|
|
|
|
return nil
|
|
|
|
} else if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// ignore the default path file
|
|
|
|
if hdr.Name == "" {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path)))
|
|
|
|
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
var mode os.FileMode = 0644
|
|
|
|
if hdr.Mode > 0 {
|
|
|
|
mode = os.FileMode(hdr.Mode)
|
|
|
|
}
|
|
|
|
dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
n, err := io.Copy(dst, tr)
|
|
|
|
dst.Close()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
} else if n != hdr.Size {
|
|
|
|
return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// DownloadFile downloads a single file into the destination directory
|
|
|
|
// if the manifest entry does not specify a file name - it will fallback
|
|
|
|
// to the hash of the file as a filename
|
|
|
|
func (c *Client) DownloadFile(hash, path, dest, credentials string) error {
|
|
|
|
hasDestinationFilename := false
|
|
|
|
if stat, err := os.Stat(dest); err == nil {
|
|
|
|
hasDestinationFilename = !stat.IsDir()
|
|
|
|
} else {
|
|
|
|
if os.IsNotExist(err) {
|
|
|
|
// does not exist - should be created
|
|
|
|
hasDestinationFilename = true
|
|
|
|
} else {
|
|
|
|
return fmt.Errorf("could not stat path: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
manifestList, err := c.List(hash, path, credentials)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
switch len(manifestList.Entries) {
|
|
|
|
case 0:
|
|
|
|
return fmt.Errorf("could not find path requested at manifest address. make sure the path you've specified is correct")
|
|
|
|
case 1:
|
|
|
|
//continue
|
|
|
|
default:
|
|
|
|
return fmt.Errorf("got too many matches for this path")
|
|
|
|
}
|
|
|
|
|
|
|
|
uri := c.Gateway + "/bzz:/" + hash + "/" + path
|
|
|
|
req, err := http.NewRequest("GET", uri, nil)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if credentials != "" {
|
|
|
|
req.SetBasicAuth("", credentials)
|
|
|
|
}
|
|
|
|
res, err := http.DefaultClient.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
switch res.StatusCode {
|
|
|
|
case http.StatusOK:
|
|
|
|
case http.StatusUnauthorized:
|
|
|
|
return ErrUnauthorized
|
|
|
|
default:
|
|
|
|
return fmt.Errorf("unexpected HTTP status: expected 200 OK, got %d", res.StatusCode)
|
|
|
|
}
|
|
|
|
filename := ""
|
|
|
|
if hasDestinationFilename {
|
|
|
|
filename = dest
|
|
|
|
} else {
|
|
|
|
// try to assert
|
|
|
|
re := regexp.MustCompile("[^/]+$") //everything after last slash
|
|
|
|
|
|
|
|
if results := re.FindAllString(path, -1); len(results) > 0 {
|
|
|
|
filename = results[len(results)-1]
|
|
|
|
} else {
|
|
|
|
if entry := manifestList.Entries[0]; entry.Path != "" && entry.Path != "/" {
|
|
|
|
filename = entry.Path
|
|
|
|
} else {
|
|
|
|
// assume hash as name if there's nothing from the command line
|
|
|
|
filename = hash
|
|
|
|
}
|
|
|
|
}
|
|
|
|
filename = filepath.Join(dest, filename)
|
|
|
|
}
|
|
|
|
filePath, err := filepath.Abs(filename)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := os.MkdirAll(filepath.Dir(filePath), 0777); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
dst, err := os.Create(filename)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer dst.Close()
|
|
|
|
|
|
|
|
_, err = io.Copy(dst, res.Body)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// UploadManifest uploads the given manifest to swarm
|
|
|
|
func (c *Client) UploadManifest(m *api.Manifest, toEncrypt bool) (string, error) {
|
|
|
|
data, err := json.Marshal(m)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return c.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt)
|
|
|
|
}
|
|
|
|
|
|
|
|
// DownloadManifest downloads a swarm manifest
|
|
|
|
func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) {
|
|
|
|
res, isEncrypted, err := c.DownloadRaw(hash)
|
|
|
|
if err != nil {
|
|
|
|
return nil, isEncrypted, err
|
|
|
|
}
|
|
|
|
defer res.Close()
|
|
|
|
var manifest api.Manifest
|
|
|
|
if err := json.NewDecoder(res).Decode(&manifest); err != nil {
|
|
|
|
return nil, isEncrypted, err
|
|
|
|
}
|
|
|
|
return &manifest, isEncrypted, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// List list files in a swarm manifest which have the given prefix, grouping
|
|
|
|
// common prefixes using "/" as a delimiter.
|
|
|
|
//
|
|
|
|
// For example, if the manifest represents the following directory structure:
|
|
|
|
//
|
|
|
|
// file1.txt
|
|
|
|
// file2.txt
|
|
|
|
// dir1/file3.txt
|
|
|
|
// dir1/dir2/file4.txt
|
|
|
|
//
|
|
|
|
// Then:
|
|
|
|
//
|
|
|
|
// - a prefix of "" would return [dir1/, file1.txt, file2.txt]
|
|
|
|
// - a prefix of "file" would return [file1.txt, file2.txt]
|
|
|
|
// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
|
|
|
|
//
|
|
|
|
// where entries ending with "/" are common prefixes.
|
|
|
|
func (c *Client) List(hash, prefix, credentials string) (*api.ManifestList, error) {
|
|
|
|
req, err := http.NewRequest(http.MethodGet, c.Gateway+"/bzz-list:/"+hash+"/"+prefix, nil)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if credentials != "" {
|
|
|
|
req.SetBasicAuth("", credentials)
|
|
|
|
}
|
|
|
|
res, err := http.DefaultClient.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
switch res.StatusCode {
|
|
|
|
case http.StatusOK:
|
|
|
|
case http.StatusUnauthorized:
|
|
|
|
return nil, ErrUnauthorized
|
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
|
|
|
|
}
|
|
|
|
var list api.ManifestList
|
|
|
|
if err := json.NewDecoder(res.Body).Decode(&list); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return &list, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Uploader uploads files to swarm using a provided UploadFn
|
|
|
|
type Uploader interface {
|
|
|
|
Upload(UploadFn) error
|
|
|
|
}
|
|
|
|
|
|
|
|
type UploaderFunc func(UploadFn) error
|
|
|
|
|
|
|
|
func (u UploaderFunc) Upload(upload UploadFn) error {
|
|
|
|
return u(upload)
|
|
|
|
}
|
|
|
|
|
|
|
|
// DirectoryUploader uploads all files in a directory, optionally uploading
|
|
|
|
// a file to the default path
|
|
|
|
type DirectoryUploader struct {
|
|
|
|
Dir string
|
|
|
|
}
|
|
|
|
|
|
|
|
// Upload performs the upload of the directory and default path
|
|
|
|
func (d *DirectoryUploader) Upload(upload UploadFn) error {
|
|
|
|
return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if f.IsDir() {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
file, err := Open(path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
relPath, err := filepath.Rel(d.Dir, path)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
file.Path = filepath.ToSlash(relPath)
|
|
|
|
return upload(file)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
// FileUploader uploads a single file
|
|
|
|
type FileUploader struct {
|
|
|
|
File *File
|
|
|
|
}
|
|
|
|
|
|
|
|
// Upload performs the upload of the file
|
|
|
|
func (f *FileUploader) Upload(upload UploadFn) error {
|
|
|
|
return upload(f.File)
|
|
|
|
}
|
|
|
|
|
|
|
|
// UploadFn is the type of function passed to an Uploader to perform the upload
|
|
|
|
// of a single file (for example, a directory uploader would call a provided
|
|
|
|
// UploadFn for each file in the directory tree)
|
|
|
|
type UploadFn func(file *File) error
|
|
|
|
|
|
|
|
// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
|
|
|
|
// returning the resulting manifest hash
|
|
|
|
func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) {
|
|
|
|
ctx, sp := spancontext.StartSpan(context.Background(), "api.client.tarupload")
|
|
|
|
defer sp.Finish()
|
|
|
|
|
|
|
|
var tn time.Time
|
|
|
|
|
|
|
|
reqR, reqW := io.Pipe()
|
|
|
|
defer reqR.Close()
|
|
|
|
addr := hash
|
|
|
|
|
|
|
|
// If there is a hash already (a manifest), then that manifest will determine if the upload has
|
|
|
|
// to be encrypted or not. If there is no manifest then the toEncrypt parameter decides if
|
|
|
|
// there is encryption or not.
|
|
|
|
if hash == "" && toEncrypt {
|
|
|
|
// This is the built-in address for the encrypted upload endpoint
|
|
|
|
addr = "encrypt"
|
|
|
|
}
|
|
|
|
req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+addr, reqR)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
trace := GetClientTrace("swarm api client - upload tar", "api.client.uploadtar", uuid.New()[:8], &tn)
|
|
|
|
|
|
|
|
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
|
|
|
|
transport := http.DefaultTransport
|
|
|
|
|
|
|
|
req.Header.Set("Content-Type", "application/x-tar")
|
|
|
|
if defaultPath != "" {
|
|
|
|
q := req.URL.Query()
|
|
|
|
q.Set("defaultpath", defaultPath)
|
|
|
|
req.URL.RawQuery = q.Encode()
|
|
|
|
}
|
|
|
|
|
|
|
|
// use 'Expect: 100-continue' so we don't send the request body if
|
|
|
|
// the server refuses the request
|
|
|
|
req.Header.Set("Expect", "100-continue")
|
|
|
|
|
|
|
|
tw := tar.NewWriter(reqW)
|
|
|
|
|
|
|
|
// define an UploadFn which adds files to the tar stream
|
|
|
|
uploadFn := func(file *File) error {
|
|
|
|
hdr := &tar.Header{
|
|
|
|
Name: file.Path,
|
|
|
|
Mode: file.Mode,
|
|
|
|
Size: file.Size,
|
|
|
|
ModTime: file.ModTime,
|
|
|
|
Xattrs: map[string]string{
|
|
|
|
"user.swarm.content-type": file.ContentType,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
if err := tw.WriteHeader(hdr); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
_, err = io.Copy(tw, file)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// run the upload in a goroutine so we can send the request headers and
|
|
|
|
// wait for a '100 Continue' response before sending the tar stream
|
|
|
|
go func() {
|
|
|
|
err := uploader.Upload(uploadFn)
|
|
|
|
if err == nil {
|
|
|
|
err = tw.Close()
|
|
|
|
}
|
|
|
|
reqW.CloseWithError(err)
|
|
|
|
}()
|
|
|
|
tn = time.Now()
|
|
|
|
res, err := transport.RoundTrip(req)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
if res.StatusCode != http.StatusOK {
|
|
|
|
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
|
|
|
|
}
|
|
|
|
data, err := ioutil.ReadAll(res.Body)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return string(data), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// MultipartUpload uses the given Uploader to upload files to swarm as a
|
|
|
|
// multipart form, returning the resulting manifest hash
|
|
|
|
func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) {
|
|
|
|
reqR, reqW := io.Pipe()
|
|
|
|
defer reqR.Close()
|
|
|
|
req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
// use 'Expect: 100-continue' so we don't send the request body if
|
|
|
|
// the server refuses the request
|
|
|
|
req.Header.Set("Expect", "100-continue")
|
|
|
|
|
|
|
|
mw := multipart.NewWriter(reqW)
|
|
|
|
req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
|
|
|
|
|
|
|
|
// define an UploadFn which adds files to the multipart form
|
|
|
|
uploadFn := func(file *File) error {
|
|
|
|
hdr := make(textproto.MIMEHeader)
|
|
|
|
hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path))
|
|
|
|
hdr.Set("Content-Type", file.ContentType)
|
|
|
|
hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10))
|
|
|
|
w, err := mw.CreatePart(hdr)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
_, err = io.Copy(w, file)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// run the upload in a goroutine so we can send the request headers and
|
|
|
|
// wait for a '100 Continue' response before sending the multipart form
|
|
|
|
go func() {
|
|
|
|
err := uploader.Upload(uploadFn)
|
|
|
|
if err == nil {
|
|
|
|
err = mw.Close()
|
|
|
|
}
|
|
|
|
reqW.CloseWithError(err)
|
|
|
|
}()
|
|
|
|
|
|
|
|
res, err := http.DefaultClient.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
if res.StatusCode != http.StatusOK {
|
|
|
|
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
|
|
|
|
}
|
|
|
|
data, err := ioutil.ReadAll(res.Body)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return string(data), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ErrNoFeedUpdatesFound is returned when Swarm cannot find updates of the given feed
|
|
|
|
var ErrNoFeedUpdatesFound = errors.New("No updates found for this feed")
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
|
|
|
|
// CreateFeedWithManifest creates a feed manifest, initializing it with the provided
|
|
|
|
// data
|
|
|
|
// Returns the resulting feed manifest address that you can use to include in an ENS Resolver (setContent)
|
|
|
|
// or reference future updates (Client.UpdateFeed)
|
|
|
|
func (c *Client) CreateFeedWithManifest(request *feed.Request) (string, error) {
|
|
|
|
responseStream, err := c.updateFeed(request, true)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
defer responseStream.Close()
|
|
|
|
|
|
|
|
body, err := ioutil.ReadAll(responseStream)
|
|
|
|
if err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
var manifestAddress string
|
|
|
|
if err = json.Unmarshal(body, &manifestAddress); err != nil {
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
return manifestAddress, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// UpdateFeed allows you to set a new version of your content
|
|
|
|
func (c *Client) UpdateFeed(request *feed.Request) error {
|
|
|
|
_, err := c.updateFeed(request, false)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *Client) updateFeed(request *feed.Request, createManifest bool) (io.ReadCloser, error) {
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
URL, err := url.Parse(c.Gateway)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
URL.Path = "/bzz-feed:/"
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
values := URL.Query()
|
|
|
|
body := request.AppendValues(values)
|
|
|
|
if createManifest {
|
|
|
|
values.Set("manifest", "1")
|
|
|
|
}
|
|
|
|
URL.RawQuery = values.Encode()
|
|
|
|
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
req, err := http.NewRequest("POST", URL.String(), bytes.NewBuffer(body))
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
res, err := http.DefaultClient.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return res.Body, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// QueryFeed returns a byte stream with the raw content of the feed update
|
|
|
|
// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver
|
|
|
|
// points to that address
|
|
|
|
func (c *Client) QueryFeed(query *feed.Query, manifestAddressOrDomain string) (io.ReadCloser, error) {
|
|
|
|
return c.queryFeed(query, manifestAddressOrDomain, false)
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
}
|
|
|
|
|
|
|
|
// queryFeed returns a byte stream with the raw content of the feed update
|
|
|
|
// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
// points to that address
|
|
|
|
// meta set to true will instruct the node return feed metainformation instead
|
|
|
|
func (c *Client) queryFeed(query *feed.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) {
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
URL, err := url.Parse(c.Gateway)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
URL.Path = "/bzz-feed:/" + manifestAddressOrDomain
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
values := URL.Query()
|
|
|
|
if query != nil {
|
|
|
|
query.AppendValues(values) //adds query parameters
|
|
|
|
}
|
|
|
|
if meta {
|
|
|
|
values.Set("meta", "1")
|
|
|
|
}
|
|
|
|
URL.RawQuery = values.Encode()
|
|
|
|
res, err := http.Get(URL.String())
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if res.StatusCode != http.StatusOK {
|
|
|
|
if res.StatusCode == http.StatusNotFound {
|
|
|
|
return nil, ErrNoFeedUpdatesFound
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
}
|
|
|
|
errorMessageBytes, err := ioutil.ReadAll(res.Body)
|
|
|
|
var errorMessage string
|
|
|
|
if err != nil {
|
|
|
|
errorMessage = "cannot retrieve error message: " + err.Error()
|
|
|
|
} else {
|
|
|
|
errorMessage = string(errorMessageBytes)
|
|
|
|
}
|
|
|
|
return nil, fmt.Errorf("Error retrieving feed updates: %s", errorMessage)
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
}
|
|
|
|
|
Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)
* swarm/storage/mru: Adaptive Frequency
swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor
swarm/api/client: better error handling in GetResource()
swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID.
Renamed ResourceID.Name to ResourceID.Topic
swarm/storage/mru: Added binarySerializer interface and test tools
swarm/storage/mru/lookup: Changed base time to time and + marshallers
swarm/storage/mru: Added ResourceID (former resourceMetadata)
swarm/storage/mru: Added ResourceViewId and serialization tests
swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals
swarm/storage/mru: Fixes as per review comments
cmd/swarm: reworded resource create/update help text regarding topic
swarm/storage/mru: Added UpdateLookup and serializer tests
swarm/storage/mru: Added UpdateHeader, serializers and tests
swarm/storage/mru: changed UpdateAddr / epoch to Base()
swarm/storage/mru: Added resourceUpdate serializer and tests
swarm/storage/mru: Added SignedResourceUpdate tests and serializers
swarm/storage/mru/lookup: fixed GetFirstEpoch bug
swarm/storage/mru: refactor, comments, cleanup
Also added tests for Topic
swarm/storage/mru: handler tests pass
swarm/storage/mru: all resource package tests pass
swarm/storage/mru: resource test pass after adding
timestamp checking support
swarm/storage/mru: Added JSON serializers to ResourceIDView structures
swarm/storage/mru: Sever, client, API test pass
swarm/storage/mru: server test pass
swarm/storage/mru: Added topic length check
swarm/storage/mru: removed some literals,
improved "previous lookup" test case
swarm/storage/mru: some fixes and comments as per review
swarm/storage/mru: first working version without metadata chunk
swarm/storage/mru: Various fixes as per review
swarm/storage/mru: client test pass
swarm/storage/mru: resource query strings and manifest-less queries
swarm/storage/mru: simplify naming
swarm/storage/mru: first autofreq working version
swarm/storage/mru: renamed ToValues to AppendValues
swarm/resource/mru: Added ToValues / FromValues for URL query strings
swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.
swarm/storage/mru: removed resourceid
swarm/storage/mru: Opened up structures
swarm/storage/mru: Merged Request and SignedResourceUpdate
swarm/storage/mru: removed initial data from CLI resource create
swarm/storage/mru: Refactor Topic as a direct fixed-length array
swarm/storage/mru/lookup: Comprehensive GetNextLevel tests
swarm/storage/mru: Added comments
Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments
swarm/storage/mru/lookup: new optimized lookup algorithm
swarm/api: moved getResourceView to api out of server
swarm/storage/mru: Lookup algorithm working
swarm/storage/mru: comments and renamed NewLookupParams
Deleted commented code
swarm/storage/mru/lookup: renamed Epoch.LaterThan to After
swarm/storage/mru/lookup: Comments and tidying naming
swarm/storage/mru: fix lookup algorithm
swarm/storage/mru: exposed lookup hint
removed updateheader
swarm/storage/mru/lookup: changed GetNextEpoch for initial values
swarm/storage/mru: resource tests pass
swarm/storage/mru: valueSerializer interface and tests
swarm/storage/mru/lookup: Comments, improvements, fixes, more tests
swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query
swarm/storage/mru: renamed query receiver var
swarm/cmd: MRU CLI tests
* cmd/swarm: remove rogue fmt
* swarm/storage/mru: Add version / header for future use
* swarm/storage/mru: Fixes/comments as per review
cmd/swarm: remove rogue fmt
swarm/storage/mru: Add version / header for future use-
* swarm/storage/mru: fix linter errors
* cmd/swarm: Speeded up TestCLIResourceUpdate
6 years ago
|
|
|
return res.Body, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// GetFeedRequest returns a structure that describes the referenced feed status
|
|
|
|
// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver
|
|
|
|
// points to that address
|
|
|
|
func (c *Client) GetFeedRequest(query *feed.Query, manifestAddressOrDomain string) (*feed.Request, error) {
|
|
|
|
|
|
|
|
responseStream, err := c.queryFeed(query, manifestAddressOrDomain, true)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer responseStream.Close()
|
|
|
|
|
|
|
|
body, err := ioutil.ReadAll(responseStream)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var metadata feed.Request
|
|
|
|
if err := metadata.UnmarshalJSON(body); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return &metadata, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func GetClientTrace(traceMsg, metricPrefix, ruid string, tn *time.Time) *httptrace.ClientTrace {
|
|
|
|
trace := &httptrace.ClientTrace{
|
|
|
|
GetConn: func(_ string) {
|
|
|
|
log.Trace(traceMsg+" - http get", "event", "GetConn", "ruid", ruid)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".getconn", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
GotConn: func(_ httptrace.GotConnInfo) {
|
|
|
|
log.Trace(traceMsg+" - http get", "event", "GotConn", "ruid", ruid)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".gotconn", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
PutIdleConn: func(err error) {
|
|
|
|
log.Trace(traceMsg+" - http get", "event", "PutIdleConn", "ruid", ruid, "err", err)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".putidle", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
GotFirstResponseByte: func() {
|
|
|
|
log.Trace(traceMsg+" - http get", "event", "GotFirstResponseByte", "ruid", ruid)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".firstbyte", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
Got100Continue: func() {
|
|
|
|
log.Trace(traceMsg, "event", "Got100Continue", "ruid", ruid)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".got100continue", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
DNSStart: func(_ httptrace.DNSStartInfo) {
|
|
|
|
log.Trace(traceMsg, "event", "DNSStart", "ruid", ruid)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsstart", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
DNSDone: func(_ httptrace.DNSDoneInfo) {
|
|
|
|
log.Trace(traceMsg, "event", "DNSDone", "ruid", ruid)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsdone", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
ConnectStart: func(network, addr string) {
|
|
|
|
log.Trace(traceMsg, "event", "ConnectStart", "ruid", ruid, "network", network, "addr", addr)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".connectstart", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
ConnectDone: func(network, addr string, err error) {
|
|
|
|
log.Trace(traceMsg, "event", "ConnectDone", "ruid", ruid, "network", network, "addr", addr, "err", err)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".connectdone", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
WroteHeaders: func() {
|
|
|
|
log.Trace(traceMsg, "event", "WroteHeaders(request)", "ruid", ruid)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".wroteheaders", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
Wait100Continue: func() {
|
|
|
|
log.Trace(traceMsg, "event", "Wait100Continue", "ruid", ruid)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".wait100continue", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
WroteRequest: func(_ httptrace.WroteRequestInfo) {
|
|
|
|
log.Trace(traceMsg, "event", "WroteRequest", "ruid", ruid)
|
|
|
|
metrics.GetOrRegisterResettingTimer(metricPrefix+".wroterequest", nil).Update(time.Since(*tn))
|
|
|
|
},
|
|
|
|
}
|
|
|
|
return trace
|
|
|
|
}
|