mirror of https://github.com/ethereum/go-ethereum
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
479 lines
13 KiB
479 lines
13 KiB
// Package storage provides clients for Microsoft Azure Storage Services.
|
|
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"net/http"
|
|
"net/url"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/Azure/go-autorest/autorest/azure"
|
|
)
|
|
|
|
const (
|
|
// DefaultBaseURL is the domain name used for storage requests in the
|
|
// public cloud when a default client is created.
|
|
DefaultBaseURL = "core.windows.net"
|
|
|
|
// DefaultAPIVersion is the Azure Storage API version string used when a
|
|
// basic client is created.
|
|
DefaultAPIVersion = "2016-05-31"
|
|
|
|
defaultUseHTTPS = true
|
|
|
|
// StorageEmulatorAccountName is the fixed storage account used by Azure Storage Emulator
|
|
StorageEmulatorAccountName = "devstoreaccount1"
|
|
|
|
// StorageEmulatorAccountKey is the the fixed storage account used by Azure Storage Emulator
|
|
StorageEmulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
|
|
|
|
blobServiceName = "blob"
|
|
tableServiceName = "table"
|
|
queueServiceName = "queue"
|
|
fileServiceName = "file"
|
|
|
|
storageEmulatorBlob = "127.0.0.1:10000"
|
|
storageEmulatorTable = "127.0.0.1:10002"
|
|
storageEmulatorQueue = "127.0.0.1:10001"
|
|
|
|
userAgentHeader = "User-Agent"
|
|
)
|
|
|
|
// Client is the object that needs to be constructed to perform
|
|
// operations on the storage account.
|
|
type Client struct {
|
|
// HTTPClient is the http.Client used to initiate API
|
|
// requests. If it is nil, http.DefaultClient is used.
|
|
HTTPClient *http.Client
|
|
|
|
accountName string
|
|
accountKey []byte
|
|
useHTTPS bool
|
|
UseSharedKeyLite bool
|
|
baseURL string
|
|
apiVersion string
|
|
userAgent string
|
|
}
|
|
|
|
type storageResponse struct {
|
|
statusCode int
|
|
headers http.Header
|
|
body io.ReadCloser
|
|
}
|
|
|
|
type odataResponse struct {
|
|
storageResponse
|
|
odata odataErrorMessage
|
|
}
|
|
|
|
// AzureStorageServiceError contains fields of the error response from
|
|
// Azure Storage Service REST API. See https://msdn.microsoft.com/en-us/library/azure/dd179382.aspx
|
|
// Some fields might be specific to certain calls.
|
|
type AzureStorageServiceError struct {
|
|
Code string `xml:"Code"`
|
|
Message string `xml:"Message"`
|
|
AuthenticationErrorDetail string `xml:"AuthenticationErrorDetail"`
|
|
QueryParameterName string `xml:"QueryParameterName"`
|
|
QueryParameterValue string `xml:"QueryParameterValue"`
|
|
Reason string `xml:"Reason"`
|
|
StatusCode int
|
|
RequestID string
|
|
}
|
|
|
|
type odataErrorMessageMessage struct {
|
|
Lang string `json:"lang"`
|
|
Value string `json:"value"`
|
|
}
|
|
|
|
type odataErrorMessageInternal struct {
|
|
Code string `json:"code"`
|
|
Message odataErrorMessageMessage `json:"message"`
|
|
}
|
|
|
|
type odataErrorMessage struct {
|
|
Err odataErrorMessageInternal `json:"odata.error"`
|
|
}
|
|
|
|
// UnexpectedStatusCodeError is returned when a storage service responds with neither an error
|
|
// nor with an HTTP status code indicating success.
|
|
type UnexpectedStatusCodeError struct {
|
|
allowed []int
|
|
got int
|
|
}
|
|
|
|
func (e UnexpectedStatusCodeError) Error() string {
|
|
s := func(i int) string { return fmt.Sprintf("%d %s", i, http.StatusText(i)) }
|
|
|
|
got := s(e.got)
|
|
expected := []string{}
|
|
for _, v := range e.allowed {
|
|
expected = append(expected, s(v))
|
|
}
|
|
return fmt.Sprintf("storage: status code from service response is %s; was expecting %s", got, strings.Join(expected, " or "))
|
|
}
|
|
|
|
// Got is the actual status code returned by Azure.
|
|
func (e UnexpectedStatusCodeError) Got() int {
|
|
return e.got
|
|
}
|
|
|
|
// NewBasicClient constructs a Client with given storage service name and
|
|
// key.
|
|
func NewBasicClient(accountName, accountKey string) (Client, error) {
|
|
if accountName == StorageEmulatorAccountName {
|
|
return NewEmulatorClient()
|
|
}
|
|
return NewClient(accountName, accountKey, DefaultBaseURL, DefaultAPIVersion, defaultUseHTTPS)
|
|
}
|
|
|
|
// NewBasicClientOnSovereignCloud constructs a Client with given storage service name and
|
|
// key in the referenced cloud.
|
|
func NewBasicClientOnSovereignCloud(accountName, accountKey string, env azure.Environment) (Client, error) {
|
|
if accountName == StorageEmulatorAccountName {
|
|
return NewEmulatorClient()
|
|
}
|
|
return NewClient(accountName, accountKey, env.StorageEndpointSuffix, DefaultAPIVersion, defaultUseHTTPS)
|
|
}
|
|
|
|
//NewEmulatorClient contructs a Client intended to only work with Azure
|
|
//Storage Emulator
|
|
func NewEmulatorClient() (Client, error) {
|
|
return NewClient(StorageEmulatorAccountName, StorageEmulatorAccountKey, DefaultBaseURL, DefaultAPIVersion, false)
|
|
}
|
|
|
|
// NewClient constructs a Client. This should be used if the caller wants
|
|
// to specify whether to use HTTPS, a specific REST API version or a custom
|
|
// storage endpoint than Azure Public Cloud.
|
|
func NewClient(accountName, accountKey, blobServiceBaseURL, apiVersion string, useHTTPS bool) (Client, error) {
|
|
var c Client
|
|
if accountName == "" {
|
|
return c, fmt.Errorf("azure: account name required")
|
|
} else if accountKey == "" {
|
|
return c, fmt.Errorf("azure: account key required")
|
|
} else if blobServiceBaseURL == "" {
|
|
return c, fmt.Errorf("azure: base storage service url required")
|
|
}
|
|
|
|
key, err := base64.StdEncoding.DecodeString(accountKey)
|
|
if err != nil {
|
|
return c, fmt.Errorf("azure: malformed storage account key: %v", err)
|
|
}
|
|
|
|
c = Client{
|
|
accountName: accountName,
|
|
accountKey: key,
|
|
useHTTPS: useHTTPS,
|
|
baseURL: blobServiceBaseURL,
|
|
apiVersion: apiVersion,
|
|
UseSharedKeyLite: false,
|
|
}
|
|
c.userAgent = c.getDefaultUserAgent()
|
|
return c, nil
|
|
}
|
|
|
|
func (c Client) getDefaultUserAgent() string {
|
|
return fmt.Sprintf("Go/%s (%s-%s) Azure-SDK-For-Go/%s storage-dataplane/%s",
|
|
runtime.Version(),
|
|
runtime.GOARCH,
|
|
runtime.GOOS,
|
|
sdkVersion,
|
|
c.apiVersion,
|
|
)
|
|
}
|
|
|
|
// AddToUserAgent adds an extension to the current user agent
|
|
func (c *Client) AddToUserAgent(extension string) error {
|
|
if extension != "" {
|
|
c.userAgent = fmt.Sprintf("%s %s", c.userAgent, extension)
|
|
return nil
|
|
}
|
|
return fmt.Errorf("Extension was empty, User Agent stayed as %s", c.userAgent)
|
|
}
|
|
|
|
// protectUserAgent is used in funcs that include extraheaders as a parameter.
|
|
// It prevents the User-Agent header to be overwritten, instead if it happens to
|
|
// be present, it gets added to the current User-Agent. Use it before getStandardHeaders
|
|
func (c *Client) protectUserAgent(extraheaders map[string]string) map[string]string {
|
|
if v, ok := extraheaders[userAgentHeader]; ok {
|
|
c.AddToUserAgent(v)
|
|
delete(extraheaders, userAgentHeader)
|
|
}
|
|
return extraheaders
|
|
}
|
|
|
|
func (c Client) getBaseURL(service string) string {
|
|
scheme := "http"
|
|
if c.useHTTPS {
|
|
scheme = "https"
|
|
}
|
|
host := ""
|
|
if c.accountName == StorageEmulatorAccountName {
|
|
switch service {
|
|
case blobServiceName:
|
|
host = storageEmulatorBlob
|
|
case tableServiceName:
|
|
host = storageEmulatorTable
|
|
case queueServiceName:
|
|
host = storageEmulatorQueue
|
|
}
|
|
} else {
|
|
host = fmt.Sprintf("%s.%s.%s", c.accountName, service, c.baseURL)
|
|
}
|
|
|
|
u := &url.URL{
|
|
Scheme: scheme,
|
|
Host: host}
|
|
return u.String()
|
|
}
|
|
|
|
func (c Client) getEndpoint(service, path string, params url.Values) string {
|
|
u, err := url.Parse(c.getBaseURL(service))
|
|
if err != nil {
|
|
// really should not be happening
|
|
panic(err)
|
|
}
|
|
|
|
// API doesn't accept path segments not starting with '/'
|
|
if !strings.HasPrefix(path, "/") {
|
|
path = fmt.Sprintf("/%v", path)
|
|
}
|
|
|
|
if c.accountName == StorageEmulatorAccountName {
|
|
path = fmt.Sprintf("/%v%v", StorageEmulatorAccountName, path)
|
|
}
|
|
|
|
u.Path = path
|
|
u.RawQuery = params.Encode()
|
|
return u.String()
|
|
}
|
|
|
|
// GetBlobService returns a BlobStorageClient which can operate on the blob
|
|
// service of the storage account.
|
|
func (c Client) GetBlobService() BlobStorageClient {
|
|
b := BlobStorageClient{
|
|
client: c,
|
|
}
|
|
b.client.AddToUserAgent(blobServiceName)
|
|
b.auth = sharedKey
|
|
if c.UseSharedKeyLite {
|
|
b.auth = sharedKeyLite
|
|
}
|
|
return b
|
|
}
|
|
|
|
// GetQueueService returns a QueueServiceClient which can operate on the queue
|
|
// service of the storage account.
|
|
func (c Client) GetQueueService() QueueServiceClient {
|
|
q := QueueServiceClient{
|
|
client: c,
|
|
}
|
|
q.client.AddToUserAgent(queueServiceName)
|
|
q.auth = sharedKey
|
|
if c.UseSharedKeyLite {
|
|
q.auth = sharedKeyLite
|
|
}
|
|
return q
|
|
}
|
|
|
|
// GetTableService returns a TableServiceClient which can operate on the table
|
|
// service of the storage account.
|
|
func (c Client) GetTableService() TableServiceClient {
|
|
t := TableServiceClient{
|
|
client: c,
|
|
}
|
|
t.client.AddToUserAgent(tableServiceName)
|
|
t.auth = sharedKeyForTable
|
|
if c.UseSharedKeyLite {
|
|
t.auth = sharedKeyLiteForTable
|
|
}
|
|
return t
|
|
}
|
|
|
|
// GetFileService returns a FileServiceClient which can operate on the file
|
|
// service of the storage account.
|
|
func (c Client) GetFileService() FileServiceClient {
|
|
f := FileServiceClient{
|
|
client: c,
|
|
}
|
|
f.client.AddToUserAgent(fileServiceName)
|
|
f.auth = sharedKey
|
|
if c.UseSharedKeyLite {
|
|
f.auth = sharedKeyLite
|
|
}
|
|
return f
|
|
}
|
|
|
|
func (c Client) getStandardHeaders() map[string]string {
|
|
return map[string]string{
|
|
userAgentHeader: c.userAgent,
|
|
"x-ms-version": c.apiVersion,
|
|
"x-ms-date": currentTimeRfc1123Formatted(),
|
|
}
|
|
}
|
|
|
|
func (c Client) exec(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*storageResponse, error) {
|
|
headers, err := c.addAuthorizationHeader(verb, url, headers, auth)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
req, err := http.NewRequest(verb, url, body)
|
|
if err != nil {
|
|
return nil, errors.New("azure/storage: error creating request: " + err.Error())
|
|
}
|
|
|
|
if clstr, ok := headers["Content-Length"]; ok {
|
|
// content length header is being signed, but completely ignored by golang.
|
|
// instead we have to use the ContentLength property on the request struct
|
|
// (see https://golang.org/src/net/http/request.go?s=18140:18370#L536 and
|
|
// https://golang.org/src/net/http/transfer.go?s=1739:2467#L49)
|
|
req.ContentLength, err = strconv.ParseInt(clstr, 10, 64)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
for k, v := range headers {
|
|
req.Header.Add(k, v)
|
|
}
|
|
|
|
httpClient := c.HTTPClient
|
|
if httpClient == nil {
|
|
httpClient = http.DefaultClient
|
|
}
|
|
resp, err := httpClient.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
statusCode := resp.StatusCode
|
|
if statusCode >= 400 && statusCode <= 505 {
|
|
var respBody []byte
|
|
respBody, err = readAndCloseBody(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
requestID := resp.Header.Get("x-ms-request-id")
|
|
if len(respBody) == 0 {
|
|
// no error in response body, might happen in HEAD requests
|
|
err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID)
|
|
} else {
|
|
// response contains storage service error object, unmarshal
|
|
storageErr, errIn := serviceErrFromXML(respBody, resp.StatusCode, requestID)
|
|
if err != nil { // error unmarshaling the error response
|
|
err = errIn
|
|
}
|
|
err = storageErr
|
|
}
|
|
return &storageResponse{
|
|
statusCode: resp.StatusCode,
|
|
headers: resp.Header,
|
|
body: ioutil.NopCloser(bytes.NewReader(respBody)), /* restore the body */
|
|
}, err
|
|
}
|
|
|
|
return &storageResponse{
|
|
statusCode: resp.StatusCode,
|
|
headers: resp.Header,
|
|
body: resp.Body}, nil
|
|
}
|
|
|
|
func (c Client) execInternalJSON(verb, url string, headers map[string]string, body io.Reader, auth authentication) (*odataResponse, error) {
|
|
headers, err := c.addAuthorizationHeader(verb, url, headers, auth)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
req, err := http.NewRequest(verb, url, body)
|
|
for k, v := range headers {
|
|
req.Header.Add(k, v)
|
|
}
|
|
|
|
httpClient := c.HTTPClient
|
|
if httpClient == nil {
|
|
httpClient = http.DefaultClient
|
|
}
|
|
|
|
resp, err := httpClient.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
respToRet := &odataResponse{}
|
|
respToRet.body = resp.Body
|
|
respToRet.statusCode = resp.StatusCode
|
|
respToRet.headers = resp.Header
|
|
|
|
statusCode := resp.StatusCode
|
|
if statusCode >= 400 && statusCode <= 505 {
|
|
var respBody []byte
|
|
respBody, err = readAndCloseBody(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if len(respBody) == 0 {
|
|
// no error in response body, might happen in HEAD requests
|
|
err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, resp.Header.Get("x-ms-request-id"))
|
|
return respToRet, err
|
|
}
|
|
// try unmarshal as odata.error json
|
|
err = json.Unmarshal(respBody, &respToRet.odata)
|
|
return respToRet, err
|
|
}
|
|
|
|
return respToRet, nil
|
|
}
|
|
|
|
func readAndCloseBody(body io.ReadCloser) ([]byte, error) {
|
|
defer body.Close()
|
|
out, err := ioutil.ReadAll(body)
|
|
if err == io.EOF {
|
|
err = nil
|
|
}
|
|
return out, err
|
|
}
|
|
|
|
func serviceErrFromXML(body []byte, statusCode int, requestID string) (AzureStorageServiceError, error) {
|
|
var storageErr AzureStorageServiceError
|
|
if err := xml.Unmarshal(body, &storageErr); err != nil {
|
|
return storageErr, err
|
|
}
|
|
storageErr.StatusCode = statusCode
|
|
storageErr.RequestID = requestID
|
|
return storageErr, nil
|
|
}
|
|
|
|
func serviceErrFromStatusCode(code int, status string, requestID string) AzureStorageServiceError {
|
|
return AzureStorageServiceError{
|
|
StatusCode: code,
|
|
Code: status,
|
|
RequestID: requestID,
|
|
Message: "no response body was available for error status code",
|
|
}
|
|
}
|
|
|
|
func (e AzureStorageServiceError) Error() string {
|
|
return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s, QueryParameterName=%s, QueryParameterValue=%s",
|
|
e.StatusCode, e.Code, e.Message, e.RequestID, e.QueryParameterName, e.QueryParameterValue)
|
|
}
|
|
|
|
// checkRespCode returns UnexpectedStatusError if the given response code is not
|
|
// one of the allowed status codes; otherwise nil.
|
|
func checkRespCode(respCode int, allowed []int) error {
|
|
for _, v := range allowed {
|
|
if respCode == v {
|
|
return nil
|
|
}
|
|
}
|
|
return UnexpectedStatusCodeError{allowed, respCode}
|
|
}
|
|
|