From 7c3281861f24b8cb8c480f9525ee60ca602f3c0c Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Tue, 3 Nov 2015 09:59:50 +0100 Subject: [PATCH] Add support for temporary URL for Swift driver Signed-off-by: Sylvain Baubeau --- docs/storage-drivers/swift.md | 64 +++++++ registry/storage/driver/swift/swift.go | 198 +++++++++++++++----- registry/storage/driver/swift/swift_test.go | 18 +- 3 files changed, 234 insertions(+), 46 deletions(-) diff --git a/docs/storage-drivers/swift.md b/docs/storage-drivers/swift.md index 123ad92e..b2f937da 100644 --- a/docs/storage-drivers/swift.md +++ b/docs/storage-drivers/swift.md @@ -93,6 +93,16 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open

+ + + trustid + + +

+ Optionally, your OpenStack trust id for Identity v3 API. +

+ + insecureskipverify @@ -133,4 +143,58 @@ An implementation of the `storagedriver.StorageDriver` interface that uses [Open

+ + + secretkey + + +

+ Optionally, the secret key used to generate temporary URLs.

+

+ + + + + accesskey + + +

+ Optionally, the access key to generate temporary URLs. It is used by HP Cloud Object Storage in addition to the `secretkey` parameter.

+

+ + + + +The features supported by the Swift server are queried by requesting the `/info` URL on the server. In case the administrator +disabled that feature, the configuration file can specify the following optional parameters : + + + + + + + + + +
+ tempurlcontainerkey + +

+ Specify whether to use container secret key to generate temporary URL when set to true, or the account secret key otherwise.

+

+
+ tempurlmethods + +

+ Array of HTTP methods that are supported by the TempURL middleware of the Swift server. Example:

+ + - tempurlmethods: + - GET + - PUT + - HEAD + - POST + - DELETE + +

+
diff --git a/registry/storage/driver/swift/swift.go b/registry/storage/driver/swift/swift.go index c9d623d3..3b2cdc53 100644 --- a/registry/storage/driver/swift/swift.go +++ b/registry/storage/driver/swift/swift.go @@ -7,9 +7,6 @@ // It supports both TempAuth authentication and Keystone authentication // (up to version 3). // -// Since Swift has no concept of directories (directories are an abstration), -// empty objects are created with the MIME type application/vnd.swift.directory. -// // As Swift has a limit on the size of a single uploaded object (by default // this is 5GB), the driver makes use of the Swift Large Object Support // (http://docs.openstack.org/developer/swift/overview_large_objects.html). @@ -24,12 +21,11 @@ import ( "crypto/sha1" "crypto/tls" "encoding/hex" - "encoding/json" "fmt" "io" "io/ioutil" "net/http" - gopath "path" + "net/url" "strconv" "strings" "time" @@ -54,22 +50,34 @@ const minChunkSize = 1 << 20 // Parameters A struct that encapsulates all of the driver parameters after all values have been set type Parameters struct { - Username string - Password string - AuthURL string - Tenant string - TenantID string - Domain string - DomainID string - TrustID string - Region string - Container string - Prefix string - InsecureSkipVerify bool - ChunkSize int + Username string + Password string + AuthURL string + Tenant string + TenantID string + Domain string + DomainID string + TrustID string + Region string + Container string + Prefix string + InsecureSkipVerify bool + ChunkSize int + SecretKey string + AccessKey string + TempURLContainerKey bool + TempURLMethods []string } -type swiftInfo map[string]interface{} +// swiftInfo maps the JSON structure returned by Swift /info endpoint +type swiftInfo struct { + Swift struct { + Version string `mapstructure:"version"` + } + Tempurl struct { + Methods []string `mapstructure:"methods"` + } +} func init() { factory.Register(driverName, &swiftDriverFactory{}) @@ -83,11 +91,15 @@ func (factory *swiftDriverFactory) Create(parameters map[string]interface{}) (st } type driver struct { - Conn swift.Connection - Container string - Prefix string - BulkDeleteSupport bool - ChunkSize int + Conn swift.Connection + Container string + Prefix string + BulkDeleteSupport bool + ChunkSize int + SecretKey string + AccessKey string + TempURLContainerKey bool + TempURLMethods []string } type baseEmbed struct { @@ -176,11 +188,65 @@ func New(params Parameters) (*Driver, error) { } d := &driver{ - Conn: ct, - Container: params.Container, - Prefix: params.Prefix, - BulkDeleteSupport: detectBulkDelete(params.AuthURL), - ChunkSize: params.ChunkSize, + Conn: ct, + Container: params.Container, + Prefix: params.Prefix, + ChunkSize: params.ChunkSize, + TempURLMethods: make([]string, 0), + AccessKey: params.AccessKey, + } + + info := swiftInfo{} + if config, err := d.Conn.QueryInfo(); err == nil { + _, d.BulkDeleteSupport = config["bulk_delete"] + + if err := mapstructure.Decode(config, &info); err == nil { + d.TempURLContainerKey = info.Swift.Version >= "2.3.0" + d.TempURLMethods = info.Tempurl.Methods + } + } else { + d.TempURLContainerKey = params.TempURLContainerKey + d.TempURLMethods = params.TempURLMethods + } + + if len(d.TempURLMethods) > 0 { + secretKey := params.SecretKey + if secretKey == "" { + secretKey, _ = generateSecret() + } + + // Since Swift 2.2.2, we can now set secret keys on containers + // in addition to the account secret keys. Use them in preference. + if d.TempURLContainerKey { + _, containerHeaders, err := d.Conn.Container(d.Container) + if err != nil { + return nil, fmt.Errorf("Failed to fetch container info %s (%s)", d.Container, err) + } + + d.SecretKey = containerHeaders["X-Container-Meta-Temp-Url-Key"] + if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) { + m := swift.Metadata{} + m["temp-url-key"] = secretKey + if d.Conn.ContainerUpdate(d.Container, m.ContainerHeaders()); err == nil { + d.SecretKey = secretKey + } + } + } else { + // Use the account secret key + _, accountHeaders, err := d.Conn.Account() + if err != nil { + return nil, fmt.Errorf("Failed to fetch account info (%s)", err) + } + + d.SecretKey = accountHeaders["X-Account-Meta-Temp-Url-Key"] + if d.SecretKey == "" || (params.SecretKey != "" && d.SecretKey != params.SecretKey) { + m := swift.Metadata{} + m["temp-url-key"] = secretKey + if err := d.Conn.AccountUpdate(m.AccountHeaders()); err == nil { + d.SecretKey = secretKey + } + } + } } return &Driver{ @@ -590,9 +656,58 @@ func (d *driver) Delete(ctx context.Context, path string) error { } // URLFor returns a URL which may be used to retrieve the content stored at the given path. -// May return an UnsupportedMethodErr in certain StorageDriver implementations. func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) { - return "", storagedriver.ErrUnsupportedMethod + if d.SecretKey == "" { + return "", storagedriver.ErrUnsupportedMethod + } + + methodString := "GET" + method, ok := options["method"] + if ok { + if methodString, ok = method.(string); !ok { + return "", storagedriver.ErrUnsupportedMethod + } + } + + if methodString == "HEAD" { + // A "HEAD" request on a temporary URL is allowed if the + // signature was generated with "GET", "POST" or "PUT" + methodString = "GET" + } + + supported := false + for _, method := range d.TempURLMethods { + if method == methodString { + supported = true + break + } + } + + if !supported { + return "", storagedriver.ErrUnsupportedMethod + } + + expiresTime := time.Now().Add(20 * time.Minute) + expires, ok := options["expiry"] + if ok { + et, ok := expires.(time.Time) + if ok { + expiresTime = et + } + } + + tempURL := d.Conn.ObjectTempUrl(d.Container, d.swiftPath(path), d.SecretKey, methodString, expiresTime) + + if d.AccessKey != "" { + // On HP Cloud, the signature must be in the form of tenant_id:access_key:signature + url, _ := url.Parse(tempURL) + query := url.Query() + query.Set("temp_url_sig", fmt.Sprintf("%s:%s:%s", d.Conn.TenantId, d.AccessKey, query.Get("temp_url_sig"))) + url.RawQuery = query.Encode() + tempURL = url.String() + } + + return tempURL, nil } func (d *driver) swiftPath(path string) string { @@ -640,19 +755,6 @@ func (d *driver) createManifest(path string, segments string) error { return nil } -func detectBulkDelete(authURL string) (bulkDelete bool) { - resp, err := http.Get(gopath.Join(authURL, "..", "..") + "/info") - if err == nil { - defer resp.Body.Close() - decoder := json.NewDecoder(resp.Body) - var infos swiftInfo - if decoder.Decode(&infos) == nil { - _, bulkDelete = infos["bulk_delete"] - } - } - return -} - func parseManifest(manifest string) (container string, prefix string) { components := strings.SplitN(manifest, "/", 2) container = components[0] @@ -661,3 +763,11 @@ func parseManifest(manifest string) (container string, prefix string) { } return container, prefix } + +func generateSecret() (string, error) { + var secretBytes [32]byte + if _, err := rand.Read(secretBytes[:]); err != nil { + return "", fmt.Errorf("could not generate random bytes for Swift secret key: %v", err) + } + return hex.EncodeToString(secretBytes[:]), nil +} diff --git a/registry/storage/driver/swift/swift_test.go b/registry/storage/driver/swift/swift_test.go index 705c2631..c4c3333c 100644 --- a/registry/storage/driver/swift/swift_test.go +++ b/registry/storage/driver/swift/swift_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "os" "strconv" + "strings" "testing" "github.com/ncw/swift/swifttest" @@ -33,8 +34,13 @@ func init() { container string region string insecureSkipVerify bool - swiftServer *swifttest.SwiftServer - err error + secretKey string + accessKey string + containerKey bool + tempURLMethods []string + + swiftServer *swifttest.SwiftServer + err error ) username = os.Getenv("SWIFT_USERNAME") password = os.Getenv("SWIFT_PASSWORD") @@ -47,6 +53,10 @@ func init() { container = os.Getenv("SWIFT_CONTAINER_NAME") region = os.Getenv("SWIFT_REGION_NAME") insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY")) + secretKey = os.Getenv("SWIFT_SECRET_KEY") + accessKey = os.Getenv("SWIFT_ACCESS_KEY") + containerKey, _ = strconv.ParseBool(os.Getenv("SWIFT_TEMPURL_CONTAINERKEY")) + tempURLMethods = strings.Split(os.Getenv("SWIFT_TEMPURL_METHODS"), ",") if username == "" || password == "" || authURL == "" || container == "" { if swiftServer, err = swifttest.NewSwiftServer("localhost"); err != nil { @@ -79,6 +89,10 @@ func init() { root, insecureSkipVerify, defaultChunkSize, + secretKey, + accessKey, + containerKey, + tempURLMethods, } return New(parameters)