Merge pull request #1401 from BrianBland/s3StorageClass

Adds "storageclass" configuration parameter for S3 driver.

docs/storage-drivers/s3.md

@@ -30,7 +30,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
       Your AWS Access Key.
     </td>
   </tr>
-    <tr>
+  <tr>
     <td>
       <code>secretkey</code>
     </td>
@@ -41,7 +41,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
       Your AWS Secret Key.
     </td>
   </tr>
-    <tr>
+  <tr>
     <td>
       <code>region</code>
     </td>
@@ -64,7 +64,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
       The bucket name in which you want to store the registry's data.
     </td>
   </tr>
-    <tr>
+  <tr>
     <td>
       <code>encrypt</code>
     </td>
@@ -76,7 +76,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
        not. A boolean value. The default is false.
     </td>
   </tr>
-    <tr>
+  <tr>
     <td>
       <code>secure</code>
     </td>
@@ -88,7 +88,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
       default is <code>true</code>.
     </td>
   </tr>
-    <tr>
+  <tr>
     <td>
       <code>v4auth</code>
     </td>
@@ -101,7 +101,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
       <code>false</code>.
     </td>
   </tr>
-    <tr>
+  <tr>
     <td>
       <code>chunksize</code>
     </td>
@@ -113,7 +113,7 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
       should be a number that is larger than 5*1024*1024.
     </td>
   </tr>
-   <tr>
+  <tr>
     <td>
       <code>rootdirectory</code>
     </td>
@@ -124,6 +124,17 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
       This is a prefix that will be applied to all S3 keys to allow you to segment data in your bucket if necessary.
     </td>
   </tr>
+  <tr>
+    <td>
+      <code>storageclass</code>
+    </td>
+    <td>
+      no
+    </td>
+    <td>
+      The S3 storage class applied to each registry file. The default value is STANDARD.
+    </td>
+  </tr>
 </table>
 
 
@@ -147,6 +158,8 @@ An implementation of the `storagedriver.StorageDriver` interface which uses Amaz
 
 `rootdirectory`: (optional) The root directory tree in which all registry files will be stored. Defaults to the empty string (bucket root).
 
+`storageclass`: (optional) The storage class applied to each registry file. Defaults to STANDARD. Valid options are STANDARD and REDUCED_REDUNDANCY.
+
 # CloudFront as Middleware with S3 backend
 
 ## Use Case

registry/storage/driver/s3/s3.go

@@ -1,7 +1,7 @@
 // Package s3 provides a storagedriver.StorageDriver implementation to
 // store blobs in Amazon S3 cloud storage.
 //
-// This package leverages the AdRoll/goamz client library for interfacing with
+// This package leverages the docker/goamz client library for interfacing with
 // s3.
 //
 // Because s3 is a key, value store the Stat call does not support last modification
@@ -59,6 +59,7 @@ type DriverParameters struct {
 	V4Auth        bool
 	ChunkSize     int64
 	RootDirectory string
+	StorageClass  s3.StorageClass
 	UserAgent     string
 }
 
@@ -79,6 +80,7 @@ type driver struct {
 	ChunkSize     int64
 	Encrypt       bool
 	RootDirectory string
+	StorageClass  s3.StorageClass
 
 	pool  sync.Pool // pool []byte buffers used for WriteStream
 	zeros []byte    // shared, zero-valued buffer used for WriteStream
@@ -183,6 +185,21 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
 		rootDirectory = ""
 	}
 
+	storageClass := s3.StandardStorage
+	storageClassParam, ok := parameters["storageclass"]
+	if ok {
+		storageClassString, ok := storageClassParam.(string)
+		if !ok {
+			return nil, fmt.Errorf("The storageclass parameter must be one of %v, %v invalid", []s3.StorageClass{s3.StandardStorage, s3.ReducedRedundancy}, storageClassParam)
+		}
+		// All valid storage class parameters are UPPERCASE, so be a bit more flexible here
+		storageClassCasted := s3.StorageClass(strings.ToUpper(storageClassString))
+		if storageClassCasted != s3.StandardStorage && storageClassCasted != s3.ReducedRedundancy {
+			return nil, fmt.Errorf("The storageclass parameter must be one of %v, %v invalid", []s3.StorageClass{s3.StandardStorage, s3.ReducedRedundancy}, storageClassParam)
+		}
+		storageClass = storageClassCasted
+	}
+
 	userAgent, ok := parameters["useragent"]
 	if !ok {
 		userAgent = ""
@@ -198,6 +215,7 @@ func FromParameters(parameters map[string]interface{}) (*Driver, error) {
 		v4AuthBool,
 		chunkSize,
 		fmt.Sprint(rootDirectory),
+		storageClass,
 		fmt.Sprint(userAgent),
 	}
 
@@ -259,6 +277,7 @@ func New(params DriverParameters) (*Driver, error) {
 		ChunkSize:     params.ChunkSize,
 		Encrypt:       params.Encrypt,
 		RootDirectory: params.RootDirectory,
+		StorageClass:  params.StorageClass,
 		zeros:         make([]byte, params.ChunkSize),
 	}
 
@@ -826,7 +845,10 @@ func hasCode(err error, code string) bool {
 }
 
 func (d *driver) getOptions() s3.Options {
-	return s3.Options{SSE: d.Encrypt}
+	return s3.Options{
+		SSE:          d.Encrypt,
+		StorageClass: d.StorageClass,
+	}
 }
 
 func getPermissions() s3.ACL {

registry/storage/driver/s3/s3_test.go

@@ -10,6 +10,7 @@ import (
 	storagedriver "github.com/docker/distribution/registry/storage/driver"
 	"github.com/docker/distribution/registry/storage/driver/testsuites"
 	"github.com/docker/goamz/aws"
+	"github.com/docker/goamz/s3"
 
 	"gopkg.in/check.v1"
 )
@@ -17,7 +18,7 @@ import (
 // Hook up gocheck into the "go test" runner.
 func Test(t *testing.T) { check.TestingT(t) }
 
-var s3DriverConstructor func(rootDirectory string) (*Driver, error)
+var s3DriverConstructor func(rootDirectory string, storageClass s3.StorageClass) (*Driver, error)
 var skipS3 func() string
 
 func init() {
@@ -34,7 +35,7 @@ func init() {
 	}
 	defer os.Remove(root)
 
-	s3DriverConstructor = func(rootDirectory string) (*Driver, error) {
+	s3DriverConstructor = func(rootDirectory string, storageClass s3.StorageClass) (*Driver, error) {
 		encryptBool := false
 		if encrypt != "" {
 			encryptBool, err = strconv.ParseBool(encrypt)
@@ -69,6 +70,7 @@ func init() {
 			v4AuthBool,
 			minChunkSize,
 			rootDirectory,
+			storageClass,
 			"",
 		}
 
@@ -84,7 +86,7 @@ func init() {
 	}
 
 	testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
-		return s3DriverConstructor(root)
+		return s3DriverConstructor(root, s3.StandardStorage)
 	}, skipS3)
 }
 
@@ -99,17 +101,17 @@ func TestEmptyRootList(t *testing.T) {
 	}
 	defer os.Remove(validRoot)
 
-	rootedDriver, err := s3DriverConstructor(validRoot)
+	rootedDriver, err := s3DriverConstructor(validRoot, s3.StandardStorage)
 	if err != nil {
 		t.Fatalf("unexpected error creating rooted driver: %v", err)
 	}
 
-	emptyRootDriver, err := s3DriverConstructor("")
+	emptyRootDriver, err := s3DriverConstructor("", s3.StandardStorage)
 	if err != nil {
 		t.Fatalf("unexpected error creating empty root driver: %v", err)
 	}
 
-	slashRootDriver, err := s3DriverConstructor("/")
+	slashRootDriver, err := s3DriverConstructor("/", s3.StandardStorage)
 	if err != nil {
 		t.Fatalf("unexpected error creating slash root driver: %v", err)
 	}
@@ -137,3 +139,63 @@ func TestEmptyRootList(t *testing.T) {
 		}
 	}
 }
+
+func TestStorageClass(t *testing.T) {
+	if skipS3() != "" {
+		t.Skip(skipS3())
+	}
+
+	rootDir, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(rootDir)
+
+	standardDriver, err := s3DriverConstructor(rootDir, s3.StandardStorage)
+	if err != nil {
+		t.Fatalf("unexpected error creating driver with standard storage: %v", err)
+	}
+
+	rrDriver, err := s3DriverConstructor(rootDir, s3.ReducedRedundancy)
+	if err != nil {
+		t.Fatalf("unexpected error creating driver with reduced redundancy storage: %v", err)
+	}
+
+	standardFilename := "/test-standard"
+	rrFilename := "/test-rr"
+	contents := []byte("contents")
+	ctx := context.Background()
+
+	err = standardDriver.PutContent(ctx, standardFilename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer standardDriver.Delete(ctx, standardFilename)
+
+	err = rrDriver.PutContent(ctx, rrFilename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer rrDriver.Delete(ctx, rrFilename)
+
+	standardDriverUnwrapped := standardDriver.Base.StorageDriver.(*driver)
+	resp, err := standardDriverUnwrapped.Bucket.GetResponse(standardDriverUnwrapped.s3Path(standardFilename))
+	if err != nil {
+		t.Fatalf("unexpected error retrieving standard storage file: %v", err)
+	}
+	defer resp.Body.Close()
+	// Amazon only populates this header value for non-standard storage classes
+	if storageClass := resp.Header.Get("x-amz-storage-class"); storageClass != "" {
+		t.Fatalf("unexpected storage class for standard file: %v", storageClass)
+	}
+
+	rrDriverUnwrapped := rrDriver.Base.StorageDriver.(*driver)
+	resp, err = rrDriverUnwrapped.Bucket.GetResponse(rrDriverUnwrapped.s3Path(rrFilename))
+	if err != nil {
+		t.Fatalf("unexpected error retrieving reduced-redundancy storage file: %v", err)
+	}
+	defer resp.Body.Close()
+	if storageClass := resp.Header.Get("x-amz-storage-class"); storageClass != string(s3.ReducedRedundancy) {
+		t.Fatalf("unexpected storage class for standard file: %v", storageClass)
+	}
+}