fcrozat/distribution
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add Openstack Swift storage driver

Browse Source 
Signed-off-by: Sylvain Baubeau <sbaubeau@redhat.com>
This commit is contained in:
Sylvain Baubeau 2015-05-11 18:11:47 +02:00
parent 5ee441cdc7
commit ea7c53df08
8 changed files with 828 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Godeps/Godeps.json generated
View File

@ -81,6 +81,9 @@
{
"ImportPath": "github.com/stevvooe/resumable",
"Rev": "51ad44105773cafcbe91927f70ac68e1bf78f8b4"
},
"ImportPath": "github.com/lebauce/swift",
"Rev": "677cb70f5d40fa1a81ddb32f872615a57bb42381"
},
{
"ImportPath": "github.com/yvasiyarov/go-metrics",

32
cmd/registry-storagedriver-swift/main.go Normal file
View File

@ -0,0 +1,32 @@
// +build ignore
package main
import (
"encoding/json"
"os"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/registry/storage/driver/ipc"
"github.com/docker/distribution/registry/storage/driver/swift"
)
// An out-of-process Swift driver, intended to be run by ipc.NewDriverClient
func main() {
parametersBytes := []byte(os.Args[1])
var parameters map[string]string
err := json.Unmarshal(parametersBytes, &parameters)
if err != nil {
panic(err)
}
driver, err := swift.FromParameters(parameters)
if err != nil {
panic(err)
}
if err := ipc.StorageDriverServer(driver); err != nil {
logrus.Fatalln(err)
}
}

1
cmd/registry/main.go
View File

@ -28,6 +28,7 @@ import (
_ "github.com/docker/distribution/registry/storage/driver/inmemory"
_ "github.com/docker/distribution/registry/storage/driver/middleware/cloudfront"
_ "github.com/docker/distribution/registry/storage/driver/s3"
_ "github.com/docker/distribution/registry/storage/driver/swift"
"github.com/docker/distribution/version"
gorhandlers "github.com/gorilla/handlers"
"github.com/yvasiyarov/gorelic"

110
docs/configuration.md
View File

@ -49,6 +49,7 @@ This section lists all the registry configuration options. Some options in
the list are mutually exclusive. So, make sure to read the detailed reference
information about each option that appears later in this page.
<<<<<<< HEAD
version: 0.1
log:
level: debug
@ -92,6 +93,14 @@ information about each option that appears later in this page.
poolname: radospool
username: radosuser
chunksize: 4194304
swift:
username: username
password: password
authurl: https://storage.myprovider.com/v2.0
tenant: tenantname
region: fr
container: containername
rootdirectory: /swift/object/name/prefix
cache:
blobdescriptor: redis
maintenance:
@ -580,6 +589,107 @@ must be set.
Note: `age` and `interval` are strings containing a number with optional fraction and a unit suffix: e.g. 45m, 2h10m, 168h (1 week).
### Openstack Swift
This storage backend uses Openstack Swift object storage.
<table>
<tr>
<th>Parameter</th>
<th>Required</th>
<th>Description</th>
</tr>
<tr>
<td>
<code>authurl</code>
</td>
<td>
yes
</td>
<td>
URL for obtaining an auth token.
</td>
</tr>
<tr>
<td>
<code>username</code>
</td>
<td>
yes
</td>
<td>
Your Openstack user name.
</td>
</tr>
<tr>
<td>
<code>password</code>
</td>
<td>
yes
</td>
<td>
Your Openstack password.
</td>
</tr>
<tr>
<td>
<code>region</code>
</td>
<td>
no
</td>
<td>
The Openstack region in which your container exists.
</td>
</tr>
<tr>
<td>
<code>container</code>
</td>
<td>
yes
</td>
<td>
The container name in which you want to store the registry's data.
</td>
</tr>
<tr>
<td>
<code>tenant</code>
</td>
<td>
no
</td>
<td>
Your Openstack tenant name.
</td>
</tr>
<tr>
<td>
<code>chunksize</code>
</td>
<td>
no
</td>
<td>
Size of the data segments for the Swift Dynamic Large Objects. This value should be a number (defaults to 5M).
</td>
</tr>
<tr>
<td>
<code>rootdirectory</code>
</td>
<td>
no
</td>
<td>
This is a prefix that will be applied to all Swift keys to allow you to segment data in your container if necessary.
</td>
</tr>
</table>
## auth
auth:

21
docs/storage-drivers/swift.md Normal file
View File

@ -0,0 +1,21 @@
# Openstack Swift storage driver
An implementation of the `storagedriver.StorageDriver` interface which uses Openstack Swift for object storage.
## Parameters
`authurl`: URL for obtaining an auth token.
`username`: Your Openstack user name.
`password`: Your Openstack password.
`container`: The name of your Swift container where you wish to store objects. An additional container - named `<container>_segments` to store the data will be used. The driver will try to create both containers during its initialization.
`tenant`: (optional) Your Openstack tenant name.
`region`: (optional) The name of the Openstack region in which you would like to store objects (for example `fr`).
`chunksize`: (optional) The segment size for Dynamic Large Objects uploads (performed by WriteStream) to swift. The default is 5 MB. You might experience better performance for larger chunk sizes depending on the speed of your connection to Swift.
`rootdirectory`: (optional) The root directory tree in which all registry files will be stored. Defaults to the empty string (container root).

1
docs/storagedrivers.md
View File

@ -23,6 +23,7 @@ This storage driver package comes bundled with several drivers:
- [s3](storage-drivers/s3.md): A driver storing objects in an Amazon Simple Storage Solution (S3) bucket.
- [azure](storage-drivers/azure.md): A driver storing objects in [Microsoft Azure Blob Storage](http://azure.microsoft.com/en-us/services/storage/).
- [rados](storage-drivers/rados.md): A driver storing objects in a [Ceph Object Storage](http://ceph.com/docs/master/rados/) pool.
- [swift](storage-drivers/swift): A driver storing objects in Openstack Swift.
## Storage Driver API

519
registry/storage/driver/swift/swift.go Normal file
View File

@ -0,0 +1,519 @@
// Package swift provides a storagedriver.StorageDriver implementation to
// store blobs in Openstack Swift object storage.
//
// This package leverages the ncw/swift client library for interfacing with
// Swift.
//
// Because Swift is a key, value store the Stat call does not support last modification
// time for directories (directories are an abstraction for key, value stores)
package swift
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
gopath "path"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/lebauce/swift"
"github.com/docker/distribution/context"
storagedriver "github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/base"
"github.com/docker/distribution/registry/storage/driver/factory"
)
const driverName = "swift"
const defaultChunkSize = 5 * 1024 * 1024
//DriverParameters A struct that encapsulates all of the driver parameters after all values have been set
type DriverParameters struct {
Username string
Password string
AuthURL string
Tenant string
Region string
Container string
Prefix string
ChunkSize int64
}
type swiftInfo map[string]interface{}
func init() {
factory.Register(driverName, &swiftDriverFactory{})
}
// swiftDriverFactory implements the factory.StorageDriverFactory interface
type swiftDriverFactory struct{}
func (factory *swiftDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
return FromParameters(parameters)
}
type driver struct {
Conn swift.Connection
Container string
Prefix string
BulkDeleteSupport bool
ChunkSize int64
}
type baseEmbed struct {
base.Base
}
// Driver is a storagedriver.StorageDriver implementation backed by Amazon Swift
// Objects are stored at absolute keys in the provided bucket.
type Driver struct {
baseEmbed
}
// FromParameters constructs a new Driver with a given parameters map
// Required parameters:
// - username
// - password
// - authurl
// - container
func FromParameters(parameters map[string]interface{}) (*Driver, error) {
username, ok := parameters["username"]
if !ok || fmt.Sprint(username) == "" {
return nil, fmt.Errorf("No username parameter provided")
}
password, ok := parameters["password"]
if !ok || fmt.Sprint(password) == "" {
return nil, fmt.Errorf("No password parameter provided")
}
authURL, ok := parameters["authurl"]
if !ok || fmt.Sprint(authURL) == "" {
return nil, fmt.Errorf("No container parameter provided")
}
container, ok := parameters["container"]
if !ok || fmt.Sprint(container) == "" {
return nil, fmt.Errorf("No container parameter provided")
}
tenant, ok := parameters["tenant"]
if !ok {
tenant = ""
}
region, ok := parameters["region"]
if !ok {
region = ""
}
rootDirectory, ok := parameters["rootdirectory"]
if !ok {
rootDirectory = ""
}
chunkSize := int64(defaultChunkSize)
chunkSizeParam, ok := parameters["chunksize"]
if ok {
chunkSize, ok = chunkSizeParam.(int64)
if !ok {
return nil, fmt.Errorf("The chunksize parameter should be a number")
}
}
params := DriverParameters{
fmt.Sprint(username),
fmt.Sprint(password),
fmt.Sprint(authURL),
fmt.Sprint(tenant),
fmt.Sprint(region),
fmt.Sprint(container),
fmt.Sprint(rootDirectory),
chunkSize,
}
return New(params)
}
// New constructs a new Driver with the given Openstack Swift credentials and container name
func New(params DriverParameters) (*Driver, error) {
ct := swift.Connection{
UserName: params.Username,
ApiKey: params.Password,
AuthUrl: params.AuthURL,
Region: params.Region,
UserAgent: "distribution",
Tenant: params.Tenant,
ConnectTimeout: 60 * time.Second,
Timeout: 15 * 60 * time.Second,
}
err := ct.Authenticate()
if err != nil {
return nil, fmt.Errorf("Swift authentication failed: %s", err)
}
if err := ct.ContainerCreate(params.Container, nil); err != nil {
return nil, fmt.Errorf("Failed to create container %s (%s)", params.Container, err)
}
if err := ct.ContainerCreate(params.Container + "_segments", nil); err != nil {
return nil, fmt.Errorf("Failed to create container %s (%s)", params.Container + "_segments", err)
}
d := &driver{
Conn: ct,
Container: params.Container,
Prefix: params.Prefix,
BulkDeleteSupport: detectBulkDelete(params.AuthURL),
ChunkSize: params.ChunkSize,
}
return &Driver{
baseEmbed: baseEmbed{
Base: base.Base{
StorageDriver: d,
},
},
}, nil
}
// Implement the storagedriver.StorageDriver interface
func (d *driver) Name() string {
return driverName
}
// GetContent retrieves the content stored at "path" as a []byte.
func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) {
content, err := d.Conn.ObjectGetBytes(d.Container, d.swiftPath(path))
if err != nil {
return nil, parseError(path, err)
}
return content, nil
}
// PutContent stores the []byte content at a location designated by "path".
func (d *driver) PutContent(ctx context.Context, path string, contents []byte) error {
if dir, err := d.createParentFolder(path); err != nil {
return parseError(dir, err)
}
err := d.Conn.ObjectPutBytes(d.Container, d.swiftPath(path),
contents, d.getContentType())
return parseError(path, err)
}
// ReadStream retrieves an io.ReadCloser for the content stored at "path" with a
// given byte offset.
func (d *driver) ReadStream(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
headers := make(swift.Headers)
headers["Range"] = "bytes=" + strconv.FormatInt(offset, 10) + "-"
file, _, err := d.Conn.ObjectOpen(d.Container, d.swiftPath(path), false, headers)
if err != nil {
if swiftErr, ok := err.(*swift.Error); ok && swiftErr.StatusCode == 416 {
return ioutil.NopCloser(bytes.NewReader(nil)), nil
}
return nil, parseError(path, err)
}
return file, nil
}
// WriteStream stores the contents of the provided io.Reader at a
// location designated by the given path. The driver will know it has
// received the full contents when the reader returns io.EOF. The number
// of successfully READ bytes will be returned, even if an error is
// returned. May be used to resume writing a stream by providing a nonzero
// offset. Offsets past the current size will write from the position
// beyond the end of the file.
func (d *driver) WriteStream(ctx context.Context, path string, offset int64, reader io.Reader) (int64, error) {
var (
segments []swift.Object
paddingReader io.Reader
)
partNumber := int64(1)
bytesRead := int64(0)
currentLength := int64(0)
zeroBuf := make([]byte, d.ChunkSize)
segmentsContainer := d.Container + "_segments"
cursor := int64(0)
getSegment := func() string {
return d.swiftPath(path) + "/" + fmt.Sprintf("%016d", partNumber)
}
max := func(a int64, b int64) int64 {
if a > b {
return a
}
return b
}
info, _, err := d.Conn.Object(d.Container, d.swiftPath(path))
if err != nil {
if swiftErr, ok := err.(*swift.Error); ok {
if swiftErr.StatusCode == 404 {
// Create a object manifest
if dir, err := d.createParentFolder(path); err != nil {
return bytesRead, parseError(dir, err)
}
headers := make(swift.Headers)
headers["X-Object-Manifest"] = segmentsContainer + "/" + d.swiftPath(path)
manifest, err := d.Conn.ObjectCreate(d.Container, d.swiftPath(path), false, "",
d.getContentType(), headers)
manifest.Close()
if err != nil {
return bytesRead, parseError(path, err)
}
} else {
return bytesRead, parseError(path, err)
}
} else {
return bytesRead, parseError(path, err)
}
} else {
// The manifest already exists. Get all the segments
currentLength = info.Bytes
headers := make(swift.Headers)
headers["Content-Type"] = "application/json"
opts := &swift.ObjectsOpts{Prefix: d.swiftPath(path), Headers: headers}
segments, err = d.Conn.Objects(d.Container + "_segments", opts)
if err != nil {
return bytesRead, parseError(path, err)
}
}
// First, we skip the existing segments that are not modified by this call
for i := range segments {
if offset < cursor + segments[i].Bytes {
break
}
cursor += segments[i].Bytes
partNumber++
}
// We reached the end of the file but we haven't reached 'offset' yet
// Therefore we add blocks of zeros
if offset >= currentLength {
for offset - currentLength >= d.ChunkSize {
// Insert a block a zero
d.Conn.ObjectPut(segmentsContainer, getSegment(),
bytes.NewReader(zeroBuf), false, "",
d.getContentType(), nil)
currentLength += d.ChunkSize
partNumber++
}
cursor = currentLength
paddingReader = bytes.NewReader(zeroBuf)
} else {
// Offset is inside the current segment : we need to read the
// data from the beginning of the segment to offset
paddingReader, _, err = d.Conn.ObjectOpen(segmentsContainer, getSegment(), false, nil)
if err != nil {
return bytesRead, parseError(getSegment(), err)
}
}
multi := io.MultiReader(
io.LimitReader(paddingReader, offset - cursor),
io.LimitReader(reader, d.ChunkSize - (offset - cursor)),
)
for {
currentSegment, err := d.Conn.ObjectCreate(segmentsContainer, getSegment(), false, "", d.getContentType(), nil)
if err != nil {
return bytesRead, parseError(path, err)
}
n, err := io.Copy(currentSegment, multi)
if err != nil {
return bytesRead, parseError(path, err)
}
if n < d.ChunkSize {
// We wrote all the data
if cursor + n < currentLength {
// Copy the end of the chunk
headers := make(swift.Headers)
headers["Range"] = "bytes=" + strconv.FormatInt(cursor + n, 10) + "-" + strconv.FormatInt(cursor + d.ChunkSize, 10)
file, _, err := d.Conn.ObjectOpen(d.Container, d.swiftPath(path), false, headers)
if err != nil {
return bytesRead, parseError(path, err)
}
io.Copy(currentSegment, file)
file.Close()
}
if n > 0 {
currentSegment.Close()
bytesRead += n - max(0, offset - cursor)
}
break
}
currentSegment.Close()
bytesRead += n - max(0, offset - cursor)
multi = io.MultiReader(io.LimitReader(reader, d.ChunkSize))
cursor += d.ChunkSize
partNumber++
}
return bytesRead, nil
}
// Stat retrieves the FileInfo for the given path, including the current size
// in bytes and the creation time.
func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
info, _, err := d.Conn.Object(d.Container, d.swiftPath(path))
if err != nil {
return nil, parseError(path, err)
}
fi := storagedriver.FileInfoFields{
Path: path,
IsDir: info.ContentType == "application/directory",
Size: info.Bytes,
ModTime: info.LastModified,
}
return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil
}
// List returns a list of the objects that are direct descendants of the given path.
func (d *driver) List(ctx context.Context, path string) ([]string, error) {
prefix := d.swiftPath(path)
if prefix != "" {
prefix += "/"
}
opts := &swift.ObjectsOpts{
Path: prefix,
Delimiter: '/',
}
files, err := d.Conn.ObjectNames(d.Container, opts)
for index, name := range files {
files[index] = "/" + strings.TrimSuffix(name, "/")
}
return files, parseError(path, err)
}
// Move moves an object stored at sourcePath to destPath, removing the original
// object.
func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error {
err := d.Conn.ObjectMove(d.Container, d.swiftPath(sourcePath),
d.Container, d.swiftPath(destPath))
if err != nil {
return parseError(sourcePath, err)
}
return nil
}
// Delete recursively deletes all objects stored at "path" and its subpaths.
func (d *driver) Delete(ctx context.Context, path string) error {
opts := swift.ObjectsOpts{
Prefix: d.swiftPath(path),
}
objects, err := d.Conn.ObjectNamesAll(d.Container, &opts)
if err != nil {
return parseError(path, err)
}
if len(objects) == 0 {
return storagedriver.PathNotFoundError{Path: path}
}
for index, name := range objects {
objects[index] = name[len(d.Prefix):]
}
var multiDelete = true
if d.BulkDeleteSupport {
_, err := d.Conn.BulkDelete(d.Container, objects)
multiDelete = err != nil
}
if multiDelete {
for _, name := range objects {
if _, headers, err := d.Conn.Object(d.Container, name); err == nil {
manifest, ok := headers["X-Object-Manifest"]
if ok {
components := strings.SplitN(manifest, "/", 2)
segContainer := components[0]
segments, err := d.Conn.ObjectNamesAll(segContainer, &swift.ObjectsOpts{ Prefix: components[1] })
if err != nil {
return parseError(name, err)
}
for _, s := range segments {
if err := d.Conn.ObjectDelete(segContainer, s); err != nil {
return parseError(s, err)
}
}
}
} else {
return parseError(name, err)
}
if err := d.Conn.ObjectDelete(d.Container, name); err != nil {
return parseError(name, err)
}
}
}
return nil
}
// URLFor returns a URL which may be used to retrieve the content stored at the given path.
// May return an UnsupportedMethodErr in certain StorageDriver implementations.
func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
return "", storagedriver.ErrUnsupportedMethod
}
func (d *driver) swiftPath(path string) string {
return strings.TrimLeft(strings.TrimRight(d.Prefix, "/")+path, "/")
}
func (d *driver) createParentFolder(path string) (string, error) {
dir := gopath.Dir(path)
if dir != "/" {
_, _, err := d.Conn.Object(d.Container, d.swiftPath(dir))
if swiftErr, ok := err.(*swift.Error); ok && swiftErr.StatusCode == 404 {
_, err := d.Conn.ObjectPut(d.Container, d.swiftPath(dir), bytes.NewReader(make([]byte, 0)),
false, "", "application/directory", nil)
if err != nil {
return dir, err
}
}
}
return dir, nil
}
func (d *driver) getContentType() string {
return "application/octet-stream"
}
func detectBulkDelete(authURL string) (bulkDelete bool) {
resp, err := http.Get(filepath.Join(authURL, "..", "..") + "/info")
if err == nil {
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
var infos swiftInfo
if decoder.Decode(&infos) == nil {
_, bulkDelete = infos["bulk_delete"]
}
}
return
}
func parseError(path string, err error) error {
if swiftErr, ok := err.(*swift.Error); ok && swiftErr.StatusCode == 404 {
return storagedriver.PathNotFoundError{Path: path}
}
return err
}

141
registry/storage/driver/swift/swift_test.go Normal file
View File

@ -0,0 +1,141 @@
package swift
import (
"io/ioutil"
"os"
"testing"
"github.com/lebauce/swift/swifttest"
"github.com/docker/distribution/context"
storagedriver "github.com/docker/distribution/registry/storage/driver"
"github.com/docker/distribution/registry/storage/driver/testsuites"
"gopkg.in/check.v1"
)
// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { check.TestingT(t) }
type SwiftDriverConstructor func(rootDirectory string) (*Driver, error)
func init() {
var (
username string
password string
authURL string
tenant string
container string
region string
prefix string
swiftServer *swifttest.SwiftServer
err error
)
if username = os.Getenv("OS_USERNAME"); username == "" {
username = os.Getenv("ST_USER")
}
if password = os.Getenv("OS_PASSWORD"); password == "" {
password = os.Getenv("ST_KEY")
}
if authURL = os.Getenv("OS_AUTH_URL"); authURL == "" {
authURL = os.Getenv("ST_AUTH")
}
tenant = os.Getenv("OS_TENANT_NAME")
container = os.Getenv("OS_CONTAINER_NAME")
region = os.Getenv("OS_REGION_NAME")
prefix = os.Getenv("OS_CONTAINER_PREFIX")
if username == "" || password == "" || authURL == "" || container == "" {
if swiftServer, err = swifttest.NewSwiftServer("localhost"); err != nil {
panic(err)
}
username = "swifttest"
password = "swifttest"
authURL = swiftServer.AuthURL
container = "test"
}
root, err := ioutil.TempDir("", "driver-")
if err != nil {
panic(err)
}
defer os.Remove(root)
swiftDriverConstructor := func(rootDirectory string) (*Driver, error) {
parameters := DriverParameters{
username,
password,
authURL,
tenant,
region,
container,
prefix,
defaultChunkSize,
}
return New(parameters)
}
skipCheck := func() string {
return ""
}
driverConstructor := func() (storagedriver.StorageDriver, error) {
return swiftDriverConstructor(root)
}
testsuites.RegisterInProcessSuite(driverConstructor, skipCheck)
RegisterSwiftDriverSuite(swiftDriverConstructor, skipCheck, swiftServer)
}
func RegisterSwiftDriverSuite(swiftDriverConstructor SwiftDriverConstructor, skipCheck testsuites.SkipCheck,
swiftServer *swifttest.SwiftServer) {
check.Suite(&SwiftDriverSuite{
Constructor: swiftDriverConstructor,
SkipCheck: skipCheck,
SwiftServer: swiftServer,
})
}
type SwiftDriverSuite struct {
Constructor SwiftDriverConstructor
SwiftServer *swifttest.SwiftServer
testsuites.SkipCheck
}
func (suite *SwiftDriverSuite) SetUpSuite(c *check.C) {
if reason := suite.SkipCheck(); reason != "" {
c.Skip(reason)
}
}
func (suite *SwiftDriverSuite) TestEmptyRootList(c *check.C) {
validRoot, err := ioutil.TempDir("", "driver-")
c.Assert(err, check.IsNil)
defer os.Remove(validRoot)
rootedDriver, err := suite.Constructor(validRoot)
c.Assert(err, check.IsNil)
emptyRootDriver, err := suite.Constructor("")
c.Assert(err, check.IsNil)
slashRootDriver, err := suite.Constructor("/")
c.Assert(err, check.IsNil)
filename := "/test"
contents := []byte("contents")
ctx := context.Background()
err = rootedDriver.PutContent(ctx, filename, contents)
c.Assert(err, check.IsNil)
defer rootedDriver.Delete(ctx, filename)
keys, err := emptyRootDriver.List(ctx, "/")
for _, path := range keys {
c.Assert(storagedriver.PathRegexp.MatchString(path), check.Equals, true)
}
keys, err = slashRootDriver.List(ctx, "/")
for _, path := range keys {
c.Assert(storagedriver.PathRegexp.MatchString(path), check.Equals, true)
}
}