9c88801a12
Back in the before time, the best practices surrounding usage of Context weren't quite worked out. We defined our own type to make usage easier. As this packaged was used elsewhere, it make it more and more challenging to integrate with the forked `Context` type. Now that it is available in the standard library, we can just use that one directly. To make usage more consistent, we now use `dcontext` when referring to the distribution context package. Signed-off-by: Stephen J Day <stephen.day@docker.com>
1273 lines
40 KiB
Go
1273 lines
40 KiB
Go
package testsuites
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha1"
|
|
"io"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
storagedriver "github.com/docker/distribution/registry/storage/driver"
|
|
"gopkg.in/check.v1"
|
|
)
|
|
|
|
// Test hooks up gocheck into the "go test" runner.
|
|
func Test(t *testing.T) { check.TestingT(t) }
|
|
|
|
// RegisterSuite registers an in-process storage driver test suite with
|
|
// the go test runner.
|
|
func RegisterSuite(driverConstructor DriverConstructor, skipCheck SkipCheck) {
|
|
check.Suite(&DriverSuite{
|
|
Constructor: driverConstructor,
|
|
SkipCheck: skipCheck,
|
|
ctx: context.Background(),
|
|
})
|
|
}
|
|
|
|
// SkipCheck is a function used to determine if a test suite should be skipped.
|
|
// If a SkipCheck returns a non-empty skip reason, the suite is skipped with
|
|
// the given reason.
|
|
type SkipCheck func() (reason string)
|
|
|
|
// NeverSkip is a default SkipCheck which never skips the suite.
|
|
var NeverSkip SkipCheck = func() string { return "" }
|
|
|
|
// DriverConstructor is a function which returns a new
|
|
// storagedriver.StorageDriver.
|
|
type DriverConstructor func() (storagedriver.StorageDriver, error)
|
|
|
|
// DriverTeardown is a function which cleans up a suite's
|
|
// storagedriver.StorageDriver.
|
|
type DriverTeardown func() error
|
|
|
|
// DriverSuite is a gocheck test suite designed to test a
|
|
// storagedriver.StorageDriver. The intended way to create a DriverSuite is
|
|
// with RegisterSuite.
|
|
type DriverSuite struct {
|
|
Constructor DriverConstructor
|
|
Teardown DriverTeardown
|
|
SkipCheck
|
|
storagedriver.StorageDriver
|
|
ctx context.Context
|
|
}
|
|
|
|
// SetUpSuite sets up the gocheck test suite.
|
|
func (suite *DriverSuite) SetUpSuite(c *check.C) {
|
|
if reason := suite.SkipCheck(); reason != "" {
|
|
c.Skip(reason)
|
|
}
|
|
d, err := suite.Constructor()
|
|
c.Assert(err, check.IsNil)
|
|
suite.StorageDriver = d
|
|
}
|
|
|
|
// TearDownSuite tears down the gocheck test suite.
|
|
func (suite *DriverSuite) TearDownSuite(c *check.C) {
|
|
if suite.Teardown != nil {
|
|
err := suite.Teardown()
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
}
|
|
|
|
// TearDownTest tears down the gocheck test.
|
|
// This causes the suite to abort if any files are left around in the storage
|
|
// driver.
|
|
func (suite *DriverSuite) TearDownTest(c *check.C) {
|
|
files, _ := suite.StorageDriver.List(suite.ctx, "/")
|
|
if len(files) > 0 {
|
|
c.Fatalf("Storage driver did not clean up properly. Offending files: %#v", files)
|
|
}
|
|
}
|
|
|
|
// TestRootExists ensures that all storage drivers have a root path by default.
|
|
func (suite *DriverSuite) TestRootExists(c *check.C) {
|
|
_, err := suite.StorageDriver.List(suite.ctx, "/")
|
|
if err != nil {
|
|
c.Fatalf(`the root path "/" should always exist: %v`, err)
|
|
}
|
|
}
|
|
|
|
// TestValidPaths checks that various valid file paths are accepted by the
|
|
// storage driver.
|
|
func (suite *DriverSuite) TestValidPaths(c *check.C) {
|
|
contents := randomContents(64)
|
|
validFiles := []string{
|
|
"/a",
|
|
"/2",
|
|
"/aa",
|
|
"/a.a",
|
|
"/0-9/abcdefg",
|
|
"/abcdefg/z.75",
|
|
"/abc/1.2.3.4.5-6_zyx/123.z/4",
|
|
"/docker/docker-registry",
|
|
"/123.abc",
|
|
"/abc./abc",
|
|
"/.abc",
|
|
"/a--b",
|
|
"/a-.b",
|
|
"/_.abc",
|
|
"/Docker/docker-registry",
|
|
"/Abc/Cba"}
|
|
|
|
for _, filename := range validFiles {
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
received, err := suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(received, check.DeepEquals, contents)
|
|
}
|
|
}
|
|
|
|
func (suite *DriverSuite) deletePath(c *check.C, path string) {
|
|
for tries := 2; tries > 0; tries-- {
|
|
err := suite.StorageDriver.Delete(suite.ctx, path)
|
|
if _, ok := err.(storagedriver.PathNotFoundError); ok {
|
|
err = nil
|
|
}
|
|
c.Assert(err, check.IsNil)
|
|
paths, err := suite.StorageDriver.List(suite.ctx, path)
|
|
if len(paths) == 0 {
|
|
break
|
|
}
|
|
time.Sleep(time.Second * 2)
|
|
}
|
|
}
|
|
|
|
// TestInvalidPaths checks that various invalid file paths are rejected by the
|
|
// storage driver.
|
|
func (suite *DriverSuite) TestInvalidPaths(c *check.C) {
|
|
contents := randomContents(64)
|
|
invalidFiles := []string{
|
|
"",
|
|
"/",
|
|
"abc",
|
|
"123.abc",
|
|
"//bcd",
|
|
"/abc_123/"}
|
|
|
|
for _, filename := range invalidFiles {
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
// only delete if file was successfully written
|
|
if err == nil {
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
}
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.InvalidPathError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.InvalidPathError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
}
|
|
|
|
// TestWriteRead1 tests a simple write-read workflow.
|
|
func (suite *DriverSuite) TestWriteRead1(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte("a")
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteRead2 tests a simple write-read workflow with unicode data.
|
|
func (suite *DriverSuite) TestWriteRead2(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte("\xc3\x9f")
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteRead3 tests a simple write-read workflow with a small string.
|
|
func (suite *DriverSuite) TestWriteRead3(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(32)
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteRead4 tests a simple write-read workflow with 1MB of data.
|
|
func (suite *DriverSuite) TestWriteRead4(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(1024 * 1024)
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadNonUTF8 tests that non-utf8 data may be written to the storage
|
|
// driver safely.
|
|
func (suite *DriverSuite) TestWriteReadNonUTF8(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte{0x80, 0x80, 0x80, 0x80}
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestTruncate tests that putting smaller contents than an original file does
|
|
// remove the excess contents.
|
|
func (suite *DriverSuite) TestTruncate(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(1024 * 1024)
|
|
suite.writeReadCompare(c, filename, contents)
|
|
|
|
contents = randomContents(1024)
|
|
suite.writeReadCompare(c, filename, contents)
|
|
}
|
|
|
|
// TestReadNonexistent tests reading content from an empty path.
|
|
func (suite *DriverSuite) TestReadNonexistent(c *check.C) {
|
|
filename := randomPath(32)
|
|
_, err := suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestWriteReadStreams1 tests a simple write-read streaming workflow.
|
|
func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte("a")
|
|
suite.writeReadCompareStreams(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadStreams2 tests a simple write-read streaming workflow with
|
|
// unicode data.
|
|
func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte("\xc3\x9f")
|
|
suite.writeReadCompareStreams(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadStreams3 tests a simple write-read streaming workflow with a
|
|
// small amount of data.
|
|
func (suite *DriverSuite) TestWriteReadStreams3(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(32)
|
|
suite.writeReadCompareStreams(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadStreams4 tests a simple write-read streaming workflow with 1MB
|
|
// of data.
|
|
func (suite *DriverSuite) TestWriteReadStreams4(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(1024 * 1024)
|
|
suite.writeReadCompareStreams(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadStreamsNonUTF8 tests that non-utf8 data may be written to the
|
|
// storage driver safely.
|
|
func (suite *DriverSuite) TestWriteReadStreamsNonUTF8(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := []byte{0x80, 0x80, 0x80, 0x80}
|
|
suite.writeReadCompareStreams(c, filename, contents)
|
|
}
|
|
|
|
// TestWriteReadLargeStreams tests that a 5GB file may be written to the storage
|
|
// driver safely.
|
|
func (suite *DriverSuite) TestWriteReadLargeStreams(c *check.C) {
|
|
if testing.Short() {
|
|
c.Skip("Skipping test in short mode")
|
|
}
|
|
|
|
filename := randomPath(32)
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
checksum := sha1.New()
|
|
var fileSize int64 = 5 * 1024 * 1024 * 1024
|
|
|
|
contents := newRandReader(fileSize)
|
|
|
|
writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
|
|
c.Assert(err, check.IsNil)
|
|
written, err := io.Copy(writer, io.TeeReader(contents, checksum))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(written, check.Equals, fileSize)
|
|
|
|
err = writer.Commit()
|
|
c.Assert(err, check.IsNil)
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
writtenChecksum := sha1.New()
|
|
io.Copy(writtenChecksum, reader)
|
|
|
|
c.Assert(writtenChecksum.Sum(nil), check.DeepEquals, checksum.Sum(nil))
|
|
}
|
|
|
|
// TestReaderWithOffset tests that the appropriate data is streamed when
|
|
// reading with a given offset.
|
|
func (suite *DriverSuite) TestReaderWithOffset(c *check.C) {
|
|
filename := randomPath(32)
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
chunkSize := int64(32)
|
|
|
|
contentsChunk1 := randomContents(chunkSize)
|
|
contentsChunk2 := randomContents(chunkSize)
|
|
contentsChunk3 := randomContents(chunkSize)
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
readContents, err := ioutil.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
c.Assert(readContents, check.DeepEquals, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...))
|
|
|
|
reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
readContents, err = ioutil.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
c.Assert(readContents, check.DeepEquals, append(contentsChunk2, contentsChunk3...))
|
|
|
|
reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize*2)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
readContents, err = ioutil.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(readContents, check.DeepEquals, contentsChunk3)
|
|
|
|
// Ensure we get invalid offest for negative offsets.
|
|
reader, err = suite.StorageDriver.Reader(suite.ctx, filename, -1)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.InvalidOffsetError{})
|
|
c.Assert(err.(storagedriver.InvalidOffsetError).Offset, check.Equals, int64(-1))
|
|
c.Assert(err.(storagedriver.InvalidOffsetError).Path, check.Equals, filename)
|
|
c.Assert(reader, check.IsNil)
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
// Read past the end of the content and make sure we get a reader that
|
|
// returns 0 bytes and io.EOF
|
|
reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize*3)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
buf := make([]byte, chunkSize)
|
|
n, err := reader.Read(buf)
|
|
c.Assert(err, check.Equals, io.EOF)
|
|
c.Assert(n, check.Equals, 0)
|
|
|
|
// Check the N-1 boundary condition, ensuring we get 1 byte then io.EOF.
|
|
reader, err = suite.StorageDriver.Reader(suite.ctx, filename, chunkSize*3-1)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
n, err = reader.Read(buf)
|
|
c.Assert(n, check.Equals, 1)
|
|
|
|
// We don't care whether the io.EOF comes on the this read or the first
|
|
// zero read, but the only error acceptable here is io.EOF.
|
|
if err != nil {
|
|
c.Assert(err, check.Equals, io.EOF)
|
|
}
|
|
|
|
// Any more reads should result in zero bytes and io.EOF
|
|
n, err = reader.Read(buf)
|
|
c.Assert(n, check.Equals, 0)
|
|
c.Assert(err, check.Equals, io.EOF)
|
|
}
|
|
|
|
// TestContinueStreamAppendLarge tests that a stream write can be appended to without
|
|
// corrupting the data with a large chunk size.
|
|
func (suite *DriverSuite) TestContinueStreamAppendLarge(c *check.C) {
|
|
suite.testContinueStreamAppend(c, int64(10*1024*1024))
|
|
}
|
|
|
|
// TestContinueStreamAppendSmall is the same as TestContinueStreamAppendLarge, but only
|
|
// with a tiny chunk size in order to test corner cases for some cloud storage drivers.
|
|
func (suite *DriverSuite) TestContinueStreamAppendSmall(c *check.C) {
|
|
suite.testContinueStreamAppend(c, int64(32))
|
|
}
|
|
|
|
func (suite *DriverSuite) testContinueStreamAppend(c *check.C, chunkSize int64) {
|
|
filename := randomPath(32)
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
contentsChunk1 := randomContents(chunkSize)
|
|
contentsChunk2 := randomContents(chunkSize)
|
|
contentsChunk3 := randomContents(chunkSize)
|
|
|
|
fullContents := append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)
|
|
|
|
writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
|
|
c.Assert(err, check.IsNil)
|
|
nn, err := io.Copy(writer, bytes.NewReader(contentsChunk1))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(nn, check.Equals, int64(len(contentsChunk1)))
|
|
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
curSize := writer.Size()
|
|
c.Assert(curSize, check.Equals, int64(len(contentsChunk1)))
|
|
|
|
writer, err = suite.StorageDriver.Writer(suite.ctx, filename, true)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(writer.Size(), check.Equals, curSize)
|
|
|
|
nn, err = io.Copy(writer, bytes.NewReader(contentsChunk2))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(nn, check.Equals, int64(len(contentsChunk2)))
|
|
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
curSize = writer.Size()
|
|
c.Assert(curSize, check.Equals, 2*chunkSize)
|
|
|
|
writer, err = suite.StorageDriver.Writer(suite.ctx, filename, true)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(writer.Size(), check.Equals, curSize)
|
|
|
|
nn, err = io.Copy(writer, bytes.NewReader(fullContents[curSize:]))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(nn, check.Equals, int64(len(fullContents[curSize:])))
|
|
|
|
err = writer.Commit()
|
|
c.Assert(err, check.IsNil)
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
received, err := suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(received, check.DeepEquals, fullContents)
|
|
}
|
|
|
|
// TestReadNonexistentStream tests that reading a stream for a nonexistent path
|
|
// fails.
|
|
func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) {
|
|
filename := randomPath(32)
|
|
|
|
_, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.Reader(suite.ctx, filename, 64)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestList checks the returned list of keys after populating a directory tree.
|
|
func (suite *DriverSuite) TestList(c *check.C) {
|
|
rootDirectory := "/" + randomFilename(int64(8+rand.Intn(8)))
|
|
defer suite.deletePath(c, rootDirectory)
|
|
|
|
doesnotexist := path.Join(rootDirectory, "nonexistent")
|
|
_, err := suite.StorageDriver.List(suite.ctx, doesnotexist)
|
|
c.Assert(err, check.Equals, storagedriver.PathNotFoundError{
|
|
Path: doesnotexist,
|
|
DriverName: suite.StorageDriver.Name(),
|
|
})
|
|
|
|
parentDirectory := rootDirectory + "/" + randomFilename(int64(8+rand.Intn(8)))
|
|
childFiles := make([]string, 50)
|
|
for i := 0; i < len(childFiles); i++ {
|
|
childFile := parentDirectory + "/" + randomFilename(int64(8+rand.Intn(8)))
|
|
childFiles[i] = childFile
|
|
err := suite.StorageDriver.PutContent(suite.ctx, childFile, randomContents(32))
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
sort.Strings(childFiles)
|
|
|
|
keys, err := suite.StorageDriver.List(suite.ctx, "/")
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(keys, check.DeepEquals, []string{rootDirectory})
|
|
|
|
keys, err = suite.StorageDriver.List(suite.ctx, rootDirectory)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(keys, check.DeepEquals, []string{parentDirectory})
|
|
|
|
keys, err = suite.StorageDriver.List(suite.ctx, parentDirectory)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
sort.Strings(keys)
|
|
c.Assert(keys, check.DeepEquals, childFiles)
|
|
|
|
// A few checks to add here (check out #819 for more discussion on this):
|
|
// 1. Ensure that all paths are absolute.
|
|
// 2. Ensure that listings only include direct children.
|
|
// 3. Ensure that we only respond to directory listings that end with a slash (maybe?).
|
|
}
|
|
|
|
// TestMove checks that a moved object no longer exists at the source path and
|
|
// does exist at the destination.
|
|
func (suite *DriverSuite) TestMove(c *check.C) {
|
|
contents := randomContents(32)
|
|
sourcePath := randomPath(32)
|
|
destPath := randomPath(32)
|
|
|
|
defer suite.deletePath(c, firstPart(sourcePath))
|
|
defer suite.deletePath(c, firstPart(destPath))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, sourcePath, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Move(suite.ctx, sourcePath, destPath)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
received, err := suite.StorageDriver.GetContent(suite.ctx, destPath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(received, check.DeepEquals, contents)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, sourcePath)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestMoveOverwrite checks that a moved object no longer exists at the source
|
|
// path and overwrites the contents at the destination.
|
|
func (suite *DriverSuite) TestMoveOverwrite(c *check.C) {
|
|
sourcePath := randomPath(32)
|
|
destPath := randomPath(32)
|
|
sourceContents := randomContents(32)
|
|
destContents := randomContents(64)
|
|
|
|
defer suite.deletePath(c, firstPart(sourcePath))
|
|
defer suite.deletePath(c, firstPart(destPath))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, sourcePath, sourceContents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, destPath, destContents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Move(suite.ctx, sourcePath, destPath)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
received, err := suite.StorageDriver.GetContent(suite.ctx, destPath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(received, check.DeepEquals, sourceContents)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, sourcePath)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestMoveNonexistent checks that moving a nonexistent key fails and does not
|
|
// delete the data at the destination path.
|
|
func (suite *DriverSuite) TestMoveNonexistent(c *check.C) {
|
|
contents := randomContents(32)
|
|
sourcePath := randomPath(32)
|
|
destPath := randomPath(32)
|
|
|
|
defer suite.deletePath(c, firstPart(destPath))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, destPath, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Move(suite.ctx, sourcePath, destPath)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
received, err := suite.StorageDriver.GetContent(suite.ctx, destPath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(received, check.DeepEquals, contents)
|
|
}
|
|
|
|
// TestMoveInvalid provides various checks for invalid moves.
|
|
func (suite *DriverSuite) TestMoveInvalid(c *check.C) {
|
|
contents := randomContents(32)
|
|
|
|
// Create a regular file.
|
|
err := suite.StorageDriver.PutContent(suite.ctx, "/notadir", contents)
|
|
c.Assert(err, check.IsNil)
|
|
defer suite.deletePath(c, "/notadir")
|
|
|
|
// Now try to move a non-existent file under it.
|
|
err = suite.StorageDriver.Move(suite.ctx, "/notadir/foo", "/notadir/bar")
|
|
c.Assert(err, check.NotNil) // non-nil error
|
|
}
|
|
|
|
// TestDelete checks that the delete operation removes data from the storage
|
|
// driver
|
|
func (suite *DriverSuite) TestDelete(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(32)
|
|
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Delete(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestURLFor checks that the URLFor method functions properly, but only if it
|
|
// is implemented
|
|
func (suite *DriverSuite) TestURLFor(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(32)
|
|
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
url, err := suite.StorageDriver.URLFor(suite.ctx, filename, nil)
|
|
if _, ok := err.(storagedriver.ErrUnsupportedMethod); ok {
|
|
return
|
|
}
|
|
c.Assert(err, check.IsNil)
|
|
|
|
response, err := http.Get(url)
|
|
c.Assert(err, check.IsNil)
|
|
defer response.Body.Close()
|
|
|
|
read, err := ioutil.ReadAll(response.Body)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(read, check.DeepEquals, contents)
|
|
|
|
url, err = suite.StorageDriver.URLFor(suite.ctx, filename, map[string]interface{}{"method": "HEAD"})
|
|
if _, ok := err.(storagedriver.ErrUnsupportedMethod); ok {
|
|
return
|
|
}
|
|
c.Assert(err, check.IsNil)
|
|
|
|
response, err = http.Head(url)
|
|
c.Assert(response.StatusCode, check.Equals, 200)
|
|
c.Assert(response.ContentLength, check.Equals, int64(32))
|
|
}
|
|
|
|
// TestDeleteNonexistent checks that removing a nonexistent key fails.
|
|
func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) {
|
|
filename := randomPath(32)
|
|
err := suite.StorageDriver.Delete(suite.ctx, filename)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestDeleteFolder checks that deleting a folder removes all child elements.
|
|
func (suite *DriverSuite) TestDeleteFolder(c *check.C) {
|
|
dirname := randomPath(32)
|
|
filename1 := randomPath(32)
|
|
filename2 := randomPath(32)
|
|
filename3 := randomPath(32)
|
|
contents := randomContents(32)
|
|
|
|
defer suite.deletePath(c, firstPart(dirname))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename1), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename2), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename3), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Delete(suite.ctx, path.Join(dirname, filename1))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename1))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename2))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename3))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Delete(suite.ctx, dirname)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename1))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename2))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename3))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
}
|
|
|
|
// TestDeleteOnlyDeletesSubpaths checks that deleting path A does not
|
|
// delete path B when A is a prefix of B but B is not a subpath of A (so that
|
|
// deleting "/a" does not delete "/ab"). This matters for services like S3 that
|
|
// do not implement directories.
|
|
func (suite *DriverSuite) TestDeleteOnlyDeletesSubpaths(c *check.C) {
|
|
dirname := randomPath(32)
|
|
filename := randomPath(32)
|
|
contents := randomContents(32)
|
|
|
|
defer suite.deletePath(c, firstPart(dirname))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, filename+"suffix"), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, dirname, filename), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, path.Join(dirname, dirname+"suffix", filename), contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Delete(suite.ctx, path.Join(dirname, filename))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, filename+"suffix"))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
err = suite.StorageDriver.Delete(suite.ctx, path.Join(dirname, dirname))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, dirname, filename))
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, path.Join(dirname, dirname+"suffix", filename))
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
|
|
// TestStatCall runs verifies the implementation of the storagedriver's Stat call.
|
|
func (suite *DriverSuite) TestStatCall(c *check.C) {
|
|
content := randomContents(4096)
|
|
dirPath := randomPath(32)
|
|
fileName := randomFilename(32)
|
|
filePath := path.Join(dirPath, fileName)
|
|
|
|
defer suite.deletePath(c, firstPart(dirPath))
|
|
|
|
// Call on non-existent file/dir, check error.
|
|
fi, err := suite.StorageDriver.Stat(suite.ctx, dirPath)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
c.Assert(fi, check.IsNil)
|
|
|
|
fi, err = suite.StorageDriver.Stat(suite.ctx, filePath)
|
|
c.Assert(err, check.NotNil)
|
|
c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{})
|
|
c.Assert(strings.Contains(err.Error(), suite.Name()), check.Equals, true)
|
|
c.Assert(fi, check.IsNil)
|
|
|
|
err = suite.StorageDriver.PutContent(suite.ctx, filePath, content)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
// Call on regular file, check results
|
|
fi, err = suite.StorageDriver.Stat(suite.ctx, filePath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(fi, check.NotNil)
|
|
c.Assert(fi.Path(), check.Equals, filePath)
|
|
c.Assert(fi.Size(), check.Equals, int64(len(content)))
|
|
c.Assert(fi.IsDir(), check.Equals, false)
|
|
createdTime := fi.ModTime()
|
|
|
|
// Sleep and modify the file
|
|
time.Sleep(time.Second * 10)
|
|
content = randomContents(4096)
|
|
err = suite.StorageDriver.PutContent(suite.ctx, filePath, content)
|
|
c.Assert(err, check.IsNil)
|
|
fi, err = suite.StorageDriver.Stat(suite.ctx, filePath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(fi, check.NotNil)
|
|
time.Sleep(time.Second * 5) // allow changes to propagate (eventual consistency)
|
|
|
|
// Check if the modification time is after the creation time.
|
|
// In case of cloud storage services, storage frontend nodes might have
|
|
// time drift between them, however that should be solved with sleeping
|
|
// before update.
|
|
modTime := fi.ModTime()
|
|
if !modTime.After(createdTime) {
|
|
c.Errorf("modtime (%s) is before the creation time (%s)", modTime, createdTime)
|
|
}
|
|
|
|
// Call on directory (do not check ModTime as dirs don't need to support it)
|
|
fi, err = suite.StorageDriver.Stat(suite.ctx, dirPath)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(fi, check.NotNil)
|
|
c.Assert(fi.Path(), check.Equals, dirPath)
|
|
c.Assert(fi.Size(), check.Equals, int64(0))
|
|
c.Assert(fi.IsDir(), check.Equals, true)
|
|
}
|
|
|
|
// TestPutContentMultipleTimes checks that if storage driver can overwrite the content
|
|
// in the subsequent puts. Validates that PutContent does not have to work
|
|
// with an offset like Writer does and overwrites the file entirely
|
|
// rather than writing the data to the [0,len(data)) of the file.
|
|
func (suite *DriverSuite) TestPutContentMultipleTimes(c *check.C) {
|
|
filename := randomPath(32)
|
|
contents := randomContents(4096)
|
|
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
contents = randomContents(2048) // upload a different, smaller file
|
|
err = suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
readContents, err := suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(readContents, check.DeepEquals, contents)
|
|
}
|
|
|
|
// TestConcurrentStreamReads checks that multiple clients can safely read from
|
|
// the same file simultaneously with various offsets.
|
|
func (suite *DriverSuite) TestConcurrentStreamReads(c *check.C) {
|
|
var filesize int64 = 128 * 1024 * 1024
|
|
|
|
if testing.Short() {
|
|
filesize = 10 * 1024 * 1024
|
|
c.Log("Reducing file size to 10MB for short mode")
|
|
}
|
|
|
|
filename := randomPath(32)
|
|
contents := randomContents(filesize)
|
|
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
readContents := func() {
|
|
defer wg.Done()
|
|
offset := rand.Int63n(int64(len(contents)))
|
|
reader, err := suite.StorageDriver.Reader(suite.ctx, filename, offset)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
readContents, err := ioutil.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(readContents, check.DeepEquals, contents[offset:])
|
|
}
|
|
|
|
wg.Add(10)
|
|
for i := 0; i < 10; i++ {
|
|
go readContents()
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
// TestConcurrentFileStreams checks that multiple *os.File objects can be passed
|
|
// in to Writer concurrently without hanging.
|
|
func (suite *DriverSuite) TestConcurrentFileStreams(c *check.C) {
|
|
numStreams := 32
|
|
|
|
if testing.Short() {
|
|
numStreams = 8
|
|
c.Log("Reducing number of streams to 8 for short mode")
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
testStream := func(size int64) {
|
|
defer wg.Done()
|
|
suite.testFileStreams(c, size)
|
|
}
|
|
|
|
wg.Add(numStreams)
|
|
for i := numStreams; i > 0; i-- {
|
|
go testStream(int64(numStreams) * 1024 * 1024)
|
|
}
|
|
|
|
wg.Wait()
|
|
}
|
|
|
|
// TODO (brianbland): evaluate the relevancy of this test
|
|
// TestEventualConsistency checks that if stat says that a file is a certain size, then
|
|
// you can freely read from the file (this is the only guarantee that the driver needs to provide)
|
|
// func (suite *DriverSuite) TestEventualConsistency(c *check.C) {
|
|
// if testing.Short() {
|
|
// c.Skip("Skipping test in short mode")
|
|
// }
|
|
//
|
|
// filename := randomPath(32)
|
|
// defer suite.deletePath(c, firstPart(filename))
|
|
//
|
|
// var offset int64
|
|
// var misswrites int
|
|
// var chunkSize int64 = 32
|
|
//
|
|
// for i := 0; i < 1024; i++ {
|
|
// contents := randomContents(chunkSize)
|
|
// read, err := suite.StorageDriver.Writer(suite.ctx, filename, offset, bytes.NewReader(contents))
|
|
// c.Assert(err, check.IsNil)
|
|
//
|
|
// fi, err := suite.StorageDriver.Stat(suite.ctx, filename)
|
|
// c.Assert(err, check.IsNil)
|
|
//
|
|
// // We are most concerned with being able to read data as soon as Stat declares
|
|
// // it is uploaded. This is the strongest guarantee that some drivers (that guarantee
|
|
// // at best eventual consistency) absolutely need to provide.
|
|
// if fi.Size() == offset+chunkSize {
|
|
// reader, err := suite.StorageDriver.Reader(suite.ctx, filename, offset)
|
|
// c.Assert(err, check.IsNil)
|
|
//
|
|
// readContents, err := ioutil.ReadAll(reader)
|
|
// c.Assert(err, check.IsNil)
|
|
//
|
|
// c.Assert(readContents, check.DeepEquals, contents)
|
|
//
|
|
// reader.Close()
|
|
// offset += read
|
|
// } else {
|
|
// misswrites++
|
|
// }
|
|
// }
|
|
//
|
|
// if misswrites > 0 {
|
|
// c.Log("There were " + string(misswrites) + " occurrences of a write not being instantly available.")
|
|
// }
|
|
//
|
|
// c.Assert(misswrites, check.Not(check.Equals), 1024)
|
|
// }
|
|
|
|
// BenchmarkPutGetEmptyFiles benchmarks PutContent/GetContent for 0B files
|
|
func (suite *DriverSuite) BenchmarkPutGetEmptyFiles(c *check.C) {
|
|
suite.benchmarkPutGetFiles(c, 0)
|
|
}
|
|
|
|
// BenchmarkPutGet1KBFiles benchmarks PutContent/GetContent for 1KB files
|
|
func (suite *DriverSuite) BenchmarkPutGet1KBFiles(c *check.C) {
|
|
suite.benchmarkPutGetFiles(c, 1024)
|
|
}
|
|
|
|
// BenchmarkPutGet1MBFiles benchmarks PutContent/GetContent for 1MB files
|
|
func (suite *DriverSuite) BenchmarkPutGet1MBFiles(c *check.C) {
|
|
suite.benchmarkPutGetFiles(c, 1024*1024)
|
|
}
|
|
|
|
// BenchmarkPutGet1GBFiles benchmarks PutContent/GetContent for 1GB files
|
|
func (suite *DriverSuite) BenchmarkPutGet1GBFiles(c *check.C) {
|
|
suite.benchmarkPutGetFiles(c, 1024*1024*1024)
|
|
}
|
|
|
|
func (suite *DriverSuite) benchmarkPutGetFiles(c *check.C, size int64) {
|
|
c.SetBytes(size)
|
|
parentDir := randomPath(8)
|
|
defer func() {
|
|
c.StopTimer()
|
|
suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
|
|
}()
|
|
|
|
for i := 0; i < c.N; i++ {
|
|
filename := path.Join(parentDir, randomPath(32))
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, randomContents(size))
|
|
c.Assert(err, check.IsNil)
|
|
|
|
_, err = suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
}
|
|
|
|
// BenchmarkStreamEmptyFiles benchmarks Writer/Reader for 0B files
|
|
func (suite *DriverSuite) BenchmarkStreamEmptyFiles(c *check.C) {
|
|
suite.benchmarkStreamFiles(c, 0)
|
|
}
|
|
|
|
// BenchmarkStream1KBFiles benchmarks Writer/Reader for 1KB files
|
|
func (suite *DriverSuite) BenchmarkStream1KBFiles(c *check.C) {
|
|
suite.benchmarkStreamFiles(c, 1024)
|
|
}
|
|
|
|
// BenchmarkStream1MBFiles benchmarks Writer/Reader for 1MB files
|
|
func (suite *DriverSuite) BenchmarkStream1MBFiles(c *check.C) {
|
|
suite.benchmarkStreamFiles(c, 1024*1024)
|
|
}
|
|
|
|
// BenchmarkStream1GBFiles benchmarks Writer/Reader for 1GB files
|
|
func (suite *DriverSuite) BenchmarkStream1GBFiles(c *check.C) {
|
|
suite.benchmarkStreamFiles(c, 1024*1024*1024)
|
|
}
|
|
|
|
func (suite *DriverSuite) benchmarkStreamFiles(c *check.C, size int64) {
|
|
c.SetBytes(size)
|
|
parentDir := randomPath(8)
|
|
defer func() {
|
|
c.StopTimer()
|
|
suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
|
|
}()
|
|
|
|
for i := 0; i < c.N; i++ {
|
|
filename := path.Join(parentDir, randomPath(32))
|
|
writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
|
|
c.Assert(err, check.IsNil)
|
|
written, err := io.Copy(writer, bytes.NewReader(randomContents(size)))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(written, check.Equals, size)
|
|
|
|
err = writer.Commit()
|
|
c.Assert(err, check.IsNil)
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
rc, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.IsNil)
|
|
rc.Close()
|
|
}
|
|
}
|
|
|
|
// BenchmarkList5Files benchmarks List for 5 small files
|
|
func (suite *DriverSuite) BenchmarkList5Files(c *check.C) {
|
|
suite.benchmarkListFiles(c, 5)
|
|
}
|
|
|
|
// BenchmarkList50Files benchmarks List for 50 small files
|
|
func (suite *DriverSuite) BenchmarkList50Files(c *check.C) {
|
|
suite.benchmarkListFiles(c, 50)
|
|
}
|
|
|
|
func (suite *DriverSuite) benchmarkListFiles(c *check.C, numFiles int64) {
|
|
parentDir := randomPath(8)
|
|
defer func() {
|
|
c.StopTimer()
|
|
suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
|
|
}()
|
|
|
|
for i := int64(0); i < numFiles; i++ {
|
|
err := suite.StorageDriver.PutContent(suite.ctx, path.Join(parentDir, randomPath(32)), nil)
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
|
|
c.ResetTimer()
|
|
for i := 0; i < c.N; i++ {
|
|
files, err := suite.StorageDriver.List(suite.ctx, parentDir)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(int64(len(files)), check.Equals, numFiles)
|
|
}
|
|
}
|
|
|
|
// BenchmarkDelete5Files benchmarks Delete for 5 small files
|
|
func (suite *DriverSuite) BenchmarkDelete5Files(c *check.C) {
|
|
suite.benchmarkDeleteFiles(c, 5)
|
|
}
|
|
|
|
// BenchmarkDelete50Files benchmarks Delete for 50 small files
|
|
func (suite *DriverSuite) BenchmarkDelete50Files(c *check.C) {
|
|
suite.benchmarkDeleteFiles(c, 50)
|
|
}
|
|
|
|
func (suite *DriverSuite) benchmarkDeleteFiles(c *check.C, numFiles int64) {
|
|
for i := 0; i < c.N; i++ {
|
|
parentDir := randomPath(8)
|
|
defer suite.deletePath(c, firstPart(parentDir))
|
|
|
|
c.StopTimer()
|
|
for j := int64(0); j < numFiles; j++ {
|
|
err := suite.StorageDriver.PutContent(suite.ctx, path.Join(parentDir, randomPath(32)), nil)
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
c.StartTimer()
|
|
|
|
// This is the operation we're benchmarking
|
|
err := suite.StorageDriver.Delete(suite.ctx, firstPart(parentDir))
|
|
c.Assert(err, check.IsNil)
|
|
}
|
|
}
|
|
|
|
func (suite *DriverSuite) testFileStreams(c *check.C, size int64) {
|
|
tf, err := ioutil.TempFile("", "tf")
|
|
c.Assert(err, check.IsNil)
|
|
defer os.Remove(tf.Name())
|
|
defer tf.Close()
|
|
|
|
filename := randomPath(32)
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
contents := randomContents(size)
|
|
|
|
_, err = tf.Write(contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
tf.Sync()
|
|
tf.Seek(0, os.SEEK_SET)
|
|
|
|
writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
|
|
c.Assert(err, check.IsNil)
|
|
nn, err := io.Copy(writer, tf)
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(nn, check.Equals, size)
|
|
|
|
err = writer.Commit()
|
|
c.Assert(err, check.IsNil)
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
readContents, err := ioutil.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
c.Assert(readContents, check.DeepEquals, contents)
|
|
}
|
|
|
|
func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents []byte) {
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
err := suite.StorageDriver.PutContent(suite.ctx, filename, contents)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
readContents, err := suite.StorageDriver.GetContent(suite.ctx, filename)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
c.Assert(readContents, check.DeepEquals, contents)
|
|
}
|
|
|
|
func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, contents []byte) {
|
|
defer suite.deletePath(c, firstPart(filename))
|
|
|
|
writer, err := suite.StorageDriver.Writer(suite.ctx, filename, false)
|
|
c.Assert(err, check.IsNil)
|
|
nn, err := io.Copy(writer, bytes.NewReader(contents))
|
|
c.Assert(err, check.IsNil)
|
|
c.Assert(nn, check.Equals, int64(len(contents)))
|
|
|
|
err = writer.Commit()
|
|
c.Assert(err, check.IsNil)
|
|
err = writer.Close()
|
|
c.Assert(err, check.IsNil)
|
|
|
|
reader, err := suite.StorageDriver.Reader(suite.ctx, filename, 0)
|
|
c.Assert(err, check.IsNil)
|
|
defer reader.Close()
|
|
|
|
readContents, err := ioutil.ReadAll(reader)
|
|
c.Assert(err, check.IsNil)
|
|
|
|
c.Assert(readContents, check.DeepEquals, contents)
|
|
}
|
|
|
|
var filenameChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789")
|
|
var separatorChars = []byte("._-")
|
|
|
|
func randomPath(length int64) string {
|
|
path := "/"
|
|
for int64(len(path)) < length {
|
|
chunkLength := rand.Int63n(length-int64(len(path))) + 1
|
|
chunk := randomFilename(chunkLength)
|
|
path += chunk
|
|
remaining := length - int64(len(path))
|
|
if remaining == 1 {
|
|
path += randomFilename(1)
|
|
} else if remaining > 1 {
|
|
path += "/"
|
|
}
|
|
}
|
|
return path
|
|
}
|
|
|
|
func randomFilename(length int64) string {
|
|
b := make([]byte, length)
|
|
wasSeparator := true
|
|
for i := range b {
|
|
if !wasSeparator && i < len(b)-1 && rand.Intn(4) == 0 {
|
|
b[i] = separatorChars[rand.Intn(len(separatorChars))]
|
|
wasSeparator = true
|
|
} else {
|
|
b[i] = filenameChars[rand.Intn(len(filenameChars))]
|
|
wasSeparator = false
|
|
}
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
// randomBytes pre-allocates all of the memory sizes needed for the test. If
|
|
// anything panics while accessing randomBytes, just make this number bigger.
|
|
var randomBytes = make([]byte, 128<<20)
|
|
|
|
func init() {
|
|
_, _ = rand.Read(randomBytes) // always returns len(randomBytes) and nil error
|
|
}
|
|
|
|
func randomContents(length int64) []byte {
|
|
return randomBytes[:length]
|
|
}
|
|
|
|
type randReader struct {
|
|
r int64
|
|
m sync.Mutex
|
|
}
|
|
|
|
func (rr *randReader) Read(p []byte) (n int, err error) {
|
|
rr.m.Lock()
|
|
defer rr.m.Unlock()
|
|
|
|
toread := int64(len(p))
|
|
if toread > rr.r {
|
|
toread = rr.r
|
|
}
|
|
n = copy(p, randomContents(toread))
|
|
rr.r -= int64(n)
|
|
|
|
if rr.r <= 0 {
|
|
err = io.EOF
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func newRandReader(n int64) *randReader {
|
|
return &randReader{r: n}
|
|
}
|
|
|
|
func firstPart(filePath string) string {
|
|
if filePath == "" {
|
|
return "/"
|
|
}
|
|
for {
|
|
if filePath[len(filePath)-1] == '/' {
|
|
filePath = filePath[:len(filePath)-1]
|
|
}
|
|
|
|
dir, file := path.Split(filePath)
|
|
if dir == "" && file == "" {
|
|
return "/"
|
|
}
|
|
if dir == "/" || dir == "" {
|
|
return "/" + file
|
|
}
|
|
if file == "" {
|
|
return dir
|
|
}
|
|
filePath = dir
|
|
}
|
|
}
|