distribution/registry/storage/driver/walk_test.go
Collin Shoop cf81f67a16 storagedriver/s3: Optimized Walk implementation + bugfix
Optimized S3 Walk impl by no longer listing files recursively. Overall gives a huge performance increase both in terms of runtime and S3 calls (up to ~500x).

Fixed a bug in WalkFallback where ErrSkipDir for was not handled as documented for non-directory.

Signed-off-by: Collin Shoop <cshoop@digitalocean.com>
2021-08-16 16:07:25 -04:00

183 lines
4.0 KiB
Go

package driver
import (
"context"
"fmt"
"strings"
"testing"
)
type changingFileSystem struct {
StorageDriver
fileset []string
keptFiles map[string]bool
}
func (cfs *changingFileSystem) List(_ context.Context, _ string) ([]string, error) {
return cfs.fileset, nil
}
func (cfs *changingFileSystem) Stat(_ context.Context, path string) (FileInfo, error) {
kept, ok := cfs.keptFiles[path]
if ok && kept {
return &FileInfoInternal{
FileInfoFields: FileInfoFields{
Path: path,
},
}, nil
}
return nil, PathNotFoundError{}
}
type fileSystem struct {
StorageDriver
// maps folder to list results
fileset map[string][]string
}
func (cfs *fileSystem) List(_ context.Context, path string) ([]string, error) {
return cfs.fileset[path], nil
}
func (cfs *fileSystem) Stat(_ context.Context, path string) (FileInfo, error) {
_, isDir := cfs.fileset[path]
return &FileInfoInternal{
FileInfoFields: FileInfoFields{
Path: path,
IsDir: isDir,
Size: int64(len(path)),
},
}, nil
}
func (cfs *fileSystem) isDir(path string) bool {
_, isDir := cfs.fileset[path]
return isDir
}
func TestWalkFileRemoved(t *testing.T) {
d := &changingFileSystem{
fileset: []string{"zoidberg", "bender"},
keptFiles: map[string]bool{
"zoidberg": true,
},
}
infos := []FileInfo{}
err := WalkFallback(context.Background(), d, "", func(fileInfo FileInfo) error {
infos = append(infos, fileInfo)
return nil
})
if len(infos) != 1 || infos[0].Path() != "zoidberg" {
t.Errorf(fmt.Sprintf("unexpected path set during walk: %s", infos))
}
if err != nil {
t.Fatalf(err.Error())
}
}
func TestWalkFallback(t *testing.T) {
d := &fileSystem{
fileset: map[string][]string{
"/": {"/file1", "/folder1", "/folder2"},
"/folder1": {"/folder1/file1"},
"/folder2": {"/folder2/file1"},
},
}
noopFn := func(fileInfo FileInfo) error { return nil }
tcs := []struct {
name string
fn WalkFn
from string
expected []string
err bool
}{
{
name: "walk all",
fn: noopFn,
expected: []string{
"/file1",
"/folder1",
"/folder1/file1",
"/folder2",
"/folder2/file1",
},
},
{
name: "skip directory",
fn: func(fileInfo FileInfo) error {
if fileInfo.Path() == "/folder1" {
return ErrSkipDir
}
if strings.Contains(fileInfo.Path(), "/folder1") {
t.Fatalf("skipped dir %s and should not walk %s", "/folder1", fileInfo.Path())
}
return nil
},
expected: []string{
"/file1",
"/folder1", // return ErrSkipDir, skip anything under /folder1
// skip /folder1/file1
"/folder2",
"/folder2/file1",
},
},
{
name: "stop early",
fn: func(fileInfo FileInfo) error {
if fileInfo.Path() == "/folder1/file1" {
return ErrSkipDir
}
return nil
},
expected: []string{
"/file1",
"/folder1",
"/folder1/file1",
// stop early
},
},
{
name: "from folder",
fn: noopFn,
expected: []string{
"/folder1/file1",
},
from: "/folder1",
},
}
for _, tc := range tcs {
var walked []string
if tc.from == "" {
tc.from = "/"
}
t.Run(tc.name, func(t *testing.T) {
err := WalkFallback(context.Background(), d, tc.from, func(fileInfo FileInfo) error {
walked = append(walked, fileInfo.Path())
if fileInfo.IsDir() != d.isDir(fileInfo.Path()) {
t.Fatalf("fileInfo isDir not matching file system: expected %t actual %t", d.isDir(fileInfo.Path()), fileInfo.IsDir())
}
return tc.fn(fileInfo)
})
if tc.err && err == nil {
t.Fatalf("expected err")
}
if !tc.err && err != nil {
t.Fatalf(err.Error())
}
compareWalked(t, tc.expected, walked)
})
}
}
func compareWalked(t *testing.T, expected, walked []string) {
if len(walked) != len(expected) {
t.Fatalf("Mismatch number of fileInfo walked %d expected %d; walked %s; expected %s;", len(walked), len(expected), walked, expected)
}
for i := range walked {
if walked[i] != expected[i] {
t.Fatalf("expected walked to come in order expected: walked %s", walked)
}
}
}