From 6fcea22b0aa01f513393033e268913dea7fef180 Mon Sep 17 00:00:00 2001 From: Mike Brown Date: Mon, 10 Jul 2017 18:09:10 -0500 Subject: [PATCH] add an ocischema manifest handler for the registry Signed-off-by: Mike Brown --- registry/storage/manifeststore.go | 2 + registry/storage/ocimanifesthandler.go | 128 ++++++++++++++++++ registry/storage/ocimanifesthandler_test.go | 136 ++++++++++++++++++++ registry/storage/registry.go | 6 + 4 files changed, 272 insertions(+) create mode 100644 registry/storage/ocimanifesthandler.go create mode 100644 registry/storage/ocimanifesthandler_test.go diff --git a/registry/storage/manifeststore.go b/registry/storage/manifeststore.go index 52bbe9da..13b4f5b5 100644 --- a/registry/storage/manifeststore.go +++ b/registry/storage/manifeststore.go @@ -121,6 +121,8 @@ func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest return ms.schema1Handler.Put(ctx, manifest, ms.skipDependencyVerification) case *schema2.DeserializedManifest: return ms.schema2Handler.Put(ctx, manifest, ms.skipDependencyVerification) + case *ocischema.DeserializedManifest: + return ms.ocischemaHandler.Put(ctx, manifest, ms.skipDependencyVerification) case *manifestlist.DeserializedManifestList: return ms.manifestListHandler.Put(ctx, manifest, ms.skipDependencyVerification) } diff --git a/registry/storage/ocimanifesthandler.go b/registry/storage/ocimanifesthandler.go new file mode 100644 index 00000000..bb658348 --- /dev/null +++ b/registry/storage/ocimanifesthandler.go @@ -0,0 +1,128 @@ +package storage + +import ( + "encoding/json" + "fmt" + "net/url" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest/ocischema" + "github.com/opencontainers/go-digest" +) + +//ocischemaManifestHandler is a ManifestHandler that covers ocischema manifests. +type ocischemaManifestHandler struct { + repository distribution.Repository + blobStore distribution.BlobStore + ctx context.Context + manifestURLs manifestURLs +} + +var _ ManifestHandler = &ocischemaManifestHandler{} + +func (ms *ocischemaManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) { + context.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Unmarshal") + + var m ocischema.DeserializedManifest + if err := json.Unmarshal(content, &m); err != nil { + return nil, err + } + + return &m, nil +} + +func (ms *ocischemaManifestHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) { + context.GetLogger(ms.ctx).Debug("(*ocischemaManifestHandler).Put") + + m, ok := manifest.(*ocischema.DeserializedManifest) + if !ok { + return "", fmt.Errorf("non-ocischema manifest put to ocischemaManifestHandler: %T", manifest) + } + + if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil { + return "", err + } + + mt, payload, err := m.Payload() + if err != nil { + return "", err + } + + revision, err := ms.blobStore.Put(ctx, mt, payload) + if err != nil { + context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err) + return "", err + } + + return revision.Digest, nil +} + +// verifyManifest ensures that the manifest content is valid from the +// perspective of the registry. As a policy, the registry only tries to store +// valid content, leaving trust policies of that content up to consumers. +func (ms *ocischemaManifestHandler) verifyManifest(ctx context.Context, mnfst ocischema.DeserializedManifest, skipDependencyVerification bool) error { + var errs distribution.ErrManifestVerification + + if skipDependencyVerification { + return nil + } + + manifestService, err := ms.repository.Manifests(ctx) + if err != nil { + return err + } + + blobsService := ms.repository.Blobs(ctx) + + for _, descriptor := range mnfst.References() { + var err error + + switch descriptor.MediaType { + case ocischema.MediaTypeForeignLayer: + // Clients download this layer from an external URL, so do not check for + // its presense. + if len(descriptor.URLs) == 0 { + err = errMissingURL + } + allow := ms.manifestURLs.allow + deny := ms.manifestURLs.deny + for _, u := range descriptor.URLs { + var pu *url.URL + pu, err = url.Parse(u) + if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" || (allow != nil && !allow.MatchString(u)) || (deny != nil && deny.MatchString(u)) { + err = errInvalidURL + break + } + } + case ocischema.MediaTypeManifest: + var exists bool + exists, err = manifestService.Exists(ctx, descriptor.Digest) + if err != nil || !exists { + err = distribution.ErrBlobUnknown // just coerce to unknown. + } + + fallthrough // double check the blob store. + default: + // forward all else to blob storage + if len(descriptor.URLs) == 0 { + _, err = blobsService.Stat(ctx, descriptor.Digest) + } + } + + if err != nil { + if err != distribution.ErrBlobUnknown { + errs = append(errs, err) + } + + // On error here, we always append unknown blob errors. + errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: descriptor.Digest}) + } + } + + if len(errs) != 0 { + return errs + } + + return nil +} diff --git a/registry/storage/ocimanifesthandler_test.go b/registry/storage/ocimanifesthandler_test.go new file mode 100644 index 00000000..e09f1ee3 --- /dev/null +++ b/registry/storage/ocimanifesthandler_test.go @@ -0,0 +1,136 @@ +package storage + +import ( + "regexp" + "testing" + + "github.com/docker/distribution" + "github.com/docker/distribution/context" + "github.com/docker/distribution/manifest" + "github.com/docker/distribution/manifest/ocischema" + "github.com/docker/distribution/registry/storage/driver/inmemory" +) + +func TestVerifyOCIManifestForeignLayer(t *testing.T) { + ctx := context.Background() + inmemoryDriver := inmemory.New() + registry := createRegistry(t, inmemoryDriver, + ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")), + ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope"))) + repo := makeRepository(t, registry, "test") + manifestService := makeManifestService(t, repo) + + config, err := repo.Blobs(ctx).Put(ctx, ocischema.MediaTypeConfig, nil) + if err != nil { + t.Fatal(err) + } + + layer, err := repo.Blobs(ctx).Put(ctx, ocischema.MediaTypeLayer, nil) + if err != nil { + t.Fatal(err) + } + + foreignLayer := distribution.Descriptor{ + Digest: "sha256:463435349086340864309863409683460843608348608934092322395278926a", + Size: 6323, + MediaType: ocischema.MediaTypeForeignLayer, + } + + template := ocischema.Manifest{ + Versioned: manifest.Versioned{ + SchemaVersion: 2, + MediaType: ocischema.MediaTypeManifest, + }, + Config: config, + } + + type testcase struct { + BaseLayer distribution.Descriptor + URLs []string + Err error + } + + cases := []testcase{ + { + foreignLayer, + nil, + errMissingURL, + }, + { + // regular layers may have foreign urls + layer, + []string{"http://foo/bar"}, + nil, + }, + { + foreignLayer, + []string{"file:///local/file"}, + errInvalidURL, + }, + { + foreignLayer, + []string{"http://foo/bar#baz"}, + errInvalidURL, + }, + { + foreignLayer, + []string{""}, + errInvalidURL, + }, + { + foreignLayer, + []string{"https://foo/bar", ""}, + errInvalidURL, + }, + { + foreignLayer, + []string{"", "https://foo/bar"}, + errInvalidURL, + }, + { + foreignLayer, + []string{"http://nope/bar"}, + errInvalidURL, + }, + { + foreignLayer, + []string{"http://foo/nope"}, + errInvalidURL, + }, + { + foreignLayer, + []string{"http://foo/bar"}, + nil, + }, + { + foreignLayer, + []string{"https://foo/bar"}, + nil, + }, + } + + for _, c := range cases { + m := template + l := c.BaseLayer + l.URLs = c.URLs + m.Layers = []distribution.Descriptor{l} + dm, err := ocischema.FromStruct(m) + if err != nil { + t.Error(err) + continue + } + + _, err = manifestService.Put(ctx, dm) + if verr, ok := err.(distribution.ErrManifestVerification); ok { + // Extract the first error + if len(verr) == 2 { + if _, ok = verr[1].(distribution.ErrManifestBlobUnknown); ok { + err = verr[0] + } + } + } + if err != c.Err { + t.Errorf("%#v: expected %v, got %v", l, c.Err, err) + } + } +} diff --git a/registry/storage/registry.go b/registry/storage/registry.go index 46b96853..f7c95a59 100644 --- a/registry/storage/registry.go +++ b/registry/storage/registry.go @@ -258,6 +258,12 @@ func (repo *repository) Manifests(ctx context.Context, options ...distribution.M repository: repo, blobStore: blobStore, }, + ocischemaHandler: &ocischemaManifestHandler{ + ctx: ctx, + repository: repo, + blobStore: blobStore, + manifestURLs: repo.registry.manifestURLs, + }, } // Apply options