From 31ece3d3b68875f0bb884deaef28833689536733 Mon Sep 17 00:00:00 2001 From: Richard Scothern Date: Thu, 24 Mar 2016 16:03:25 -0700 Subject: [PATCH] Fix signature handling with GC. If a schema 1 manifest is uploaded with the `disablesignaturestore` option set to true, then no signatures will exist. Handle this case. If a schema 1 manifest is pushed, deleted, garbage collected and pushed again, the repository will contain signature links from the first version, but the blobs will not exist. Disable the signature store in the garbage-collect command so signatures are not fetched. Signed-off-by: Richard Scothern --- docs/garbagecollect.go | 43 ++++++++++++++++++++++------------- docs/garbagecollect_test.go | 6 ++--- docs/storage/manifeststore.go | 12 ++++++++-- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/docs/garbagecollect.go b/docs/garbagecollect.go index cfeee078..8df956b9 100644 --- a/docs/garbagecollect.go +++ b/docs/garbagecollect.go @@ -13,20 +13,18 @@ import ( "github.com/docker/distribution/registry/storage" "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/factory" + "github.com/docker/libtrust" "github.com/spf13/cobra" ) -func emit(ctx context.Context, s string) { +func emit(format string, a ...interface{}) { if dryRun { - context.GetLogger(ctx).Infof("gc: %s", s) + fmt.Printf(format, a...) + fmt.Println("") } } -func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error { - registry, err := storage.NewRegistry(ctx, storageDriver) - if err != nil { - return fmt.Errorf("failed to construct registry: %v", err) - } +func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace) error { repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator) if !ok { @@ -35,8 +33,8 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error // mark markSet := make(map[digest.Digest]struct{}) - err = repositoryEnumerator.Enumerate(ctx, func(repoName string) error { - emit(ctx, fmt.Sprint(repoName)) + err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error { + emit(repoName) var err error named, err := reference.ParseNamed(repoName) @@ -59,8 +57,8 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error } err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error { - // Mark the manifest's blo - emit(ctx, fmt.Sprintf("%s: adding manifest %s ", repoName, dgst)) + // Mark the manifest's blob + emit("%s: marking manifest %s ", repoName, dgst) markSet[dgst] = struct{}{} manifest, err := manifestService.Get(ctx, dgst) @@ -71,7 +69,7 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error descriptors := manifest.References() for _, descriptor := range descriptors { markSet[descriptor.Digest] = struct{}{} - emit(ctx, fmt.Sprintf("%s: marking blob %v", repoName, descriptor)) + emit("%s: marking blob %s", repoName, descriptor.Digest) } switch manifest.(type) { @@ -85,13 +83,13 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error return fmt.Errorf("failed to get signatures for signed manifest: %v", err) } for _, signatureDigest := range signatures { - emit(ctx, fmt.Sprintf("%s: marking signature %s", repoName, signatureDigest)) + emit("%s: marking signature %s", repoName, signatureDigest) markSet[signatureDigest] = struct{}{} } break case *schema2.DeserializedManifest: config := manifest.(*schema2.DeserializedManifest).Config - emit(ctx, fmt.Sprintf("%s: marking configuration %s", repoName, config.Digest)) + emit("%s: marking configuration %s", repoName, config.Digest) markSet[config.Digest] = struct{}{} break } @@ -120,11 +118,12 @@ func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver) error return fmt.Errorf("error enumerating blobs: %v", err) } + emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet)) // Construct vacuum vacuum := storage.NewVacuum(ctx, storageDriver) for dgst := range deleteSet { if dryRun { - emit(ctx, fmt.Sprintf("deleting %s", dgst)) + emit("deleting %s", dgst) continue } err = vacuum.RemoveBlob(string(dgst)) @@ -168,7 +167,19 @@ var GCCmd = &cobra.Command{ os.Exit(1) } - err = markAndSweep(ctx, driver) + k, err := libtrust.GenerateECP256PrivateKey() + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(1) + } + + registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k)) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err) + os.Exit(1) + } + + err = markAndSweep(ctx, driver, registry) if err != nil { fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err) os.Exit(1) diff --git a/docs/garbagecollect_test.go b/docs/garbagecollect_test.go index 6096e758..dd5fadd5 100644 --- a/docs/garbagecollect_test.go +++ b/docs/garbagecollect_test.go @@ -161,7 +161,7 @@ func TestNoDeletionNoEffect(t *testing.T) { } // Run GC - err = markAndSweep(context.Background(), inmemoryDriver) + err = markAndSweep(context.Background(), inmemoryDriver, registry) if err != nil { t.Fatalf("Failed mark and sweep: %v", err) } @@ -193,7 +193,7 @@ func TestDeletionHasEffect(t *testing.T) { manifests.Delete(ctx, image3.manifestDigest) // Run GC - err = markAndSweep(context.Background(), inmemoryDriver) + err = markAndSweep(context.Background(), inmemoryDriver, registry) if err != nil { t.Fatalf("Failed mark and sweep: %v", err) } @@ -327,7 +327,7 @@ func TestOrphanBlobDeleted(t *testing.T) { uploadRandomSchema2Image(t, repo) // Run GC - err = markAndSweep(context.Background(), inmemoryDriver) + err = markAndSweep(context.Background(), inmemoryDriver, registry) if err != nil { t.Fatalf("Failed mark and sweep: %v", err) } diff --git a/docs/storage/manifeststore.go b/docs/storage/manifeststore.go index e0b82309..5a9165f9 100644 --- a/docs/storage/manifeststore.go +++ b/docs/storage/manifeststore.go @@ -12,6 +12,7 @@ import ( "github.com/docker/distribution/manifest/manifestlist" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" + "github.com/docker/distribution/registry/storage/driver" ) // A ManifestHandler gets and puts manifests of a particular type. @@ -161,13 +162,20 @@ func (ms *manifestStore) GetSignatures(ctx context.Context, manifestDigest diges return nil, err } + var digests []digest.Digest alg := string(digest.SHA256) signaturePaths, err := ms.blobStore.driver.List(ctx, path.Join(signaturesPath, alg)) - if err != nil { + + switch err.(type) { + case nil: + break + case driver.PathNotFoundError: + // Manifest may have been pushed with signature store disabled + return digests, nil + default: return nil, err } - var digests []digest.Digest for _, sigPath := range signaturePaths { sigdigest, err := digest.ParseDigest(alg + ":" + path.Base(sigPath)) if err != nil {