diff --git a/cmd/registry/config.yml b/cmd/registry/config.yml index 5dd39cb3..b1a8f48d 100644 --- a/cmd/registry/config.yml +++ b/cmd/registry/config.yml @@ -9,6 +9,9 @@ storage: layerinfo: inmemory filesystem: rootdirectory: /tmp/registry-dev + maintenance: + uploadpurging: + enabled: false http: addr: :5000 secret: asecretforlocaldevelopment @@ -39,3 +42,4 @@ notifications: threshold: 10 backoff: 1s disabled: true + \ No newline at end of file diff --git a/configuration/configuration.go b/configuration/configuration.go index 3d302e1c..074471b4 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -188,6 +188,8 @@ func (storage Storage) Type() string { // Return only key in this map for k := range storage { switch k { + case "maintenance": + // allow configuration of maintenance case "cache": // allow configuration of caching default: @@ -217,6 +219,8 @@ func (storage *Storage) UnmarshalYAML(unmarshal func(interface{}) error) error { types := make([]string, 0, len(storageMap)) for k := range storageMap { switch k { + case "maintenance": + // allow for configuration of maintenance case "cache": // allow configuration of caching default: diff --git a/docs/configuration.md b/docs/configuration.md index 0b59f41b..ab783af3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -43,6 +43,12 @@ storage: rootdirectory: /s3/object/name/prefix cache: layerinfo: inmemory + maintenance: + uploadpurging: + enabled: true + age: 168h + interval: 24h + dryrun: false auth: silly: realm: silly-realm @@ -221,6 +227,12 @@ storage: rootdirectory: /s3/object/name/prefix cache: layerinfo: inmemory + maintenance: + uploadpurging: + enabled: true + age: 168h + interval: 24h + dryrun: false ``` The storage option is **required** and defines which storage backend is in use. @@ -410,6 +422,27 @@ This storage backend uses Amazon's Simple Storage Service (S3). +### Maintenance + +Currently the registry can perform one maintenance function: upload purging. This and future +maintenance functions which are related to storage can be configured under the maintenance section. + +### Upload Purging + +Upload purging is a background process that periodically removes orphaned files from the upload +directories of the registry. Upload purging is enabled by default. To + configure upload directory purging, the following parameters +must be set. + + +| Parameter | Required | Description + --------- | -------- | ----------- +`enabled` | yes | Set to true to enable upload purging. Default=true. | +`age` | yes | Upload directories which are older than this age will be deleted. Default=168h (1 week) +`interval` | yes | The interval between upload directory purging. Default=24h. +`dryrun` | yes | dryrun can be set to true to obtain a summary of what directories will be deleted. Default=false. + +Note: `age` and `interval` are strings containing a number with optional fraction and a unit suffix: e.g. 45m, 2h10m, 168h (1 week). ## auth @@ -1139,7 +1172,8 @@ Configure the behavior of the Redis connection pool. - + + ## Example: Development configuration The following is a simple example you can use for local development: diff --git a/docs/deploying.md b/docs/deploying.md index c7d3d17c..b26499da 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -205,6 +205,9 @@ storage: layerinfo: inmemory filesystem: rootdirectory: /tmp/registry-dev + maintenance: + uploadpurging: + enabled: false http: addr: :5000 secret: asecretforlocaldevelopment diff --git a/registry/handlers/app.go b/registry/handlers/app.go index e35d8633..3cc360c6 100644 --- a/registry/handlers/app.go +++ b/registry/handlers/app.go @@ -81,7 +81,18 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App panic(err) } - startUploadPurger(app.driver, ctxu.GetLogger(app)) + purgeConfig := uploadPurgeDefaultConfig() + if mc, ok := configuration.Storage["maintenance"]; ok { + for k, v := range mc { + switch k { + case "uploadpurging": + purgeConfig = v.(map[interface{}]interface{}) + } + } + + } + + startUploadPurger(app.driver, ctxu.GetLogger(app), purgeConfig) app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"]) if err != nil { @@ -568,26 +579,82 @@ func applyStorageMiddleware(driver storagedriver.StorageDriver, middlewares []co return driver, nil } +// uploadPurgeDefaultConfig provides a default configuration for upload +// purging to be used in the absence of configuration in the +// confifuration file +func uploadPurgeDefaultConfig() map[interface{}]interface{} { + config := map[interface{}]interface{}{} + config["enabled"] = true + config["age"] = "168h" + config["interval"] = "24h" + config["dryrun"] = false + return config +} + +func badPurgeUploadConfig(reason string) { + panic(fmt.Sprintf("Unable to parse upload purge configuration: %s", reason)) +} + // startUploadPurger schedules a goroutine which will periodically // check upload directories for old files and delete them -func startUploadPurger(storageDriver storagedriver.StorageDriver, log ctxu.Logger) { - rand.Seed(time.Now().Unix()) - jitter := time.Duration(rand.Int()%60) * time.Minute +func startUploadPurger(storageDriver storagedriver.StorageDriver, log ctxu.Logger, config map[interface{}]interface{}) { + if config["enabled"] == false { + return + } - // Start with reasonable defaults - // TODO:(richardscothern) make configurable - purgeAge := time.Duration(7 * 24 * time.Hour) - timeBetweenPurges := time.Duration(1 * 24 * time.Hour) + var purgeAgeDuration time.Duration + var err error + purgeAge, ok := config["age"] + if ok { + ageStr, ok := purgeAge.(string) + if !ok { + badPurgeUploadConfig("age is not a string") + } + purgeAgeDuration, err = time.ParseDuration(ageStr) + if err != nil { + badPurgeUploadConfig(fmt.Sprintf("Cannot parse duration: %s", err.Error())) + } + } else { + badPurgeUploadConfig("age missing") + } + + var intervalDuration time.Duration + interval, ok := config["interval"] + if ok { + intervalStr, ok := interval.(string) + if !ok { + badPurgeUploadConfig("interval is not a string") + } + + intervalDuration, err = time.ParseDuration(intervalStr) + if err != nil { + badPurgeUploadConfig(fmt.Sprintf("Cannot parse interval: %s", err.Error())) + } + } else { + badPurgeUploadConfig("interval missing") + } + + var dryRunBool bool + dryRun, ok := config["dryrun"] + if ok { + dryRunBool, ok = dryRun.(bool) + if !ok { + badPurgeUploadConfig("cannot parse dryrun") + } + } else { + badPurgeUploadConfig("dryrun missing") + } go func() { + rand.Seed(time.Now().Unix()) + jitter := time.Duration(rand.Int()%60) * time.Minute log.Infof("Starting upload purge in %s", jitter) time.Sleep(jitter) for { - storage.PurgeUploads(storageDriver, time.Now().Add(-purgeAge), true) - log.Infof("Starting upload purge in %s", timeBetweenPurges) - time.Sleep(timeBetweenPurges) + storage.PurgeUploads(storageDriver, time.Now().Add(-purgeAgeDuration), !dryRunBool) + log.Infof("Starting upload purge in %s", intervalDuration) + time.Sleep(intervalDuration) } }() - }