diff --git a/configuration/README.md b/configuration/README.md index 03ac8ab3..1219051e 100644 --- a/configuration/README.md +++ b/configuration/README.md @@ -68,6 +68,6 @@ Any configuration field other than version can be replaced by providing an envir For example, to change the loglevel to `error`, one can provide `REGISTRY_LOGLEVEL=error`, and to change the s3 storage driver's region parameter to `us-west-1`, one can provide `REGISTRY_STORAGE_S3_LOGLEVEL=us-west-1`. ### Notes -If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, changing the storage type will remove all parameters related to the old storage type. +If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, specifying the storage type in the environment will remove all parameters related to the old storage configuration. By restricting all keys in the configuration file to lowercase letters and numbers, we can avoid any potential environment variable mapping ambiguity. diff --git a/configuration/configuration.go b/configuration/configuration.go index 43405279..8e68190d 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -4,12 +4,8 @@ import ( "fmt" "io" "io/ioutil" - "os" - "regexp" - "strconv" + "reflect" "strings" - - "gopkg.in/BrianBland/yaml.v2" ) // Configuration is a versioned registry configuration, intended to be provided by a yaml file, and @@ -39,40 +35,6 @@ type Configuration struct { // This is currently aliased to Configuration, as it is the current version type v0_1Configuration Configuration -// Version is a major/minor version pair of the form Major.Minor -// Major version upgrades indicate structure or type changes -// Minor version upgrades should be strictly additive -type Version string - -// MajorMinorVersion constructs a Version from its Major and Minor components -func MajorMinorVersion(major, minor uint) Version { - return Version(fmt.Sprintf("%d.%d", major, minor)) -} - -func (version Version) major() (uint, error) { - majorPart := strings.Split(string(version), ".")[0] - major, err := strconv.ParseUint(majorPart, 10, 0) - return uint(major), err -} - -// Major returns the major version portion of a Version -func (version Version) Major() uint { - major, _ := version.major() - return major -} - -func (version Version) minor() (uint, error) { - minorPart := strings.Split(string(version), ".")[1] - minor, err := strconv.ParseUint(minorPart, 10, 0) - return uint(minor), err -} - -// Minor returns the minor version portion of a Version -func (version Version) Minor() uint { - minor, _ := version.minor() - return minor -} - // UnmarshalYAML implements the yaml.Unmarshaler interface // Unmarshals a string of the form X.Y into a Version, validating that X and Y can represent uints func (version *Version) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -141,7 +103,7 @@ func (storage Storage) Parameters() Parameters { } // setParameter changes the parameter at the provided key to the new value -func (storage Storage) setParameter(key, value string) { +func (storage Storage) setParameter(key string, value interface{}) { storage[storage.Type()][key] = value } @@ -181,7 +143,7 @@ func (storage Storage) MarshalYAML() (interface{}, error) { } // Parameters defines a key-value parameters mapping -type Parameters map[string]string +type Parameters map[string]interface{} // Reporting defines error reporting methods. type Reporting struct { @@ -223,109 +185,30 @@ func Parse(rd io.Reader) (*Configuration, error) { return nil, err } - var untypedConfig struct { - Version Version - } - var config *Configuration + p := NewParser("registry", []VersionedParseInfo{ + { + Version: MajorMinorVersion(0, 1), + ParseAs: reflect.TypeOf(v0_1Configuration{}), + ConversionFunc: func(c interface{}) (interface{}, error) { + if v0_1, ok := c.(*v0_1Configuration); ok { + if v0_1.Loglevel == Loglevel("") { + v0_1.Loglevel = Loglevel("info") + } + if v0_1.Storage.Type() == "" { + return nil, fmt.Errorf("No storage configuration provided") + } + return (*Configuration)(v0_1), nil + } + return nil, fmt.Errorf("Expected *v0_1Configuration, received %#v", c) + }, + }, + }) - if err := yaml.Unmarshal(in, &untypedConfig); err != nil { + config := new(Configuration) + err = p.Parse(in, config) + if err != nil { return nil, err } - if untypedConfig.Version == "" { - return nil, fmt.Errorf("Please specify a configuration version. Current version is %s", CurrentVersion) - } - - // Parse the remainder of the configuration depending on the provided version - switch untypedConfig.Version { - case "0.1": - config, err = parseV0_1Registry(in) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("Unsupported configuration version %s Current version is %s", untypedConfig.Version, CurrentVersion) - } - return config, nil } - -// parseV0_1Registry parses a registry Configuration for Version 0.1 -func parseV0_1Registry(in []byte) (*Configuration, error) { - envMap := getEnvMap() - - var config v0_1Configuration - err := yaml.Unmarshal(in, &config) - if err != nil { - return nil, err - } - - // Override config.Loglevel if environment variable is provided - if loglevel, ok := envMap["REGISTRY_LOGLEVEL"]; ok { - var newLoglevel Loglevel - err := yaml.Unmarshal([]byte(loglevel), &newLoglevel) - if err != nil { - return nil, err - } - config.Loglevel = newLoglevel - } - - // Override config.Storage if environment variable is provided - if storageType, ok := envMap["REGISTRY_STORAGE"]; ok { - if storageType != config.Storage.Type() { - // Reset the storage parameters because we're using a different storage type - config.Storage = Storage{storageType: Parameters{}} - } - } - - if config.Storage.Type() == "" { - return nil, fmt.Errorf("Must provide exactly one storage type, optionally with parameters. Provided: %v", config.Storage) - } - - // Override storage parameters with all environment variables of the format: - // REGISTRY_STORAGE__ - storageParamsRegexp, err := regexp.Compile(fmt.Sprintf("^REGISTRY_STORAGE_%s_([A-Z0-9]+)$", strings.ToUpper(config.Storage.Type()))) - if err != nil { - return nil, err - } - for k, v := range envMap { - if submatches := storageParamsRegexp.FindStringSubmatch(k); submatches != nil { - config.Storage.setParameter(strings.ToLower(submatches[1]), v) - } - } - - if bugsnagAPIKey, ok := envMap["REGISTRY_REPORTING_BUGSNAG_APIKEY"]; ok { - config.Reporting.Bugsnag.APIKey = bugsnagAPIKey - } - if bugsnagReleaseStage, ok := envMap["REGISTRY_REPORTING_BUGSNAG_RELEASESTAGE"]; ok { - config.Reporting.Bugsnag.ReleaseStage = bugsnagReleaseStage - } - if bugsnagEndpoint, ok := envMap["REGISTRY_REPORTING_BUGSNAG_ENDPOINT"]; ok { - config.Reporting.Bugsnag.Endpoint = bugsnagEndpoint - } - - if newRelicLicenseKey, ok := envMap["REGISTRY_REPORTING_NEWRELIC_LICENSEKEY"]; ok { - config.Reporting.NewRelic.LicenseKey = newRelicLicenseKey - } - if newRelicName, ok := envMap["REGISTRY_REPORTING_NEWRELIC_NAME"]; ok { - config.Reporting.NewRelic.Name = newRelicName - } - - if httpAddr, ok := envMap["REGISTRY_HTTP_ADDR"]; ok { - config.HTTP.Addr = httpAddr - } - - return (*Configuration)(&config), nil -} - -// getEnvMap reads the current environment variables and converts these into a key/value map -// This is used to distinguish between empty strings returned by os.GetEnv(key) because of undefined -// environment variables and explicitly empty ones -func getEnvMap() map[string]string { - envMap := make(map[string]string) - for _, env := range os.Environ() { - envParts := strings.SplitN(env, "=", 2) - envMap[envParts[0]] = envParts[1] - } - return envMap -} diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 5c9ec9e7..e6fd6295 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -21,12 +21,12 @@ var configStruct = Configuration{ "region": "us-east-1", "bucket": "my-bucket", "rootpath": "/registry", - "encrypt": "true", - "secure": "false", + "encrypt": true, + "secure": false, "accesskey": "SAMPLEACCESSKEY", "secretkey": "SUPERSECRET", - "host": "", - "port": "", + "host": nil, + "port": 42, }, }, Reporting: Reporting{ @@ -50,14 +50,14 @@ storage: accesskey: SAMPLEACCESSKEY secretkey: SUPERSECRET host: ~ - port: ~ + port: 42 reporting: bugsnag: apikey: BugsnagApiKey ` -// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory storage driver with -// no parameters +// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory +// storage driver with no parameters var inmemoryConfigYamlV0_1 = ` version: 0.1 loglevel: info @@ -75,8 +75,8 @@ func (suite *ConfigSuite) SetUpTest(c *C) { suite.expectedConfig = copyConfig(configStruct) } -// TestMarshalRoundtrip validates that configStruct can be marshaled and unmarshaled without -// changing any parameters +// TestMarshalRoundtrip validates that configStruct can be marshaled and +// unmarshaled without changing any parameters func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) { configBytes, err := yaml.Marshal(suite.expectedConfig) c.Assert(err, IsNil) @@ -85,15 +85,16 @@ func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) { c.Assert(config, DeepEquals, suite.expectedConfig) } -// TestParseSimple validates that configYamlV0_1 can be parsed into a struct matching configStruct +// TestParseSimple validates that configYamlV0_1 can be parsed into a struct +// matching configStruct func (suite *ConfigSuite) TestParseSimple(c *C) { config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } -// TestParseInmemory validates that configuration yaml with storage provided as a string can be -// parsed into a Configuration struct with no storage parameters +// TestParseInmemory validates that configuration yaml with storage provided as +// a string can be parsed into a Configuration struct with no storage parameters func (suite *ConfigSuite) TestParseInmemory(c *C) { suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}} suite.expectedConfig.Reporting = Reporting{} @@ -103,9 +104,31 @@ func (suite *ConfigSuite) TestParseInmemory(c *C) { c.Assert(config, DeepEquals, suite.expectedConfig) } -// TestParseWithSameEnvStorage validates that providing environment variables that match the given -// storage type and parameters will not alter the parsed Configuration struct +// TestParseIncomplete validates that an incomplete yaml configuration cannot +// be parsed without providing environment variables to fill in the missing +// components. +func (suite *ConfigSuite) TestParseIncomplete(c *C) { + incompleteConfigYaml := "version: 0.1" + _, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml))) + c.Assert(err, NotNil) + + suite.expectedConfig.Storage = Storage{"filesystem": Parameters{"rootdirectory": "/tmp/testroot"}} + suite.expectedConfig.Reporting = Reporting{} + + os.Setenv("REGISTRY_STORAGE", "filesystem") + os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot") + + config, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml))) + c.Assert(err, IsNil) + c.Assert(config, DeepEquals, suite.expectedConfig) +} + +// TestParseWithSameEnvStorage validates that providing environment variables +// that match the given storage type will only include environment-defined +// parameters and remove yaml-defined parameters func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) { + suite.expectedConfig.Storage = Storage{"s3": Parameters{"region": "us-east-1"}} + os.Setenv("REGISTRY_STORAGE", "s3") os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-east-1") @@ -119,7 +142,7 @@ func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) { // Configuration struct func (suite *ConfigSuite) TestParseWithDifferentEnvStorageParams(c *C) { suite.expectedConfig.Storage.setParameter("region", "us-west-1") - suite.expectedConfig.Storage.setParameter("secure", "true") + suite.expectedConfig.Storage.setParameter("secure", true) suite.expectedConfig.Storage.setParameter("newparam", "some Value") os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-west-1") @@ -180,6 +203,22 @@ func (suite *ConfigSuite) TestParseWithDifferentEnvLoglevel(c *C) { c.Assert(config, DeepEquals, suite.expectedConfig) } +// TestParseInvalidLoglevel validates that the parser will fail to parse a +// configuration if the loglevel is malformed +func (suite *ConfigSuite) TestParseInvalidLoglevel(c *C) { + invalidConfigYaml := "version: 0.1\nloglevel: derp\nstorage: inmemory" + _, err := Parse(bytes.NewReader([]byte(invalidConfigYaml))) + c.Assert(err, NotNil) + + os.Setenv("REGISTRY_LOGLEVEL", "derp") + + _, err = Parse(bytes.NewReader([]byte(configYamlV0_1))) + c.Assert(err, NotNil) + +} + +// TestParseWithDifferentEnvReporting validates that environment variables +// properly override reporting parameters func (suite *ConfigSuite) TestParseWithDifferentEnvReporting(c *C) { suite.expectedConfig.Reporting.Bugsnag.APIKey = "anotherBugsnagApiKey" suite.expectedConfig.Reporting.Bugsnag.Endpoint = "localhost:8080" diff --git a/configuration/parser.go b/configuration/parser.go new file mode 100644 index 00000000..ca5f9afd --- /dev/null +++ b/configuration/parser.go @@ -0,0 +1,203 @@ +package configuration + +import ( + "fmt" + "os" + "reflect" + "regexp" + "strconv" + "strings" + + "gopkg.in/BrianBland/yaml.v2" +) + +// Version is a major/minor version pair of the form Major.Minor +// Major version upgrades indicate structure or type changes +// Minor version upgrades should be strictly additive +type Version string + +// MajorMinorVersion constructs a Version from its Major and Minor components +func MajorMinorVersion(major, minor uint) Version { + return Version(fmt.Sprintf("%d.%d", major, minor)) +} + +func (version Version) major() (uint, error) { + majorPart := strings.Split(string(version), ".")[0] + major, err := strconv.ParseUint(majorPart, 10, 0) + return uint(major), err +} + +// Major returns the major version portion of a Version +func (version Version) Major() uint { + major, _ := version.major() + return major +} + +func (version Version) minor() (uint, error) { + minorPart := strings.Split(string(version), ".")[1] + minor, err := strconv.ParseUint(minorPart, 10, 0) + return uint(minor), err +} + +// Minor returns the minor version portion of a Version +func (version Version) Minor() uint { + minor, _ := version.minor() + return minor +} + +// VersionedParseInfo defines how a specific version of a configuration should +// be parsed into the current version +type VersionedParseInfo struct { + // Version is the version which this parsing information relates to + Version Version + // ParseAs defines the type which a configuration file of this version + // should be parsed into + ParseAs reflect.Type + // ConversionFunc defines a method for converting the parsed configuration + // (of type ParseAs) into the current configuration version + // Note: this method signature is very unclear with the absence of generics + ConversionFunc func(interface{}) (interface{}, error) +} + +// Parser can be used to parse a configuration file and environment of a defined +// version into a unified output structure +type Parser struct { + prefix string + mapping map[Version]VersionedParseInfo + env map[string]string +} + +// NewParser returns a *Parser with the given environment prefix which handles +// versioned configurations which match the given parseInfos +func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser { + p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo), env: make(map[string]string)} + + for _, parseInfo := range parseInfos { + p.mapping[parseInfo.Version] = parseInfo + } + + for _, env := range os.Environ() { + envParts := strings.SplitN(env, "=", 2) + p.env[envParts[0]] = envParts[1] + } + + return &p +} + +// Parse reads in the given []byte and environment and writes the resulting +// configuration into the input v +// +// Environment variables may be used to override configuration parameters other +// than version, following the scheme below: +// v.Abc may be replaced by the value of PREFIX_ABC, +// v.Abc.Xyz may be replaced by the value of PREFIX_ABC_XYZ, and so forth +func (p *Parser) Parse(in []byte, v interface{}) error { + var versionedStruct struct { + Version Version + } + + if err := yaml.Unmarshal(in, &versionedStruct); err != nil { + return err + } + + parseInfo, ok := p.mapping[versionedStruct.Version] + if !ok { + return fmt.Errorf("Unsupported version: %q", versionedStruct.Version) + } + + parseAs := reflect.New(parseInfo.ParseAs) + err := yaml.Unmarshal(in, parseAs.Interface()) + if err != nil { + return err + } + + err = p.overwriteFields(parseAs, p.prefix) + if err != nil { + return err + } + + c, err := parseInfo.ConversionFunc(parseAs.Interface()) + if err != nil { + return err + } + reflect.ValueOf(v).Elem().Set(reflect.Indirect(reflect.ValueOf(c))) + return nil +} + +func (p *Parser) overwriteFields(v reflect.Value, prefix string) error { + for v.Kind() == reflect.Ptr { + v = reflect.Indirect(v) + } + switch v.Kind() { + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + sf := v.Type().Field(i) + fieldPrefix := strings.ToUpper(prefix + "_" + sf.Name) + if e, ok := p.env[fieldPrefix]; ok { + fieldVal := reflect.New(sf.Type) + err := yaml.Unmarshal([]byte(e), fieldVal.Interface()) + if err != nil { + return err + } + v.Field(i).Set(reflect.Indirect(fieldVal)) + } + err := p.overwriteFields(v.Field(i), fieldPrefix) + if err != nil { + return err + } + } + case reflect.Map: + p.overwriteMap(v, prefix) + } + return nil +} + +func (p *Parser) overwriteMap(m reflect.Value, prefix string) error { + switch m.Type().Elem().Kind() { + case reflect.Struct: + for _, k := range m.MapKeys() { + err := p.overwriteFields(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k))) + if err != nil { + return err + } + } + envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix))) + if err != nil { + return err + } + for key, val := range p.env { + if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil { + mapValue := reflect.New(m.Type().Elem()) + err := yaml.Unmarshal([]byte(val), mapValue.Interface()) + if err != nil { + return err + } + m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue)) + } + } + case reflect.Map: + for _, k := range m.MapKeys() { + err := p.overwriteMap(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k))) + if err != nil { + return err + } + } + default: + envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix))) + if err != nil { + return err + } + + for key, val := range p.env { + if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil { + mapValue := reflect.New(m.Type().Elem()) + err := yaml.Unmarshal([]byte(val), mapValue.Interface()) + if err != nil { + return err + } + m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue)) + } + } + } + return nil +} diff --git a/storagedriver/factory/factory.go b/storagedriver/factory/factory.go index 0f8ca001..254cd9bb 100644 --- a/storagedriver/factory/factory.go +++ b/storagedriver/factory/factory.go @@ -16,7 +16,7 @@ type StorageDriverFactory interface { // Create returns a new storagedriver.StorageDriver with the given parameters // Parameters will vary by driver and may be ignored // Each parameter key must only consist of lowercase letters and numbers - Create(parameters map[string]string) (storagedriver.StorageDriver, error) + Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) } // Register makes a storage driver available by the provided name. @@ -37,7 +37,7 @@ func Register(name string, factory StorageDriverFactory) { // To run in-process, the StorageDriverFactory must first be registered with the given name // If no in-process drivers are found with the given name, this attempts to create an IPC driver // If no in-process or external drivers are found, an InvalidStorageDriverError is returned -func Create(name string, parameters map[string]string) (storagedriver.StorageDriver, error) { +func Create(name string, parameters map[string]interface{}) (storagedriver.StorageDriver, error) { driverFactory, ok := driverFactories[name] if !ok { return nil, InvalidStorageDriverError{name} diff --git a/storagedriver/filesystem/driver.go b/storagedriver/filesystem/driver.go index 3c3c950f..635c13a2 100644 --- a/storagedriver/filesystem/driver.go +++ b/storagedriver/filesystem/driver.go @@ -23,7 +23,7 @@ func init() { // filesystemDriverFactory implements the factory.StorageDriverFactory interface type filesystemDriverFactory struct{} -func (factory *filesystemDriverFactory) Create(parameters map[string]string) (storagedriver.StorageDriver, error) { +func (factory *filesystemDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) { return FromParameters(parameters), nil } @@ -36,12 +36,12 @@ type Driver struct { // FromParameters constructs a new Driver with a given parameters map // Optional Parameters: // - rootdirectory -func FromParameters(parameters map[string]string) *Driver { +func FromParameters(parameters map[string]interface{}) *Driver { var rootDirectory = defaultRootDirectory if parameters != nil { rootDir, ok := parameters["rootdirectory"] if ok { - rootDirectory = rootDir + rootDirectory = fmt.Sprint(rootDir) } } return New(rootDirectory) diff --git a/storagedriver/inmemory/driver.go b/storagedriver/inmemory/driver.go index 7481c472..2e23c758 100644 --- a/storagedriver/inmemory/driver.go +++ b/storagedriver/inmemory/driver.go @@ -21,7 +21,7 @@ func init() { // inMemoryDriverFacotry implements the factory.StorageDriverFactory interface. type inMemoryDriverFactory struct{} -func (factory *inMemoryDriverFactory) Create(parameters map[string]string) (storagedriver.StorageDriver, error) { +func (factory *inMemoryDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) { return New(), nil }