Genericizes the yaml+environment versioned configuration parser

Registry configuration parsing uses the new parser with a single version
declaration and an environment prefix of "REGISTRY"

configuration/README.md

@@ -68,6 +68,6 @@ Any configuration field other than version can be replaced by providing an envir
 For example, to change the loglevel to `error`, one can provide `REGISTRY_LOGLEVEL=error`, and to change the s3 storage driver's region parameter to `us-west-1`, one can provide `REGISTRY_STORAGE_S3_LOGLEVEL=us-west-1`.
 
 ### Notes
-If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, changing the storage type will remove all parameters related to the old storage type.
+If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, specifying the storage type in the environment will remove all parameters related to the old storage configuration.
 
 By restricting all keys in the configuration file to lowercase letters and numbers, we can avoid any potential environment variable mapping ambiguity.

configuration/configuration.go

@@ -4,12 +4,8 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"os"
+	"reflect"
-	"regexp"
-	"strconv"
 	"strings"
-
-	"gopkg.in/BrianBland/yaml.v2"
 )
 
 // Configuration is a versioned registry configuration, intended to be provided by a yaml file, and
@@ -39,40 +35,6 @@ type Configuration struct {
 // This is currently aliased to Configuration, as it is the current version
 type v0_1Configuration Configuration
 
-// Version is a major/minor version pair of the form Major.Minor
-// Major version upgrades indicate structure or type changes
-// Minor version upgrades should be strictly additive
-type Version string
-
-// MajorMinorVersion constructs a Version from its Major and Minor components
-func MajorMinorVersion(major, minor uint) Version {
-	return Version(fmt.Sprintf("%d.%d", major, minor))
-}
-
-func (version Version) major() (uint, error) {
-	majorPart := strings.Split(string(version), ".")[0]
-	major, err := strconv.ParseUint(majorPart, 10, 0)
-	return uint(major), err
-}
-
-// Major returns the major version portion of a Version
-func (version Version) Major() uint {
-	major, _ := version.major()
-	return major
-}
-
-func (version Version) minor() (uint, error) {
-	minorPart := strings.Split(string(version), ".")[1]
-	minor, err := strconv.ParseUint(minorPart, 10, 0)
-	return uint(minor), err
-}
-
-// Minor returns the minor version portion of a Version
-func (version Version) Minor() uint {
-	minor, _ := version.minor()
-	return minor
-}
-
 // UnmarshalYAML implements the yaml.Unmarshaler interface
 // Unmarshals a string of the form X.Y into a Version, validating that X and Y can represent uints
 func (version *Version) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -223,109 +185,30 @@ func Parse(rd io.Reader) (*Configuration, error) {
 		return nil, err
 	}
 
-	var untypedConfig struct {
+	p := NewParser("registry", []VersionedParseInfo{
-		Version Version
+		{
+			Version: MajorMinorVersion(0, 1),
+			ParseAs: reflect.TypeOf(v0_1Configuration{}),
+			ConversionFunc: func(c interface{}) (interface{}, error) {
+				if v0_1, ok := c.(*v0_1Configuration); ok {
+					if v0_1.Loglevel == Loglevel("") {
+						v0_1.Loglevel = Loglevel("info")
 					}
-	var config *Configuration
+					if v0_1.Storage.Type() == "" {
-
+						return nil, fmt.Errorf("No storage configuration provided")
-	if err := yaml.Unmarshal(in, &untypedConfig); err != nil {
-		return nil, err
 					}
-
+					return (*Configuration)(v0_1), nil
-	if untypedConfig.Version == "" {
-		return nil, fmt.Errorf("Please specify a configuration version. Current version is %s", CurrentVersion)
 				}
+				return nil, fmt.Errorf("Expected *v0_1Configuration, received %#v", c)
+			},
+		},
+	})
 
-	// Parse the remainder of the configuration depending on the provided version
+	config := new(Configuration)
-	switch untypedConfig.Version {
+	err = p.Parse(in, config)
-	case "0.1":
-		config, err = parseV0_1Registry(in)
 	if err != nil {
 		return nil, err
 	}
-	default:
-		return nil, fmt.Errorf("Unsupported configuration version %s Current version is %s", untypedConfig.Version, CurrentVersion)
-	}
 
 	return config, nil
 }
-
-// parseV0_1Registry parses a registry Configuration for Version 0.1
-func parseV0_1Registry(in []byte) (*Configuration, error) {
-	envMap := getEnvMap()
-
-	var config v0_1Configuration
-	err := yaml.Unmarshal(in, &config)
-	if err != nil {
-		return nil, err
-	}
-
-	// Override config.Loglevel if environment variable is provided
-	if loglevel, ok := envMap["REGISTRY_LOGLEVEL"]; ok {
-		var newLoglevel Loglevel
-		err := yaml.Unmarshal([]byte(loglevel), &newLoglevel)
-		if err != nil {
-			return nil, err
-		}
-		config.Loglevel = newLoglevel
-	}
-
-	// Override config.Storage if environment variable is provided
-	if storageType, ok := envMap["REGISTRY_STORAGE"]; ok {
-		if storageType != config.Storage.Type() {
-			// Reset the storage parameters because we're using a different storage type
-			config.Storage = Storage{storageType: Parameters{}}
-		}
-	}
-
-	if config.Storage.Type() == "" {
-		return nil, fmt.Errorf("Must provide exactly one storage type, optionally with parameters. Provided: %v", config.Storage)
-	}
-
-	// Override storage parameters with all environment variables of the format:
-	// REGISTRY_STORAGE_<storage driver type>_<parameter name>
-	storageParamsRegexp, err := regexp.Compile(fmt.Sprintf("^REGISTRY_STORAGE_%s_([A-Z0-9]+)$", strings.ToUpper(config.Storage.Type())))
-	if err != nil {
-		return nil, err
-	}
-	for k, v := range envMap {
-		if submatches := storageParamsRegexp.FindStringSubmatch(k); submatches != nil {
-			config.Storage.setParameter(strings.ToLower(submatches[1]), v)
-		}
-	}
-
-	if bugsnagAPIKey, ok := envMap["REGISTRY_REPORTING_BUGSNAG_APIKEY"]; ok {
-		config.Reporting.Bugsnag.APIKey = bugsnagAPIKey
-	}
-	if bugsnagReleaseStage, ok := envMap["REGISTRY_REPORTING_BUGSNAG_RELEASESTAGE"]; ok {
-		config.Reporting.Bugsnag.ReleaseStage = bugsnagReleaseStage
-	}
-	if bugsnagEndpoint, ok := envMap["REGISTRY_REPORTING_BUGSNAG_ENDPOINT"]; ok {
-		config.Reporting.Bugsnag.Endpoint = bugsnagEndpoint
-	}
-
-	if newRelicLicenseKey, ok := envMap["REGISTRY_REPORTING_NEWRELIC_LICENSEKEY"]; ok {
-		config.Reporting.NewRelic.LicenseKey = newRelicLicenseKey
-	}
-	if newRelicName, ok := envMap["REGISTRY_REPORTING_NEWRELIC_NAME"]; ok {
-		config.Reporting.NewRelic.Name = newRelicName
-	}
-
-	if httpAddr, ok := envMap["REGISTRY_HTTP_ADDR"]; ok {
-		config.HTTP.Addr = httpAddr
-	}
-
-	return (*Configuration)(&config), nil
-}
-
-// getEnvMap reads the current environment variables and converts these into a key/value map
-// This is used to distinguish between empty strings returned by os.GetEnv(key) because of undefined
-// environment variables and explicitly empty ones
-func getEnvMap() map[string]string {
-	envMap := make(map[string]string)
-	for _, env := range os.Environ() {
-		envParts := strings.SplitN(env, "=", 2)
-		envMap[envParts[0]] = envParts[1]
-	}
-	return envMap
-}

configuration/configuration_test.go

@@ -56,8 +56,8 @@ reporting:
     apikey: BugsnagApiKey
 `
 
-// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory storage driver with
+// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory
-// no parameters
+// storage driver with no parameters
 var inmemoryConfigYamlV0_1 = `
 version: 0.1
 loglevel: info
@@ -75,8 +75,8 @@ func (suite *ConfigSuite) SetUpTest(c *C) {
 	suite.expectedConfig = copyConfig(configStruct)
 }
 
-// TestMarshalRoundtrip validates that configStruct can be marshaled and unmarshaled without
+// TestMarshalRoundtrip validates that configStruct can be marshaled and
-// changing any parameters
+// unmarshaled without changing any parameters
 func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) {
 	configBytes, err := yaml.Marshal(suite.expectedConfig)
 	c.Assert(err, IsNil)
@@ -85,15 +85,16 @@ func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) {
 	c.Assert(config, DeepEquals, suite.expectedConfig)
 }
 
-// TestParseSimple validates that configYamlV0_1 can be parsed into a struct matching configStruct
+// TestParseSimple validates that configYamlV0_1 can be parsed into a struct
+// matching configStruct
 func (suite *ConfigSuite) TestParseSimple(c *C) {
 	config, err := Parse(bytes.NewReader([]byte(configYamlV0_1)))
 	c.Assert(err, IsNil)
 	c.Assert(config, DeepEquals, suite.expectedConfig)
 }
 
-// TestParseInmemory validates that configuration yaml with storage provided as a string can be
+// TestParseInmemory validates that configuration yaml with storage provided as
-// parsed into a Configuration struct with no storage parameters
+// a string can be parsed into a Configuration struct with no storage parameters
 func (suite *ConfigSuite) TestParseInmemory(c *C) {
 	suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}}
 	suite.expectedConfig.Reporting = Reporting{}
@@ -103,9 +104,31 @@ func (suite *ConfigSuite) TestParseInmemory(c *C) {
 	c.Assert(config, DeepEquals, suite.expectedConfig)
 }
 
-// TestParseWithSameEnvStorage validates that providing environment variables that match the given
+// TestParseIncomplete validates that an incomplete yaml configuration cannot
-// storage type and parameters will not alter the parsed Configuration struct
+// be parsed without providing environment variables to fill in the missing
+// components.
+func (suite *ConfigSuite) TestParseIncomplete(c *C) {
+	incompleteConfigYaml := "version: 0.1"
+	_, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml)))
+	c.Assert(err, NotNil)
+
+	suite.expectedConfig.Storage = Storage{"filesystem": Parameters{"rootdirectory": "/tmp/testroot"}}
+	suite.expectedConfig.Reporting = Reporting{}
+
+	os.Setenv("REGISTRY_STORAGE", "filesystem")
+	os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot")
+
+	config, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml)))
+	c.Assert(err, IsNil)
+	c.Assert(config, DeepEquals, suite.expectedConfig)
+}
+
+// TestParseWithSameEnvStorage validates that providing environment variables
+// that match the given storage type will only include environment-defined
+// parameters and remove yaml-defined parameters
 func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) {
+	suite.expectedConfig.Storage = Storage{"s3": Parameters{"region": "us-east-1"}}
+
 	os.Setenv("REGISTRY_STORAGE", "s3")
 	os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-east-1")
 
@@ -180,6 +203,22 @@ func (suite *ConfigSuite) TestParseWithDifferentEnvLoglevel(c *C) {
 	c.Assert(config, DeepEquals, suite.expectedConfig)
 }
 
+// TestParseInvalidLoglevel validates that the parser will fail to parse a
+// configuration if the loglevel is malformed
+func (suite *ConfigSuite) TestParseInvalidLoglevel(c *C) {
+	invalidConfigYaml := "version: 0.1\nloglevel: derp\nstorage: inmemory"
+	_, err := Parse(bytes.NewReader([]byte(invalidConfigYaml)))
+	c.Assert(err, NotNil)
+
+	os.Setenv("REGISTRY_LOGLEVEL", "derp")
+
+	_, err = Parse(bytes.NewReader([]byte(configYamlV0_1)))
+	c.Assert(err, NotNil)
+
+}
+
+// TestParseWithDifferentEnvReporting validates that environment variables
+// properly override reporting parameters
 func (suite *ConfigSuite) TestParseWithDifferentEnvReporting(c *C) {
 	suite.expectedConfig.Reporting.Bugsnag.APIKey = "anotherBugsnagApiKey"
 	suite.expectedConfig.Reporting.Bugsnag.Endpoint = "localhost:8080"

configuration/parser.go

@@ -0,0 +1,203 @@
+package configuration
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"gopkg.in/BrianBland/yaml.v2"
+)
+
+// Version is a major/minor version pair of the form Major.Minor
+// Major version upgrades indicate structure or type changes
+// Minor version upgrades should be strictly additive
+type Version string
+
+// MajorMinorVersion constructs a Version from its Major and Minor components
+func MajorMinorVersion(major, minor uint) Version {
+	return Version(fmt.Sprintf("%d.%d", major, minor))
+}
+
+func (version Version) major() (uint, error) {
+	majorPart := strings.Split(string(version), ".")[0]
+	major, err := strconv.ParseUint(majorPart, 10, 0)
+	return uint(major), err
+}
+
+// Major returns the major version portion of a Version
+func (version Version) Major() uint {
+	major, _ := version.major()
+	return major
+}
+
+func (version Version) minor() (uint, error) {
+	minorPart := strings.Split(string(version), ".")[1]
+	minor, err := strconv.ParseUint(minorPart, 10, 0)
+	return uint(minor), err
+}
+
+// Minor returns the minor version portion of a Version
+func (version Version) Minor() uint {
+	minor, _ := version.minor()
+	return minor
+}
+
+// VersionedParseInfo defines how a specific version of a configuration should
+// be parsed into the current version
+type VersionedParseInfo struct {
+	// Version is the version which this parsing information relates to
+	Version Version
+	// ParseAs defines the type which a configuration file of this version
+	// should be parsed into
+	ParseAs reflect.Type
+	// ConversionFunc defines a method for converting the parsed configuration
+	// (of type ParseAs) into the current configuration version
+	// Note: this method signature is very unclear with the absence of generics
+	ConversionFunc func(interface{}) (interface{}, error)
+}
+
+// Parser can be used to parse a configuration file and environment of a defined
+// version into a unified output structure
+type Parser struct {
+	prefix  string
+	mapping map[Version]VersionedParseInfo
+	env     map[string]string
+}
+
+// NewParser returns a *Parser with the given environment prefix which handles
+// versioned configurations which match the given parseInfos
+func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser {
+	p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo), env: make(map[string]string)}
+
+	for _, parseInfo := range parseInfos {
+		p.mapping[parseInfo.Version] = parseInfo
+	}
+
+	for _, env := range os.Environ() {
+		envParts := strings.SplitN(env, "=", 2)
+		p.env[envParts[0]] = envParts[1]
+	}
+
+	return &p
+}
+
+// Parse reads in the given []byte and environment and writes the resulting
+// configuration into the input v
+//
+// Environment variables may be used to override configuration parameters other
+// than version, following the scheme below:
+// v.Abc may be replaced by the value of PREFIX_ABC,
+// v.Abc.Xyz may be replaced by the value of PREFIX_ABC_XYZ, and so forth
+func (p *Parser) Parse(in []byte, v interface{}) error {
+	var versionedStruct struct {
+		Version Version
+	}
+
+	if err := yaml.Unmarshal(in, &versionedStruct); err != nil {
+		return err
+	}
+
+	parseInfo, ok := p.mapping[versionedStruct.Version]
+	if !ok {
+		return fmt.Errorf("Unsupported version: %q", versionedStruct.Version)
+	}
+
+	parseAs := reflect.New(parseInfo.ParseAs)
+	err := yaml.Unmarshal(in, parseAs.Interface())
+	if err != nil {
+		return err
+	}
+
+	err = p.overwriteFields(parseAs, p.prefix)
+	if err != nil {
+		return err
+	}
+
+	c, err := parseInfo.ConversionFunc(parseAs.Interface())
+	if err != nil {
+		return err
+	}
+	reflect.ValueOf(v).Elem().Set(reflect.Indirect(reflect.ValueOf(c)))
+	return nil
+}
+
+func (p *Parser) overwriteFields(v reflect.Value, prefix string) error {
+	for v.Kind() == reflect.Ptr {
+		v = reflect.Indirect(v)
+	}
+	switch v.Kind() {
+	case reflect.Struct:
+		for i := 0; i < v.NumField(); i++ {
+			sf := v.Type().Field(i)
+			fieldPrefix := strings.ToUpper(prefix + "_" + sf.Name)
+			if e, ok := p.env[fieldPrefix]; ok {
+				fieldVal := reflect.New(sf.Type)
+				err := yaml.Unmarshal([]byte(e), fieldVal.Interface())
+				if err != nil {
+					return err
+				}
+				v.Field(i).Set(reflect.Indirect(fieldVal))
+			}
+			err := p.overwriteFields(v.Field(i), fieldPrefix)
+			if err != nil {
+				return err
+			}
+		}
+	case reflect.Map:
+		p.overwriteMap(v, prefix)
+	}
+	return nil
+}
+
+func (p *Parser) overwriteMap(m reflect.Value, prefix string) error {
+	switch m.Type().Elem().Kind() {
+	case reflect.Struct:
+		for _, k := range m.MapKeys() {
+			err := p.overwriteFields(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k)))
+			if err != nil {
+				return err
+			}
+		}
+		envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix)))
+		if err != nil {
+			return err
+		}
+		for key, val := range p.env {
+			if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil {
+				mapValue := reflect.New(m.Type().Elem())
+				err := yaml.Unmarshal([]byte(val), mapValue.Interface())
+				if err != nil {
+					return err
+				}
+				m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue))
+			}
+		}
+	case reflect.Map:
+		for _, k := range m.MapKeys() {
+			err := p.overwriteMap(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k)))
+			if err != nil {
+				return err
+			}
+		}
+	default:
+		envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix)))
+		if err != nil {
+			return err
+		}
+
+		for key, val := range p.env {
+			if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil {
+				mapValue := reflect.New(m.Type().Elem())
+				err := yaml.Unmarshal([]byte(val), mapValue.Interface())
+				if err != nil {
+					return err
+				}
+				m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue))
+			}
+		}
+	}
+	return nil
+}