From f9b119974d4f38834b9fe7521ee40a5f47fbb045 Mon Sep 17 00:00:00 2001 From: Brian Bland Date: Wed, 17 Dec 2014 14:18:58 -0800 Subject: [PATCH] Genericizes the yaml+environment versioned configuration parser Registry configuration parsing uses the new parser with a single version declaration and an environment prefix of "REGISTRY" --- configuration/README.md | 2 +- configuration/configuration.go | 161 +++------------------- configuration/configuration_test.go | 57 ++++++-- configuration/parser.go | 203 ++++++++++++++++++++++++++++ 4 files changed, 274 insertions(+), 149 deletions(-) create mode 100644 configuration/parser.go diff --git a/configuration/README.md b/configuration/README.md index 03ac8ab3..1219051e 100644 --- a/configuration/README.md +++ b/configuration/README.md @@ -68,6 +68,6 @@ Any configuration field other than version can be replaced by providing an envir For example, to change the loglevel to `error`, one can provide `REGISTRY_LOGLEVEL=error`, and to change the s3 storage driver's region parameter to `us-west-1`, one can provide `REGISTRY_STORAGE_S3_LOGLEVEL=us-west-1`. ### Notes -If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, changing the storage type will remove all parameters related to the old storage type. +If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, specifying the storage type in the environment will remove all parameters related to the old storage configuration. By restricting all keys in the configuration file to lowercase letters and numbers, we can avoid any potential environment variable mapping ambiguity. diff --git a/configuration/configuration.go b/configuration/configuration.go index 43405279..bbb88a0e 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -4,12 +4,8 @@ import ( "fmt" "io" "io/ioutil" - "os" - "regexp" - "strconv" + "reflect" "strings" - - "gopkg.in/BrianBland/yaml.v2" ) // Configuration is a versioned registry configuration, intended to be provided by a yaml file, and @@ -39,40 +35,6 @@ type Configuration struct { // This is currently aliased to Configuration, as it is the current version type v0_1Configuration Configuration -// Version is a major/minor version pair of the form Major.Minor -// Major version upgrades indicate structure or type changes -// Minor version upgrades should be strictly additive -type Version string - -// MajorMinorVersion constructs a Version from its Major and Minor components -func MajorMinorVersion(major, minor uint) Version { - return Version(fmt.Sprintf("%d.%d", major, minor)) -} - -func (version Version) major() (uint, error) { - majorPart := strings.Split(string(version), ".")[0] - major, err := strconv.ParseUint(majorPart, 10, 0) - return uint(major), err -} - -// Major returns the major version portion of a Version -func (version Version) Major() uint { - major, _ := version.major() - return major -} - -func (version Version) minor() (uint, error) { - minorPart := strings.Split(string(version), ".")[1] - minor, err := strconv.ParseUint(minorPart, 10, 0) - return uint(minor), err -} - -// Minor returns the minor version portion of a Version -func (version Version) Minor() uint { - minor, _ := version.minor() - return minor -} - // UnmarshalYAML implements the yaml.Unmarshaler interface // Unmarshals a string of the form X.Y into a Version, validating that X and Y can represent uints func (version *Version) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -223,109 +185,30 @@ func Parse(rd io.Reader) (*Configuration, error) { return nil, err } - var untypedConfig struct { - Version Version - } - var config *Configuration + p := NewParser("registry", []VersionedParseInfo{ + { + Version: MajorMinorVersion(0, 1), + ParseAs: reflect.TypeOf(v0_1Configuration{}), + ConversionFunc: func(c interface{}) (interface{}, error) { + if v0_1, ok := c.(*v0_1Configuration); ok { + if v0_1.Loglevel == Loglevel("") { + v0_1.Loglevel = Loglevel("info") + } + if v0_1.Storage.Type() == "" { + return nil, fmt.Errorf("No storage configuration provided") + } + return (*Configuration)(v0_1), nil + } + return nil, fmt.Errorf("Expected *v0_1Configuration, received %#v", c) + }, + }, + }) - if err := yaml.Unmarshal(in, &untypedConfig); err != nil { + config := new(Configuration) + err = p.Parse(in, config) + if err != nil { return nil, err } - if untypedConfig.Version == "" { - return nil, fmt.Errorf("Please specify a configuration version. Current version is %s", CurrentVersion) - } - - // Parse the remainder of the configuration depending on the provided version - switch untypedConfig.Version { - case "0.1": - config, err = parseV0_1Registry(in) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("Unsupported configuration version %s Current version is %s", untypedConfig.Version, CurrentVersion) - } - return config, nil } - -// parseV0_1Registry parses a registry Configuration for Version 0.1 -func parseV0_1Registry(in []byte) (*Configuration, error) { - envMap := getEnvMap() - - var config v0_1Configuration - err := yaml.Unmarshal(in, &config) - if err != nil { - return nil, err - } - - // Override config.Loglevel if environment variable is provided - if loglevel, ok := envMap["REGISTRY_LOGLEVEL"]; ok { - var newLoglevel Loglevel - err := yaml.Unmarshal([]byte(loglevel), &newLoglevel) - if err != nil { - return nil, err - } - config.Loglevel = newLoglevel - } - - // Override config.Storage if environment variable is provided - if storageType, ok := envMap["REGISTRY_STORAGE"]; ok { - if storageType != config.Storage.Type() { - // Reset the storage parameters because we're using a different storage type - config.Storage = Storage{storageType: Parameters{}} - } - } - - if config.Storage.Type() == "" { - return nil, fmt.Errorf("Must provide exactly one storage type, optionally with parameters. Provided: %v", config.Storage) - } - - // Override storage parameters with all environment variables of the format: - // REGISTRY_STORAGE__ - storageParamsRegexp, err := regexp.Compile(fmt.Sprintf("^REGISTRY_STORAGE_%s_([A-Z0-9]+)$", strings.ToUpper(config.Storage.Type()))) - if err != nil { - return nil, err - } - for k, v := range envMap { - if submatches := storageParamsRegexp.FindStringSubmatch(k); submatches != nil { - config.Storage.setParameter(strings.ToLower(submatches[1]), v) - } - } - - if bugsnagAPIKey, ok := envMap["REGISTRY_REPORTING_BUGSNAG_APIKEY"]; ok { - config.Reporting.Bugsnag.APIKey = bugsnagAPIKey - } - if bugsnagReleaseStage, ok := envMap["REGISTRY_REPORTING_BUGSNAG_RELEASESTAGE"]; ok { - config.Reporting.Bugsnag.ReleaseStage = bugsnagReleaseStage - } - if bugsnagEndpoint, ok := envMap["REGISTRY_REPORTING_BUGSNAG_ENDPOINT"]; ok { - config.Reporting.Bugsnag.Endpoint = bugsnagEndpoint - } - - if newRelicLicenseKey, ok := envMap["REGISTRY_REPORTING_NEWRELIC_LICENSEKEY"]; ok { - config.Reporting.NewRelic.LicenseKey = newRelicLicenseKey - } - if newRelicName, ok := envMap["REGISTRY_REPORTING_NEWRELIC_NAME"]; ok { - config.Reporting.NewRelic.Name = newRelicName - } - - if httpAddr, ok := envMap["REGISTRY_HTTP_ADDR"]; ok { - config.HTTP.Addr = httpAddr - } - - return (*Configuration)(&config), nil -} - -// getEnvMap reads the current environment variables and converts these into a key/value map -// This is used to distinguish between empty strings returned by os.GetEnv(key) because of undefined -// environment variables and explicitly empty ones -func getEnvMap() map[string]string { - envMap := make(map[string]string) - for _, env := range os.Environ() { - envParts := strings.SplitN(env, "=", 2) - envMap[envParts[0]] = envParts[1] - } - return envMap -} diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go index 5c9ec9e7..75bec818 100644 --- a/configuration/configuration_test.go +++ b/configuration/configuration_test.go @@ -56,8 +56,8 @@ reporting: apikey: BugsnagApiKey ` -// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory storage driver with -// no parameters +// inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory +// storage driver with no parameters var inmemoryConfigYamlV0_1 = ` version: 0.1 loglevel: info @@ -75,8 +75,8 @@ func (suite *ConfigSuite) SetUpTest(c *C) { suite.expectedConfig = copyConfig(configStruct) } -// TestMarshalRoundtrip validates that configStruct can be marshaled and unmarshaled without -// changing any parameters +// TestMarshalRoundtrip validates that configStruct can be marshaled and +// unmarshaled without changing any parameters func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) { configBytes, err := yaml.Marshal(suite.expectedConfig) c.Assert(err, IsNil) @@ -85,15 +85,16 @@ func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) { c.Assert(config, DeepEquals, suite.expectedConfig) } -// TestParseSimple validates that configYamlV0_1 can be parsed into a struct matching configStruct +// TestParseSimple validates that configYamlV0_1 can be parsed into a struct +// matching configStruct func (suite *ConfigSuite) TestParseSimple(c *C) { config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } -// TestParseInmemory validates that configuration yaml with storage provided as a string can be -// parsed into a Configuration struct with no storage parameters +// TestParseInmemory validates that configuration yaml with storage provided as +// a string can be parsed into a Configuration struct with no storage parameters func (suite *ConfigSuite) TestParseInmemory(c *C) { suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}} suite.expectedConfig.Reporting = Reporting{} @@ -103,9 +104,31 @@ func (suite *ConfigSuite) TestParseInmemory(c *C) { c.Assert(config, DeepEquals, suite.expectedConfig) } -// TestParseWithSameEnvStorage validates that providing environment variables that match the given -// storage type and parameters will not alter the parsed Configuration struct +// TestParseIncomplete validates that an incomplete yaml configuration cannot +// be parsed without providing environment variables to fill in the missing +// components. +func (suite *ConfigSuite) TestParseIncomplete(c *C) { + incompleteConfigYaml := "version: 0.1" + _, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml))) + c.Assert(err, NotNil) + + suite.expectedConfig.Storage = Storage{"filesystem": Parameters{"rootdirectory": "/tmp/testroot"}} + suite.expectedConfig.Reporting = Reporting{} + + os.Setenv("REGISTRY_STORAGE", "filesystem") + os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot") + + config, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml))) + c.Assert(err, IsNil) + c.Assert(config, DeepEquals, suite.expectedConfig) +} + +// TestParseWithSameEnvStorage validates that providing environment variables +// that match the given storage type will only include environment-defined +// parameters and remove yaml-defined parameters func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) { + suite.expectedConfig.Storage = Storage{"s3": Parameters{"region": "us-east-1"}} + os.Setenv("REGISTRY_STORAGE", "s3") os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-east-1") @@ -180,6 +203,22 @@ func (suite *ConfigSuite) TestParseWithDifferentEnvLoglevel(c *C) { c.Assert(config, DeepEquals, suite.expectedConfig) } +// TestParseInvalidLoglevel validates that the parser will fail to parse a +// configuration if the loglevel is malformed +func (suite *ConfigSuite) TestParseInvalidLoglevel(c *C) { + invalidConfigYaml := "version: 0.1\nloglevel: derp\nstorage: inmemory" + _, err := Parse(bytes.NewReader([]byte(invalidConfigYaml))) + c.Assert(err, NotNil) + + os.Setenv("REGISTRY_LOGLEVEL", "derp") + + _, err = Parse(bytes.NewReader([]byte(configYamlV0_1))) + c.Assert(err, NotNil) + +} + +// TestParseWithDifferentEnvReporting validates that environment variables +// properly override reporting parameters func (suite *ConfigSuite) TestParseWithDifferentEnvReporting(c *C) { suite.expectedConfig.Reporting.Bugsnag.APIKey = "anotherBugsnagApiKey" suite.expectedConfig.Reporting.Bugsnag.Endpoint = "localhost:8080" diff --git a/configuration/parser.go b/configuration/parser.go new file mode 100644 index 00000000..ca5f9afd --- /dev/null +++ b/configuration/parser.go @@ -0,0 +1,203 @@ +package configuration + +import ( + "fmt" + "os" + "reflect" + "regexp" + "strconv" + "strings" + + "gopkg.in/BrianBland/yaml.v2" +) + +// Version is a major/minor version pair of the form Major.Minor +// Major version upgrades indicate structure or type changes +// Minor version upgrades should be strictly additive +type Version string + +// MajorMinorVersion constructs a Version from its Major and Minor components +func MajorMinorVersion(major, minor uint) Version { + return Version(fmt.Sprintf("%d.%d", major, minor)) +} + +func (version Version) major() (uint, error) { + majorPart := strings.Split(string(version), ".")[0] + major, err := strconv.ParseUint(majorPart, 10, 0) + return uint(major), err +} + +// Major returns the major version portion of a Version +func (version Version) Major() uint { + major, _ := version.major() + return major +} + +func (version Version) minor() (uint, error) { + minorPart := strings.Split(string(version), ".")[1] + minor, err := strconv.ParseUint(minorPart, 10, 0) + return uint(minor), err +} + +// Minor returns the minor version portion of a Version +func (version Version) Minor() uint { + minor, _ := version.minor() + return minor +} + +// VersionedParseInfo defines how a specific version of a configuration should +// be parsed into the current version +type VersionedParseInfo struct { + // Version is the version which this parsing information relates to + Version Version + // ParseAs defines the type which a configuration file of this version + // should be parsed into + ParseAs reflect.Type + // ConversionFunc defines a method for converting the parsed configuration + // (of type ParseAs) into the current configuration version + // Note: this method signature is very unclear with the absence of generics + ConversionFunc func(interface{}) (interface{}, error) +} + +// Parser can be used to parse a configuration file and environment of a defined +// version into a unified output structure +type Parser struct { + prefix string + mapping map[Version]VersionedParseInfo + env map[string]string +} + +// NewParser returns a *Parser with the given environment prefix which handles +// versioned configurations which match the given parseInfos +func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser { + p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo), env: make(map[string]string)} + + for _, parseInfo := range parseInfos { + p.mapping[parseInfo.Version] = parseInfo + } + + for _, env := range os.Environ() { + envParts := strings.SplitN(env, "=", 2) + p.env[envParts[0]] = envParts[1] + } + + return &p +} + +// Parse reads in the given []byte and environment and writes the resulting +// configuration into the input v +// +// Environment variables may be used to override configuration parameters other +// than version, following the scheme below: +// v.Abc may be replaced by the value of PREFIX_ABC, +// v.Abc.Xyz may be replaced by the value of PREFIX_ABC_XYZ, and so forth +func (p *Parser) Parse(in []byte, v interface{}) error { + var versionedStruct struct { + Version Version + } + + if err := yaml.Unmarshal(in, &versionedStruct); err != nil { + return err + } + + parseInfo, ok := p.mapping[versionedStruct.Version] + if !ok { + return fmt.Errorf("Unsupported version: %q", versionedStruct.Version) + } + + parseAs := reflect.New(parseInfo.ParseAs) + err := yaml.Unmarshal(in, parseAs.Interface()) + if err != nil { + return err + } + + err = p.overwriteFields(parseAs, p.prefix) + if err != nil { + return err + } + + c, err := parseInfo.ConversionFunc(parseAs.Interface()) + if err != nil { + return err + } + reflect.ValueOf(v).Elem().Set(reflect.Indirect(reflect.ValueOf(c))) + return nil +} + +func (p *Parser) overwriteFields(v reflect.Value, prefix string) error { + for v.Kind() == reflect.Ptr { + v = reflect.Indirect(v) + } + switch v.Kind() { + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + sf := v.Type().Field(i) + fieldPrefix := strings.ToUpper(prefix + "_" + sf.Name) + if e, ok := p.env[fieldPrefix]; ok { + fieldVal := reflect.New(sf.Type) + err := yaml.Unmarshal([]byte(e), fieldVal.Interface()) + if err != nil { + return err + } + v.Field(i).Set(reflect.Indirect(fieldVal)) + } + err := p.overwriteFields(v.Field(i), fieldPrefix) + if err != nil { + return err + } + } + case reflect.Map: + p.overwriteMap(v, prefix) + } + return nil +} + +func (p *Parser) overwriteMap(m reflect.Value, prefix string) error { + switch m.Type().Elem().Kind() { + case reflect.Struct: + for _, k := range m.MapKeys() { + err := p.overwriteFields(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k))) + if err != nil { + return err + } + } + envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix))) + if err != nil { + return err + } + for key, val := range p.env { + if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil { + mapValue := reflect.New(m.Type().Elem()) + err := yaml.Unmarshal([]byte(val), mapValue.Interface()) + if err != nil { + return err + } + m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue)) + } + } + case reflect.Map: + for _, k := range m.MapKeys() { + err := p.overwriteMap(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k))) + if err != nil { + return err + } + } + default: + envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix))) + if err != nil { + return err + } + + for key, val := range p.env { + if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil { + mapValue := reflect.New(m.Type().Elem()) + err := yaml.Unmarshal([]byte(val), mapValue.Interface()) + if err != nil { + return err + } + m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue)) + } + } + } + return nil +}