From 760ddf39e69bf376266ed8d6669b8bfd03847b83 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Thu, 16 Aug 2018 21:46:05 -0500 Subject: [PATCH] osclib/conf: decouple from StagingAPI and always fetch remote config. As the remote config is no longer optional for SLE and is utilized by openSUSE to the point were it is dangerous not to load the remote config it should be required. Currently only certain users call apply_remote() while this will make it built-in during construction and thus makes the usage consistent and no longer require StagingAPI. --- ReviewBot.py | 4 +--- devel-project.py | 7 ++++--- metrics.py | 6 +++--- openqa-comments.py | 6 +++--- osc-staging.py | 4 +--- osclib/conf.py | 17 ++++++++--------- pkglistgen.py | 5 ++--- suppkg_rebuild.py | 2 +- tests/accept_tests.py | 2 +- tests/api_tests.py | 2 +- tests/check_tests.py | 2 +- tests/config_tests.py | 29 ++++++++++++++++------------- tests/freeze_tests.py | 2 +- tests/obslock_tests.py | 2 +- tests/select_tests.py | 2 +- tests/unselect_tests.py | 2 +- totest-manager.py | 2 +- update_crawler.py | 2 +- 18 files changed, 48 insertions(+), 50 deletions(-) diff --git a/ReviewBot.py b/ReviewBot.py index fe0ecde9..6b73266e 100644 --- a/ReviewBot.py +++ b/ReviewBot.py @@ -141,10 +141,8 @@ class ReviewBot(object): def staging_api(self, project): if project not in self.staging_apis: - config = Config(project) + Config(self.apiurl, project) self.staging_apis[project] = StagingAPI(self.apiurl, project) - - config.apply_remote(self.staging_apis[project]) self.staging_config[project] = conf.config[project].copy() return self.staging_apis[project] diff --git a/devel-project.py b/devel-project.py index 7d19b73c..62c5a7b1 100755 --- a/devel-project.py +++ b/devel-project.py @@ -40,8 +40,9 @@ osc.core._search = osc.core.search osc.core.search = search def staging_api(args): - Config(args.project) - return StagingAPI(osc.conf.config['apiurl'], args.project) + apiurl = osc.conf.config['apiurl'] + Config(apiurl, args.project) + return StagingAPI(apiurl, args.project) def devel_projects_get(apiurl, project): """ @@ -116,7 +117,7 @@ def notify(args): maintainer_map.setdefault(userid, set()) maintainer_map[userid].add(devel_package_identifier) - Config(args.project) # Ensure mail-* options are loaded for mail_send(). + Config(apiurl, args.project) # Ensure mail-* options are loaded for mail_send(). subject = 'Packages you maintain are present in {}'.format(args.project) for userid, package_identifiers in maintainer_map.items(): email = entity_email(apiurl, userid) diff --git a/metrics.py b/metrics.py index 065d0b68..20d23a81 100755 --- a/metrics.py +++ b/metrics.py @@ -549,9 +549,9 @@ def main(args): Cache.PATTERNS['/source/[^/]+/dashboard/[^/]+\?rev=.*'] = sys.maxint Cache.init() - config = Config(args.project) - api = StagingAPI(osc.conf.config['apiurl'], args.project) - config.apply_remote(api) + apiurl = osc.conf.config['apiurl'] + Config(apiurl, args.project) + api = StagingAPI(apiurl, args.project) print('dashboard: wrote {:,} points'.format(ingest_dashboard(api))) diff --git a/openqa-comments.py b/openqa-comments.py index 24f797be..f930f32c 100755 --- a/openqa-comments.py +++ b/openqa-comments.py @@ -211,9 +211,9 @@ if __name__ == '__main__': if args.force: MARGIN_HOURS = 0 - config = Config(args.project) - api = StagingAPI(osc.conf.config['apiurl'], args.project) - config.apply_remote(api) + apiurl = osc.conf.config['apiurl'] + Config(apiurl, args.project) + api = StagingAPI(apiurl, args.project) openQA = OpenQAReport(api) if args.staging: diff --git a/osc-staging.py b/osc-staging.py index a22bf292..ee11b33c 100644 --- a/osc-staging.py +++ b/osc-staging.py @@ -407,7 +407,7 @@ def do_staging(self, subcmd, opts, *args): opts.project = self._full_project_name(opts.project) opts.apiurl = self.get_api_url() opts.verbose = False - config = Config(opts.project) + Config(opts.apiurl, opts.project) colorama.init(autoreset=True, strip=(opts.no_color or not bool(int(conf.config.get('staging.color', True))))) @@ -423,8 +423,6 @@ def do_staging(self, subcmd, opts, *args): Cache.delete_all() api = StagingAPI(opts.apiurl, opts.project) - config.apply_remote(api) - needed = lock_needed(cmd, opts) with OBSLock(opts.apiurl, opts.project, reason=cmd, needed=needed) as lock: diff --git a/osclib/conf.py b/osclib/conf.py index de195643..8c6379eb 100644 --- a/osclib/conf.py +++ b/osclib/conf.py @@ -171,12 +171,12 @@ def str2bool(v): class Config(object): """Helper class to configuration file.""" - def __init__(self, project): + def __init__(self, apiurl, project): self.project = project + self.remote_values = self.fetch_remote(apiurl) conf_file = conf.config.get('conffile', os.environ.get('OSC_CONFIG', '~/.oscrc')) self.conf_file = os.path.expanduser(conf_file) - self.remote_values = None # Populate the configuration dictionary self.populate_conf() @@ -186,7 +186,7 @@ class Config(object): return conf def populate_conf(self): - """Add sane default into the configuration.""" + """Add sane default into the configuration and layer (defaults, remote, ~/.oscrc).""" defaults = {} default_ordered = OrderedDict(sorted(DEFAULT.items(), key=lambda i: int(i[1].get('_priority', 99)))) for prj_pattern in default_ordered: @@ -232,13 +232,12 @@ class Config(object): else: return defaults - def apply_remote(self, api): - """Fetch remote config and re-process (defaults, remote, .oscrc).""" - - config = attribute_value_load(api.apiurl, self.project, 'Config') + def fetch_remote(self, apiurl): + config = attribute_value_load(apiurl, self.project, 'Config') if config: cp = ConfigParser() config = '[remote]\n' + config cp.readfp(io.BytesIO(config)) - self.remote_values = dict(cp.items('remote')) - self.populate_conf() + return dict(cp.items('remote')) + + return None diff --git a/pkglistgen.py b/pkglistgen.py index 1e4e89de..122045cb 100755 --- a/pkglistgen.py +++ b/pkglistgen.py @@ -1159,10 +1159,9 @@ class CommandLineInterface(ToolBase.CommandLineInterface): # Store target project as opts.project will contain subprojects. target_project = opts.project - config = Config(target_project) apiurl = conf.config['apiurl'] + config = Config(apiurl, target_project) api = StagingAPI(apiurl, target_project) - config.apply_remote(api) target_config = conf.config[target_project] if opts.scope == 'ports': @@ -1387,7 +1386,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface): project_family.extend(project_list_family(apiurl, family_include)) for project in project_family: - config = Config(project) + config = Config(apiurl, project) project_config = conf.config[project] baseurl = project_config.get('download-baseurl') diff --git a/suppkg_rebuild.py b/suppkg_rebuild.py index b7497028..65564575 100755 --- a/suppkg_rebuild.py +++ b/suppkg_rebuild.py @@ -47,7 +47,7 @@ class StagingHelper(object): def __init__(self, project): self.project = project self.apiurl = osc.conf.config['apiurl'] - Config(self.project) + Config(self.apiurl, self.project) self.api = StagingAPI(self.apiurl, self.project) def get_support_package_list(self, project, repository): diff --git a/tests/accept_tests.py b/tests/accept_tests.py index 59286dd5..dee82064 100644 --- a/tests/accept_tests.py +++ b/tests/accept_tests.py @@ -31,7 +31,7 @@ class TestAccept(unittest.TestCase): Initialize the configuration """ self.obs = OBS() - Config('openSUSE:Factory') + Config(APIURL, 'openSUSE:Factory') self.api = StagingAPI(APIURL, 'openSUSE:Factory') def test_accept_comments(self): diff --git a/tests/api_tests.py b/tests/api_tests.py index 4607205c..7014a228 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -43,7 +43,7 @@ class TestApiCalls(unittest.TestCase): """ self.obs = OBS() - Config('openSUSE:Factory') + Config(APIURL, 'openSUSE:Factory') self.api = StagingAPI(APIURL, 'openSUSE:Factory') def tearDown(self): diff --git a/tests/check_tests.py b/tests/check_tests.py index 93584102..8e44078a 100644 --- a/tests/check_tests.py +++ b/tests/check_tests.py @@ -63,7 +63,7 @@ class TestCheckCommand(unittest.TestCase): """Initialize the configuration.""" self.obs = OBS() - Config('openSUSE:Factory') + Config(APIURL, 'openSUSE:Factory') self.stagingapi = StagingAPI(APIURL, 'openSUSE:Factory') self.checkcommand = CheckCommand(self.stagingapi) diff --git a/tests/config_tests.py b/tests/config_tests.py index d610ef46..c6a2fb87 100644 --- a/tests/config_tests.py +++ b/tests/config_tests.py @@ -2,6 +2,8 @@ import unittest from osc import conf from osclib.conf import DEFAULT from osclib.conf import Config +#from osclib.core import attribute_value_load +from osclib.core import attribute_value_save from osclib.stagingapi import StagingAPI from obs import APIURL @@ -12,31 +14,32 @@ from obs import OBS class TestConfig(unittest.TestCase): def setUp(self): self.obs = OBS() - self.config = Config(PROJECT) + self.load_config() self.api = StagingAPI(APIURL, PROJECT) + def load_config(self, project=PROJECT): + self.config = Config(APIURL, project) + def test_basic(self): self.assertEqual('openSUSE', conf.config[PROJECT]['lock-ns']) def test_remote(self): - self.assertEqual('local', conf.config[PROJECT]['overridden-by-local']) - self.assertIsNone(conf.config[PROJECT].get('remote-only')) - - self.config.apply_remote(self.api) - + # Initial config present in fixtures/oscrc and obs.py attribute default. + # Local config fixture contains overridden-by-local and should win over + # the remote config value. self.assertEqual('local', conf.config[PROJECT]['overridden-by-local']) self.assertEqual('remote-indeed', conf.config[PROJECT]['remote-only']) - self.api.attribute_value_save('Config', 'remote-only = nope') - self.config.apply_remote(self.api) + # Change remote value. + attribute_value_save(APIURL, PROJECT, 'Config', 'remote-only = new value\n') + self.load_config() self.assertEqual('local', conf.config[PROJECT]['overridden-by-local']) - self.assertEqual('nope', conf.config[PROJECT]['remote-only']) + self.assertEqual('new value', conf.config[PROJECT]['remote-only']) def test_remote_none(self): - self.api.attribute_value_save('Config', '') - # don't crash - self.config.apply_remote(self.api) + self.load_config('not_real_project') + self.assertTrue(True) # Did not crash! def test_pattern_order(self): # Add pattern to defaults in order to identify which was matched. @@ -56,7 +59,7 @@ class TestConfig(unittest.TestCase): # Ensure each pattern is match instead of catch-all pattern. patterns = set() for project in projects: - config = Config(project) + config = Config(APIURL, project) patterns.add(conf.config[project]['pattern']) self.assertEqual(len(patterns), len(DEFAULT)) diff --git a/tests/freeze_tests.py b/tests/freeze_tests.py index 09ece9bc..7c06c8c2 100644 --- a/tests/freeze_tests.py +++ b/tests/freeze_tests.py @@ -33,7 +33,7 @@ class TestFreeze(unittest.TestCase): Initialize the configuration """ self.obs = OBS() - Config('openSUSE:Factory') + Config(APIURL, 'openSUSE:Factory') self.api = StagingAPI(APIURL, 'openSUSE:Factory') def _get_fixture_path(self, filename): diff --git a/tests/obslock_tests.py b/tests/obslock_tests.py index 869543ae..663f0712 100644 --- a/tests/obslock_tests.py +++ b/tests/obslock_tests.py @@ -11,7 +11,7 @@ from obs import OBS class TestOBSLock(unittest.TestCase): def setUp(self): self.obs = OBS() - Config(PROJECT) + Config(APIURL, PROJECT) def obs_lock(self, reason='list'): return OBSLock(APIURL, PROJECT, reason=reason) diff --git a/tests/select_tests.py b/tests/select_tests.py index 95bca7ec..a81f62cf 100644 --- a/tests/select_tests.py +++ b/tests/select_tests.py @@ -32,7 +32,7 @@ class TestSelect(unittest.TestCase): Initialize the configuration """ self.obs = OBS() - Config('openSUSE:Factory') + Config(APIURL, 'openSUSE:Factory') self.api = StagingAPI(APIURL, 'openSUSE:Factory') def test_old_frozen(self): diff --git a/tests/unselect_tests.py b/tests/unselect_tests.py index f7462d7b..7ed04699 100644 --- a/tests/unselect_tests.py +++ b/tests/unselect_tests.py @@ -11,7 +11,7 @@ from obs import OBS class TestUnselect(unittest.TestCase): def setUp(self): self.obs = OBS() - Config(PROJECT) + Config(APIURL, PROJECT) self.api = StagingAPI(APIURL, PROJECT) def test_cleanup_filter(self): diff --git a/totest-manager.py b/totest-manager.py index 33d33faf..4795b1ec 100755 --- a/totest-manager.py +++ b/totest-manager.py @@ -1040,7 +1040,7 @@ class CommandlineInterface(cmdln.Cmdln): if not self.options.obs_api_url: self.options.obs_api_url = self.api_url[project_base] - Config(project) + Config(self.options.obs_api_url, project) if project not in self.totest_class: msg = 'Project %s not recognized. Possible values [%s]' % ( project, ', '.join(self.totest_class)) diff --git a/update_crawler.py b/update_crawler.py index 4c164a9f..a34c641a 100755 --- a/update_crawler.py +++ b/update_crawler.py @@ -329,7 +329,7 @@ def main(args): osc.conf.config['debug'] = args.osc_debug # initialize stagingapi config - Config(args.to_prj) + Config(osc.conf.config['apiurl'], args.to_prj) uc = UpdateCrawler(args.from_prj, args.to_prj) uc.caching = args.cache_requests