From 60f996efed5e99e153f70f1345babf67a47e875a Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Thu, 12 Sep 2019 09:48:52 -0500 Subject: [PATCH 1/3] osclib/origin_listener: provide origin_updatable_map(). --- osclib/origin_listener.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/osclib/origin_listener.py b/osclib/origin_listener.py index 7f066a93..0b8918bd 100644 --- a/osclib/origin_listener.py +++ b/osclib/origin_listener.py @@ -32,11 +32,11 @@ class OriginSourceChangeListener(PubSubConsumer): raise Exception('Unrequested message: {}'.format(method.routing_key)) def on_message_package_update(self, payload): - origins = origin_updatable_map(self.apiurl) + origins = self.origin_updatable_map() self.update_consider(origins, payload['project'], payload['package']) def on_message_request_create(self, payload): - origins = origin_updatable_map(self.apiurl, pending=True) + origins = self.origin_updatable_map(pending=True) for action in payload['actions']: # The following code demonstrates the quality of the data structure. # The base structure is inconsistent enough and yet the event data @@ -68,6 +68,9 @@ class OriginSourceChangeListener(PubSubConsumer): self.update_consider(origins, project, package) + def origin_updatable_map(self, pending=None): + return origin_updatable_map(self.apiurl, pending=pending) + def update_consider(self, origins, origin_project, package): if origin_project not in origins: self.logger.info('skipped irrelevant project: {}'.format(origin_project)) From 791d4497773fde6c49c059addc8ec5e436c83eb3 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Thu, 12 Sep 2019 09:49:48 -0500 Subject: [PATCH 2/3] osclib/origin_listener: change skipping log message to mention origin. --- osclib/origin_listener.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osclib/origin_listener.py b/osclib/origin_listener.py index 0b8918bd..0ba2b3c8 100644 --- a/osclib/origin_listener.py +++ b/osclib/origin_listener.py @@ -73,7 +73,7 @@ class OriginSourceChangeListener(PubSubConsumer): def update_consider(self, origins, origin_project, package): if origin_project not in origins: - self.logger.info('skipped irrelevant project: {}'.format(origin_project)) + self.logger.info('skipped irrelevant origin: {}'.format(origin_project)) return for project in origins[origin_project]: From ac6e61a15e0f25259ad4caff58e14c07637113d0 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Thu, 12 Sep 2019 09:52:28 -0500 Subject: [PATCH 3/3] osclib/origin_listener: start additional listeners for remote origins. For example, SLE-15-SP2 has openSUSE.org:openSUSE:Factory as an origin. The events for that project are not included on the IBS message bus and thus package updates to that project will be missed. When origins contain a remote prefix another listener needs to be started pointing at the remote OBS instance message bus. The resulting messages need to be prefixed before being considered by the primary listener. --- osclib/origin_listener.py | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/osclib/origin_listener.py b/osclib/origin_listener.py index 0ba2b3c8..1e6de28e 100644 --- a/osclib/origin_listener.py +++ b/osclib/origin_listener.py @@ -1,8 +1,10 @@ import json from osclib.core import package_kind +from osclib.core import project_remote_list from osclib.origin import origin_updatable_map from osclib.origin import origin_update from osclib.PubSubConsumer import PubSubConsumer +import threading class OriginSourceChangeListener(PubSubConsumer): @@ -10,10 +12,28 @@ class OriginSourceChangeListener(PubSubConsumer): self.apiurl = apiurl self.project = project self.dry = dry + self.listeners = {} amqp_prefix = 'suse' if self.apiurl.endswith('suse.de') else 'opensuse' super().__init__(amqp_prefix, logger) + def run(self, runtime=None): + super().run(runtime=runtime) + + for listener in self.listeners.values(): + listener.run(runtime=runtime) + + def stop(self): + super().stop() + + for listener in self.listeners.values(): + listener.stop() + + def start_consuming(self): + super().start_consuming() + + self.check_remotes() + def routing_keys(self): return [self._prefix + k for k in [ '.obs.package.update', @@ -68,6 +88,16 @@ class OriginSourceChangeListener(PubSubConsumer): self.update_consider(origins, project, package) + def check_remotes(self): + origins = self.origin_updatable_map() + remotes = project_remote_list(self.apiurl) + for remote, apiurl in remotes.items(): + for origin in origins: + if origin.startswith(remote + ':') and apiurl not in self.listeners: + self.logger.info('starting remote listener due to {} origin'.format(origin)) + self.listeners[apiurl] = OriginSourceChangeListenerRemote(apiurl, self, remote) + threading.Thread(target=self.listeners[apiurl].run).start() + def origin_updatable_map(self, pending=None): return origin_updatable_map(self.apiurl, pending=pending) @@ -89,3 +119,21 @@ class OriginSourceChangeListener(PubSubConsumer): else: # This eliminates the possibility for deletes by listener. self.logger.info('skipped updating non-existant package {}/{}'.format(project, package)) + +class OriginSourceChangeListenerRemote(OriginSourceChangeListener): + def __init__(self, apiurl, parent, prefix): + self.parent = parent + self.prefix = prefix + + super().__init__(apiurl, self.parent.logger) + self._run_until = self.parent._run_until + + def check_remotes(self): + pass + + def origin_updatable_map(self, pending=None): + return self.parent.origin_updatable_map(pending=pending) + + def update_consider(self, origins, origin_project, package): + origin_project = '{}:{}'.format(self.prefix, origin_project) + self.parent.update_consider(origins, origin_project, package)