From 92fad0708ec5307f7e252a57b8732e2abbbd1334 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Mon, 21 Mar 2022 14:11:38 +0100 Subject: [PATCH] Prepare applying pkglistgen diffs on staging accept We will check if there are approved pkglistgen comments and if so, apply them to the target. For this we need to parse the generated markdown, so refactor both in a common class. There diff is applied, but not yet commited in this commit --- osclib/accept_command.py | 3 + osclib/pkglistgen_comments.py | 220 ++++++++++++++++++++++++++++++++++ pkglistgen/tool.py | 110 +---------------- 3 files changed, 226 insertions(+), 107 deletions(-) create mode 100644 osclib/pkglistgen_comments.py diff --git a/osclib/accept_command.py b/osclib/accept_command.py index a956a554..f0ad2285 100644 --- a/osclib/accept_command.py +++ b/osclib/accept_command.py @@ -12,6 +12,7 @@ from osclib.core import attribute_value_save from osclib.core import attribute_value_load from osclib.core import source_file_load from osclib.core import source_file_save +from osclib.pkglistgen_comments import PkglistComments from datetime import date @@ -19,6 +20,7 @@ class AcceptCommand(object): def __init__(self, api): self.api = api self.config = conf.config[self.api.project] + self.pkglist_comments = PkglistComments(self.api.apiurl) def find_new_requests(self, project): match = f"state/@name='new' and action/target/@project='{project}'" @@ -133,6 +135,7 @@ class AcceptCommand(object): self.api.staging_deactivate(project) + self.pkglist_comments.check_staging_accept(project, self.api.project) self.reset_rebuild_data(project) if cleanup: diff --git a/osclib/pkglistgen_comments.py b/osclib/pkglistgen_comments.py new file mode 100644 index 00000000..ab545b04 --- /dev/null +++ b/osclib/pkglistgen_comments.py @@ -0,0 +1,220 @@ +import textwrap +import re +import tempfile +import logging +import sys +from osclib.comments import CommentAPI +from osc.core import checkout_package + +MARKER = 'PackageListDiff' + + +class PkglistComments(object): + """Handling staging comments of diffs""" + + def __init__(self, apiurl): + self.apiurl = apiurl + self.comment = CommentAPI(apiurl) + + def read_summary_file(self, file): + ret = dict() + with open(file, 'r') as f: + for line in f: + pkg, group = line.strip().split(':') + ret.setdefault(pkg, []) + ret[pkg].append(group) + return ret + + def write_summary_file(self, file, content): + output = [] + for pkg in sorted(content): + for group in sorted(content[pkg]): + output.append(f"{pkg}:{group}") + + with open(file, 'w') as f: + for line in sorted(output): + f.write(line + '\n') + + def calculcate_package_diff(self, old_file, new_file): + old_file = self.read_summary_file(old_file) + new_file = self.read_summary_file(new_file) + + # remove common part + keys = list(old_file.keys()) + for key in keys: + if new_file.get(key, []) == old_file[key]: + del new_file[key] + del old_file[key] + + if not old_file and not new_file: + return None + + removed = dict() + for pkg in old_file: + old_groups = old_file[pkg] + if new_file.get(pkg): + continue + removekey = ','.join(old_groups) + removed.setdefault(removekey, []) + removed[removekey].append(pkg) + + report = '' + for rm in sorted(removed.keys()): + report += f"**Remove from {rm}**\n\n```\n" + paragraph = ', '.join(removed[rm]) + report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) + report += "\n```\n\n" + + moved = dict() + for pkg in old_file: + old_groups = old_file[pkg] + new_groups = new_file.get(pkg) + if not new_groups: + continue + movekey = ','.join(old_groups) + ' to ' + ','.join(new_groups) + moved.setdefault(movekey, []) + moved[movekey].append(pkg) + + for move in sorted(moved.keys()): + report += f"**Move from {move}**\n\n```\n" + paragraph = ', '.join(moved[move]) + report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) + report += "\n```\n\n" + + added = dict() + for pkg in new_file: + if pkg in old_file: + continue + addkey = ','.join(new_file[pkg]) + added.setdefault(addkey, []) + added[addkey].append(pkg) + + for group in sorted(added): + report += f"**Add to {group}**\n\n```\n" + paragraph = ', '.join(added[group]) + report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) + report += "\n```\n\n" + + return report.strip() + + def handle_package_diff(self, project, old_file, new_file): + comments = self.comment.get_comments(project_name=project) + comment, _ = self.comment.comment_find(comments, MARKER) + + report = self.calculcate_package_diff(old_file, new_file) + if not report: + if comment: + self.comment.delete(comment['id']) + return 0 + report = self.comment.add_marker(report, MARKER) + + if comment: + write_comment = report != comment['comment'] + else: + write_comment = True + if write_comment: + if comment: + self.comment.delete(comment['id']) + self.comment.add_comment(project_name=project, comment=report) + else: + for c in comments.values(): + if c['parent'] == comment['id']: + ct = c['comment'] + if ct.startswith('ignore ') or ct == 'ignore': + print(c) + return 0 + + return 1 + + def is_approved(self, comment, comments): + for c in comments.values(): + if c['parent'] == comment['id']: + ct = c['comment'] + if ct.startswith('approve ') or ct == 'approve': + return True + return False + + def parse_title(self, line): + m = re.match(r'\*\*Add to (.*)\*\*', line) + if m: + return {'cmd': 'add', 'to': m.group(1).split(','), 'pkgs': []} + m = re.match(r'\*\*Move from (.*) to (.*)\*\*', line) + if m: + return {'cmd': 'move', 'from': m.group(1).split(','), 'to': m.group(2).split(','), 'pkgs': []} + m = re.match(r'\*\*Remove from (.*)\*\*', line) + if m: + return {'cmd': 'remove', 'from': m.group(1).split(','), 'pkgs': []} + return None + + def parse_sections(self, comment): + current_section = None + sections = [] + in_quote = False + for line in comment.split('\n'): + if line.startswith('**'): + if current_section: + sections.append(current_section) + current_section = self.parse_title(line) + continue + if line.startswith("```"): + in_quote = not in_quote + continue + if in_quote: + for pkg in line.split(','): + pkg = pkg.strip() + if pkg: + current_section['pkgs'].append(pkg) + if current_section: + sections.append(current_section) + return sections + + def apply_move(self, content, section): + for pkg in section['pkgs']: + pkg_content = content[pkg] + for group in section['from']: + try: + pkg_content.remove(group) + except ValueError: + logging.error(f"Can't remove {pkg} from {group}, not there. Mismatch.") + sys.exit(1) + for group in section['to']: + pkg_content.append(group) + content[pkg] = pkg_content + + def apply_add(self, content, section): + for pkg in section['pkgs']: + content.setdefault(pkg, []) + content[pkg] += section['to'] + + def apply_remove(self, content, section): + for pkg in section['pkgs']: + pkg_content = content[pkg] + for group in section['from']: + try: + pkg_content.remove(group) + except ValueError: + logging.error(f"Can't remove {pkg} from {group}, not there. Mismatch.") + sys.exit(1) + content[pkg] = pkg_content + + def apply_commands(self, filename, sections): + content = self.read_summary_file(filename) + for section in sections: + if section['cmd'] == 'move': + self.apply_move(content, section) + elif section['cmd'] == 'add': + self.apply_add(content, section) + elif section['cmd'] == 'remove': + self.apply_remove(content, section) + self.write_summary_file(filename, content) + + def check_staging_accept(self, project, target): + comments = self.comment.get_comments(project_name=project) + comment, _ = self.comment.comment_find(comments, MARKER) + if not comment or not self.is_approved(comment, comments): + return + sections = self.parse_sections(comment['comment']) + with tempfile.TemporaryDirectory() as tmpdirname: + checkout_package(self.apiurl, target, '000package-groups', expand_link=True, outdir=tmpdirname) + self.apply_commands(tmpdirname + '/summary-staging.txt', sections) + raise RuntimeError("Not implemented commit") diff --git a/pkglistgen/tool.py b/pkglistgen/tool.py index e3d93407..82cacd39 100644 --- a/pkglistgen/tool.py +++ b/pkglistgen/tool.py @@ -7,7 +7,6 @@ import solv import shutil import subprocess import yaml -import textwrap from lxml import etree as ET @@ -23,7 +22,7 @@ from osclib.conf import str2bool from osclib.core import repository_path_expand from osclib.core import repository_arch_state from osclib.cache_manager import CacheManager -from osclib.comments import CommentAPI +from osclib.pkglistgen_comments import PkglistComments from urllib.parse import urlparse @@ -33,7 +32,6 @@ from pkglistgen.group import Group SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product' -MARKER = 'PackageListDiff' # share header cache with repochecker CACHEDIR = CacheManager.directory('repository-meta') @@ -48,7 +46,7 @@ class PkgListGen(ToolBase.ToolBase): def __init__(self): ToolBase.ToolBase.__init__(self) self.logger = logging.getLogger(__name__) - self.comment = CommentAPI(self.apiurl) + self.comment = PkglistComments(self.apiurl) self.reset() def reset(self): @@ -508,106 +506,6 @@ class PkgListGen(ToolBase.ToolBase): print('%endif', file=output) output.flush() - def read_summary_file(self, file): - ret = dict() - with open(file, 'r') as f: - for line in f: - pkg, group = line.strip().split(':') - ret.setdefault(pkg, []) - ret[pkg].append(group) - return ret - - def calculcate_package_diff(self, old_file, new_file): - old_file = self.read_summary_file(old_file) - new_file = self.read_summary_file(new_file) - - # remove common part - keys = list(old_file.keys()) - for key in keys: - if new_file.get(key, []) == old_file[key]: - del new_file[key] - del old_file[key] - - if not old_file and not new_file: - return None - - removed = dict() - for pkg in old_file: - old_groups = old_file[pkg] - if new_file.get(pkg): - continue - removekey = ','.join(old_groups) - removed.setdefault(removekey, []) - removed[removekey].append(pkg) - - report = '' - for rm in sorted(removed.keys()): - report += f"**Remove from {rm}**\n\n```\n" - paragraph = ', '.join(removed[rm]) - report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) - report += "\n```\n\n" - - moved = dict() - for pkg in old_file: - old_groups = old_file[pkg] - new_groups = new_file.get(pkg) - if not new_groups: - continue - movekey = ','.join(old_groups) + ' to ' + ','.join(new_groups) - moved.setdefault(movekey, []) - moved[movekey].append(pkg) - - for move in sorted(moved.keys()): - report += f"**Move from {move}**\n\n```\n" - paragraph = ', '.join(moved[move]) - report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) - report += "\n```\n\n" - - added = dict() - for pkg in new_file: - if pkg in old_file: - continue - addkey = ','.join(new_file[pkg]) - added.setdefault(addkey, []) - added[addkey].append(pkg) - - for group in sorted(added): - report += f"**Add to {group}**\n\n```\n" - paragraph = ', '.join(added[group]) - report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) - report += "\n```\n\n" - - return report.strip() - - def handle_package_diff(self, project, old_file, new_file): - comments = self.comment.get_comments(project_name=project) - comment, _ = self.comment.comment_find(comments, MARKER) - - report = self.calculcate_package_diff(old_file, new_file) - if not report: - if comment: - self.comment.delete(comment['id']) - return 0 - report = self.comment.add_marker(report, MARKER) - - if comment: - write_comment = report != comment['comment'] - else: - write_comment = True - if write_comment: - if comment: - self.comment.delete(comment['id']) - self.comment.add_comment(project_name=project, comment=report) - else: - for c in comments.values(): - if c['parent'] == comment['id']: - ct = c['comment'] - if ct.startswith('ignore ') or ct == 'ignore': - print(c) - return 0 - - return 1 - def solve_project(self, ignore_unresolvable=False, ignore_recommended=False, locale=None, locales_from=None): self.load_all_groups() if not self.output: @@ -766,8 +664,6 @@ class PkgListGen(ToolBase.ToolBase): checkout_package(api.apiurl, project, package, expand_link=True, prj_dir=cache_dir, outdir=os.path.join(cache_dir, package)) - # print('RET', self.handle_package_diff(project, f"{group_dir}/summary-staging.txt", f"{product_dir}/summary-staging.txt")) - file_utils.unlink_all_except(release_dir, ['weakremovers.inc']) if not only_release_packages: file_utils.unlink_all_except(product_dir) @@ -860,4 +756,4 @@ class PkgListGen(ToolBase.ToolBase): self.commit_package(product_dir) if os.path.isfile(reference_summary): - return self.handle_package_diff(project, reference_summary, summary_file) + return self.comment.handle_package_diff(project, reference_summary, summary_file)