diff --git a/osclib/accept_command.py b/osclib/accept_command.py index a956a554..f0ad2285 100644 --- a/osclib/accept_command.py +++ b/osclib/accept_command.py @@ -12,6 +12,7 @@ from osclib.core import attribute_value_save from osclib.core import attribute_value_load from osclib.core import source_file_load from osclib.core import source_file_save +from osclib.pkglistgen_comments import PkglistComments from datetime import date @@ -19,6 +20,7 @@ class AcceptCommand(object): def __init__(self, api): self.api = api self.config = conf.config[self.api.project] + self.pkglist_comments = PkglistComments(self.api.apiurl) def find_new_requests(self, project): match = f"state/@name='new' and action/target/@project='{project}'" @@ -133,6 +135,7 @@ class AcceptCommand(object): self.api.staging_deactivate(project) + self.pkglist_comments.check_staging_accept(project, self.api.project) self.reset_rebuild_data(project) if cleanup: diff --git a/osclib/pkglistgen_comments.py b/osclib/pkglistgen_comments.py new file mode 100644 index 00000000..ab545b04 --- /dev/null +++ b/osclib/pkglistgen_comments.py @@ -0,0 +1,220 @@ +import textwrap +import re +import tempfile +import logging +import sys +from osclib.comments import CommentAPI +from osc.core import checkout_package + +MARKER = 'PackageListDiff' + + +class PkglistComments(object): + """Handling staging comments of diffs""" + + def __init__(self, apiurl): + self.apiurl = apiurl + self.comment = CommentAPI(apiurl) + + def read_summary_file(self, file): + ret = dict() + with open(file, 'r') as f: + for line in f: + pkg, group = line.strip().split(':') + ret.setdefault(pkg, []) + ret[pkg].append(group) + return ret + + def write_summary_file(self, file, content): + output = [] + for pkg in sorted(content): + for group in sorted(content[pkg]): + output.append(f"{pkg}:{group}") + + with open(file, 'w') as f: + for line in sorted(output): + f.write(line + '\n') + + def calculcate_package_diff(self, old_file, new_file): + old_file = self.read_summary_file(old_file) + new_file = self.read_summary_file(new_file) + + # remove common part + keys = list(old_file.keys()) + for key in keys: + if new_file.get(key, []) == old_file[key]: + del new_file[key] + del old_file[key] + + if not old_file and not new_file: + return None + + removed = dict() + for pkg in old_file: + old_groups = old_file[pkg] + if new_file.get(pkg): + continue + removekey = ','.join(old_groups) + removed.setdefault(removekey, []) + removed[removekey].append(pkg) + + report = '' + for rm in sorted(removed.keys()): + report += f"**Remove from {rm}**\n\n```\n" + paragraph = ', '.join(removed[rm]) + report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) + report += "\n```\n\n" + + moved = dict() + for pkg in old_file: + old_groups = old_file[pkg] + new_groups = new_file.get(pkg) + if not new_groups: + continue + movekey = ','.join(old_groups) + ' to ' + ','.join(new_groups) + moved.setdefault(movekey, []) + moved[movekey].append(pkg) + + for move in sorted(moved.keys()): + report += f"**Move from {move}**\n\n```\n" + paragraph = ', '.join(moved[move]) + report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) + report += "\n```\n\n" + + added = dict() + for pkg in new_file: + if pkg in old_file: + continue + addkey = ','.join(new_file[pkg]) + added.setdefault(addkey, []) + added[addkey].append(pkg) + + for group in sorted(added): + report += f"**Add to {group}**\n\n```\n" + paragraph = ', '.join(added[group]) + report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) + report += "\n```\n\n" + + return report.strip() + + def handle_package_diff(self, project, old_file, new_file): + comments = self.comment.get_comments(project_name=project) + comment, _ = self.comment.comment_find(comments, MARKER) + + report = self.calculcate_package_diff(old_file, new_file) + if not report: + if comment: + self.comment.delete(comment['id']) + return 0 + report = self.comment.add_marker(report, MARKER) + + if comment: + write_comment = report != comment['comment'] + else: + write_comment = True + if write_comment: + if comment: + self.comment.delete(comment['id']) + self.comment.add_comment(project_name=project, comment=report) + else: + for c in comments.values(): + if c['parent'] == comment['id']: + ct = c['comment'] + if ct.startswith('ignore ') or ct == 'ignore': + print(c) + return 0 + + return 1 + + def is_approved(self, comment, comments): + for c in comments.values(): + if c['parent'] == comment['id']: + ct = c['comment'] + if ct.startswith('approve ') or ct == 'approve': + return True + return False + + def parse_title(self, line): + m = re.match(r'\*\*Add to (.*)\*\*', line) + if m: + return {'cmd': 'add', 'to': m.group(1).split(','), 'pkgs': []} + m = re.match(r'\*\*Move from (.*) to (.*)\*\*', line) + if m: + return {'cmd': 'move', 'from': m.group(1).split(','), 'to': m.group(2).split(','), 'pkgs': []} + m = re.match(r'\*\*Remove from (.*)\*\*', line) + if m: + return {'cmd': 'remove', 'from': m.group(1).split(','), 'pkgs': []} + return None + + def parse_sections(self, comment): + current_section = None + sections = [] + in_quote = False + for line in comment.split('\n'): + if line.startswith('**'): + if current_section: + sections.append(current_section) + current_section = self.parse_title(line) + continue + if line.startswith("```"): + in_quote = not in_quote + continue + if in_quote: + for pkg in line.split(','): + pkg = pkg.strip() + if pkg: + current_section['pkgs'].append(pkg) + if current_section: + sections.append(current_section) + return sections + + def apply_move(self, content, section): + for pkg in section['pkgs']: + pkg_content = content[pkg] + for group in section['from']: + try: + pkg_content.remove(group) + except ValueError: + logging.error(f"Can't remove {pkg} from {group}, not there. Mismatch.") + sys.exit(1) + for group in section['to']: + pkg_content.append(group) + content[pkg] = pkg_content + + def apply_add(self, content, section): + for pkg in section['pkgs']: + content.setdefault(pkg, []) + content[pkg] += section['to'] + + def apply_remove(self, content, section): + for pkg in section['pkgs']: + pkg_content = content[pkg] + for group in section['from']: + try: + pkg_content.remove(group) + except ValueError: + logging.error(f"Can't remove {pkg} from {group}, not there. Mismatch.") + sys.exit(1) + content[pkg] = pkg_content + + def apply_commands(self, filename, sections): + content = self.read_summary_file(filename) + for section in sections: + if section['cmd'] == 'move': + self.apply_move(content, section) + elif section['cmd'] == 'add': + self.apply_add(content, section) + elif section['cmd'] == 'remove': + self.apply_remove(content, section) + self.write_summary_file(filename, content) + + def check_staging_accept(self, project, target): + comments = self.comment.get_comments(project_name=project) + comment, _ = self.comment.comment_find(comments, MARKER) + if not comment or not self.is_approved(comment, comments): + return + sections = self.parse_sections(comment['comment']) + with tempfile.TemporaryDirectory() as tmpdirname: + checkout_package(self.apiurl, target, '000package-groups', expand_link=True, outdir=tmpdirname) + self.apply_commands(tmpdirname + '/summary-staging.txt', sections) + raise RuntimeError("Not implemented commit") diff --git a/pkglistgen/tool.py b/pkglistgen/tool.py index e3d93407..82cacd39 100644 --- a/pkglistgen/tool.py +++ b/pkglistgen/tool.py @@ -7,7 +7,6 @@ import solv import shutil import subprocess import yaml -import textwrap from lxml import etree as ET @@ -23,7 +22,7 @@ from osclib.conf import str2bool from osclib.core import repository_path_expand from osclib.core import repository_arch_state from osclib.cache_manager import CacheManager -from osclib.comments import CommentAPI +from osclib.pkglistgen_comments import PkglistComments from urllib.parse import urlparse @@ -33,7 +32,6 @@ from pkglistgen.group import Group SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__)) PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product' -MARKER = 'PackageListDiff' # share header cache with repochecker CACHEDIR = CacheManager.directory('repository-meta') @@ -48,7 +46,7 @@ class PkgListGen(ToolBase.ToolBase): def __init__(self): ToolBase.ToolBase.__init__(self) self.logger = logging.getLogger(__name__) - self.comment = CommentAPI(self.apiurl) + self.comment = PkglistComments(self.apiurl) self.reset() def reset(self): @@ -508,106 +506,6 @@ class PkgListGen(ToolBase.ToolBase): print('%endif', file=output) output.flush() - def read_summary_file(self, file): - ret = dict() - with open(file, 'r') as f: - for line in f: - pkg, group = line.strip().split(':') - ret.setdefault(pkg, []) - ret[pkg].append(group) - return ret - - def calculcate_package_diff(self, old_file, new_file): - old_file = self.read_summary_file(old_file) - new_file = self.read_summary_file(new_file) - - # remove common part - keys = list(old_file.keys()) - for key in keys: - if new_file.get(key, []) == old_file[key]: - del new_file[key] - del old_file[key] - - if not old_file and not new_file: - return None - - removed = dict() - for pkg in old_file: - old_groups = old_file[pkg] - if new_file.get(pkg): - continue - removekey = ','.join(old_groups) - removed.setdefault(removekey, []) - removed[removekey].append(pkg) - - report = '' - for rm in sorted(removed.keys()): - report += f"**Remove from {rm}**\n\n```\n" - paragraph = ', '.join(removed[rm]) - report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) - report += "\n```\n\n" - - moved = dict() - for pkg in old_file: - old_groups = old_file[pkg] - new_groups = new_file.get(pkg) - if not new_groups: - continue - movekey = ','.join(old_groups) + ' to ' + ','.join(new_groups) - moved.setdefault(movekey, []) - moved[movekey].append(pkg) - - for move in sorted(moved.keys()): - report += f"**Move from {move}**\n\n```\n" - paragraph = ', '.join(moved[move]) - report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) - report += "\n```\n\n" - - added = dict() - for pkg in new_file: - if pkg in old_file: - continue - addkey = ','.join(new_file[pkg]) - added.setdefault(addkey, []) - added[addkey].append(pkg) - - for group in sorted(added): - report += f"**Add to {group}**\n\n```\n" - paragraph = ', '.join(added[group]) - report += "\n".join(textwrap.wrap(paragraph, width=90, break_long_words=False, break_on_hyphens=False)) - report += "\n```\n\n" - - return report.strip() - - def handle_package_diff(self, project, old_file, new_file): - comments = self.comment.get_comments(project_name=project) - comment, _ = self.comment.comment_find(comments, MARKER) - - report = self.calculcate_package_diff(old_file, new_file) - if not report: - if comment: - self.comment.delete(comment['id']) - return 0 - report = self.comment.add_marker(report, MARKER) - - if comment: - write_comment = report != comment['comment'] - else: - write_comment = True - if write_comment: - if comment: - self.comment.delete(comment['id']) - self.comment.add_comment(project_name=project, comment=report) - else: - for c in comments.values(): - if c['parent'] == comment['id']: - ct = c['comment'] - if ct.startswith('ignore ') or ct == 'ignore': - print(c) - return 0 - - return 1 - def solve_project(self, ignore_unresolvable=False, ignore_recommended=False, locale=None, locales_from=None): self.load_all_groups() if not self.output: @@ -766,8 +664,6 @@ class PkgListGen(ToolBase.ToolBase): checkout_package(api.apiurl, project, package, expand_link=True, prj_dir=cache_dir, outdir=os.path.join(cache_dir, package)) - # print('RET', self.handle_package_diff(project, f"{group_dir}/summary-staging.txt", f"{product_dir}/summary-staging.txt")) - file_utils.unlink_all_except(release_dir, ['weakremovers.inc']) if not only_release_packages: file_utils.unlink_all_except(product_dir) @@ -860,4 +756,4 @@ class PkgListGen(ToolBase.ToolBase): self.commit_package(product_dir) if os.path.isfile(reference_summary): - return self.handle_package_diff(project, reference_summary, summary_file) + return self.comment.handle_package_diff(project, reference_summary, summary_file)