From b5a5143da6a0608d19bfaab56fe2bf466264a8cb Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Mon, 15 Apr 2024 13:56:56 +0200 Subject: [PATCH 1/7] Move core.File to obs_scm.File --- osc/core.py | 55 +-------------------------------------- osc/obs_scm/__init__.py | 1 + osc/obs_scm/file.py | 57 +++++++++++++++++++++++++++++++++++++++++ setup.cfg | 1 + 4 files changed, 60 insertions(+), 54 deletions(-) create mode 100644 osc/obs_scm/__init__.py create mode 100644 osc/obs_scm/file.py diff --git a/osc/core.py b/osc/core.py index 1af0f94c..ebefc721 100644 --- a/osc/core.py +++ b/osc/core.py @@ -53,6 +53,7 @@ from . import oscerr from . import output from . import store as osc_store from .connection import http_request, http_GET, http_POST, http_PUT, http_DELETE +from .obs_scm import File from .output import sanitize_text from .store import Store from .util import xdg @@ -263,60 +264,6 @@ def revision_is_empty(rev: Union[None, str, int]): return rev in (None, "") -@total_ordering -class File: - """represent a file, including its metadata""" - - def __init__(self, name, md5, size, mtime, skipped=False): - self.name = name - self.md5 = md5 - self.size = size - self.mtime = mtime - self.skipped = skipped - - def __repr__(self): - return self.name - - def __str__(self): - return self.name - - def __eq__(self, other): - if isinstance(other, str): - return self.name == other - self_data = (self.name, self.md5, self.size, self.mtime, self.skipped) - other_data = (other.name, other.md5, other.size, other.mtime, other.skipped) - return self_data == other_data - - def __lt__(self, other): - self_data = (self.name, self.md5, self.size, self.mtime, self.skipped) - other_data = (other.name, other.md5, other.size, other.mtime, other.skipped) - return self_data < other_data - - @classmethod - def from_xml_node(cls, node): - assert node.tag == "entry" - kwargs = { - "name": node.get("name"), - "md5": node.get("md5"), - "size": int(node.get("size")), - "mtime": int(node.get("mtime")), - "skipped": "skipped" in node.attrib, - } - return cls(**kwargs) - - def to_xml_node(self, parent_node): - attributes = { - "name": self.name, - "md5": self.md5, - "size": str(int(self.size)), - "mtime": str(int(self.mtime)), - } - if self.skipped: - attributes["skipped"] = "true" - new_node = ET.SubElement(parent_node, "entry", attributes) - return new_node - - class Serviceinfo: """Source service content """ diff --git a/osc/obs_scm/__init__.py b/osc/obs_scm/__init__.py new file mode 100644 index 00000000..401f3300 --- /dev/null +++ b/osc/obs_scm/__init__.py @@ -0,0 +1 @@ +from .file import File diff --git a/osc/obs_scm/file.py b/osc/obs_scm/file.py new file mode 100644 index 00000000..cd2fd3c6 --- /dev/null +++ b/osc/obs_scm/file.py @@ -0,0 +1,57 @@ +from functools import total_ordering + +from ..util.xml import ET + + +@total_ordering +class File: + """represent a file, including its metadata""" + + def __init__(self, name, md5, size, mtime, skipped=False): + self.name = name + self.md5 = md5 + self.size = size + self.mtime = mtime + self.skipped = skipped + + def __repr__(self): + return self.name + + def __str__(self): + return self.name + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + self_data = (self.name, self.md5, self.size, self.mtime, self.skipped) + other_data = (other.name, other.md5, other.size, other.mtime, other.skipped) + return self_data == other_data + + def __lt__(self, other): + self_data = (self.name, self.md5, self.size, self.mtime, self.skipped) + other_data = (other.name, other.md5, other.size, other.mtime, other.skipped) + return self_data < other_data + + @classmethod + def from_xml_node(cls, node): + assert node.tag == "entry" + kwargs = { + "name": node.get("name"), + "md5": node.get("md5"), + "size": int(node.get("size")), + "mtime": int(node.get("mtime")), + "skipped": "skipped" in node.attrib, + } + return cls(**kwargs) + + def to_xml_node(self, parent_node): + attributes = { + "name": self.name, + "md5": self.md5, + "size": str(int(self.size)), + "mtime": str(int(self.mtime)), + } + if self.skipped: + attributes["skipped"] = "true" + new_node = ET.SubElement(parent_node, "entry", attributes) + return new_node diff --git a/setup.cfg b/setup.cfg index 5dfb3129..39d24a0d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,7 @@ packages = osc.commands osc.git_scm osc.obs_api + osc.obs_scm osc.output osc.util install_requires = From 7d05d74456f7a43e341f6754a06c4f5a867c8b37 Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Mon, 15 Apr 2024 14:06:41 +0200 Subject: [PATCH 2/7] Move core.Serviceinfo to obs_scm.Serviceinfo --- osc/core.py | 233 +--------------------------------- osc/obs_scm/__init__.py | 1 + osc/obs_scm/serviceinfo.py | 252 +++++++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+), 232 deletions(-) create mode 100644 osc/obs_scm/serviceinfo.py diff --git a/osc/core.py b/osc/core.py index ebefc721..5eafceac 100644 --- a/osc/core.py +++ b/osc/core.py @@ -54,6 +54,7 @@ from . import output from . import store as osc_store from .connection import http_request, http_GET, http_POST, http_PUT, http_DELETE from .obs_scm import File +from .obs_scm import Serviceinfo from .output import sanitize_text from .store import Store from .util import xdg @@ -264,238 +265,6 @@ def revision_is_empty(rev: Union[None, str, int]): return rev in (None, "") -class Serviceinfo: - """Source service content - """ - - def __init__(self): - """creates an empty serviceinfo instance""" - self.services = [] - self.apiurl: Optional[str] = None - self.project: Optional[str] = None - self.package: Optional[str] = None - - def read(self, serviceinfo_node, append=False): - """read in the source services ```` element passed as - elementtree node. - """ - def error(msg, xml): - data = f'invalid service format:\n{ET.tostring(xml, encoding=ET_ENCODING)}' - raise ValueError(f"{data}\n\n{msg}") - - if serviceinfo_node is None: - return - if not append: - self.services = [] - services = serviceinfo_node.findall('service') - - for service in services: - name = service.get('name') - if name is None: - error("invalid service definition. Attribute name missing.", service) - if len(name) < 3 or '/' in name: - error(f"invalid service name: {name}", service) - mode = service.get('mode', '') - data = {'name': name, 'mode': mode} - command = [name] - for param in service.findall('param'): - option = param.get('name') - if option is None: - error(f"{name}: a parameter requires a name", service) - value = '' - if param.text: - value = param.text - command.append('--' + option) - # hmm is this reasonable or do we want to allow real - # options (e.g., "--force" (without an argument)) as well? - command.append(value) - data['command'] = command - self.services.append(data) - - def getProjectGlobalServices(self, apiurl: str, project: str, package: str): - self.apiurl = apiurl - # get all project wide services in one file, we don't store it yet - u = makeurl(apiurl, ["source", project, package], query={"cmd": "getprojectservices"}) - try: - f = http_POST(u) - root = ET.parse(f).getroot() - self.read(root, True) - self.project = project - self.package = package - except HTTPError as e: - if e.code == 404 and package != '_project': - self.getProjectGlobalServices(apiurl, project, '_project') - self.package = package - elif e.code != 403 and e.code != 400: - raise e - - def addVerifyFile(self, serviceinfo_node, filename: str): - f = open(filename, 'rb') - digest = hashlib.sha256(f.read()).hexdigest() - f.close() - - r = serviceinfo_node - s = ET.Element("service", name="verify_file") - ET.SubElement(s, "param", name="file").text = filename - ET.SubElement(s, "param", name="verifier").text = "sha256" - ET.SubElement(s, "param", name="checksum").text = digest - - r.append(s) - return r - - def addDownloadUrl(self, serviceinfo_node, url_string: str): - url = urlparse(url_string) - protocol = url.scheme - host = url.netloc - path = url.path - - r = serviceinfo_node - s = ET.Element("service", name="download_url") - ET.SubElement(s, "param", name="protocol").text = protocol - ET.SubElement(s, "param", name="host").text = host - ET.SubElement(s, "param", name="path").text = path - - r.append(s) - return r - - def addSetVersion(self, serviceinfo_node): - r = serviceinfo_node - s = ET.Element("service", name="set_version", mode="buildtime") - r.append(s) - return r - - def addGitUrl(self, serviceinfo_node, url_string: Optional[str]): - r = serviceinfo_node - s = ET.Element("service", name="obs_scm") - ET.SubElement(s, "param", name="url").text = url_string - ET.SubElement(s, "param", name="scm").text = "git" - r.append(s) - return r - - def addTarUp(self, serviceinfo_node): - r = serviceinfo_node - s = ET.Element("service", name="tar", mode="buildtime") - r.append(s) - return r - - def addRecompressTar(self, serviceinfo_node): - r = serviceinfo_node - s = ET.Element("service", name="recompress", mode="buildtime") - ET.SubElement(s, "param", name="file").text = "*.tar" - ET.SubElement(s, "param", name="compression").text = "xz" - r.append(s) - return r - - def execute(self, dir, callmode: Optional[str] = None, singleservice=None, verbose: Optional[bool] = None): - old_dir = os.path.join(dir, '.old') - - # if 2 osc instances are executed at a time one, of them fails on .old file existence - # sleep up to 10 seconds until we can create the directory - for i in reversed(range(10)): - try: - os.mkdir(old_dir) - break - except FileExistsError: - time.sleep(1) - - if i == 0: - msg = f'"{old_dir}" exists, please remove it' - raise oscerr.OscIOError(None, msg) - - try: - result = self._execute(dir, old_dir, callmode, singleservice, verbose) - finally: - shutil.rmtree(old_dir) - return result - - def _execute( - self, dir, old_dir, callmode: Optional[str] = None, singleservice=None, verbose: Optional[bool] = None - ): - # cleanup existing generated files - for filename in os.listdir(dir): - if filename.startswith('_service:') or filename.startswith('_service_'): - os.rename(os.path.join(dir, filename), - os.path.join(old_dir, filename)) - - allservices = self.services or [] - service_names = [s['name'] for s in allservices] - if singleservice and singleservice not in service_names: - # set array to the manual specified singleservice, if it is not part of _service file - data = {'name': singleservice, 'command': [singleservice], 'mode': callmode} - allservices = [data] - elif singleservice: - allservices = [s for s in allservices if s['name'] == singleservice] - # set the right called mode or the service would be skipped below - for s in allservices: - s['mode'] = callmode - - if not allservices: - # short-circuit to avoid a potential http request in vc_export_env - # (if there are no services to execute this http request is - # useless) - return 0 - - # services can detect that they run via osc this way - os.putenv("OSC_VERSION", get_osc_version()) - - # set environment when using OBS 2.3 or later - if self.project is not None: - # These need to be kept in sync with bs_service - os.putenv("OBS_SERVICE_APIURL", self.apiurl) - os.putenv("OBS_SERVICE_PROJECT", self.project) - os.putenv("OBS_SERVICE_PACKAGE", self.package) - # also export vc env vars (some services (like obs_scm) use them) - vc_export_env(self.apiurl) - - # recreate files - ret = 0 - for service in allservices: - if callmode != "all": - if service['mode'] == "buildtime": - continue - if service['mode'] == "serveronly" and callmode != "local": - continue - if service['mode'] == "manual" and callmode != "manual": - continue - if service['mode'] != "manual" and callmode == "manual": - continue - if service['mode'] == "disabled" and callmode != "disabled": - continue - if service['mode'] != "disabled" and callmode == "disabled": - continue - if service['mode'] != "trylocal" and service['mode'] != "localonly" and callmode == "trylocal": - continue - temp_dir = None - try: - temp_dir = tempfile.mkdtemp(dir=dir, suffix=f".{service['name']}.service") - cmd = service['command'] - if not os.path.exists("/usr/lib/obs/service/" + cmd[0]): - raise oscerr.PackageNotInstalled(f"obs-service-{cmd[0]}") - cmd[0] = "/usr/lib/obs/service/" + cmd[0] - cmd = cmd + ["--outdir", temp_dir] - output.print_msg("Run source service:", " ".join(cmd), print_to="verbose") - r = run_external(*cmd) - - if r != 0: - print("Aborting: service call failed: ", ' '.join(cmd)) - # FIXME: addDownloadUrlService calls si.execute after - # updating _services. - return r - - if service['mode'] == "manual" or service['mode'] == "disabled" or service['mode'] == "trylocal" or service['mode'] == "localonly" or callmode == "local" or callmode == "trylocal" or callmode == "all": - for filename in os.listdir(temp_dir): - os.rename(os.path.join(temp_dir, filename), os.path.join(dir, filename)) - else: - name = service['name'] - for filename in os.listdir(temp_dir): - os.rename(os.path.join(temp_dir, filename), os.path.join(dir, "_service:" + name + ":" + filename)) - finally: - if temp_dir is not None: - shutil.rmtree(temp_dir) - - return 0 - - class Linkinfo: """linkinfo metadata (which is part of the xml representing a directory) """ diff --git a/osc/obs_scm/__init__.py b/osc/obs_scm/__init__.py index 401f3300..8ce0a462 100644 --- a/osc/obs_scm/__init__.py +++ b/osc/obs_scm/__init__.py @@ -1 +1,2 @@ from .file import File +from .serviceinfo import Serviceinfo \ No newline at end of file diff --git a/osc/obs_scm/serviceinfo.py b/osc/obs_scm/serviceinfo.py new file mode 100644 index 00000000..58eca87f --- /dev/null +++ b/osc/obs_scm/serviceinfo.py @@ -0,0 +1,252 @@ +import hashlib +import os +import shutil +import tempfile +import time +from typing import Optional +from urllib.error import HTTPError +from urllib.parse import urlparse + +from .. import oscerr +from .. import output +from ..util.xml import ET + + +class Serviceinfo: + """Source service content + """ + + def __init__(self): + """creates an empty serviceinfo instance""" + self.services = [] + self.apiurl: Optional[str] = None + self.project: Optional[str] = None + self.package: Optional[str] = None + + def read(self, serviceinfo_node, append=False): + """read in the source services ```` element passed as + elementtree node. + """ + def error(msg, xml): + from ..core import ET_ENCODING + data = f'invalid service format:\n{ET.tostring(xml, encoding=ET_ENCODING)}' + raise ValueError(f"{data}\n\n{msg}") + + if serviceinfo_node is None: + return + if not append: + self.services = [] + services = serviceinfo_node.findall('service') + + for service in services: + name = service.get('name') + if name is None: + error("invalid service definition. Attribute name missing.", service) + if len(name) < 3 or '/' in name: + error(f"invalid service name: {name}", service) + mode = service.get('mode', '') + data = {'name': name, 'mode': mode} + command = [name] + for param in service.findall('param'): + option = param.get('name') + if option is None: + error(f"{name}: a parameter requires a name", service) + value = '' + if param.text: + value = param.text + command.append('--' + option) + # hmm is this reasonable or do we want to allow real + # options (e.g., "--force" (without an argument)) as well? + command.append(value) + data['command'] = command + self.services.append(data) + + def getProjectGlobalServices(self, apiurl: str, project: str, package: str): + from ..core import http_POST + from ..core import makeurl + + self.apiurl = apiurl + # get all project wide services in one file, we don't store it yet + u = makeurl(apiurl, ["source", project, package], query={"cmd": "getprojectservices"}) + try: + f = http_POST(u) + root = ET.parse(f).getroot() + self.read(root, True) + self.project = project + self.package = package + except HTTPError as e: + if e.code == 404 and package != '_project': + self.getProjectGlobalServices(apiurl, project, '_project') + self.package = package + elif e.code != 403 and e.code != 400: + raise e + + def addVerifyFile(self, serviceinfo_node, filename: str): + f = open(filename, 'rb') + digest = hashlib.sha256(f.read()).hexdigest() + f.close() + + r = serviceinfo_node + s = ET.Element("service", name="verify_file") + ET.SubElement(s, "param", name="file").text = filename + ET.SubElement(s, "param", name="verifier").text = "sha256" + ET.SubElement(s, "param", name="checksum").text = digest + + r.append(s) + return r + + def addDownloadUrl(self, serviceinfo_node, url_string: str): + url = urlparse(url_string) + protocol = url.scheme + host = url.netloc + path = url.path + + r = serviceinfo_node + s = ET.Element("service", name="download_url") + ET.SubElement(s, "param", name="protocol").text = protocol + ET.SubElement(s, "param", name="host").text = host + ET.SubElement(s, "param", name="path").text = path + + r.append(s) + return r + + def addSetVersion(self, serviceinfo_node): + r = serviceinfo_node + s = ET.Element("service", name="set_version", mode="buildtime") + r.append(s) + return r + + def addGitUrl(self, serviceinfo_node, url_string: Optional[str]): + r = serviceinfo_node + s = ET.Element("service", name="obs_scm") + ET.SubElement(s, "param", name="url").text = url_string + ET.SubElement(s, "param", name="scm").text = "git" + r.append(s) + return r + + def addTarUp(self, serviceinfo_node): + r = serviceinfo_node + s = ET.Element("service", name="tar", mode="buildtime") + r.append(s) + return r + + def addRecompressTar(self, serviceinfo_node): + r = serviceinfo_node + s = ET.Element("service", name="recompress", mode="buildtime") + ET.SubElement(s, "param", name="file").text = "*.tar" + ET.SubElement(s, "param", name="compression").text = "xz" + r.append(s) + return r + + def execute(self, dir, callmode: Optional[str] = None, singleservice=None, verbose: Optional[bool] = None): + old_dir = os.path.join(dir, '.old') + + # if 2 osc instances are executed at a time one, of them fails on .old file existence + # sleep up to 10 seconds until we can create the directory + for i in reversed(range(10)): + try: + os.mkdir(old_dir) + break + except FileExistsError: + time.sleep(1) + + if i == 0: + msg = f'"{old_dir}" exists, please remove it' + raise oscerr.OscIOError(None, msg) + + try: + result = self._execute(dir, old_dir, callmode, singleservice, verbose) + finally: + shutil.rmtree(old_dir) + return result + + def _execute( + self, dir, old_dir, callmode: Optional[str] = None, singleservice=None, verbose: Optional[bool] = None + ): + from ..core import get_osc_version + from ..core import run_external + from ..core import vc_export_env + + # cleanup existing generated files + for filename in os.listdir(dir): + if filename.startswith('_service:') or filename.startswith('_service_'): + os.rename(os.path.join(dir, filename), + os.path.join(old_dir, filename)) + + allservices = self.services or [] + service_names = [s['name'] for s in allservices] + if singleservice and singleservice not in service_names: + # set array to the manual specified singleservice, if it is not part of _service file + data = {'name': singleservice, 'command': [singleservice], 'mode': callmode} + allservices = [data] + elif singleservice: + allservices = [s for s in allservices if s['name'] == singleservice] + # set the right called mode or the service would be skipped below + for s in allservices: + s['mode'] = callmode + + if not allservices: + # short-circuit to avoid a potential http request in vc_export_env + # (if there are no services to execute this http request is + # useless) + return 0 + + # services can detect that they run via osc this way + os.putenv("OSC_VERSION", get_osc_version()) + + # set environment when using OBS 2.3 or later + if self.project is not None: + # These need to be kept in sync with bs_service + os.putenv("OBS_SERVICE_APIURL", self.apiurl) + os.putenv("OBS_SERVICE_PROJECT", self.project) + os.putenv("OBS_SERVICE_PACKAGE", self.package) + # also export vc env vars (some services (like obs_scm) use them) + vc_export_env(self.apiurl) + + # recreate files + ret = 0 + for service in allservices: + if callmode != "all": + if service['mode'] == "buildtime": + continue + if service['mode'] == "serveronly" and callmode != "local": + continue + if service['mode'] == "manual" and callmode != "manual": + continue + if service['mode'] != "manual" and callmode == "manual": + continue + if service['mode'] == "disabled" and callmode != "disabled": + continue + if service['mode'] != "disabled" and callmode == "disabled": + continue + if service['mode'] != "trylocal" and service['mode'] != "localonly" and callmode == "trylocal": + continue + temp_dir = None + try: + temp_dir = tempfile.mkdtemp(dir=dir, suffix=f".{service['name']}.service") + cmd = service['command'] + if not os.path.exists("/usr/lib/obs/service/" + cmd[0]): + raise oscerr.PackageNotInstalled(f"obs-service-{cmd[0]}") + cmd[0] = "/usr/lib/obs/service/" + cmd[0] + cmd = cmd + ["--outdir", temp_dir] + output.print_msg("Run source service:", " ".join(cmd), print_to="verbose") + r = run_external(*cmd) + + if r != 0: + print("Aborting: service call failed: ", ' '.join(cmd)) + # FIXME: addDownloadUrlService calls si.execute after + # updating _services. + return r + + if service['mode'] == "manual" or service['mode'] == "disabled" or service['mode'] == "trylocal" or service['mode'] == "localonly" or callmode == "local" or callmode == "trylocal" or callmode == "all": + for filename in os.listdir(temp_dir): + os.rename(os.path.join(temp_dir, filename), os.path.join(dir, filename)) + else: + name = service['name'] + for filename in os.listdir(temp_dir): + os.rename(os.path.join(temp_dir, filename), os.path.join(dir, "_service:" + name + ":" + filename)) + finally: + if temp_dir is not None: + shutil.rmtree(temp_dir) + + return 0 From c8999c9b3309fd94c16f770954d9179b36ce37fe Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Mon, 15 Apr 2024 14:08:25 +0200 Subject: [PATCH 3/7] Move core.Linkinfo to obs_scm.Linkinfo --- osc/core.py | 66 +---------------------------------------- osc/obs_scm/__init__.py | 1 + osc/obs_scm/linkinfo.py | 63 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 65 deletions(-) create mode 100644 osc/obs_scm/linkinfo.py diff --git a/osc/core.py b/osc/core.py index 5eafceac..f30135ba 100644 --- a/osc/core.py +++ b/osc/core.py @@ -54,6 +54,7 @@ from . import output from . import store as osc_store from .connection import http_request, http_GET, http_POST, http_PUT, http_DELETE from .obs_scm import File +from .obs_scm import Linkinfo from .obs_scm import Serviceinfo from .output import sanitize_text from .store import Store @@ -265,71 +266,6 @@ def revision_is_empty(rev: Union[None, str, int]): return rev in (None, "") -class Linkinfo: - """linkinfo metadata (which is part of the xml representing a directory) - """ - - def __init__(self): - """creates an empty linkinfo instance""" - self.project = None - self.package = None - self.xsrcmd5 = None - self.lsrcmd5 = None - self.srcmd5 = None - self.error = None - self.rev = None - self.baserev = None - - def read(self, linkinfo_node): - """read in the linkinfo metadata from the ```` element passed as - elementtree node. - If the passed element is ``None``, the method does nothing. - """ - if linkinfo_node is None: - return - self.project = linkinfo_node.get('project') - self.package = linkinfo_node.get('package') - self.xsrcmd5 = linkinfo_node.get('xsrcmd5') - self.lsrcmd5 = linkinfo_node.get('lsrcmd5') - self.srcmd5 = linkinfo_node.get('srcmd5') - self.error = linkinfo_node.get('error') - self.rev = linkinfo_node.get('rev') - self.baserev = linkinfo_node.get('baserev') - - def islink(self): - """:return: ``True`` if the linkinfo is not empty, otherwise ``False``""" - if self.xsrcmd5 or self.lsrcmd5 or self.error is not None: - return True - return False - - def isexpanded(self): - """:return: ``True`` if the package is an expanded link""" - if self.lsrcmd5 and not self.xsrcmd5: - return True - return False - - def haserror(self): - """:return: ``True`` if the link is in error state (could not be applied)""" - if self.error: - return True - return False - - def __str__(self): - """return an informatory string representation""" - if self.islink() and not self.isexpanded(): - return 'project %s, package %s, xsrcmd5 %s, rev %s' \ - % (self.project, self.package, self.xsrcmd5, self.rev) - elif self.islink() and self.isexpanded(): - if self.haserror(): - return 'broken link to project %s, package %s, srcmd5 %s, lsrcmd5 %s: %s' \ - % (self.project, self.package, self.srcmd5, self.lsrcmd5, self.error) - else: - return 'expanded link to project %s, package %s, srcmd5 %s, lsrcmd5 %s' \ - % (self.project, self.package, self.srcmd5, self.lsrcmd5) - else: - return 'None' - - class DirectoryServiceinfo: def __init__(self): self.code = None diff --git a/osc/obs_scm/__init__.py b/osc/obs_scm/__init__.py index 8ce0a462..403f952d 100644 --- a/osc/obs_scm/__init__.py +++ b/osc/obs_scm/__init__.py @@ -1,2 +1,3 @@ from .file import File +from .linkinfo import Linkinfo from .serviceinfo import Serviceinfo \ No newline at end of file diff --git a/osc/obs_scm/linkinfo.py b/osc/obs_scm/linkinfo.py new file mode 100644 index 00000000..11e94bbb --- /dev/null +++ b/osc/obs_scm/linkinfo.py @@ -0,0 +1,63 @@ +class Linkinfo: + """linkinfo metadata (which is part of the xml representing a directory) + """ + + def __init__(self): + """creates an empty linkinfo instance""" + self.project = None + self.package = None + self.xsrcmd5 = None + self.lsrcmd5 = None + self.srcmd5 = None + self.error = None + self.rev = None + self.baserev = None + + def read(self, linkinfo_node): + """read in the linkinfo metadata from the ```` element passed as + elementtree node. + If the passed element is ``None``, the method does nothing. + """ + if linkinfo_node is None: + return + self.project = linkinfo_node.get('project') + self.package = linkinfo_node.get('package') + self.xsrcmd5 = linkinfo_node.get('xsrcmd5') + self.lsrcmd5 = linkinfo_node.get('lsrcmd5') + self.srcmd5 = linkinfo_node.get('srcmd5') + self.error = linkinfo_node.get('error') + self.rev = linkinfo_node.get('rev') + self.baserev = linkinfo_node.get('baserev') + + def islink(self): + """:return: ``True`` if the linkinfo is not empty, otherwise ``False``""" + if self.xsrcmd5 or self.lsrcmd5 or self.error is not None: + return True + return False + + def isexpanded(self): + """:return: ``True`` if the package is an expanded link""" + if self.lsrcmd5 and not self.xsrcmd5: + return True + return False + + def haserror(self): + """:return: ``True`` if the link is in error state (could not be applied)""" + if self.error: + return True + return False + + def __str__(self): + """return an informatory string representation""" + if self.islink() and not self.isexpanded(): + return 'project %s, package %s, xsrcmd5 %s, rev %s' \ + % (self.project, self.package, self.xsrcmd5, self.rev) + elif self.islink() and self.isexpanded(): + if self.haserror(): + return 'broken link to project %s, package %s, srcmd5 %s, lsrcmd5 %s: %s' \ + % (self.project, self.package, self.srcmd5, self.lsrcmd5, self.error) + else: + return 'expanded link to project %s, package %s, srcmd5 %s, lsrcmd5 %s' \ + % (self.project, self.package, self.srcmd5, self.lsrcmd5) + else: + return 'None' From 354f4caca6fbd9f2d5b9df1428b4de68af8e1a69 Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Mon, 15 Apr 2024 14:37:01 +0200 Subject: [PATCH 4/7] Move store.Store to obs_scm.Store --- osc/obs_scm/__init__.py | 3 +- osc/obs_scm/store.py | 319 ++++++++++++++++++++++++++++++++++++++++ osc/store.py | 307 +------------------------------------- 3 files changed, 322 insertions(+), 307 deletions(-) create mode 100644 osc/obs_scm/store.py diff --git a/osc/obs_scm/__init__.py b/osc/obs_scm/__init__.py index 403f952d..fb5792e5 100644 --- a/osc/obs_scm/__init__.py +++ b/osc/obs_scm/__init__.py @@ -1,3 +1,4 @@ from .file import File from .linkinfo import Linkinfo -from .serviceinfo import Serviceinfo \ No newline at end of file +from .serviceinfo import Serviceinfo +from .store import Store diff --git a/osc/obs_scm/store.py b/osc/obs_scm/store.py new file mode 100644 index 00000000..2efc3573 --- /dev/null +++ b/osc/obs_scm/store.py @@ -0,0 +1,319 @@ +""" +Store class wraps access to files in the '.osc' directory. +It is meant to be used as an implementation detail of Project and Package classes +and shouldn't be used in any code outside osc. +""" + + +import os + +from .. import oscerr +from .._private import api +from ..util.xml import ET + + +class Store: + STORE_DIR = ".osc" + STORE_VERSION = "1.0" + + @classmethod + def is_project_dir(cls, path): + try: + store = cls(path) + except oscerr.NoWorkingCopy: + return False + return store.is_project + + @classmethod + def is_package_dir(cls, path): + try: + store = cls(path) + except oscerr.NoWorkingCopy: + return False + return store.is_package + + def __init__(self, path, check=True): + self.path = path + self.abspath = os.path.abspath(self.path) + + self.is_project = self.exists("_project") and not self.exists("_package") + self.is_package = self.exists("_project") and self.exists("_package") + + if check and not any([self.is_project, self.is_package]): + msg = f"Directory '{self.path}' is not an OBS SCM working copy" + raise oscerr.NoWorkingCopy(msg) + + def __contains__(self, fn): + return self.exists(fn) + + def __iter__(self): + path = os.path.join(self.abspath, self.STORE_DIR) + yield from os.listdir(path) + + def assert_is_project(self): + if not self.is_project: + msg = f"Directory '{self.path}' is not an OBS SCM working copy of a project" + raise oscerr.NoWorkingCopy(msg) + + def assert_is_package(self): + if not self.is_package: + msg = f"Directory '{self.path}' is not an OBS SCM working copy of a package" + raise oscerr.NoWorkingCopy(msg) + + def get_path(self, fn, subdir=None): + # sanitize input to ensure that joining path works as expected + fn = fn.lstrip("/") + if subdir: + subdir = subdir.lstrip("/") + return os.path.join(self.abspath, self.STORE_DIR, subdir, fn) + return os.path.join(self.abspath, self.STORE_DIR, fn) + + def exists(self, fn, subdir=None): + return os.path.exists(self.get_path(fn, subdir=subdir)) + + def unlink(self, fn, subdir=None): + try: + os.unlink(self.get_path(fn, subdir=subdir)) + except FileNotFoundError: + pass + + def read_file(self, fn, subdir=None): + if not self.exists(fn, subdir=subdir): + return None + with open(self.get_path(fn, subdir=subdir), encoding="utf-8") as f: + return f.read() + + def write_file(self, fn, value, subdir=None): + if value is None: + self.unlink(fn, subdir=subdir) + return + try: + if subdir: + os.makedirs(self.get_path(subdir)) + else: + os.makedirs(self.get_path("")) + except FileExistsError: + pass + + old = self.get_path(fn, subdir=subdir) + new = self.get_path(f"{fn}.new", subdir=subdir) + try: + with open(new, "w", encoding="utf-8") as f: + f.write(value) + os.rename(new, old) + except: + if os.path.exists(new): + os.unlink(new) + raise + + def read_list(self, fn, subdir=None): + if not self.exists(fn, subdir=subdir): + return None + with open(self.get_path(fn, subdir=subdir), encoding="utf-8") as f: + return [line.rstrip("\n") for line in f] + + def write_list(self, fn, value, subdir=None): + if value is None: + self.unlink(fn, subdir=subdir) + return + if not isinstance(value, (list, tuple)): + msg = f"The argument `value` should be list, not {type(value).__name__}" + raise TypeError(msg) + value = "".join((f"{line or ''}\n" for line in value)) + self.write_file(fn, value, subdir=subdir) + + def read_string(self, fn, subdir=None): + if not self.exists(fn, subdir=subdir): + return None + with open(self.get_path(fn, subdir=subdir), encoding="utf-8") as f: + return f.readline().strip() + + def write_string(self, fn, value, subdir=None): + if value is None: + self.unlink(fn, subdir=subdir) + return + if isinstance(value, bytes): + value = value.decode("utf-8") + if not isinstance(value, str): + msg = f"The argument `value` should be str, not {type(value).__name__}" + raise TypeError(msg) + self.write_file(fn, f"{value}\n", subdir=subdir) + + def read_int(self, fn): + if not self.exists(fn): + return None + result = self.read_string(fn) + if not result.isdigit(): + return None + return int(result) + + def write_int(self, fn, value, subdir=None): + if value is None: + self.unlink(fn, subdir=subdir) + return + if not isinstance(value, int): + msg = f"The argument `value` should be int, not {type(value).__name__}" + raise TypeError(msg) + value = str(value) + self.write_string(fn, value, subdir=subdir) + + def read_xml_node(self, fn, node_name, subdir=None): + path = self.get_path(fn, subdir=subdir) + try: + tree = ET.parse(path) + except SyntaxError as e: + msg = f"Unable to parse '{path}': {e}" + raise oscerr.NoWorkingCopy(msg) + root = tree.getroot() + assert root.tag == node_name + # TODO: return root? + return tree + + def write_xml_node(self, fn, node_name, node, subdir=None): + path = self.get_path(fn, subdir=subdir) + assert node.tag == node_name + api.write_xml_node_to_file(node, path) + + def _sanitize_apiurl(self, value): + # apiurl shouldn't end with a slash, strip it so we can use apiurl without modifications + # in config['api_host_options'][apiurl] and other places + if isinstance(value, str): + value = value.strip("/") + elif isinstance(value, bytes): + value = value.strip(b"/") + return value + + @property + def apiurl(self): + return self._sanitize_apiurl(self.read_string("_apiurl")) + + @apiurl.setter + def apiurl(self, value): + self.write_string("_apiurl", self._sanitize_apiurl(value)) + + @property + def project(self): + return self.read_string("_project") + + @project.setter + def project(self, value): + self.write_string("_project", value) + + @property + def package(self): + return self.read_string("_package") + + @package.setter + def package(self, value): + self.write_string("_package", value) + + @property + def scmurl(self): + return self.read_string("_scm") + + @scmurl.setter + def scmurl(self, value): + return self.write_string("_scm", value) + + @property + def size_limit(self): + return self.read_int("_size_limit") + + @size_limit.setter + def size_limit(self, value): + return self.write_int("_size_limit", value) + + @property + def to_be_added(self): + self.assert_is_package() + return self.read_list("_to_be_added") or [] + + @to_be_added.setter + def to_be_added(self, value): + self.assert_is_package() + return self.write_list("_to_be_added", value) + + @property + def to_be_deleted(self): + self.assert_is_package() + return self.read_list("_to_be_deleted") or [] + + @to_be_deleted.setter + def to_be_deleted(self, value): + self.assert_is_package() + return self.write_list("_to_be_deleted", value) + + @property + def in_conflict(self): + self.assert_is_package() + return self.read_list("_in_conflict") or [] + + @in_conflict.setter + def in_conflict(self, value): + self.assert_is_package() + return self.write_list("_in_conflict", value) + + @property + def osclib_version(self): + return self.read_string("_osclib_version") + + @property + def files(self): + from .. import core as osc_core + + self.assert_is_package() + if self.exists("_scm"): + msg = "Package '{self.path}' is managed via SCM" + raise oscerr.NoWorkingCopy(msg) + if not self.exists("_files"): + msg = "Package '{self.path}' doesn't contain _files metadata" + raise oscerr.NoWorkingCopy(msg) + result = [] + directory_node = self.read_xml_node("_files", "directory").getroot() + for entry_node in api.find_nodes(directory_node, "directory", "entry"): + result.append(osc_core.File.from_xml_node(entry_node)) + return result + + @files.setter + def files(self, value): + if not isinstance(value, (list, tuple)): + msg = f"The argument `value` should be list, not {type(value).__name__}" + raise TypeError(msg) + + root = ET.Element("directory") + for file_obj in sorted(value): + file_obj.to_xml_node(root) + self.write_xml_node("_files", "directory", root) + + @property + def last_buildroot(self): + self.assert_is_package() + items = self.read_list("_last_buildroot") + if items is None: + return items + + if len(items) != 3: + msg = f"Package '{self.path}' contains _last_buildroot metadata that doesn't contain 3 lines: [repo, arch, vm_type]" + raise oscerr.NoWorkingCopy(msg) + + if items[2] in ("", "None"): + items[2] = None + + return items + + @last_buildroot.setter + def last_buildroot(self, value): + self.assert_is_package() + if len(value) != 3: + raise ValueError("A list with exactly 3 items is expected: [repo, arch, vm_type]") + self.write_list("_last_buildroot", value) + + @property + def _meta_node(self): + if not self.exists("_meta"): + return None + if self.is_package: + root = self.read_xml_node("_meta", "package").getroot() + else: + root = self.read_xml_node("_meta", "project").getroot() + return root diff --git a/osc/store.py b/osc/store.py index 91b569d8..cf4d8bc3 100644 --- a/osc/store.py +++ b/osc/store.py @@ -9,313 +9,8 @@ import os from xml.etree import ElementTree as ET from . import oscerr -from ._private import api from . import git_scm - -class Store: - STORE_DIR = ".osc" - STORE_VERSION = "1.0" - - @classmethod - def is_project_dir(cls, path): - try: - store = cls(path) - except oscerr.NoWorkingCopy: - return False - return store.is_project - - @classmethod - def is_package_dir(cls, path): - try: - store = cls(path) - except oscerr.NoWorkingCopy: - return False - return store.is_package - - def __init__(self, path, check=True): - self.path = path - self.abspath = os.path.abspath(self.path) - - self.is_project = self.exists("_project") and not self.exists("_package") - self.is_package = self.exists("_project") and self.exists("_package") - - if check and not any([self.is_project, self.is_package]): - msg = f"Directory '{self.path}' is not an OBS SCM working copy" - raise oscerr.NoWorkingCopy(msg) - - def __contains__(self, fn): - return self.exists(fn) - - def __iter__(self): - path = os.path.join(self.abspath, self.STORE_DIR) - yield from os.listdir(path) - - def assert_is_project(self): - if not self.is_project: - msg = f"Directory '{self.path}' is not an OBS SCM working copy of a project" - raise oscerr.NoWorkingCopy(msg) - - def assert_is_package(self): - if not self.is_package: - msg = f"Directory '{self.path}' is not an OBS SCM working copy of a package" - raise oscerr.NoWorkingCopy(msg) - - def get_path(self, fn, subdir=None): - # sanitize input to ensure that joining path works as expected - fn = fn.lstrip("/") - if subdir: - subdir = subdir.lstrip("/") - return os.path.join(self.abspath, self.STORE_DIR, subdir, fn) - return os.path.join(self.abspath, self.STORE_DIR, fn) - - def exists(self, fn, subdir=None): - return os.path.exists(self.get_path(fn, subdir=subdir)) - - def unlink(self, fn, subdir=None): - try: - os.unlink(self.get_path(fn, subdir=subdir)) - except FileNotFoundError: - pass - - def read_file(self, fn, subdir=None): - if not self.exists(fn, subdir=subdir): - return None - with open(self.get_path(fn, subdir=subdir), encoding="utf-8") as f: - return f.read() - - def write_file(self, fn, value, subdir=None): - if value is None: - self.unlink(fn, subdir=subdir) - return - try: - if subdir: - os.makedirs(self.get_path(subdir)) - else: - os.makedirs(self.get_path("")) - except FileExistsError: - pass - - old = self.get_path(fn, subdir=subdir) - new = self.get_path(f"{fn}.new", subdir=subdir) - try: - with open(new, "w", encoding="utf-8") as f: - f.write(value) - os.rename(new, old) - except: - if os.path.exists(new): - os.unlink(new) - raise - - def read_list(self, fn, subdir=None): - if not self.exists(fn, subdir=subdir): - return None - with open(self.get_path(fn, subdir=subdir), encoding="utf-8") as f: - return [line.rstrip("\n") for line in f] - - def write_list(self, fn, value, subdir=None): - if value is None: - self.unlink(fn, subdir=subdir) - return - if not isinstance(value, (list, tuple)): - msg = f"The argument `value` should be list, not {type(value).__name__}" - raise TypeError(msg) - value = "".join((f"{line or ''}\n" for line in value)) - self.write_file(fn, value, subdir=subdir) - - def read_string(self, fn, subdir=None): - if not self.exists(fn, subdir=subdir): - return None - with open(self.get_path(fn, subdir=subdir), encoding="utf-8") as f: - return f.readline().strip() - - def write_string(self, fn, value, subdir=None): - if value is None: - self.unlink(fn, subdir=subdir) - return - if isinstance(value, bytes): - value = value.decode("utf-8") - if not isinstance(value, str): - msg = f"The argument `value` should be str, not {type(value).__name__}" - raise TypeError(msg) - self.write_file(fn, f"{value}\n", subdir=subdir) - - def read_int(self, fn): - if not self.exists(fn): - return None - result = self.read_string(fn) - if not result.isdigit(): - return None - return int(result) - - def write_int(self, fn, value, subdir=None): - if value is None: - self.unlink(fn, subdir=subdir) - return - if not isinstance(value, int): - msg = f"The argument `value` should be int, not {type(value).__name__}" - raise TypeError(msg) - value = str(value) - self.write_string(fn, value, subdir=subdir) - - def read_xml_node(self, fn, node_name, subdir=None): - path = self.get_path(fn, subdir=subdir) - try: - tree = ET.parse(path) - except SyntaxError as e: - msg = f"Unable to parse '{path}': {e}" - raise oscerr.NoWorkingCopy(msg) - root = tree.getroot() - assert root.tag == node_name - # TODO: return root? - return tree - - def write_xml_node(self, fn, node_name, node, subdir=None): - path = self.get_path(fn, subdir=subdir) - assert node.tag == node_name - api.write_xml_node_to_file(node, path) - - def _sanitize_apiurl(self, value): - # apiurl shouldn't end with a slash, strip it so we can use apiurl without modifications - # in config['api_host_options'][apiurl] and other places - if isinstance(value, str): - value = value.strip("/") - elif isinstance(value, bytes): - value = value.strip(b"/") - return value - - @property - def apiurl(self): - return self._sanitize_apiurl(self.read_string("_apiurl")) - - @apiurl.setter - def apiurl(self, value): - self.write_string("_apiurl", self._sanitize_apiurl(value)) - - @property - def project(self): - return self.read_string("_project") - - @project.setter - def project(self, value): - self.write_string("_project", value) - - @property - def package(self): - return self.read_string("_package") - - @package.setter - def package(self, value): - self.write_string("_package", value) - - @property - def scmurl(self): - return self.read_string("_scm") - - @scmurl.setter - def scmurl(self, value): - return self.write_string("_scm", value) - - @property - def size_limit(self): - return self.read_int("_size_limit") - - @size_limit.setter - def size_limit(self, value): - return self.write_int("_size_limit", value) - - @property - def to_be_added(self): - self.assert_is_package() - return self.read_list("_to_be_added") or [] - - @to_be_added.setter - def to_be_added(self, value): - self.assert_is_package() - return self.write_list("_to_be_added", value) - - @property - def to_be_deleted(self): - self.assert_is_package() - return self.read_list("_to_be_deleted") or [] - - @to_be_deleted.setter - def to_be_deleted(self, value): - self.assert_is_package() - return self.write_list("_to_be_deleted", value) - - @property - def in_conflict(self): - self.assert_is_package() - return self.read_list("_in_conflict") or [] - - @in_conflict.setter - def in_conflict(self, value): - self.assert_is_package() - return self.write_list("_in_conflict", value) - - @property - def osclib_version(self): - return self.read_string("_osclib_version") - - @property - def files(self): - self.assert_is_package() - if self.exists("_scm"): - msg = "Package '{self.path}' is managed via SCM" - raise oscerr.NoWorkingCopy(msg) - if not self.exists("_files"): - msg = "Package '{self.path}' doesn't contain _files metadata" - raise oscerr.NoWorkingCopy(msg) - result = [] - directory_node = self.read_xml_node("_files", "directory").getroot() - from . import core as osc_core - for entry_node in api.find_nodes(directory_node, "directory", "entry"): - result.append(osc_core.File.from_xml_node(entry_node)) - return result - - @files.setter - def files(self, value): - if not isinstance(value, (list, tuple)): - msg = f"The argument `value` should be list, not {type(value).__name__}" - raise TypeError(msg) - - root = ET.Element("directory") - for file_obj in sorted(value): - file_obj.to_xml_node(root) - self.write_xml_node("_files", "directory", root) - - @property - def last_buildroot(self): - self.assert_is_package() - items = self.read_list("_last_buildroot") - if items is None: - return items - - if len(items) != 3: - msg = f"Package '{self.path}' contains _last_buildroot metadata that doesn't contain 3 lines: [repo, arch, vm_type]" - raise oscerr.NoWorkingCopy(msg) - - if items[2] in ("", "None"): - items[2] = None - - return items - - @last_buildroot.setter - def last_buildroot(self, value): - self.assert_is_package() - if len(value) != 3: - raise ValueError("A list with exactly 3 items is expected: [repo, arch, vm_type]") - self.write_list("_last_buildroot", value) - - @property - def _meta_node(self): - if not self.exists("_meta"): - return None - if self.is_package: - root = self.read_xml_node("_meta", "package").getroot() - else: - root = self.read_xml_node("_meta", "project").getroot() - return root +from .obs_scm import Store def get_store(path, check=True, print_warnings=False): From 59f530c793f0dada8568a6115de2874ce0926a77 Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Mon, 15 Apr 2024 15:42:20 +0200 Subject: [PATCH 5/7] Move functions manipulating store from core to obs_scm.store --- osc/core.py | 267 ++++--------------------------------------- osc/obs_scm/store.py | 247 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+), 242 deletions(-) diff --git a/osc/core.py b/osc/core.py index f30135ba..3fd1887b 100644 --- a/osc/core.py +++ b/osc/core.py @@ -4,12 +4,6 @@ # either version 2, or version 3 (at your option). -# __store_version__ is to be incremented when the format of the working copy -# "store" changes in an incompatible way. Please add any needed migration -# functionality to check_store_version(). -__store_version__ = '1.0' - - import codecs import copy import datetime @@ -56,8 +50,32 @@ from .connection import http_request, http_GET, http_POST, http_PUT, http_DELETE from .obs_scm import File from .obs_scm import Linkinfo from .obs_scm import Serviceinfo +from .obs_scm import Store +from .obs_scm.store import __store_version__ +from .obs_scm.store import check_store_version +from .obs_scm.store import delete_storedir +from .obs_scm.store import is_package_dir +from .obs_scm.store import is_project_dir +from .obs_scm.store import read_inconflict +from .obs_scm.store import read_filemeta +from .obs_scm.store import read_sizelimit +from .obs_scm.store import read_tobeadded +from .obs_scm.store import read_tobedeleted +from .obs_scm.store import store +from .obs_scm.store import store_read_apiurl +from .obs_scm.store import store_read_file +from .obs_scm.store import store_read_last_buildroot +from .obs_scm.store import store_readlist +from .obs_scm.store import store_read_package +from .obs_scm.store import store_read_project +from .obs_scm.store import store_read_scmurl +from .obs_scm.store import store_unlink_file +from .obs_scm.store import store_write_apiurl +from .obs_scm.store import store_write_initial_packages +from .obs_scm.store import store_write_last_buildroot +from .obs_scm.store import store_write_project +from .obs_scm.store import store_write_string from .output import sanitize_text -from .store import Store from .util import xdg from .util.helper import decode_list, decode_it, raw_input, _html_escape from .util.xml import xml_indent_compat as xmlindent @@ -76,7 +94,6 @@ def cmp(a, b): DISTURL_RE = re.compile(r"^(?P.*)://(?P.*?)/(?P.*?)/(?P.*?)/(?P.*)-(?P.*)$") BUILDLOGURL_RE = re.compile(r"^(?Phttps?://.*?)/build/(?P.*?)/(?P.*?)/(?P.*?)/(?P.*?)/_log$") BUFSIZE = 1024 * 1024 -store = '.osc' new_project_templ = """\ @@ -3047,20 +3064,6 @@ def shorttime(t): return time.strftime('%b %d %Y', time.gmtime(t)) -def is_project_dir(d): - global store - - return os.path.exists(os.path.join(d, store, '_project')) and not \ - os.path.exists(os.path.join(d, store, '_package')) - - -def is_package_dir(d): - global store - - return os.path.exists(os.path.join(d, store, '_project')) and \ - os.path.exists(os.path.join(d, store, '_package')) - - def parse_disturl(disturl: str): """Parse a disturl, returns tuple (apiurl, project, source, repository, revision), else raises an oscerr.WrongArgs exception @@ -3166,62 +3169,6 @@ def findpacs(files, progress_obj=None, fatal=True): return Package.from_paths_nofail(files, progress_obj) -def read_filemeta(dir): - global store - - msg = f'\'{dir}\' is not a valid working copy.' - filesmeta = os.path.join(dir, store, '_files') - if not is_package_dir(dir): - raise oscerr.NoWorkingCopy(msg) - if os.path.isfile(os.path.join(dir, store, '_scm')): - raise oscerr.NoWorkingCopy("Is managed via scm") - if not os.path.isfile(filesmeta): - raise oscerr.NoWorkingCopy(f'{msg} ({filesmeta} does not exist)') - - try: - r = ET.parse(filesmeta) - except SyntaxError as e: - raise oscerr.NoWorkingCopy(f'{msg}\nWhen parsing .osc/_files, the following error was encountered:\n{e}') - return r - - -def store_readlist(dir, name): - global store - - r = [] - if os.path.exists(os.path.join(dir, store, name)): - with open(os.path.join(dir, store, name)) as f: - r = [line.rstrip('\n') for line in f] - return r - - -def read_tobeadded(dir): - return store_readlist(dir, '_to_be_added') - - -def read_tobedeleted(dir): - return store_readlist(dir, '_to_be_deleted') - - -def read_sizelimit(dir): - global store - - r = None - fname = os.path.join(dir, store, '_size_limit') - - if os.path.exists(fname): - with open(fname) as f: - r = f.readline().strip() - - if r is None or not r.isdigit(): - return None - return int(r) - - -def read_inconflict(dir): - return store_readlist(dir, '_in_conflict') - - def parseargs(list_of_args): """Convenience method osc's commandline argument parsing. @@ -3313,35 +3260,6 @@ def makeurl(apiurl: str, path: List[str], query: Optional[dict] = None): return urlunsplit((apiurl_scheme, apiurl_netloc, path_str, query_str, "")) -def check_store_version(dir): - global store - - versionfile = os.path.join(dir, store, '_osclib_version') - try: - with open(versionfile) as f: - v = f.read().strip() - except: - v = '' - - if v == '': - msg = f'Error: "{os.path.abspath(dir)}" is not an osc package working copy.' - if os.path.exists(os.path.join(dir, '.svn')): - msg = msg + '\nTry svn instead of osc.' - raise oscerr.NoWorkingCopy(msg) - - if v != __store_version__: - if v in ['0.2', '0.3', '0.4', '0.5', '0.6', '0.7', '0.8', '0.9', '0.95', '0.96', '0.97', '0.98', '0.99']: - # version is fine, no migration needed - f = open(versionfile, 'w') - f.write(__store_version__ + '\n') - f.close() - return - msg = f'The osc metadata of your working copy "{dir}"' - msg += f'\nhas __store_version__ = {v}, but it should be {__store_version__}' - msg += '\nPlease do a fresh checkout or update your client. Sorry about the inconvenience.' - raise oscerr.WorkingCopyWrongVersion(msg) - - def meta_get_packagelist(apiurl: str, prj, deleted=None, expand=False): query = {} @@ -6984,132 +6902,6 @@ def rebuild(apiurl: str, prj: str, package: str, repo: str, arch: str, code=None return root.get('code') -def store_read_project(dir): - global store - - try: - with open(os.path.join(dir, store, '_project')) as f: - p = f.readline().strip() - except OSError: - msg = f'Error: \'{os.path.abspath(dir)}\' is not an osc project dir or working copy' - if os.path.exists(os.path.join(dir, '.svn')): - msg += '\nTry svn instead of osc.' - raise oscerr.NoWorkingCopy(msg) - return p - - -def store_read_package(dir): - global store - - try: - with open(os.path.join(dir, store, '_package')) as f: - p = f.readline().strip() - except OSError: - msg = f'Error: \'{os.path.abspath(dir)}\' is not an osc package working copy' - if os.path.exists(os.path.join(dir, '.svn')): - msg += '\nTry svn instead of osc.' - raise oscerr.NoWorkingCopy(msg) - return p - - -def store_read_scmurl(dir): - import warnings - warnings.warn( - "osc.core.store_read_scmurl() is deprecated. " - "You should be using high-level classes such as Store, Project or Package instead.", - DeprecationWarning - ) - return Store(dir).scmurl - - -def store_read_apiurl(dir, defaulturl=True): - import warnings - warnings.warn( - "osc.core.store_read_apiurl() is deprecated. " - "You should be using high-level classes such as Store, Project or Package instead.", - DeprecationWarning - ) - return Store(dir).apiurl - - -def store_read_last_buildroot(dir): - global store - - fname = os.path.join(dir, store, '_last_buildroot') - if os.path.exists(fname): - lines = open(fname).read().splitlines() - if len(lines) == 3: - return lines - - return - - -def store_write_string(dir, file, string, subdir=''): - global store - - if subdir and not os.path.isdir(os.path.join(dir, store, subdir)): - os.mkdir(os.path.join(dir, store, subdir)) - fname = os.path.join(dir, store, subdir, file) - try: - f = open(fname + '.new', 'w') - if not isinstance(string, str): - string = decode_it(string) - f.write(string) - f.close() - os.rename(fname + '.new', fname) - except: - if os.path.exists(fname + '.new'): - os.unlink(fname + '.new') - raise - - -def store_write_project(dir, project): - store_write_string(dir, '_project', project + '\n') - - -def store_write_apiurl(dir, apiurl): - import warnings - warnings.warn( - "osc.core.store_write_apiurl() is deprecated. " - "You should be using high-level classes such as Store, Project or Package instead.", - DeprecationWarning - ) - Store(dir).apiurl = apiurl - - -def store_write_last_buildroot(dir, repo, arch, vm_type): - store_write_string(dir, '_last_buildroot', repo + '\n' + arch + '\n' + vm_type + '\n') - - -def store_unlink_file(dir, file): - global store - - try: - os.unlink(os.path.join(dir, store, file)) - except: - pass - - -def store_read_file(dir, file): - global store - - try: - with open(os.path.join(dir, store, file)) as f: - return f.read() - except: - return None - - -def store_write_initial_packages(dir, project, subelements): - global store - - fname = os.path.join(dir, store, '_packages') - root = ET.Element('project', name=project) - for elem in subelements: - root.append(elem) - ET.ElementTree(root).write(fname) - - def get_osc_version(): return __version__ @@ -7443,15 +7235,6 @@ def delete_dir(dir): os.rmdir(dir) -def delete_storedir(store_dir): - """ - This method deletes a store dir. - """ - head, tail = os.path.split(store_dir) - if tail == '.osc': - delete_dir(store_dir) - - def unpack_srcrpm(srpm, dir, *files): """ This method unpacks the passed srpm into the diff --git a/osc/obs_scm/store.py b/osc/obs_scm/store.py index 2efc3573..e8437e5d 100644 --- a/osc/obs_scm/store.py +++ b/osc/obs_scm/store.py @@ -12,6 +12,12 @@ from .._private import api from ..util.xml import ET +# __store_version__ is to be incremented when the format of the working copy +# "store" changes in an incompatible way. Please add any needed migration +# functionality to check_store_version(). +__store_version__ = '1.0' + + class Store: STORE_DIR = ".osc" STORE_VERSION = "1.0" @@ -317,3 +323,244 @@ class Store: else: root = self.read_xml_node("_meta", "project").getroot() return root + + +store = '.osc' + + +def check_store_version(dir): + global store + + versionfile = os.path.join(dir, store, '_osclib_version') + try: + with open(versionfile) as f: + v = f.read().strip() + except: + v = '' + + if v == '': + msg = f'Error: "{os.path.abspath(dir)}" is not an osc package working copy.' + if os.path.exists(os.path.join(dir, '.svn')): + msg = msg + '\nTry svn instead of osc.' + raise oscerr.NoWorkingCopy(msg) + + if v != __store_version__: + if v in ['0.2', '0.3', '0.4', '0.5', '0.6', '0.7', '0.8', '0.9', '0.95', '0.96', '0.97', '0.98', '0.99']: + # version is fine, no migration needed + f = open(versionfile, 'w') + f.write(__store_version__ + '\n') + f.close() + return + msg = f'The osc metadata of your working copy "{dir}"' + msg += f'\nhas __store_version__ = {v}, but it should be {__store_version__}' + msg += '\nPlease do a fresh checkout or update your client. Sorry about the inconvenience.' + raise oscerr.WorkingCopyWrongVersion(msg) + + +def is_project_dir(d): + global store + + return os.path.exists(os.path.join(d, store, '_project')) and not \ + os.path.exists(os.path.join(d, store, '_package')) + + +def is_package_dir(d): + global store + + return os.path.exists(os.path.join(d, store, '_project')) and \ + os.path.exists(os.path.join(d, store, '_package')) + + +def read_filemeta(dir): + global store + + msg = f'\'{dir}\' is not a valid working copy.' + filesmeta = os.path.join(dir, store, '_files') + if not is_package_dir(dir): + raise oscerr.NoWorkingCopy(msg) + if os.path.isfile(os.path.join(dir, store, '_scm')): + raise oscerr.NoWorkingCopy("Is managed via scm") + if not os.path.isfile(filesmeta): + raise oscerr.NoWorkingCopy(f'{msg} ({filesmeta} does not exist)') + + try: + r = ET.parse(filesmeta) + except SyntaxError as e: + raise oscerr.NoWorkingCopy(f'{msg}\nWhen parsing .osc/_files, the following error was encountered:\n{e}') + return r + + +def store_readlist(dir, name): + global store + + r = [] + if os.path.exists(os.path.join(dir, store, name)): + with open(os.path.join(dir, store, name)) as f: + r = [line.rstrip('\n') for line in f] + return r + + +def read_tobeadded(dir): + return store_readlist(dir, '_to_be_added') + + +def read_tobedeleted(dir): + return store_readlist(dir, '_to_be_deleted') + + +def read_sizelimit(dir): + global store + + r = None + fname = os.path.join(dir, store, '_size_limit') + + if os.path.exists(fname): + with open(fname) as f: + r = f.readline().strip() + + if r is None or not r.isdigit(): + return None + return int(r) + + +def read_inconflict(dir): + return store_readlist(dir, '_in_conflict') + + +def store_read_project(dir): + global store + + try: + with open(os.path.join(dir, store, '_project')) as f: + p = f.readline().strip() + except OSError: + msg = f'Error: \'{os.path.abspath(dir)}\' is not an osc project dir or working copy' + if os.path.exists(os.path.join(dir, '.svn')): + msg += '\nTry svn instead of osc.' + raise oscerr.NoWorkingCopy(msg) + return p + + +def store_read_package(dir): + global store + + try: + with open(os.path.join(dir, store, '_package')) as f: + p = f.readline().strip() + except OSError: + msg = f'Error: \'{os.path.abspath(dir)}\' is not an osc package working copy' + if os.path.exists(os.path.join(dir, '.svn')): + msg += '\nTry svn instead of osc.' + raise oscerr.NoWorkingCopy(msg) + return p + + +def store_read_scmurl(dir): + import warnings + warnings.warn( + "osc.core.store_read_scmurl() is deprecated. " + "You should be using high-level classes such as Store, Project or Package instead.", + DeprecationWarning + ) + return Store(dir).scmurl + + +def store_read_apiurl(dir, defaulturl=True): + import warnings + warnings.warn( + "osc.core.store_read_apiurl() is deprecated. " + "You should be using high-level classes such as Store, Project or Package instead.", + DeprecationWarning + ) + return Store(dir).apiurl + + +def store_read_last_buildroot(dir): + global store + + fname = os.path.join(dir, store, '_last_buildroot') + if os.path.exists(fname): + lines = open(fname).read().splitlines() + if len(lines) == 3: + return lines + + return + + +def store_write_string(dir, file, string, subdir=''): + from ..core import decode_it + + global store + + if subdir and not os.path.isdir(os.path.join(dir, store, subdir)): + os.mkdir(os.path.join(dir, store, subdir)) + fname = os.path.join(dir, store, subdir, file) + try: + f = open(fname + '.new', 'w') + if not isinstance(string, str): + string = decode_it(string) + f.write(string) + f.close() + os.rename(fname + '.new', fname) + except: + if os.path.exists(fname + '.new'): + os.unlink(fname + '.new') + raise + + +def store_write_project(dir, project): + store_write_string(dir, '_project', project + '\n') + + +def store_write_apiurl(dir, apiurl): + import warnings + warnings.warn( + "osc.core.store_write_apiurl() is deprecated. " + "You should be using high-level classes such as Store, Project or Package instead.", + DeprecationWarning + ) + Store(dir).apiurl = apiurl + + +def store_write_last_buildroot(dir, repo, arch, vm_type): + store_write_string(dir, '_last_buildroot', repo + '\n' + arch + '\n' + vm_type + '\n') + + +def store_unlink_file(dir, file): + global store + + try: + os.unlink(os.path.join(dir, store, file)) + except: + pass + + +def store_read_file(dir, file): + global store + + try: + with open(os.path.join(dir, store, file)) as f: + return f.read() + except: + return None + + +def store_write_initial_packages(dir, project, subelements): + global store + + fname = os.path.join(dir, store, '_packages') + root = ET.Element('project', name=project) + for elem in subelements: + root.append(elem) + ET.ElementTree(root).write(fname) + + +def delete_storedir(store_dir): + """ + This method deletes a store dir. + """ + from ..core import delete_dir + + head, tail = os.path.split(store_dir) + if tail == '.osc': + delete_dir(store_dir) From 45ea1b698e2b6de4764dfd7246ec70bad652e621 Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Mon, 15 Apr 2024 16:04:05 +0200 Subject: [PATCH 6/7] Move core.Project to obs_scm.Project --- osc/core.py | 566 +----------------------------------- osc/obs_scm/__init__.py | 1 + osc/obs_scm/project.py | 626 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 628 insertions(+), 565 deletions(-) create mode 100644 osc/obs_scm/project.py diff --git a/osc/core.py b/osc/core.py index 3fd1887b..f9784929 100644 --- a/osc/core.py +++ b/osc/core.py @@ -49,6 +49,7 @@ from . import store as osc_store from .connection import http_request, http_GET, http_POST, http_PUT, http_DELETE from .obs_scm import File from .obs_scm import Linkinfo +from .obs_scm import Project from .obs_scm import Serviceinfo from .obs_scm import Store from .obs_scm.store import __store_version__ @@ -310,571 +311,6 @@ class DirectoryServiceinfo: return self.error is not None -class Project: - """ - Represent a checked out project directory, holding packages. - - :Attributes: - ``dir`` - The directory path containing the project. - - ``name`` - The name of the project. - - ``apiurl`` - The endpoint URL of the API server. - - ``pacs_available`` - List of names of packages available server-side. - This is only populated if ``getPackageList`` is set - to ``True`` in the constructor. - - ``pacs_have`` - List of names of packages which exist server-side - and exist in the local project working copy (if - 'do_package_tracking' is disabled). - If 'do_package_tracking' is enabled it represents the - list names of packages which are tracked in the project - working copy (that is it might contain packages which - exist on the server as well as packages which do not - exist on the server (for instance if the local package - was added or if the package was removed on the server-side)). - - ``pacs_excluded`` - List of names of packages in the local project directory - which are excluded by the `exclude_glob` configuration - variable. Only set if `do_package_tracking` is enabled. - - ``pacs_unvers`` - List of names of packages in the local project directory - which are not tracked. Only set if `do_package_tracking` - is enabled. - - ``pacs_broken`` - List of names of packages which are tracked but do not - exist in the local project working copy. Only set if - `do_package_tracking` is enabled. - - ``pacs_missing`` - List of names of packages which exist server-side but - are not expected to exist in the local project directory. - """ - - REQ_STOREFILES = ('_project', '_apiurl') - - def __init__(self, dir, getPackageList=True, progress_obj=None, wc_check=True): - """ - Constructor. - - :Parameters: - `dir` : str - The directory path containing the checked out project. - - `getPackageList` : bool - Set to `False` if you want to skip retrieval from the - server of the list of packages in the project . - - `wc_check` : bool - """ - self.dir = Path(dir) - self.absdir = os.path.abspath(dir) - self.store = Store(dir) - self.progress_obj = progress_obj - - self.name = store_read_project(self.dir) - self.scm_url = self.store.scmurl - self.apiurl = self.store.apiurl - - dirty_files = [] - if wc_check: - dirty_files = self.wc_check() - if dirty_files: - msg = 'Your working copy \'%s\' is in an inconsistent state.\n' \ - 'Please run \'osc repairwc %s\' and check the state\n' \ - 'of the working copy afterwards (via \'osc status %s\')' % (self.dir, self.dir, self.dir) - raise oscerr.WorkingCopyInconsistent(self.name, None, dirty_files, msg) - - if getPackageList: - self.pacs_available = meta_get_packagelist(self.apiurl, self.name) - else: - self.pacs_available = [] - - if conf.config['do_package_tracking']: - self.pac_root = self.read_packages().getroot() - self.pacs_have = [pac.get('name') for pac in self.pac_root.findall('package')] - self.pacs_excluded = [i for i in os.listdir(self.dir) - for j in conf.config['exclude_glob'] - if fnmatch.fnmatch(i, j)] - self.pacs_unvers = [i for i in os.listdir(self.dir) if i not in self.pacs_have and i not in self.pacs_excluded] - # store all broken packages (e.g. packages which where removed by a non-osc cmd) - # in the self.pacs_broken list - self.pacs_broken = [] - for p in self.pacs_have: - if not os.path.isdir(os.path.join(self.absdir, p)): - # all states will be replaced with the '!'-state - # (except it is already marked as deleted ('D'-state)) - self.pacs_broken.append(p) - else: - self.pacs_have = [i for i in os.listdir(self.dir) if i in self.pacs_available] - - self.pacs_missing = [i for i in self.pacs_available if i not in self.pacs_have] - - def wc_check(self): - global store - dirty_files = [] - req_storefiles = Project.REQ_STOREFILES - if conf.config['do_package_tracking'] and self.scm_url is None: - req_storefiles += ('_packages',) - for fname in req_storefiles: - if not os.path.exists(os.path.join(self.absdir, store, fname)): - dirty_files.append(fname) - return dirty_files - - def wc_repair(self, apiurl: Optional[str] = None): - store = Store(self.dir) - store.assert_is_project() - if not store.exists("_apiurl") or apiurl: - if apiurl is None: - msg = 'cannot repair wc: the \'_apiurl\' file is missing but ' \ - 'no \'apiurl\' was passed to wc_repair' - # hmm should we raise oscerr.WrongArgs? - raise oscerr.WorkingCopyInconsistent(self.name, None, [], msg) - # sanity check - conf.parse_apisrv_url(None, apiurl) - store.apiurl = apiurl - self.apiurl = apiurl - - def checkout_missing_pacs(self, sinfos, expand_link=False, unexpand_link=False): - for pac in self.pacs_missing: - if conf.config['do_package_tracking'] and pac in self.pacs_unvers: - # pac is not under version control but a local file/dir exists - msg = f'can\'t add package \'{pac}\': Object already exists' - raise oscerr.PackageExists(self.name, pac, msg) - - if not (expand_link or unexpand_link): - sinfo = sinfos.get(pac) - if sinfo is None: - # should never happen... - continue - linked = sinfo.find('linked') - if linked is not None and linked.get('project') == self.name: - # hmm what about a linkerror (sinfo.get('lsrcmd5') is None)? - # Should we skip the package as well or should we it out? - # let's skip it for now - print(f"Skipping {pac} (link to package {linked.get('package')})") - continue - - print(f'checking out new package {pac}') - checkout_package(self.apiurl, self.name, pac, - pathname=getTransActPath(os.path.join(self.dir, pac)), - prj_obj=self, prj_dir=self.dir, - expand_link=expand_link or not unexpand_link, progress_obj=self.progress_obj) - - def status(self, pac: str): - exists = os.path.exists(os.path.join(self.absdir, pac)) - st = self.get_state(pac) - if st is None and exists: - return '?' - elif st is None: - raise oscerr.OscIOError(None, f'osc: \'{pac}\' is not under version control') - elif st in ('A', ' ') and not exists: - return '!' - elif st == 'D' and not exists: - return 'D' - else: - return st - - def get_status(self, *exclude_states): - res = [] - for pac in self.pacs_have: - st = self.status(pac) - if st not in exclude_states: - res.append((st, pac)) - if '?' not in exclude_states: - res.extend([('?', pac) for pac in self.pacs_unvers]) - return res - - def get_pacobj(self, pac, *pac_args, **pac_kwargs): - try: - st = self.status(pac) - if st in ('?', '!') or st == 'D' and not os.path.exists(os.path.join(self.dir, pac)): - return None - return Package(os.path.join(self.dir, pac), *pac_args, **pac_kwargs) - except oscerr.OscIOError: - return None - - def set_state(self, pac, state): - node = self.get_package_node(pac) - if node is None: - self.new_package_entry(pac, state) - else: - node.set('state', state) - - def get_package_node(self, pac: str): - for node in self.pac_root.findall('package'): - if pac == node.get('name'): - return node - return None - - def del_package_node(self, pac): - for node in self.pac_root.findall('package'): - if pac == node.get('name'): - self.pac_root.remove(node) - - def get_state(self, pac: str): - node = self.get_package_node(pac) - if node is not None: - return node.get('state') - else: - return None - - def new_package_entry(self, name, state): - ET.SubElement(self.pac_root, 'package', name=name, state=state) - - def read_packages(self): - """ - Returns an ``xml.etree.ElementTree`` object representing the - parsed contents of the project's ``.osc/_packages`` XML file. - """ - global store - - packages_file = os.path.join(self.absdir, store, '_packages') - if os.path.isfile(packages_file) and os.path.getsize(packages_file): - try: - result = ET.parse(packages_file) - except: - msg = f'Cannot read package file \'{packages_file}\'. ' - msg += 'You can try to remove it and then run osc repairwc.' - raise oscerr.OscIOError(None, msg) - return result - else: - # scan project for existing packages and migrate them - cur_pacs = [] - for data in os.listdir(self.dir): - pac_dir = os.path.join(self.absdir, data) - # we cannot use self.pacs_available because we cannot guarantee that the package list - # was fetched from the server - if data in meta_get_packagelist(self.apiurl, self.name) and is_package_dir(pac_dir) \ - and Package(pac_dir).name == data: - cur_pacs.append(ET.Element('package', name=data, state=' ')) - store_write_initial_packages(self.absdir, self.name, cur_pacs) - return ET.parse(os.path.join(self.absdir, store, '_packages')) - - def write_packages(self): - xmlindent(self.pac_root) - store_write_string(self.absdir, '_packages', ET.tostring(self.pac_root, encoding=ET_ENCODING)) - - def addPackage(self, pac): - for i in conf.config['exclude_glob']: - if fnmatch.fnmatch(pac, i): - msg = f'invalid package name: \'{pac}\' (see \'exclude_glob\' config option)' - raise oscerr.OscIOError(None, msg) - state = self.get_state(pac) - if state is None or state == 'D': - self.new_package_entry(pac, 'A') - self.write_packages() - # sometimes the new pac doesn't exist in the list because - # it would take too much time to update all data structs regularly - if pac in self.pacs_unvers: - self.pacs_unvers.remove(pac) - else: - raise oscerr.PackageExists(self.name, pac, f'package \'{pac}\' is already under version control') - - def delPackage(self, pac, force=False): - state = self.get_state(pac.name) - can_delete = True - if state == ' ' or state == 'D': - del_files = [] - for filename in pac.filenamelist + pac.filenamelist_unvers: - filestate = pac.status(filename) - if filestate == 'M' or filestate == 'C' or \ - filestate == 'A' or filestate == '?': - can_delete = False - else: - del_files.append(filename) - if can_delete or force: - for filename in del_files: - pac.delete_localfile(filename) - if pac.status(filename) != '?': - # this is not really necessary - pac.put_on_deletelist(filename) - print(statfrmt('D', getTransActPath(os.path.join(pac.dir, filename)))) - print(statfrmt('D', getTransActPath(os.path.join(pac.dir, os.pardir, pac.name)))) - pac.write_deletelist() - self.set_state(pac.name, 'D') - self.write_packages() - else: - print(f'package \'{pac.name}\' has local modifications (see osc st for details)') - elif state == 'A': - if force: - delete_dir(pac.absdir) - self.del_package_node(pac.name) - self.write_packages() - print(statfrmt('D', pac.name)) - else: - print(f'package \'{pac.name}\' has local modifications (see osc st for details)') - elif state is None: - print('package is not under version control') - else: - print('unsupported state') - - def update(self, pacs=(), expand_link=False, unexpand_link=False, service_files=False): - if pacs: - for pac in pacs: - Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj).update() - else: - # we need to make sure that the _packages file will be written (even if an exception - # occurs) - try: - # update complete project - # packages which no longer exists upstream - upstream_del = [pac for pac in self.pacs_have if pac not in self.pacs_available and self.get_state(pac) != 'A'] - sinfo_pacs = [pac for pac in self.pacs_have if self.get_state(pac) in (' ', 'D') and pac not in self.pacs_broken] - sinfo_pacs.extend(self.pacs_missing) - sinfos = get_project_sourceinfo(self.apiurl, self.name, True, *sinfo_pacs) - - for pac in upstream_del: - if self.status(pac) != '!': - p = Package(os.path.join(self.dir, pac)) - self.delPackage(p, force=True) - delete_storedir(p.storedir) - try: - os.rmdir(pac) - except: - pass - self.pac_root.remove(self.get_package_node(pac)) - self.pacs_have.remove(pac) - - for pac in self.pacs_have: - state = self.get_state(pac) - if pac in self.pacs_broken: - if self.get_state(pac) != 'A': - checkout_package(self.apiurl, self.name, pac, - pathname=getTransActPath(os.path.join(self.dir, pac)), prj_obj=self, - prj_dir=self.dir, expand_link=not unexpand_link, progress_obj=self.progress_obj) - elif state == ' ': - # do a simple update - p = Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj) - rev = None - needs_update = True - if p.scm_url is not None: - # git managed. - print("Skipping git managed package ", pac) - continue - elif expand_link and p.islink() and not p.isexpanded(): - if p.haslinkerror(): - try: - rev = show_upstream_xsrcmd5(p.apiurl, p.prjname, p.name, revision=p.rev) - except: - rev = show_upstream_xsrcmd5(p.apiurl, p.prjname, p.name, revision=p.rev, linkrev="base") - p.mark_frozen() - else: - rev = p.linkinfo.xsrcmd5 - print('Expanding to rev', rev) - elif unexpand_link and p.islink() and p.isexpanded(): - rev = p.linkinfo.lsrcmd5 - print('Unexpanding to rev', rev) - elif p.islink() and p.isexpanded(): - needs_update = p.update_needed(sinfos[p.name]) - if needs_update: - rev = p.latest_rev() - elif p.hasserviceinfo() and p.serviceinfo.isexpanded() and not service_files: - # FIXME: currently, do_update does not propagate the --server-side-source-service-files - # option to this method. Consequence: an expanded service is always unexpanded during - # an update (TODO: discuss if this is a reasonable behavior (at least this the default - # behavior for a while)) - needs_update = True - else: - needs_update = p.update_needed(sinfos[p.name]) - print(f'Updating {p.name}') - if needs_update: - p.update(rev, service_files) - else: - print(f'At revision {p.rev}.') - if unexpand_link: - p.unmark_frozen() - elif state == 'D': - # pac exists (the non-existent pac case was handled in the first if block) - p = Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj) - if p.update_needed(sinfos[p.name]): - p.update() - elif state == 'A' and pac in self.pacs_available: - # file/dir called pac already exists and is under version control - msg = f'can\'t add package \'{pac}\': Object already exists' - raise oscerr.PackageExists(self.name, pac, msg) - elif state == 'A': - # do nothing - pass - else: - print(f'unexpected state.. package \'{pac}\'') - - self.checkout_missing_pacs(sinfos, expand_link, unexpand_link) - finally: - self.write_packages() - - def commit(self, pacs=(), msg='', files=None, verbose=False, skip_local_service_run=False, can_branch=False, force=False): - files = files or {} - if pacs: - try: - for pac in pacs: - todo = [] - if pac in files: - todo = files[pac] - state = self.get_state(pac) - if state == 'A': - self.commitNewPackage(pac, msg, todo, verbose=verbose, skip_local_service_run=skip_local_service_run) - elif state == 'D': - self.commitDelPackage(pac, force=force) - elif state == ' ': - # display the correct dir when sending the changes - if os_path_samefile(os.path.join(self.dir, pac), os.getcwd()): - p = Package('.') - else: - p = Package(os.path.join(self.dir, pac)) - p.todo = todo - p.commit(msg, verbose=verbose, skip_local_service_run=skip_local_service_run, can_branch=can_branch, force=force) - elif pac in self.pacs_unvers and not is_package_dir(os.path.join(self.dir, pac)): - print(f'osc: \'{pac}\' is not under version control') - elif pac in self.pacs_broken or not os.path.exists(os.path.join(self.dir, pac)): - print(f'osc: \'{pac}\' package not found') - elif state is None: - self.commitExtPackage(pac, msg, todo, verbose=verbose, skip_local_service_run=skip_local_service_run) - finally: - self.write_packages() - else: - # if we have packages marked as '!' we cannot commit - for pac in self.pacs_broken: - if self.get_state(pac) != 'D': - msg = f'commit failed: package \'{pac}\' is missing' - raise oscerr.PackageMissing(self.name, pac, msg) - try: - for pac in self.pacs_have: - state = self.get_state(pac) - if state == ' ': - # do a simple commit - Package(os.path.join(self.dir, pac)).commit(msg, verbose=verbose, skip_local_service_run=skip_local_service_run) - elif state == 'D': - self.commitDelPackage(pac, force=force) - elif state == 'A': - self.commitNewPackage(pac, msg, verbose=verbose, skip_local_service_run=skip_local_service_run) - finally: - self.write_packages() - - def commitNewPackage(self, pac, msg='', files=None, verbose=False, skip_local_service_run=False): - """creates and commits a new package if it does not exist on the server""" - files = files or [] - if pac in self.pacs_available: - print(f'package \'{pac}\' already exists') - else: - user = conf.get_apiurl_usr(self.apiurl) - edit_meta(metatype='pkg', - path_args=(self.name, pac), - template_args=({ - 'name': pac, - 'user': user}), - apiurl=self.apiurl) - # display the correct dir when sending the changes - olddir = os.getcwd() - if os_path_samefile(os.path.join(self.dir, pac), os.curdir): - os.chdir(os.pardir) - p = Package(pac) - else: - p = Package(os.path.join(self.dir, pac)) - p.todo = files - print(statfrmt('Sending', os.path.normpath(p.dir))) - p.commit(msg=msg, verbose=verbose, skip_local_service_run=skip_local_service_run) - self.set_state(pac, ' ') - os.chdir(olddir) - - def commitDelPackage(self, pac, force=False): - """deletes a package on the server and in the working copy""" - try: - # display the correct dir when sending the changes - if os_path_samefile(os.path.join(self.dir, pac), os.curdir): - pac_dir = pac - else: - pac_dir = os.path.join(self.dir, pac) - p = Package(os.path.join(self.dir, pac)) - # print statfrmt('Deleting', os.path.normpath(os.path.join(p.dir, os.pardir, pac))) - delete_storedir(p.storedir) - try: - os.rmdir(p.dir) - except: - pass - except OSError: - pac_dir = os.path.join(self.dir, pac) - except (oscerr.NoWorkingCopy, oscerr.WorkingCopyOutdated, oscerr.PackageError): - pass - # print statfrmt('Deleting', getTransActPath(os.path.join(self.dir, pac))) - print(statfrmt('Deleting', getTransActPath(pac_dir))) - delete_package(self.apiurl, self.name, pac, force=force) - self.del_package_node(pac) - - def commitExtPackage(self, pac, msg, files=None, verbose=False, skip_local_service_run=False): - """commits a package from an external project""" - files = files or [] - if os_path_samefile(os.path.join(self.dir, pac), os.getcwd()): - pac_path = '.' - else: - pac_path = os.path.join(self.dir, pac) - - store = Store(pac_path) - project = store_read_project(pac_path) - package = store_read_package(pac_path) - apiurl = store.apiurl - if not meta_exists(metatype='pkg', - path_args=(project, package), - template_args=None, create_new=False, apiurl=apiurl): - user = conf.get_apiurl_usr(self.apiurl) - edit_meta(metatype='pkg', - path_args=(project, package), - template_args=({'name': pac, 'user': user}), apiurl=apiurl) - p = Package(pac_path) - p.todo = files - p.commit(msg=msg, verbose=verbose, skip_local_service_run=skip_local_service_run) - - def __str__(self): - r = [] - r.append('*****************************************************') - r.append(f'Project {self.name} (dir={self.dir}, absdir={self.absdir})') - r.append(f"have pacs:\n{', '.join(self.pacs_have)}") - r.append(f"missing pacs:\n{', '.join(self.pacs_missing)}") - r.append('*****************************************************') - return '\n'.join(r) - - @staticmethod - def init_project( - apiurl: str, - dir: Path, - project, - package_tracking=True, - getPackageList=True, - progress_obj=None, - wc_check=True, - scm_url=None, - ): - global store - - if not os.path.exists(dir): - # use makedirs (checkout_no_colon config option might be enabled) - os.makedirs(dir) - elif not os.path.isdir(dir): - raise oscerr.OscIOError(None, f'error: \'{dir}\' is no directory') - if os.path.exists(os.path.join(dir, store)): - raise oscerr.OscIOError(None, f'error: \'{dir}\' is already an initialized osc working copy') - else: - os.mkdir(os.path.join(dir, store)) - - store_write_project(dir, project) - Store(dir).apiurl = apiurl - if scm_url: - Store(dir).scmurl = scm_url - package_tracking = None - if package_tracking: - store_write_initial_packages(dir, project, []) - return Project(dir, getPackageList, progress_obj, wc_check) - - @total_ordering class Package: """represent a package (its directory) and read/keep/write its metadata""" diff --git a/osc/obs_scm/__init__.py b/osc/obs_scm/__init__.py index fb5792e5..f2a3c86e 100644 --- a/osc/obs_scm/__init__.py +++ b/osc/obs_scm/__init__.py @@ -1,4 +1,5 @@ from .file import File from .linkinfo import Linkinfo +from .project import Project from .serviceinfo import Serviceinfo from .store import Store diff --git a/osc/obs_scm/project.py b/osc/obs_scm/project.py new file mode 100644 index 00000000..91332be2 --- /dev/null +++ b/osc/obs_scm/project.py @@ -0,0 +1,626 @@ +import fnmatch +import os +from pathlib import Path +from typing import Optional + +from .. import conf +from .. import oscerr +from ..util.xml import ET +from .store import Store +from .store import delete_storedir +from .store import store +from .store import store_read_package +from .store import store_read_project +from .store import store_write_initial_packages +from .store import store_write_project +from .store import store_write_string +from .store import is_package_dir + + +class Project: + """ + Represent a checked out project directory, holding packages. + + :Attributes: + ``dir`` + The directory path containing the project. + + ``name`` + The name of the project. + + ``apiurl`` + The endpoint URL of the API server. + + ``pacs_available`` + List of names of packages available server-side. + This is only populated if ``getPackageList`` is set + to ``True`` in the constructor. + + ``pacs_have`` + List of names of packages which exist server-side + and exist in the local project working copy (if + 'do_package_tracking' is disabled). + If 'do_package_tracking' is enabled it represents the + list names of packages which are tracked in the project + working copy (that is it might contain packages which + exist on the server as well as packages which do not + exist on the server (for instance if the local package + was added or if the package was removed on the server-side)). + + ``pacs_excluded`` + List of names of packages in the local project directory + which are excluded by the `exclude_glob` configuration + variable. Only set if `do_package_tracking` is enabled. + + ``pacs_unvers`` + List of names of packages in the local project directory + which are not tracked. Only set if `do_package_tracking` + is enabled. + + ``pacs_broken`` + List of names of packages which are tracked but do not + exist in the local project working copy. Only set if + `do_package_tracking` is enabled. + + ``pacs_missing`` + List of names of packages which exist server-side but + are not expected to exist in the local project directory. + """ + + REQ_STOREFILES = ('_project', '_apiurl') + + def __init__(self, dir, getPackageList=True, progress_obj=None, wc_check=True): + """ + Constructor. + + :Parameters: + `dir` : str + The directory path containing the checked out project. + + `getPackageList` : bool + Set to `False` if you want to skip retrieval from the + server of the list of packages in the project . + + `wc_check` : bool + """ + from ..core import meta_get_packagelist + + self.dir = Path(dir) + self.absdir = os.path.abspath(dir) + self.store = Store(dir) + self.progress_obj = progress_obj + + self.name = store_read_project(self.dir) + self.scm_url = self.store.scmurl + self.apiurl = self.store.apiurl + + dirty_files = [] + if wc_check: + dirty_files = self.wc_check() + if dirty_files: + msg = 'Your working copy \'%s\' is in an inconsistent state.\n' \ + 'Please run \'osc repairwc %s\' and check the state\n' \ + 'of the working copy afterwards (via \'osc status %s\')' % (self.dir, self.dir, self.dir) + raise oscerr.WorkingCopyInconsistent(self.name, None, dirty_files, msg) + + if getPackageList: + self.pacs_available = meta_get_packagelist(self.apiurl, self.name) + else: + self.pacs_available = [] + + if conf.config['do_package_tracking']: + self.pac_root = self.read_packages().getroot() + self.pacs_have = [pac.get('name') for pac in self.pac_root.findall('package')] + self.pacs_excluded = [i for i in os.listdir(self.dir) + for j in conf.config['exclude_glob'] + if fnmatch.fnmatch(i, j)] + self.pacs_unvers = [i for i in os.listdir(self.dir) if i not in self.pacs_have and i not in self.pacs_excluded] + # store all broken packages (e.g. packages which where removed by a non-osc cmd) + # in the self.pacs_broken list + self.pacs_broken = [] + for p in self.pacs_have: + if not os.path.isdir(os.path.join(self.absdir, p)): + # all states will be replaced with the '!'-state + # (except it is already marked as deleted ('D'-state)) + self.pacs_broken.append(p) + else: + self.pacs_have = [i for i in os.listdir(self.dir) if i in self.pacs_available] + + self.pacs_missing = [i for i in self.pacs_available if i not in self.pacs_have] + + def wc_check(self): + global store + dirty_files = [] + req_storefiles = Project.REQ_STOREFILES + if conf.config['do_package_tracking'] and self.scm_url is None: + req_storefiles += ('_packages',) + for fname in req_storefiles: + if not os.path.exists(os.path.join(self.absdir, store, fname)): + dirty_files.append(fname) + return dirty_files + + def wc_repair(self, apiurl: Optional[str] = None): + store = Store(self.dir) + store.assert_is_project() + if not store.exists("_apiurl") or apiurl: + if apiurl is None: + msg = 'cannot repair wc: the \'_apiurl\' file is missing but ' \ + 'no \'apiurl\' was passed to wc_repair' + # hmm should we raise oscerr.WrongArgs? + raise oscerr.WorkingCopyInconsistent(self.name, None, [], msg) + # sanity check + conf.parse_apisrv_url(None, apiurl) + store.apiurl = apiurl + self.apiurl = apiurl + + def checkout_missing_pacs(self, sinfos, expand_link=False, unexpand_link=False): + from ..core import checkout_package + from ..core import getTransActPath + + for pac in self.pacs_missing: + if conf.config['do_package_tracking'] and pac in self.pacs_unvers: + # pac is not under version control but a local file/dir exists + msg = f'can\'t add package \'{pac}\': Object already exists' + raise oscerr.PackageExists(self.name, pac, msg) + + if not (expand_link or unexpand_link): + sinfo = sinfos.get(pac) + if sinfo is None: + # should never happen... + continue + linked = sinfo.find('linked') + if linked is not None and linked.get('project') == self.name: + # hmm what about a linkerror (sinfo.get('lsrcmd5') is None)? + # Should we skip the package as well or should we it out? + # let's skip it for now + print(f"Skipping {pac} (link to package {linked.get('package')})") + continue + + print(f'checking out new package {pac}') + checkout_package(self.apiurl, self.name, pac, + pathname=getTransActPath(os.path.join(self.dir, pac)), + prj_obj=self, prj_dir=self.dir, + expand_link=expand_link or not unexpand_link, progress_obj=self.progress_obj) + + def status(self, pac: str): + exists = os.path.exists(os.path.join(self.absdir, pac)) + st = self.get_state(pac) + if st is None and exists: + return '?' + elif st is None: + raise oscerr.OscIOError(None, f'osc: \'{pac}\' is not under version control') + elif st in ('A', ' ') and not exists: + return '!' + elif st == 'D' and not exists: + return 'D' + else: + return st + + def get_status(self, *exclude_states): + res = [] + for pac in self.pacs_have: + st = self.status(pac) + if st not in exclude_states: + res.append((st, pac)) + if '?' not in exclude_states: + res.extend([('?', pac) for pac in self.pacs_unvers]) + return res + + def get_pacobj(self, pac, *pac_args, **pac_kwargs): + from ..core import Package + + try: + st = self.status(pac) + if st in ('?', '!') or st == 'D' and not os.path.exists(os.path.join(self.dir, pac)): + return None + return Package(os.path.join(self.dir, pac), *pac_args, **pac_kwargs) + except oscerr.OscIOError: + return None + + def set_state(self, pac, state): + node = self.get_package_node(pac) + if node is None: + self.new_package_entry(pac, state) + else: + node.set('state', state) + + def get_package_node(self, pac: str): + for node in self.pac_root.findall('package'): + if pac == node.get('name'): + return node + return None + + def del_package_node(self, pac): + for node in self.pac_root.findall('package'): + if pac == node.get('name'): + self.pac_root.remove(node) + + def get_state(self, pac: str): + node = self.get_package_node(pac) + if node is not None: + return node.get('state') + else: + return None + + def new_package_entry(self, name, state): + ET.SubElement(self.pac_root, 'package', name=name, state=state) + + def read_packages(self): + """ + Returns an ``xml.etree.ElementTree`` object representing the + parsed contents of the project's ``.osc/_packages`` XML file. + """ + from ..core import Package + from ..core import meta_get_packagelist + + global store + + packages_file = os.path.join(self.absdir, store, '_packages') + if os.path.isfile(packages_file) and os.path.getsize(packages_file): + try: + result = ET.parse(packages_file) + except: + msg = f'Cannot read package file \'{packages_file}\'. ' + msg += 'You can try to remove it and then run osc repairwc.' + raise oscerr.OscIOError(None, msg) + return result + else: + # scan project for existing packages and migrate them + cur_pacs = [] + for data in os.listdir(self.dir): + pac_dir = os.path.join(self.absdir, data) + # we cannot use self.pacs_available because we cannot guarantee that the package list + # was fetched from the server + if data in meta_get_packagelist(self.apiurl, self.name) and is_package_dir(pac_dir) \ + and Package(pac_dir).name == data: + cur_pacs.append(ET.Element('package', name=data, state=' ')) + store_write_initial_packages(self.absdir, self.name, cur_pacs) + return ET.parse(os.path.join(self.absdir, store, '_packages')) + + def write_packages(self): + from ..core import ET_ENCODING + from ..core import xmlindent + + xmlindent(self.pac_root) + store_write_string(self.absdir, '_packages', ET.tostring(self.pac_root, encoding=ET_ENCODING)) + + def addPackage(self, pac): + for i in conf.config['exclude_glob']: + if fnmatch.fnmatch(pac, i): + msg = f'invalid package name: \'{pac}\' (see \'exclude_glob\' config option)' + raise oscerr.OscIOError(None, msg) + state = self.get_state(pac) + if state is None or state == 'D': + self.new_package_entry(pac, 'A') + self.write_packages() + # sometimes the new pac doesn't exist in the list because + # it would take too much time to update all data structs regularly + if pac in self.pacs_unvers: + self.pacs_unvers.remove(pac) + else: + raise oscerr.PackageExists(self.name, pac, f'package \'{pac}\' is already under version control') + + def delPackage(self, pac, force=False): + from ..core import delete_dir + from ..core import getTransActPath + from ..core import statfrmt + + state = self.get_state(pac.name) + can_delete = True + if state == ' ' or state == 'D': + del_files = [] + for filename in pac.filenamelist + pac.filenamelist_unvers: + filestate = pac.status(filename) + if filestate == 'M' or filestate == 'C' or \ + filestate == 'A' or filestate == '?': + can_delete = False + else: + del_files.append(filename) + if can_delete or force: + for filename in del_files: + pac.delete_localfile(filename) + if pac.status(filename) != '?': + # this is not really necessary + pac.put_on_deletelist(filename) + print(statfrmt('D', getTransActPath(os.path.join(pac.dir, filename)))) + print(statfrmt('D', getTransActPath(os.path.join(pac.dir, os.pardir, pac.name)))) + pac.write_deletelist() + self.set_state(pac.name, 'D') + self.write_packages() + else: + print(f'package \'{pac.name}\' has local modifications (see osc st for details)') + elif state == 'A': + if force: + delete_dir(pac.absdir) + self.del_package_node(pac.name) + self.write_packages() + print(statfrmt('D', pac.name)) + else: + print(f'package \'{pac.name}\' has local modifications (see osc st for details)') + elif state is None: + print('package is not under version control') + else: + print('unsupported state') + + def update(self, pacs=(), expand_link=False, unexpand_link=False, service_files=False): + from ..core import Package + from ..core import checkout_package + from ..core import get_project_sourceinfo + from ..core import getTransActPath + from ..core import show_upstream_xsrcmd5 + + if pacs: + for pac in pacs: + Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj).update() + else: + # we need to make sure that the _packages file will be written (even if an exception + # occurs) + try: + # update complete project + # packages which no longer exists upstream + upstream_del = [pac for pac in self.pacs_have if pac not in self.pacs_available and self.get_state(pac) != 'A'] + sinfo_pacs = [pac for pac in self.pacs_have if self.get_state(pac) in (' ', 'D') and pac not in self.pacs_broken] + sinfo_pacs.extend(self.pacs_missing) + sinfos = get_project_sourceinfo(self.apiurl, self.name, True, *sinfo_pacs) + + for pac in upstream_del: + if self.status(pac) != '!': + p = Package(os.path.join(self.dir, pac)) + self.delPackage(p, force=True) + delete_storedir(p.storedir) + try: + os.rmdir(pac) + except: + pass + self.pac_root.remove(self.get_package_node(pac)) + self.pacs_have.remove(pac) + + for pac in self.pacs_have: + state = self.get_state(pac) + if pac in self.pacs_broken: + if self.get_state(pac) != 'A': + checkout_package(self.apiurl, self.name, pac, + pathname=getTransActPath(os.path.join(self.dir, pac)), prj_obj=self, + prj_dir=self.dir, expand_link=not unexpand_link, progress_obj=self.progress_obj) + elif state == ' ': + # do a simple update + p = Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj) + rev = None + needs_update = True + if p.scm_url is not None: + # git managed. + print("Skipping git managed package ", pac) + continue + elif expand_link and p.islink() and not p.isexpanded(): + if p.haslinkerror(): + try: + rev = show_upstream_xsrcmd5(p.apiurl, p.prjname, p.name, revision=p.rev) + except: + rev = show_upstream_xsrcmd5(p.apiurl, p.prjname, p.name, revision=p.rev, linkrev="base") + p.mark_frozen() + else: + rev = p.linkinfo.xsrcmd5 + print('Expanding to rev', rev) + elif unexpand_link and p.islink() and p.isexpanded(): + rev = p.linkinfo.lsrcmd5 + print('Unexpanding to rev', rev) + elif p.islink() and p.isexpanded(): + needs_update = p.update_needed(sinfos[p.name]) + if needs_update: + rev = p.latest_rev() + elif p.hasserviceinfo() and p.serviceinfo.isexpanded() and not service_files: + # FIXME: currently, do_update does not propagate the --server-side-source-service-files + # option to this method. Consequence: an expanded service is always unexpanded during + # an update (TODO: discuss if this is a reasonable behavior (at least this the default + # behavior for a while)) + needs_update = True + else: + needs_update = p.update_needed(sinfos[p.name]) + print(f'Updating {p.name}') + if needs_update: + p.update(rev, service_files) + else: + print(f'At revision {p.rev}.') + if unexpand_link: + p.unmark_frozen() + elif state == 'D': + # pac exists (the non-existent pac case was handled in the first if block) + p = Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj) + if p.update_needed(sinfos[p.name]): + p.update() + elif state == 'A' and pac in self.pacs_available: + # file/dir called pac already exists and is under version control + msg = f'can\'t add package \'{pac}\': Object already exists' + raise oscerr.PackageExists(self.name, pac, msg) + elif state == 'A': + # do nothing + pass + else: + print(f'unexpected state.. package \'{pac}\'') + + self.checkout_missing_pacs(sinfos, expand_link, unexpand_link) + finally: + self.write_packages() + + def commit(self, pacs=(), msg='', files=None, verbose=False, skip_local_service_run=False, can_branch=False, force=False): + from ..core import Package + from ..core import os_path_samefile + + files = files or {} + if pacs: + try: + for pac in pacs: + todo = [] + if pac in files: + todo = files[pac] + state = self.get_state(pac) + if state == 'A': + self.commitNewPackage(pac, msg, todo, verbose=verbose, skip_local_service_run=skip_local_service_run) + elif state == 'D': + self.commitDelPackage(pac, force=force) + elif state == ' ': + # display the correct dir when sending the changes + if os_path_samefile(os.path.join(self.dir, pac), os.getcwd()): + p = Package('.') + else: + p = Package(os.path.join(self.dir, pac)) + p.todo = todo + p.commit(msg, verbose=verbose, skip_local_service_run=skip_local_service_run, can_branch=can_branch, force=force) + elif pac in self.pacs_unvers and not is_package_dir(os.path.join(self.dir, pac)): + print(f'osc: \'{pac}\' is not under version control') + elif pac in self.pacs_broken or not os.path.exists(os.path.join(self.dir, pac)): + print(f'osc: \'{pac}\' package not found') + elif state is None: + self.commitExtPackage(pac, msg, todo, verbose=verbose, skip_local_service_run=skip_local_service_run) + finally: + self.write_packages() + else: + # if we have packages marked as '!' we cannot commit + for pac in self.pacs_broken: + if self.get_state(pac) != 'D': + msg = f'commit failed: package \'{pac}\' is missing' + raise oscerr.PackageMissing(self.name, pac, msg) + try: + for pac in self.pacs_have: + state = self.get_state(pac) + if state == ' ': + # do a simple commit + Package(os.path.join(self.dir, pac)).commit(msg, verbose=verbose, skip_local_service_run=skip_local_service_run) + elif state == 'D': + self.commitDelPackage(pac, force=force) + elif state == 'A': + self.commitNewPackage(pac, msg, verbose=verbose, skip_local_service_run=skip_local_service_run) + finally: + self.write_packages() + + def commitNewPackage(self, pac, msg='', files=None, verbose=False, skip_local_service_run=False): + """creates and commits a new package if it does not exist on the server""" + from ..core import Package + from ..core import edit_meta + from ..core import os_path_samefile + from ..core import statfrmt + + files = files or [] + if pac in self.pacs_available: + print(f'package \'{pac}\' already exists') + else: + user = conf.get_apiurl_usr(self.apiurl) + edit_meta(metatype='pkg', + path_args=(self.name, pac), + template_args=({ + 'name': pac, + 'user': user}), + apiurl=self.apiurl) + # display the correct dir when sending the changes + olddir = os.getcwd() + if os_path_samefile(os.path.join(self.dir, pac), os.curdir): + os.chdir(os.pardir) + p = Package(pac) + else: + p = Package(os.path.join(self.dir, pac)) + p.todo = files + print(statfrmt('Sending', os.path.normpath(p.dir))) + p.commit(msg=msg, verbose=verbose, skip_local_service_run=skip_local_service_run) + self.set_state(pac, ' ') + os.chdir(olddir) + + def commitDelPackage(self, pac, force=False): + """deletes a package on the server and in the working copy""" + + from ..core import Package + from ..core import delete_package + from ..core import getTransActPath + from ..core import os_path_samefile + from ..core import statfrmt + + try: + # display the correct dir when sending the changes + if os_path_samefile(os.path.join(self.dir, pac), os.curdir): + pac_dir = pac + else: + pac_dir = os.path.join(self.dir, pac) + p = Package(os.path.join(self.dir, pac)) + # print statfrmt('Deleting', os.path.normpath(os.path.join(p.dir, os.pardir, pac))) + delete_storedir(p.storedir) + try: + os.rmdir(p.dir) + except: + pass + except OSError: + pac_dir = os.path.join(self.dir, pac) + except (oscerr.NoWorkingCopy, oscerr.WorkingCopyOutdated, oscerr.PackageError): + pass + # print statfrmt('Deleting', getTransActPath(os.path.join(self.dir, pac))) + print(statfrmt('Deleting', getTransActPath(pac_dir))) + delete_package(self.apiurl, self.name, pac, force=force) + self.del_package_node(pac) + + def commitExtPackage(self, pac, msg, files=None, verbose=False, skip_local_service_run=False): + """commits a package from an external project""" + + from ..core import Package + from ..core import edit_meta + from ..core import meta_exists + from ..core import os_path_samefile + + files = files or [] + if os_path_samefile(os.path.join(self.dir, pac), os.getcwd()): + pac_path = '.' + else: + pac_path = os.path.join(self.dir, pac) + + store = Store(pac_path) + project = store_read_project(pac_path) + package = store_read_package(pac_path) + apiurl = store.apiurl + if not meta_exists(metatype='pkg', + path_args=(project, package), + template_args=None, create_new=False, apiurl=apiurl): + user = conf.get_apiurl_usr(self.apiurl) + edit_meta(metatype='pkg', + path_args=(project, package), + template_args=({'name': pac, 'user': user}), apiurl=apiurl) + p = Package(pac_path) + p.todo = files + p.commit(msg=msg, verbose=verbose, skip_local_service_run=skip_local_service_run) + + def __str__(self): + r = [] + r.append('*****************************************************') + r.append(f'Project {self.name} (dir={self.dir}, absdir={self.absdir})') + r.append(f"have pacs:\n{', '.join(self.pacs_have)}") + r.append(f"missing pacs:\n{', '.join(self.pacs_missing)}") + r.append('*****************************************************') + return '\n'.join(r) + + @staticmethod + def init_project( + apiurl: str, + dir: Path, + project, + package_tracking=True, + getPackageList=True, + progress_obj=None, + wc_check=True, + scm_url=None, + ): + global store + + if not os.path.exists(dir): + # use makedirs (checkout_no_colon config option might be enabled) + os.makedirs(dir) + elif not os.path.isdir(dir): + raise oscerr.OscIOError(None, f'error: \'{dir}\' is no directory') + if os.path.exists(os.path.join(dir, store)): + raise oscerr.OscIOError(None, f'error: \'{dir}\' is already an initialized osc working copy') + else: + os.mkdir(os.path.join(dir, store)) + + store_write_project(dir, project) + Store(dir).apiurl = apiurl + if scm_url: + Store(dir).scmurl = scm_url + package_tracking = None + if package_tracking: + store_write_initial_packages(dir, project, []) + return Project(dir, getPackageList, progress_obj, wc_check) From 281f59c842381db4e3a68dedf9e3290fcde7bf33 Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Mon, 15 Apr 2024 16:31:08 +0200 Subject: [PATCH 7/7] Move core.Package to obs_scm.Package --- osc/core.py | 1497 +----------------------------------- osc/obs_scm/__init__.py | 1 + osc/obs_scm/package.py | 1588 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 1590 insertions(+), 1496 deletions(-) create mode 100644 osc/obs_scm/package.py diff --git a/osc/core.py b/osc/core.py index f9784929..a20fc4dc 100644 --- a/osc/core.py +++ b/osc/core.py @@ -49,6 +49,7 @@ from . import store as osc_store from .connection import http_request, http_GET, http_POST, http_PUT, http_DELETE from .obs_scm import File from .obs_scm import Linkinfo +from .obs_scm import Package from .obs_scm import Project from .obs_scm import Serviceinfo from .obs_scm import Store @@ -311,1502 +312,6 @@ class DirectoryServiceinfo: return self.error is not None -@total_ordering -class Package: - """represent a package (its directory) and read/keep/write its metadata""" - - # should _meta be a required file? - REQ_STOREFILES = ('_project', '_package', '_apiurl', '_files', '_osclib_version') - OPT_STOREFILES = ('_to_be_added', '_to_be_deleted', '_in_conflict', '_in_update', - '_in_commit', '_meta', '_meta_mode', '_frozenlink', '_pulled', '_linkrepair', - '_size_limit', '_commit_msg', '_last_buildroot') - - def __init__(self, workingdir, progress_obj=None, size_limit=None, wc_check=True): - global store - - self.todo = [] - if os.path.isfile(workingdir) or not os.path.exists(workingdir): - # workingdir is a file - # workingdir doesn't exist -> it points to a non-existing file in a working dir (e.g. during mv) - workingdir, todo_entry = os.path.split(workingdir) - self.todo.append(todo_entry) - - self.dir = workingdir or "." - self.absdir = os.path.abspath(self.dir) - self.store = osc_store.get_store(self.dir) - self.store.assert_is_package() - self.storedir = os.path.join(self.absdir, store) - self.progress_obj = progress_obj - self.size_limit = size_limit - self.scm_url = self.store.scmurl - if size_limit and size_limit == 0: - self.size_limit = None - - self.prjname = self.store.project - self.name = self.store.package - self.apiurl = self.store.apiurl - - self.update_datastructs() - dirty_files = [] - if wc_check: - dirty_files = self.wc_check() - if dirty_files: - msg = 'Your working copy \'%s\' is in an inconsistent state.\n' \ - 'Please run \'osc repairwc %s\' (Note this might _remove_\n' \ - 'files from the .osc/ dir). Please check the state\n' \ - 'of the working copy afterwards (via \'osc status %s\')' % (self.dir, self.dir, self.dir) - raise oscerr.WorkingCopyInconsistent(self.prjname, self.name, dirty_files, msg) - - def __repr__(self): - return super().__repr__() + f"({self.prjname}/{self.name})" - - def __hash__(self): - return hash((self.name, self.prjname, self.apiurl)) - - def __eq__(self, other): - return (self.name, self.prjname, self.apiurl) == (other.name, other.prjname, other.apiurl) - - def __lt__(self, other): - return (self.name, self.prjname, self.apiurl) < (other.name, other.prjname, other.apiurl) - - @classmethod - def from_paths(cls, paths, progress_obj=None): - """ - Return a list of Package objects from working copies in given paths. - """ - packages = [] - for path in paths: - package = cls(path, progress_obj) - seen_package = None - try: - # re-use an existing package - seen_package_index = packages.index(package) - seen_package = packages[seen_package_index] - except ValueError: - pass - - if seen_package: - # merge package into seen_package - if seen_package.absdir != package.absdir: - raise oscerr.PackageExists(package.prjname, package.name, "Duplicate package") - seen_package.merge(package) - else: - # use the new package instance - packages.append(package) - - return packages - - @classmethod - def from_paths_nofail(cls, paths, progress_obj=None): - """ - Return a list of Package objects from working copies in given paths - and a list of strings with paths that do not contain Package working copies. - """ - packages = [] - failed_to_load = [] - for path in paths: - try: - package = cls(path, progress_obj) - except oscerr.NoWorkingCopy: - failed_to_load.append(path) - continue - - # the following code is identical to from_paths() - seen_package = None - try: - # re-use an existing package - seen_package_index = packages.index(package) - seen_package = packages[seen_package_index] - except ValueError: - pass - - if seen_package: - # merge package into seen_package - if seen_package.absdir != package.absdir: - raise oscerr.PackageExists(package.prjname, package.name, "Duplicate package") - seen_package.merge(package) - else: - # use the new package instance - packages.append(package) - - return packages, failed_to_load - - def wc_check(self): - dirty_files = [] - if self.scm_url: - return dirty_files - for fname in self.filenamelist: - if not os.path.exists(os.path.join(self.storedir, fname)) and fname not in self.skipped: - dirty_files.append(fname) - for fname in Package.REQ_STOREFILES: - if not os.path.isfile(os.path.join(self.storedir, fname)): - dirty_files.append(fname) - for fname in os.listdir(self.storedir): - if fname in Package.REQ_STOREFILES or fname in Package.OPT_STOREFILES or \ - fname.startswith('_build'): - continue - elif fname in self.filenamelist and fname in self.skipped: - dirty_files.append(fname) - elif fname not in self.filenamelist: - dirty_files.append(fname) - for fname in self.to_be_deleted[:]: - if fname not in self.filenamelist: - dirty_files.append(fname) - for fname in self.in_conflict[:]: - if fname not in self.filenamelist: - dirty_files.append(fname) - return dirty_files - - def wc_repair(self, apiurl: Optional[str] = None): - store = Store(self.dir) - store.assert_is_package() - if not store.exists("_apiurl") or apiurl: - if apiurl is None: - msg = 'cannot repair wc: the \'_apiurl\' file is missing but ' \ - 'no \'apiurl\' was passed to wc_repair' - # hmm should we raise oscerr.WrongArgs? - raise oscerr.WorkingCopyInconsistent(self.prjname, self.name, [], msg) - # sanity check - conf.parse_apisrv_url(None, apiurl) - store.apiurl = apiurl - self.apiurl = apiurl - - # all files which are present in the filelist have to exist in the storedir - for f in self.filelist: - # XXX: should we also check the md5? - if not os.path.exists(os.path.join(self.storedir, f.name)) and f.name not in self.skipped: - # if get_source_file fails we're screwed up... - get_source_file(self.apiurl, self.prjname, self.name, f.name, - targetfilename=os.path.join(self.storedir, f.name), revision=self.rev, - mtime=f.mtime) - - for fname in store: - if fname in Package.REQ_STOREFILES or fname in Package.OPT_STOREFILES or \ - fname.startswith('_build'): - continue - elif fname not in self.filenamelist or fname in self.skipped: - # this file does not belong to the storedir so remove it - store.unlink(fname) - - for fname in self.to_be_deleted[:]: - if fname not in self.filenamelist: - self.to_be_deleted.remove(fname) - self.write_deletelist() - - for fname in self.in_conflict[:]: - if fname not in self.filenamelist: - self.in_conflict.remove(fname) - self.write_conflictlist() - - def info(self): - source_url = makeurl(self.apiurl, ['source', self.prjname, self.name]) - r = info_templ % (self.prjname, self.name, self.absdir, self.apiurl, source_url, self.srcmd5, self.rev, self.linkinfo) - return r - - def addfile(self, n): - if not os.path.exists(os.path.join(self.absdir, n)): - raise oscerr.OscIOError(None, f'error: file \'{n}\' does not exist') - if n in self.to_be_deleted: - self.to_be_deleted.remove(n) -# self.delete_storefile(n) - self.write_deletelist() - elif n in self.filenamelist or n in self.to_be_added: - raise oscerr.PackageFileConflict(self.prjname, self.name, n, f'osc: warning: \'{n}\' is already under version control') -# shutil.copyfile(os.path.join(self.dir, n), os.path.join(self.storedir, n)) - if self.dir != '.': - pathname = os.path.join(self.dir, n) - else: - pathname = n - self.to_be_added.append(n) - self.write_addlist() - print(statfrmt('A', pathname)) - - def delete_file(self, n, force=False): - """deletes a file if possible and marks the file as deleted""" - state = '?' - try: - state = self.status(n) - except OSError as ioe: - if not force: - raise ioe - if state in ['?', 'A', 'M', 'R', 'C'] and not force: - return (False, state) - # special handling for skipped files: if file exists, simply delete it - if state == 'S': - exists = os.path.exists(os.path.join(self.dir, n)) - self.delete_localfile(n) - return (exists, 'S') - - self.delete_localfile(n) - was_added = n in self.to_be_added - if state in ('A', 'R') or state == '!' and was_added: - self.to_be_added.remove(n) - self.write_addlist() - elif state == 'C': - # don't remove "merge files" (*.mine, *.new...) - # that's why we don't use clear_from_conflictlist - self.in_conflict.remove(n) - self.write_conflictlist() - if state not in ('A', '?') and not (state == '!' and was_added): - self.put_on_deletelist(n) - self.write_deletelist() - return (True, state) - - def delete_storefile(self, n): - try: - os.unlink(os.path.join(self.storedir, n)) - except: - pass - - def delete_localfile(self, n): - try: - os.unlink(os.path.join(self.dir, n)) - except: - pass - - def put_on_deletelist(self, n): - if n not in self.to_be_deleted: - self.to_be_deleted.append(n) - - def put_on_conflictlist(self, n): - if n not in self.in_conflict: - self.in_conflict.append(n) - - def put_on_addlist(self, n): - if n not in self.to_be_added: - self.to_be_added.append(n) - - def clear_from_conflictlist(self, n): - """delete an entry from the file, and remove the file if it would be empty""" - if n in self.in_conflict: - - filename = os.path.join(self.dir, n) - storefilename = os.path.join(self.storedir, n) - myfilename = os.path.join(self.dir, n + '.mine') - upfilename = os.path.join(self.dir, n + '.new') - - try: - os.unlink(myfilename) - os.unlink(upfilename) - if self.islinkrepair() or self.ispulled(): - os.unlink(os.path.join(self.dir, n + '.old')) - except: - pass - - self.in_conflict.remove(n) - - self.write_conflictlist() - - # XXX: this isn't used at all - def write_meta_mode(self): - # XXX: the "elif" is somehow a contradiction (with current and the old implementation - # it's not possible to "leave" the metamode again) (except if you modify pac.meta - # which is really ugly:) ) - if self.meta: - store_write_string(self.absdir, '_meta_mode', '') - elif self.ismetamode(): - os.unlink(os.path.join(self.storedir, '_meta_mode')) - - def write_sizelimit(self): - if self.size_limit and self.size_limit <= 0: - try: - os.unlink(os.path.join(self.storedir, '_size_limit')) - except: - pass - else: - store_write_string(self.absdir, '_size_limit', str(self.size_limit) + '\n') - - def write_addlist(self): - self.__write_storelist('_to_be_added', self.to_be_added) - - def write_deletelist(self): - self.__write_storelist('_to_be_deleted', self.to_be_deleted) - - def delete_source_file(self, n): - """delete local a source file""" - self.delete_localfile(n) - self.delete_storefile(n) - - def delete_remote_source_file(self, n): - """delete a remote source file (e.g. from the server)""" - query = {"rev": "upload"} - u = makeurl(self.apiurl, ['source', self.prjname, self.name, n], query=query) - http_DELETE(u) - - def put_source_file(self, n, tdir, copy_only=False): - query = {"rev": "repository"} - tfilename = os.path.join(tdir, n) - shutil.copyfile(os.path.join(self.dir, n), tfilename) - # escaping '+' in the URL path (note: not in the URL query string) is - # only a workaround for ruby on rails, which swallows it otherwise - if not copy_only: - u = makeurl(self.apiurl, ['source', self.prjname, self.name, n], query=query) - http_PUT(u, file=tfilename) - if n in self.to_be_added: - self.to_be_added.remove(n) - - def __commit_update_store(self, tdir): - """move files from transaction directory into the store""" - for filename in os.listdir(tdir): - os.rename(os.path.join(tdir, filename), os.path.join(self.storedir, filename)) - - def __generate_commitlist(self, todo_send): - root = ET.Element('directory') - for i in sorted(todo_send.keys()): - ET.SubElement(root, 'entry', name=i, md5=todo_send[i]) - return root - - @staticmethod - def commit_filelist(apiurl: str, project: str, package: str, filelist, msg="", user=None, **query): - """send the commitlog and the local filelist to the server""" - if user is None: - user = conf.get_apiurl_usr(apiurl) - query.update({'cmd': 'commitfilelist', 'user': user, 'comment': msg}) - u = makeurl(apiurl, ['source', project, package], query=query) - f = http_POST(u, data=ET.tostring(filelist, encoding=ET_ENCODING)) - root = ET.parse(f).getroot() - return root - - @staticmethod - def commit_get_missing(filelist): - """returns list of missing files (filelist is the result of commit_filelist)""" - error = filelist.get('error') - if error is None: - return [] - elif error != 'missing': - raise oscerr.APIError('commit_get_missing_files: ' - 'unexpected \'error\' attr: \'%s\'' % error) - todo = [] - for n in filelist.findall('entry'): - name = n.get('name') - if name is None: - raise oscerr.APIError('missing \'name\' attribute:\n%s\n' - % ET.tostring(filelist, encoding=ET_ENCODING)) - todo.append(n.get('name')) - return todo - - def __send_commitlog(self, msg, local_filelist, validate=False): - """send the commitlog and the local filelist to the server""" - query = {} - if self.islink() and self.isexpanded(): - query['keeplink'] = '1' - if conf.config['linkcontrol'] or self.isfrozen(): - query['linkrev'] = self.linkinfo.srcmd5 - if self.ispulled(): - query['repairlink'] = '1' - query['linkrev'] = self.get_pulled_srcmd5() - if self.islinkrepair(): - query['repairlink'] = '1' - if validate: - query['withvalidate'] = '1' - return self.commit_filelist(self.apiurl, self.prjname, self.name, - local_filelist, msg, **query) - - def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch=False, force=False): - # commit only if the upstream revision is the same as the working copy's - upstream_rev = self.latest_rev() - if self.rev != upstream_rev: - raise oscerr.WorkingCopyOutdated((self.absdir, self.rev, upstream_rev)) - - if not skip_local_service_run: - r = self.run_source_services(mode="trylocal", verbose=verbose) - if r != 0: - # FIXME: it is better to raise this in Serviceinfo.execute with more - # information (like which service/command failed) - raise oscerr.ServiceRuntimeError('A service failed with error: %d' % r) - - # check if it is a link, if so, branch the package - if self.is_link_to_different_project(): - if can_branch: - orgprj = self.get_local_origin_project() - print(f"Branching {self.name} from {orgprj} to {self.prjname}") - exists, targetprj, targetpkg, srcprj, srcpkg = branch_pkg( - self.apiurl, orgprj, self.name, target_project=self.prjname) - # update _meta and _files to sychronize the local package - # to the new branched one in OBS - self.update_local_pacmeta() - self.update_local_filesmeta() - else: - print(f"{self.name} Not commited because is link to a different project") - return 1 - - if not self.todo: - self.todo = [i for i in self.to_be_added if i not in self.filenamelist] + self.filenamelist - - pathn = getTransActPath(self.dir) - - todo_send = {} - todo_delete = [] - real_send = [] - sha256sums = {} - for filename in self.filenamelist + [i for i in self.to_be_added if i not in self.filenamelist]: - if filename.startswith('_service:') or filename.startswith('_service_'): - continue - st = self.status(filename) - if st == 'C': - print('Please resolve all conflicts before committing using "osc resolved FILE"!') - return 1 - elif filename in self.todo: - if st in ('A', 'R', 'M'): - todo_send[filename] = dgst(os.path.join(self.absdir, filename)) - sha256sums[filename] = sha256_dgst(os.path.join(self.absdir, filename)) - real_send.append(filename) - print(statfrmt('Sending', os.path.join(pathn, filename))) - elif st in (' ', '!', 'S'): - if st == '!' and filename in self.to_be_added: - print(f'file \'{filename}\' is marked as \'A\' but does not exist') - return 1 - f = self.findfilebyname(filename) - if f is None: - raise oscerr.PackageInternalError(self.prjname, self.name, - 'error: file \'%s\' with state \'%s\' is not known by meta' - % (filename, st)) - todo_send[filename] = f.md5 - elif st == 'D': - todo_delete.append(filename) - print(statfrmt('Deleting', os.path.join(pathn, filename))) - elif st in ('R', 'M', 'D', ' ', '!', 'S'): - # ignore missing new file (it's not part of the current commit) - if st == '!' and filename in self.to_be_added: - continue - f = self.findfilebyname(filename) - if f is None: - raise oscerr.PackageInternalError(self.prjname, self.name, - 'error: file \'%s\' with state \'%s\' is not known by meta' - % (filename, st)) - todo_send[filename] = f.md5 - if ((self.ispulled() or self.islinkrepair() or self.isfrozen()) - and st != 'A' and filename not in sha256sums): - # Ignore files with state 'A': if we should consider it, - # it would have been in pac.todo, which implies that it is - # in sha256sums. - # The storefile is guaranteed to exist (since we have a - # pulled/linkrepair wc, the file cannot have state 'S') - storefile = os.path.join(self.storedir, filename) - sha256sums[filename] = sha256_dgst(storefile) - - if not force and not real_send and not todo_delete and not self.islinkrepair() and not self.ispulled(): - print(f'nothing to do for package {self.name}') - return 1 - - print('Transmitting file data', end=' ') - filelist = self.__generate_commitlist(todo_send) - sfilelist = self.__send_commitlog(msg, filelist, validate=True) - hash_entries = [e for e in sfilelist.findall('entry') if e.get('hash') is not None] - if sfilelist.get('error') and hash_entries: - name2elem = {e.get('name'): e for e in filelist.findall('entry')} - for entry in hash_entries: - filename = entry.get('name') - fileelem = name2elem.get(filename) - if filename not in sha256sums: - msg = 'There is no sha256 sum for file %s.\n' \ - 'This could be due to an outdated working copy.\n' \ - 'Please update your working copy with osc update and\n' \ - 'commit again afterwards.' - print(msg % filename) - return 1 - fileelem.set('hash', f'sha256:{sha256sums[filename]}') - sfilelist = self.__send_commitlog(msg, filelist) - send = self.commit_get_missing(sfilelist) - real_send = [i for i in real_send if i not in send] - # abort after 3 tries - tries = 3 - tdir = None - try: - tdir = os.path.join(self.storedir, '_in_commit') - if os.path.isdir(tdir): - shutil.rmtree(tdir) - os.mkdir(tdir) - while send and tries: - for filename in send[:]: - sys.stdout.write('.') - sys.stdout.flush() - self.put_source_file(filename, tdir) - send.remove(filename) - tries -= 1 - sfilelist = self.__send_commitlog(msg, filelist) - send = self.commit_get_missing(sfilelist) - if send: - raise oscerr.PackageInternalError(self.prjname, self.name, - 'server does not accept filelist:\n%s\nmissing:\n%s\n' - % (ET.tostring(filelist, encoding=ET_ENCODING), ET.tostring(sfilelist, encoding=ET_ENCODING))) - # these files already exist on the server - for filename in real_send: - self.put_source_file(filename, tdir, copy_only=True) - # update store with the committed files - self.__commit_update_store(tdir) - finally: - if tdir is not None and os.path.isdir(tdir): - shutil.rmtree(tdir) - self.rev = sfilelist.get('rev') - print() - print(f'Committed revision {self.rev}.') - - if self.ispulled(): - os.unlink(os.path.join(self.storedir, '_pulled')) - if self.islinkrepair(): - os.unlink(os.path.join(self.storedir, '_linkrepair')) - self.linkrepair = False - # XXX: mark package as invalid? - print('The source link has been repaired. This directory can now be removed.') - - if self.islink() and self.isexpanded(): - li = Linkinfo() - li.read(sfilelist.find('linkinfo')) - if li.xsrcmd5 is None: - raise oscerr.APIError(f'linkinfo has no xsrcmd5 attr:\n{ET.tostring(sfilelist, encoding=ET_ENCODING)}\n') - sfilelist = ET.fromstring(self.get_files_meta(revision=li.xsrcmd5)) - for i in sfilelist.findall('entry'): - if i.get('name') in self.skipped: - i.set('skipped', 'true') - store_write_string(self.absdir, '_files', ET.tostring(sfilelist, encoding=ET_ENCODING) + '\n') - for filename in todo_delete: - self.to_be_deleted.remove(filename) - self.delete_storefile(filename) - self.write_deletelist() - self.write_addlist() - self.update_datastructs() - - print_request_list(self.apiurl, self.prjname, self.name) - - # FIXME: add testcases for this codepath - sinfo = sfilelist.find('serviceinfo') - if sinfo is not None: - print('Waiting for server side source service run') - u = makeurl(self.apiurl, ['source', self.prjname, self.name]) - while sinfo is not None and sinfo.get('code') == 'running': - sys.stdout.write('.') - sys.stdout.flush() - # does it make sense to add some delay? - sfilelist = ET.fromstring(http_GET(u).read()) - # if sinfo is None another commit might have occured in the "meantime" - sinfo = sfilelist.find('serviceinfo') - print('') - rev = self.latest_rev() - self.update(rev=rev) - elif self.get_local_meta() is None: - # if this was a newly added package there is no _meta - # file - self.update_local_pacmeta() - - def __write_storelist(self, name, data): - if len(data) == 0: - try: - os.unlink(os.path.join(self.storedir, name)) - except: - pass - else: - store_write_string(self.absdir, name, '%s\n' % '\n'.join(data)) - - def write_conflictlist(self): - self.__write_storelist('_in_conflict', self.in_conflict) - - def updatefile(self, n, revision, mtime=None): - filename = os.path.join(self.dir, n) - storefilename = os.path.join(self.storedir, n) - origfile_tmp = os.path.join(self.storedir, '_in_update', f'{n}.copy') - origfile = os.path.join(self.storedir, '_in_update', n) - if os.path.isfile(filename): - shutil.copyfile(filename, origfile_tmp) - os.rename(origfile_tmp, origfile) - else: - origfile = None - - get_source_file(self.apiurl, self.prjname, self.name, n, targetfilename=storefilename, - revision=revision, progress_obj=self.progress_obj, mtime=mtime, meta=self.meta) - - shutil.copyfile(storefilename, filename) - if mtime: - utime(filename, (-1, mtime)) - if origfile is not None: - os.unlink(origfile) - - def mergefile(self, n, revision, mtime=None): - filename = os.path.join(self.dir, n) - storefilename = os.path.join(self.storedir, n) - myfilename = os.path.join(self.dir, n + '.mine') - upfilename = os.path.join(self.dir, n + '.new') - origfile_tmp = os.path.join(self.storedir, '_in_update', f'{n}.copy') - origfile = os.path.join(self.storedir, '_in_update', n) - shutil.copyfile(filename, origfile_tmp) - os.rename(origfile_tmp, origfile) - os.rename(filename, myfilename) - - get_source_file(self.apiurl, self.prjname, self.name, n, - revision=revision, targetfilename=upfilename, - progress_obj=self.progress_obj, mtime=mtime, meta=self.meta) - - if binary_file(myfilename) or binary_file(upfilename): - # don't try merging - shutil.copyfile(upfilename, filename) - shutil.copyfile(upfilename, storefilename) - os.unlink(origfile) - self.in_conflict.append(n) - self.write_conflictlist() - return 'C' - else: - # try merging - # diff3 OPTIONS... MINE OLDER YOURS - ret = -1 - with open(filename, 'w') as f: - args = ('-m', '-E', myfilename, storefilename, upfilename) - ret = run_external('diff3', *args, stdout=f) - - # "An exit status of 0 means `diff3' was successful, 1 means some - # conflicts were found, and 2 means trouble." - if ret == 0: - # merge was successful... clean up - shutil.copyfile(upfilename, storefilename) - os.unlink(upfilename) - os.unlink(myfilename) - os.unlink(origfile) - return 'G' - elif ret == 1: - # unsuccessful merge - shutil.copyfile(upfilename, storefilename) - os.unlink(origfile) - self.in_conflict.append(n) - self.write_conflictlist() - return 'C' - else: - merge_cmd = 'diff3 ' + ' '.join(args) - raise oscerr.ExtRuntimeError(f'diff3 failed with exit code: {ret}', merge_cmd) - - def update_local_filesmeta(self, revision=None): - """ - Update the local _files file in the store. - It is replaced with the version pulled from upstream. - """ - meta = self.get_files_meta(revision=revision) - store_write_string(self.absdir, '_files', meta + '\n') - - def get_files_meta(self, revision='latest', skip_service=True): - fm = show_files_meta(self.apiurl, self.prjname, self.name, revision=revision, meta=self.meta) - # look for "too large" files according to size limit and mark them - root = ET.fromstring(fm) - for e in root.findall('entry'): - size = e.get('size') - if size and self.size_limit and int(size) > self.size_limit \ - or skip_service and (e.get('name').startswith('_service:') or e.get('name').startswith('_service_')): - e.set('skipped', 'true') - continue - - if conf.config["exclude_files"]: - exclude = False - for pattern in conf.config["exclude_files"]: - if fnmatch.fnmatch(e.get("name"), pattern): - exclude = True - break - if exclude: - e.set("skipped", "true") - continue - - if conf.config["include_files"]: - include = False - for pattern in conf.config["include_files"]: - if fnmatch.fnmatch(e.get("name"), pattern): - include = True - break - if not include: - e.set("skipped", "true") - continue - - return ET.tostring(root, encoding=ET_ENCODING) - - def get_local_meta(self): - """Get the local _meta file for the package.""" - meta = store_read_file(self.absdir, '_meta') - return meta - - def get_local_origin_project(self): - """Get the originproject from the _meta file.""" - # if the wc was checked out via some old osc version - # there might be no meta file: in this case we assume - # that the origin project is equal to the wc's project - meta = self.get_local_meta() - if meta is None: - return self.prjname - root = ET.fromstring(meta) - return root.get('project') - - def is_link_to_different_project(self): - """Check if the package is a link to a different project.""" - if self.name == "_project": - return False - orgprj = self.get_local_origin_project() - return self.prjname != orgprj - - def update_datastructs(self): - """ - Update the internal data structures if the local _files - file has changed (e.g. update_local_filesmeta() has been - called). - """ - if self.scm_url: - self.filenamelist = [] - self.filelist = [] - self.skipped = [] - self.to_be_added = [] - self.to_be_deleted = [] - self.in_conflict = [] - self.linkrepair = None - self.rev = None - self.srcmd5 = None - self.linkinfo = Linkinfo() - self.serviceinfo = DirectoryServiceinfo() - self.size_limit = None - self.meta = None - self.excluded = [] - self.filenamelist_unvers = [] - return - - files_tree = read_filemeta(self.dir) - files_tree_root = files_tree.getroot() - - self.rev = files_tree_root.get('rev') - self.srcmd5 = files_tree_root.get('srcmd5') - - self.linkinfo = Linkinfo() - self.linkinfo.read(files_tree_root.find('linkinfo')) - self.serviceinfo = DirectoryServiceinfo() - self.serviceinfo.read(files_tree_root.find('serviceinfo')) - self.filenamelist = [] - self.filelist = [] - self.skipped = [] - - for node in files_tree_root.findall('entry'): - try: - f = File(node.get('name'), - node.get('md5'), - int(node.get('size')), - int(node.get('mtime'))) - if node.get('skipped'): - self.skipped.append(f.name) - f.skipped = True - except: - # okay, a very old version of _files, which didn't contain any metadata yet... - f = File(node.get('name'), '', 0, 0) - self.filelist.append(f) - self.filenamelist.append(f.name) - - self.to_be_added = read_tobeadded(self.absdir) - self.to_be_deleted = read_tobedeleted(self.absdir) - self.in_conflict = read_inconflict(self.absdir) - self.linkrepair = os.path.isfile(os.path.join(self.storedir, '_linkrepair')) - self.size_limit = read_sizelimit(self.dir) - self.meta = self.ismetamode() - - # gather unversioned files, but ignore some stuff - self.excluded = [] - for i in os.listdir(self.dir): - for j in conf.config['exclude_glob']: - if fnmatch.fnmatch(i, j): - self.excluded.append(i) - break - self.filenamelist_unvers = [i for i in os.listdir(self.dir) - if i not in self.excluded - if i not in self.filenamelist] - - def islink(self): - """tells us if the package is a link (has 'linkinfo'). - A package with linkinfo is a package which links to another package. - Returns ``True`` if the package is a link, otherwise ``False``.""" - return self.linkinfo.islink() - - def isexpanded(self): - """tells us if the package is a link which is expanded. - Returns ``True`` if the package is expanded, otherwise ``False``.""" - return self.linkinfo.isexpanded() - - def islinkrepair(self): - """tells us if we are repairing a broken source link.""" - return self.linkrepair - - def ispulled(self): - """tells us if we have pulled a link.""" - return os.path.isfile(os.path.join(self.storedir, '_pulled')) - - def isfrozen(self): - """tells us if the link is frozen.""" - return os.path.isfile(os.path.join(self.storedir, '_frozenlink')) - - def ismetamode(self): - """tells us if the package is in meta mode""" - return os.path.isfile(os.path.join(self.storedir, '_meta_mode')) - - def get_pulled_srcmd5(self): - pulledrev = None - for line in open(os.path.join(self.storedir, '_pulled')): - pulledrev = line.strip() - return pulledrev - - def haslinkerror(self): - """ - Returns ``True`` if the link is broken otherwise ``False``. - If the package is not a link it returns ``False``. - """ - return self.linkinfo.haserror() - - def linkerror(self): - """ - Returns an error message if the link is broken otherwise ``None``. - If the package is not a link it returns ``None``. - """ - return self.linkinfo.error - - def hasserviceinfo(self): - """ - Returns ``True``, if this package contains services. - """ - return self.serviceinfo.lsrcmd5 is not None or self.serviceinfo.xsrcmd5 is not None - - def update_local_pacmeta(self): - """ - Update the local _meta file in the store. - It is replaced with the version pulled from upstream. - """ - meta = show_package_meta(self.apiurl, self.prjname, self.name) - if meta != "": - # is empty for _project for example - meta = b''.join(meta) - store_write_string(self.absdir, '_meta', meta + b'\n') - - def findfilebyname(self, n): - for i in self.filelist: - if i.name == n: - return i - - def get_status(self, excluded=False, *exclude_states): - global store - todo = self.todo - if not todo: - todo = self.filenamelist + self.to_be_added + \ - [i for i in self.filenamelist_unvers if not os.path.isdir(os.path.join(self.absdir, i))] - if excluded: - todo.extend([i for i in self.excluded if i != store]) - todo = set(todo) - res = [] - for fname in sorted(todo): - st = self.status(fname) - if st not in exclude_states: - res.append((st, fname)) - return res - - def status(self, n): - """ - status can be:: - - file storefile file present STATUS - exists exists in _files - x - - 'A' and listed in _to_be_added - x x - 'R' and listed in _to_be_added - x x x ' ' if digest differs: 'M' - and if in conflicts file: 'C' - x - - '?' - - x x 'D' and listed in _to_be_deleted - x x x 'D' and listed in _to_be_deleted (e.g. if deleted file was modified) - x x x 'C' and listed in _in_conflict - x - x 'S' and listed in self.skipped - - - x 'S' and listed in self.skipped - - x x '!' - - - - NOT DEFINED - """ - - known_by_meta = False - exists = False - exists_in_store = False - localfile = os.path.join(self.absdir, n) - if n in self.filenamelist: - known_by_meta = True - if os.path.exists(localfile): - exists = True - if os.path.exists(os.path.join(self.storedir, n)): - exists_in_store = True - - if n in self.to_be_deleted: - state = 'D' - elif n in self.in_conflict: - state = 'C' - elif n in self.skipped: - state = 'S' - elif n in self.to_be_added and exists and exists_in_store: - state = 'R' - elif n in self.to_be_added and exists: - state = 'A' - elif exists and exists_in_store and known_by_meta: - filemeta = self.findfilebyname(n) - state = ' ' - if conf.config['status_mtime_heuristic']: - if os.path.getmtime(localfile) != filemeta.mtime and dgst(localfile) != filemeta.md5: - state = 'M' - elif dgst(localfile) != filemeta.md5: - state = 'M' - elif n in self.to_be_added and not exists: - state = '!' - elif not exists and exists_in_store and known_by_meta and n not in self.to_be_deleted: - state = '!' - elif exists and not exists_in_store and not known_by_meta: - state = '?' - elif not exists_in_store and known_by_meta: - # XXX: this codepath shouldn't be reached (we restore the storefile - # in update_datastructs) - raise oscerr.PackageInternalError(self.prjname, self.name, - 'error: file \'%s\' is known by meta but no storefile exists.\n' - 'This might be caused by an old wc format. Please backup your current\n' - 'wc and checkout the package again. Afterwards copy all files (except the\n' - '.osc/ dir) into the new package wc.' % n) - elif os.path.islink(localfile): - # dangling symlink, whose name is _not_ tracked: treat it - # as unversioned - state = '?' - else: - # this case shouldn't happen (except there was a typo in the filename etc.) - raise oscerr.OscIOError(None, f'osc: \'{n}\' is not under version control') - - return state - - def get_diff(self, revision=None, ignoreUnversioned=False): - diff_hdr = b'Index: %s\n' - diff_hdr += b'===================================================================\n' - kept = [] - added = [] - deleted = [] - - def diff_add_delete(fname, add, revision): - diff = [] - diff.append(diff_hdr % fname.encode()) - origname = fname - if add: - diff.append(b'--- %s\t(revision 0)\n' % fname.encode()) - rev = 'revision 0' - if not revision_is_empty(revision) and fname not in self.to_be_added: - rev = 'working copy' - diff.append(b'+++ %s\t(%s)\n' % (fname.encode(), rev.encode())) - fname = os.path.join(self.absdir, fname) - if not os.path.isfile(fname): - raise oscerr.OscIOError(None, 'file \'%s\' is marked as \'A\' but does not exist\n' - '(either add the missing file or revert it)' % fname) - else: - if not revision_is_empty(revision): - b_revision = str(revision).encode() - else: - b_revision = self.rev.encode() - diff.append(b'--- %s\t(revision %s)\n' % (fname.encode(), b_revision)) - diff.append(b'+++ %s\t(working copy)\n' % fname.encode()) - fname = os.path.join(self.storedir, fname) - - fd = None - tmpfile = None - try: - if not revision_is_empty(revision) and not add: - (fd, tmpfile) = tempfile.mkstemp(prefix='osc_diff') - get_source_file(self.apiurl, self.prjname, self.name, origname, tmpfile, revision) - fname = tmpfile - if binary_file(fname): - what = b'added' - if not add: - what = b'deleted' - diff = diff[:1] - diff.append(b'Binary file \'%s\' %s.\n' % (origname.encode(), what)) - return diff - tmpl = b'+%s' - ltmpl = b'@@ -0,0 +1,%d @@\n' - if not add: - tmpl = b'-%s' - ltmpl = b'@@ -1,%d +0,0 @@\n' - with open(fname, 'rb') as f: - lines = [tmpl % i for i in f.readlines()] - if len(lines): - diff.append(ltmpl % len(lines)) - if not lines[-1].endswith(b'\n'): - lines.append(b'\n\\ No newline at end of file\n') - diff.extend(lines) - finally: - if fd is not None: - os.close(fd) - if tmpfile is not None and os.path.exists(tmpfile): - os.unlink(tmpfile) - return diff - - if revision is None: - todo = self.todo or [i for i in self.filenamelist if i not in self.to_be_added] + self.to_be_added - for fname in todo: - if fname in self.to_be_added and self.status(fname) == 'A': - added.append(fname) - elif fname in self.to_be_deleted: - deleted.append(fname) - elif fname in self.filenamelist: - kept.append(self.findfilebyname(fname)) - elif fname in self.to_be_added and self.status(fname) == '!': - raise oscerr.OscIOError(None, 'file \'%s\' is marked as \'A\' but does not exist\n' - '(either add the missing file or revert it)' % fname) - elif not ignoreUnversioned: - raise oscerr.OscIOError(None, f'file \'{fname}\' is not under version control') - else: - fm = self.get_files_meta(revision=revision) - root = ET.fromstring(fm) - rfiles = self.__get_files(root) - # swap added and deleted - kept, deleted, added, services = self.__get_rev_changes(rfiles) - added = [f.name for f in added] - added.extend([f for f in self.to_be_added if f not in kept]) - deleted = [f.name for f in deleted] - deleted.extend(self.to_be_deleted) - for f in added[:]: - if f in deleted: - added.remove(f) - deleted.remove(f) -# print kept, added, deleted - for f in kept: - state = self.status(f.name) - if state in ('S', '?', '!'): - continue - elif state == ' ' and revision is None: - continue - elif not revision_is_empty(revision) and self.findfilebyname(f.name).md5 == f.md5 and state != 'M': - continue - yield [diff_hdr % f.name.encode()] - if revision is None: - yield get_source_file_diff(self.absdir, f.name, self.rev) - else: - fd = None - tmpfile = None - diff = [] - try: - (fd, tmpfile) = tempfile.mkstemp(prefix='osc_diff') - get_source_file(self.apiurl, self.prjname, self.name, f.name, tmpfile, revision) - diff = get_source_file_diff(self.absdir, f.name, revision, - os.path.basename(tmpfile), os.path.dirname(tmpfile), f.name) - finally: - if fd is not None: - os.close(fd) - if tmpfile is not None and os.path.exists(tmpfile): - os.unlink(tmpfile) - yield diff - - for f in added: - yield diff_add_delete(f, True, revision) - for f in deleted: - yield diff_add_delete(f, False, revision) - - def merge(self, otherpac): - for todo_entry in otherpac.todo: - if todo_entry not in self.todo: - self.todo.append(todo_entry) - - def __str__(self): - r = """ -name: %s -prjname: %s -workingdir: %s -localfilelist: %s -linkinfo: %s -rev: %s -'todo' files: %s -""" % (self.name, - self.prjname, - self.dir, - '\n '.join(self.filenamelist), - self.linkinfo, - self.rev, - self.todo) - - return r - - def read_meta_from_spec(self, spec=None): - if spec: - specfile = spec - else: - # scan for spec files - speclist = glob.glob(os.path.join(self.dir, '*.spec')) - if len(speclist) == 1: - specfile = speclist[0] - elif len(speclist) > 1: - print('the following specfiles were found:') - for filename in speclist: - print(filename) - print('please specify one with --specfile') - sys.exit(1) - else: - print('no specfile was found - please specify one ' - 'with --specfile') - sys.exit(1) - - data = read_meta_from_spec(specfile, 'Summary', 'Url', '%description') - self.summary = data.get('Summary', '') - self.url = data.get('Url', '') - self.descr = data.get('%description', '') - - def update_package_meta(self, force=False): - """ - for the updatepacmetafromspec subcommand - argument force supress the confirm question - """ - from . import obs_api - from .output import get_user_input - - package_obj = obs_api.Package.from_api(self.apiurl, self.prjname, self.name) - old = package_obj.to_string() - package_obj.title = self.summary.strip() - package_obj.description = "".join(self.descr).strip() - package_obj.url = self.url.strip() - new = package_obj.to_string() - - if not package_obj.has_changed(): - return - - if force: - reply = "y" - else: - while True: - print("\n".join(difflib.unified_diff(old.splitlines(), new.splitlines(), fromfile="old", tofile="new"))) - print() - - reply = get_user_input( - "Write?", - answers={"y": "yes", "n": "no", "e": "edit"}, - ) - if reply == "y": - break - if reply == "n": - break - if reply == "e": - _, _, edited_obj = package_obj.do_edit() - package_obj.do_update(edited_obj) - new = package_obj.to_string() - continue - - if reply == "y": - package_obj.to_api(self.apiurl) - - def mark_frozen(self): - store_write_string(self.absdir, '_frozenlink', '') - print() - print(f"The link in this package (\"{self.name}\") is currently broken. Checking") - print("out the last working version instead; please use 'osc pull'") - print("to merge the conflicts.") - print() - - def unmark_frozen(self): - if os.path.exists(os.path.join(self.storedir, '_frozenlink')): - os.unlink(os.path.join(self.storedir, '_frozenlink')) - - def latest_rev(self, include_service_files=False, expand=False): - # if expand is True the xsrcmd5 will be returned (even if the wc is unexpanded) - if self.islinkrepair(): - upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrepair=1, meta=self.meta, include_service_files=include_service_files) - elif self.islink() and (self.isexpanded() or expand): - if self.isfrozen() or self.ispulled(): - upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrev=self.linkinfo.srcmd5, meta=self.meta, include_service_files=include_service_files) - else: - try: - upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, meta=self.meta, include_service_files=include_service_files) - except: - try: - upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrev=self.linkinfo.srcmd5, meta=self.meta, include_service_files=include_service_files) - except: - upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrev="base", meta=self.meta, include_service_files=include_service_files) - self.mark_frozen() - elif not self.islink() and expand: - upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, meta=self.meta, include_service_files=include_service_files) - else: - upstream_rev = show_upstream_rev(self.apiurl, self.prjname, self.name, meta=self.meta, include_service_files=include_service_files) - return upstream_rev - - def __get_files(self, fmeta_root): - f = [] - if fmeta_root.get('rev') is None and len(fmeta_root.findall('entry')) > 0: - raise oscerr.APIError(f"missing rev attribute in _files:\n{''.join(ET.tostring(fmeta_root, encoding=ET_ENCODING))}") - for i in fmeta_root.findall('entry'): - error = i.get('error') - if error is not None: - raise oscerr.APIError(f'broken files meta: {error}') - skipped = i.get('skipped') is not None - f.append(File(i.get('name'), i.get('md5'), - int(i.get('size')), int(i.get('mtime')), skipped)) - return f - - def __get_rev_changes(self, revfiles): - kept = [] - added = [] - deleted = [] - services = [] - revfilenames = [] - for f in revfiles: - revfilenames.append(f.name) - # treat skipped like deleted files - if f.skipped: - if f.name.startswith('_service:'): - services.append(f) - else: - deleted.append(f) - continue - # treat skipped like added files - # problem: this overwrites existing files during the update - # (because skipped files aren't in self.filenamelist_unvers) - if f.name in self.filenamelist and f.name not in self.skipped: - kept.append(f) - else: - added.append(f) - for f in self.filelist: - if f.name not in revfilenames: - deleted.append(f) - - return kept, added, deleted, services - - def update_needed(self, sinfo): - # this method might return a false-positive (that is a True is returned, - # even though no update is needed) (for details, see comments below) - if self.islink(): - if self.isexpanded(): - # check if both revs point to the same expanded sources - # Note: if the package contains a _service file, sinfo.srcmd5's lsrcmd5 - # points to the "expanded" services (xservicemd5) => chances - # for a false-positive are high, because osc usually works on the - # "unexpanded" services. - # Once the srcserver supports something like noservice=1, we can get rid of - # this false-positives (patch was already sent to the ml) (but this also - # requires some slight changes in osc) - return sinfo.get('srcmd5') != self.srcmd5 - elif self.hasserviceinfo(): - # check if we have expanded or unexpanded services - if self.serviceinfo.isexpanded(): - return sinfo.get('lsrcmd5') != self.srcmd5 - else: - # again, we might have a false-positive here, because - # a mismatch of the "xservicemd5"s does not neccessarily - # imply a change in the "unexpanded" services. - return sinfo.get('lsrcmd5') != self.serviceinfo.xsrcmd5 - # simple case: unexpanded sources and no services - # self.srcmd5 should also work - return sinfo.get('lsrcmd5') != self.linkinfo.lsrcmd5 - elif self.hasserviceinfo(): - if self.serviceinfo.isexpanded(): - return sinfo.get('srcmd5') != self.srcmd5 - else: - # cannot handle this case, because the sourceinfo does not contain - # information about the lservicemd5. Once the srcserver supports - # a noservice=1 query parameter, we can handle this case. - return True - return sinfo.get('srcmd5') != self.srcmd5 - - def update(self, rev=None, service_files=False, size_limit=None): - rfiles = [] - # size_limit is only temporary for this update - old_size_limit = self.size_limit - if size_limit is not None: - self.size_limit = int(size_limit) - if os.path.isfile(os.path.join(self.storedir, '_in_update', '_files')): - print('resuming broken update...') - root = ET.parse(os.path.join(self.storedir, '_in_update', '_files')).getroot() - rfiles = self.__get_files(root) - kept, added, deleted, services = self.__get_rev_changes(rfiles) - # check if we aborted in the middle of a file update - broken_file = os.listdir(os.path.join(self.storedir, '_in_update')) - broken_file.remove('_files') - if len(broken_file) == 1: - origfile = os.path.join(self.storedir, '_in_update', broken_file[0]) - wcfile = os.path.join(self.absdir, broken_file[0]) - origfile_md5 = dgst(origfile) - origfile_meta = self.findfilebyname(broken_file[0]) - if origfile.endswith('.copy'): - # ok it seems we aborted at some point during the copy process - # (copy process == copy wcfile to the _in_update dir). remove file+continue - os.unlink(origfile) - elif self.findfilebyname(broken_file[0]) is None: - # should we remove this file from _in_update? if we don't - # the user has no chance to continue without removing the file manually - raise oscerr.PackageInternalError(self.prjname, self.name, - '\'%s\' is not known by meta but exists in \'_in_update\' dir') - elif os.path.isfile(wcfile) and dgst(wcfile) != origfile_md5: - (fd, tmpfile) = tempfile.mkstemp(dir=self.absdir, prefix=broken_file[0] + '.') - os.close(fd) - os.rename(wcfile, tmpfile) - os.rename(origfile, wcfile) - print('warning: it seems you modified \'%s\' after the broken ' - 'update. Restored original file and saved modified version ' - 'to \'%s\'.' % (wcfile, tmpfile)) - elif not os.path.isfile(wcfile): - # this is strange... because it existed before the update. restore it - os.rename(origfile, wcfile) - else: - # everything seems to be ok - os.unlink(origfile) - elif len(broken_file) > 1: - raise oscerr.PackageInternalError(self.prjname, self.name, 'too many files in \'_in_update\' dir') - tmp = rfiles[:] - for f in tmp: - if os.path.exists(os.path.join(self.storedir, f.name)): - if dgst(os.path.join(self.storedir, f.name)) == f.md5: - if f in kept: - kept.remove(f) - elif f in added: - added.remove(f) - # this can't happen - elif f in deleted: - deleted.remove(f) - if not service_files: - services = [] - self.__update(kept, added, deleted, services, ET.tostring(root, encoding=ET_ENCODING), root.get('rev')) - os.unlink(os.path.join(self.storedir, '_in_update', '_files')) - os.rmdir(os.path.join(self.storedir, '_in_update')) - # ok everything is ok (hopefully)... - fm = self.get_files_meta(revision=rev) - root = ET.fromstring(fm) - rfiles = self.__get_files(root) - store_write_string(self.absdir, '_files', fm + '\n', subdir='_in_update') - kept, added, deleted, services = self.__get_rev_changes(rfiles) - if not service_files: - services = [] - self.__update(kept, added, deleted, services, fm, root.get('rev')) - os.unlink(os.path.join(self.storedir, '_in_update', '_files')) - if os.path.isdir(os.path.join(self.storedir, '_in_update')): - os.rmdir(os.path.join(self.storedir, '_in_update')) - self.size_limit = old_size_limit - - def __update(self, kept, added, deleted, services, fm, rev): - pathn = getTransActPath(self.dir) - # check for conflicts with existing files - for f in added: - if f.name in self.filenamelist_unvers: - raise oscerr.PackageFileConflict(self.prjname, self.name, f.name, - f'failed to add file \'{f.name}\' file/dir with the same name already exists') - # ok, the update can't fail due to existing files - for f in added: - self.updatefile(f.name, rev, f.mtime) - print(statfrmt('A', os.path.join(pathn, f.name))) - for f in deleted: - # if the storefile doesn't exist we're resuming an aborted update: - # the file was already deleted but we cannot know this - # OR we're processing a _service: file (simply keep the file) - if os.path.isfile(os.path.join(self.storedir, f.name)) and self.status(f.name) not in ('M', 'C'): - # if self.status(f.name) != 'M': - self.delete_localfile(f.name) - self.delete_storefile(f.name) - print(statfrmt('D', os.path.join(pathn, f.name))) - if f.name in self.to_be_deleted: - self.to_be_deleted.remove(f.name) - self.write_deletelist() - elif f.name in self.in_conflict: - self.in_conflict.remove(f.name) - self.write_conflictlist() - - for f in kept: - state = self.status(f.name) -# print f.name, state - if state == 'M' and self.findfilebyname(f.name).md5 == f.md5: - # remote file didn't change - pass - elif state == 'M': - # try to merge changes - merge_status = self.mergefile(f.name, rev, f.mtime) - print(statfrmt(merge_status, os.path.join(pathn, f.name))) - elif state == '!': - self.updatefile(f.name, rev, f.mtime) - print(f'Restored \'{os.path.join(pathn, f.name)}\'') - elif state == 'C': - get_source_file(self.apiurl, self.prjname, self.name, f.name, - targetfilename=os.path.join(self.storedir, f.name), revision=rev, - progress_obj=self.progress_obj, mtime=f.mtime, meta=self.meta) - print(f'skipping \'{f.name}\' (this is due to conflicts)') - elif state == 'D' and self.findfilebyname(f.name).md5 != f.md5: - # XXX: in the worst case we might end up with f.name being - # in _to_be_deleted and in _in_conflict... this needs to be checked - if os.path.exists(os.path.join(self.absdir, f.name)): - merge_status = self.mergefile(f.name, rev, f.mtime) - print(statfrmt(merge_status, os.path.join(pathn, f.name))) - if merge_status == 'C': - # state changes from delete to conflict - self.to_be_deleted.remove(f.name) - self.write_deletelist() - else: - # XXX: we cannot recover this case because we've no file - # to backup - self.updatefile(f.name, rev, f.mtime) - print(statfrmt('U', os.path.join(pathn, f.name))) - elif state == ' ' and self.findfilebyname(f.name).md5 != f.md5: - self.updatefile(f.name, rev, f.mtime) - print(statfrmt('U', os.path.join(pathn, f.name))) - - # checkout service files - for f in services: - get_source_file(self.apiurl, self.prjname, self.name, f.name, - targetfilename=os.path.join(self.absdir, f.name), revision=rev, - progress_obj=self.progress_obj, mtime=f.mtime, meta=self.meta) - print(statfrmt('A', os.path.join(pathn, f.name))) - store_write_string(self.absdir, '_files', fm + '\n') - if not self.meta: - self.update_local_pacmeta() - self.update_datastructs() - - print(f'At revision {self.rev}.') - - def run_source_services(self, mode=None, singleservice=None, verbose=None): - if self.name.startswith("_"): - return 0 - curdir = os.getcwd() - os.chdir(self.absdir) # e.g. /usr/lib/obs/service/verify_file fails if not inside the project dir. - si = Serviceinfo() - if os.path.exists('_service'): - try: - service = ET.parse(os.path.join(self.absdir, '_service')).getroot() - except ET.ParseError as v: - line, column = v.position - print(f'XML error in _service file on line {line}, column {column}') - sys.exit(1) - si.read(service) - si.getProjectGlobalServices(self.apiurl, self.prjname, self.name) - r = si.execute(self.absdir, mode, singleservice, verbose) - os.chdir(curdir) - return r - - def revert(self, filename): - if filename not in self.filenamelist and filename not in self.to_be_added: - raise oscerr.OscIOError(None, f'file \'{filename}\' is not under version control') - elif filename in self.skipped: - raise oscerr.OscIOError(None, f'file \'{filename}\' is marked as skipped and cannot be reverted') - if filename in self.filenamelist and not os.path.exists(os.path.join(self.storedir, filename)): - msg = f"file '{filename}' is listed in filenamelist but no storefile exists" - raise oscerr.PackageInternalError(self.prjname, self.name, msg) - state = self.status(filename) - if not (state == 'A' or state == '!' and filename in self.to_be_added): - shutil.copyfile(os.path.join(self.storedir, filename), os.path.join(self.absdir, filename)) - if state == 'D': - self.to_be_deleted.remove(filename) - self.write_deletelist() - elif state == 'C': - self.clear_from_conflictlist(filename) - elif state in ('A', 'R') or state == '!' and filename in self.to_be_added: - self.to_be_added.remove(filename) - self.write_addlist() - - @staticmethod - def init_package(apiurl: str, project, package, dir, size_limit=None, meta=False, progress_obj=None, scm_url=None): - global store - - if not os.path.exists(dir): - os.mkdir(dir) - elif not os.path.isdir(dir): - raise oscerr.OscIOError(None, f'error: \'{dir}\' is no directory') - if os.path.exists(os.path.join(dir, store)): - raise oscerr.OscIOError(None, f'error: \'{dir}\' is already an initialized osc working copy') - else: - os.mkdir(os.path.join(dir, store)) - store_write_project(dir, project) - store_write_string(dir, '_package', package + '\n') - Store(dir).apiurl = apiurl - if meta: - store_write_string(dir, '_meta_mode', '') - if size_limit: - store_write_string(dir, '_size_limit', str(size_limit) + '\n') - if scm_url: - Store(dir).scmurl = scm_url - else: - store_write_string(dir, '_files', '' + '\n') - store_write_string(dir, '_osclib_version', __store_version__ + '\n') - return Package(dir, progress_obj=progress_obj, size_limit=size_limit) - - class AbstractState: """ Base class which represents state-like objects (````, ````). diff --git a/osc/obs_scm/__init__.py b/osc/obs_scm/__init__.py index f2a3c86e..d7d1b091 100644 --- a/osc/obs_scm/__init__.py +++ b/osc/obs_scm/__init__.py @@ -1,5 +1,6 @@ from .file import File from .linkinfo import Linkinfo +from .package import Package from .project import Project from .serviceinfo import Serviceinfo from .store import Store diff --git a/osc/obs_scm/package.py b/osc/obs_scm/package.py new file mode 100644 index 00000000..36c06815 --- /dev/null +++ b/osc/obs_scm/package.py @@ -0,0 +1,1588 @@ +import difflib +import fnmatch +import glob +import shutil +import os +import sys +import tempfile +from functools import total_ordering +from typing import Optional + +from .. import conf +from .. import oscerr +from ..util.xml import ET +from .file import File +from .linkinfo import Linkinfo +from .serviceinfo import Serviceinfo +from .store import __store_version__ +from .store import Store +from .store import read_inconflict +from .store import read_filemeta +from .store import read_sizelimit +from .store import read_tobeadded +from .store import read_tobedeleted +from .store import store +from .store import store_read_file +from .store import store_write_project +from .store import store_write_string + + +@total_ordering +class Package: + """represent a package (its directory) and read/keep/write its metadata""" + + # should _meta be a required file? + REQ_STOREFILES = ('_project', '_package', '_apiurl', '_files', '_osclib_version') + OPT_STOREFILES = ('_to_be_added', '_to_be_deleted', '_in_conflict', '_in_update', + '_in_commit', '_meta', '_meta_mode', '_frozenlink', '_pulled', '_linkrepair', + '_size_limit', '_commit_msg', '_last_buildroot') + + def __init__(self, workingdir, progress_obj=None, size_limit=None, wc_check=True): + from .. import store as osc_store + + global store + + self.todo = [] + if os.path.isfile(workingdir) or not os.path.exists(workingdir): + # workingdir is a file + # workingdir doesn't exist -> it points to a non-existing file in a working dir (e.g. during mv) + workingdir, todo_entry = os.path.split(workingdir) + self.todo.append(todo_entry) + + self.dir = workingdir or "." + self.absdir = os.path.abspath(self.dir) + self.store = osc_store.get_store(self.dir) + self.store.assert_is_package() + self.storedir = os.path.join(self.absdir, store) + self.progress_obj = progress_obj + self.size_limit = size_limit + self.scm_url = self.store.scmurl + if size_limit and size_limit == 0: + self.size_limit = None + + self.prjname = self.store.project + self.name = self.store.package + self.apiurl = self.store.apiurl + + self.update_datastructs() + dirty_files = [] + if wc_check: + dirty_files = self.wc_check() + if dirty_files: + msg = 'Your working copy \'%s\' is in an inconsistent state.\n' \ + 'Please run \'osc repairwc %s\' (Note this might _remove_\n' \ + 'files from the .osc/ dir). Please check the state\n' \ + 'of the working copy afterwards (via \'osc status %s\')' % (self.dir, self.dir, self.dir) + raise oscerr.WorkingCopyInconsistent(self.prjname, self.name, dirty_files, msg) + + def __repr__(self): + return super().__repr__() + f"({self.prjname}/{self.name})" + + def __hash__(self): + return hash((self.name, self.prjname, self.apiurl)) + + def __eq__(self, other): + return (self.name, self.prjname, self.apiurl) == (other.name, other.prjname, other.apiurl) + + def __lt__(self, other): + return (self.name, self.prjname, self.apiurl) < (other.name, other.prjname, other.apiurl) + + @classmethod + def from_paths(cls, paths, progress_obj=None): + """ + Return a list of Package objects from working copies in given paths. + """ + packages = [] + for path in paths: + package = cls(path, progress_obj) + seen_package = None + try: + # re-use an existing package + seen_package_index = packages.index(package) + seen_package = packages[seen_package_index] + except ValueError: + pass + + if seen_package: + # merge package into seen_package + if seen_package.absdir != package.absdir: + raise oscerr.PackageExists(package.prjname, package.name, "Duplicate package") + seen_package.merge(package) + else: + # use the new package instance + packages.append(package) + + return packages + + @classmethod + def from_paths_nofail(cls, paths, progress_obj=None): + """ + Return a list of Package objects from working copies in given paths + and a list of strings with paths that do not contain Package working copies. + """ + packages = [] + failed_to_load = [] + for path in paths: + try: + package = cls(path, progress_obj) + except oscerr.NoWorkingCopy: + failed_to_load.append(path) + continue + + # the following code is identical to from_paths() + seen_package = None + try: + # re-use an existing package + seen_package_index = packages.index(package) + seen_package = packages[seen_package_index] + except ValueError: + pass + + if seen_package: + # merge package into seen_package + if seen_package.absdir != package.absdir: + raise oscerr.PackageExists(package.prjname, package.name, "Duplicate package") + seen_package.merge(package) + else: + # use the new package instance + packages.append(package) + + return packages, failed_to_load + + def wc_check(self): + dirty_files = [] + if self.scm_url: + return dirty_files + for fname in self.filenamelist: + if not os.path.exists(os.path.join(self.storedir, fname)) and fname not in self.skipped: + dirty_files.append(fname) + for fname in Package.REQ_STOREFILES: + if not os.path.isfile(os.path.join(self.storedir, fname)): + dirty_files.append(fname) + for fname in os.listdir(self.storedir): + if fname in Package.REQ_STOREFILES or fname in Package.OPT_STOREFILES or \ + fname.startswith('_build'): + continue + elif fname in self.filenamelist and fname in self.skipped: + dirty_files.append(fname) + elif fname not in self.filenamelist: + dirty_files.append(fname) + for fname in self.to_be_deleted[:]: + if fname not in self.filenamelist: + dirty_files.append(fname) + for fname in self.in_conflict[:]: + if fname not in self.filenamelist: + dirty_files.append(fname) + return dirty_files + + def wc_repair(self, apiurl: Optional[str] = None): + from ..core import get_source_file + + store = Store(self.dir) + store.assert_is_package() + if not store.exists("_apiurl") or apiurl: + if apiurl is None: + msg = 'cannot repair wc: the \'_apiurl\' file is missing but ' \ + 'no \'apiurl\' was passed to wc_repair' + # hmm should we raise oscerr.WrongArgs? + raise oscerr.WorkingCopyInconsistent(self.prjname, self.name, [], msg) + # sanity check + conf.parse_apisrv_url(None, apiurl) + store.apiurl = apiurl + self.apiurl = apiurl + + # all files which are present in the filelist have to exist in the storedir + for f in self.filelist: + # XXX: should we also check the md5? + if not os.path.exists(os.path.join(self.storedir, f.name)) and f.name not in self.skipped: + # if get_source_file fails we're screwed up... + get_source_file(self.apiurl, self.prjname, self.name, f.name, + targetfilename=os.path.join(self.storedir, f.name), revision=self.rev, + mtime=f.mtime) + + for fname in store: + if fname in Package.REQ_STOREFILES or fname in Package.OPT_STOREFILES or \ + fname.startswith('_build'): + continue + elif fname not in self.filenamelist or fname in self.skipped: + # this file does not belong to the storedir so remove it + store.unlink(fname) + + for fname in self.to_be_deleted[:]: + if fname not in self.filenamelist: + self.to_be_deleted.remove(fname) + self.write_deletelist() + + for fname in self.in_conflict[:]: + if fname not in self.filenamelist: + self.in_conflict.remove(fname) + self.write_conflictlist() + + def info(self): + from ..core import info_templ + from ..core import makeurl + + source_url = makeurl(self.apiurl, ['source', self.prjname, self.name]) + r = info_templ % (self.prjname, self.name, self.absdir, self.apiurl, source_url, self.srcmd5, self.rev, self.linkinfo) + return r + + def addfile(self, n): + from ..core import statfrmt + + if not os.path.exists(os.path.join(self.absdir, n)): + raise oscerr.OscIOError(None, f'error: file \'{n}\' does not exist') + if n in self.to_be_deleted: + self.to_be_deleted.remove(n) +# self.delete_storefile(n) + self.write_deletelist() + elif n in self.filenamelist or n in self.to_be_added: + raise oscerr.PackageFileConflict(self.prjname, self.name, n, f'osc: warning: \'{n}\' is already under version control') +# shutil.copyfile(os.path.join(self.dir, n), os.path.join(self.storedir, n)) + if self.dir != '.': + pathname = os.path.join(self.dir, n) + else: + pathname = n + self.to_be_added.append(n) + self.write_addlist() + print(statfrmt('A', pathname)) + + def delete_file(self, n, force=False): + """deletes a file if possible and marks the file as deleted""" + state = '?' + try: + state = self.status(n) + except OSError as ioe: + if not force: + raise ioe + if state in ['?', 'A', 'M', 'R', 'C'] and not force: + return (False, state) + # special handling for skipped files: if file exists, simply delete it + if state == 'S': + exists = os.path.exists(os.path.join(self.dir, n)) + self.delete_localfile(n) + return (exists, 'S') + + self.delete_localfile(n) + was_added = n in self.to_be_added + if state in ('A', 'R') or state == '!' and was_added: + self.to_be_added.remove(n) + self.write_addlist() + elif state == 'C': + # don't remove "merge files" (*.mine, *.new...) + # that's why we don't use clear_from_conflictlist + self.in_conflict.remove(n) + self.write_conflictlist() + if state not in ('A', '?') and not (state == '!' and was_added): + self.put_on_deletelist(n) + self.write_deletelist() + return (True, state) + + def delete_storefile(self, n): + try: + os.unlink(os.path.join(self.storedir, n)) + except: + pass + + def delete_localfile(self, n): + try: + os.unlink(os.path.join(self.dir, n)) + except: + pass + + def put_on_deletelist(self, n): + if n not in self.to_be_deleted: + self.to_be_deleted.append(n) + + def put_on_conflictlist(self, n): + if n not in self.in_conflict: + self.in_conflict.append(n) + + def put_on_addlist(self, n): + if n not in self.to_be_added: + self.to_be_added.append(n) + + def clear_from_conflictlist(self, n): + """delete an entry from the file, and remove the file if it would be empty""" + if n in self.in_conflict: + + filename = os.path.join(self.dir, n) + storefilename = os.path.join(self.storedir, n) + myfilename = os.path.join(self.dir, n + '.mine') + upfilename = os.path.join(self.dir, n + '.new') + + try: + os.unlink(myfilename) + os.unlink(upfilename) + if self.islinkrepair() or self.ispulled(): + os.unlink(os.path.join(self.dir, n + '.old')) + except: + pass + + self.in_conflict.remove(n) + + self.write_conflictlist() + + # XXX: this isn't used at all + def write_meta_mode(self): + # XXX: the "elif" is somehow a contradiction (with current and the old implementation + # it's not possible to "leave" the metamode again) (except if you modify pac.meta + # which is really ugly:) ) + if self.meta: + store_write_string(self.absdir, '_meta_mode', '') + elif self.ismetamode(): + os.unlink(os.path.join(self.storedir, '_meta_mode')) + + def write_sizelimit(self): + if self.size_limit and self.size_limit <= 0: + try: + os.unlink(os.path.join(self.storedir, '_size_limit')) + except: + pass + else: + store_write_string(self.absdir, '_size_limit', str(self.size_limit) + '\n') + + def write_addlist(self): + self.__write_storelist('_to_be_added', self.to_be_added) + + def write_deletelist(self): + self.__write_storelist('_to_be_deleted', self.to_be_deleted) + + def delete_source_file(self, n): + """delete local a source file""" + self.delete_localfile(n) + self.delete_storefile(n) + + def delete_remote_source_file(self, n): + """delete a remote source file (e.g. from the server)""" + from ..core import http_DELETE + from ..core import makeurl + + query = {"rev": "upload"} + u = makeurl(self.apiurl, ['source', self.prjname, self.name, n], query=query) + http_DELETE(u) + + def put_source_file(self, n, tdir, copy_only=False): + from ..core import http_PUT + from ..core import makeurl + + query = {"rev": "repository"} + tfilename = os.path.join(tdir, n) + shutil.copyfile(os.path.join(self.dir, n), tfilename) + # escaping '+' in the URL path (note: not in the URL query string) is + # only a workaround for ruby on rails, which swallows it otherwise + if not copy_only: + u = makeurl(self.apiurl, ['source', self.prjname, self.name, n], query=query) + http_PUT(u, file=tfilename) + if n in self.to_be_added: + self.to_be_added.remove(n) + + def __commit_update_store(self, tdir): + """move files from transaction directory into the store""" + for filename in os.listdir(tdir): + os.rename(os.path.join(tdir, filename), os.path.join(self.storedir, filename)) + + def __generate_commitlist(self, todo_send): + root = ET.Element('directory') + for i in sorted(todo_send.keys()): + ET.SubElement(root, 'entry', name=i, md5=todo_send[i]) + return root + + @staticmethod + def commit_filelist(apiurl: str, project: str, package: str, filelist, msg="", user=None, **query): + """send the commitlog and the local filelist to the server""" + from ..core import ET_ENCODING + from ..core import http_POST + from ..core import makeurl + + if user is None: + user = conf.get_apiurl_usr(apiurl) + query.update({'cmd': 'commitfilelist', 'user': user, 'comment': msg}) + u = makeurl(apiurl, ['source', project, package], query=query) + f = http_POST(u, data=ET.tostring(filelist, encoding=ET_ENCODING)) + root = ET.parse(f).getroot() + return root + + @staticmethod + def commit_get_missing(filelist): + """returns list of missing files (filelist is the result of commit_filelist)""" + from ..core import ET_ENCODING + + error = filelist.get('error') + if error is None: + return [] + elif error != 'missing': + raise oscerr.APIError('commit_get_missing_files: ' + 'unexpected \'error\' attr: \'%s\'' % error) + todo = [] + for n in filelist.findall('entry'): + name = n.get('name') + if name is None: + raise oscerr.APIError('missing \'name\' attribute:\n%s\n' + % ET.tostring(filelist, encoding=ET_ENCODING)) + todo.append(n.get('name')) + return todo + + def __send_commitlog(self, msg, local_filelist, validate=False): + """send the commitlog and the local filelist to the server""" + query = {} + if self.islink() and self.isexpanded(): + query['keeplink'] = '1' + if conf.config['linkcontrol'] or self.isfrozen(): + query['linkrev'] = self.linkinfo.srcmd5 + if self.ispulled(): + query['repairlink'] = '1' + query['linkrev'] = self.get_pulled_srcmd5() + if self.islinkrepair(): + query['repairlink'] = '1' + if validate: + query['withvalidate'] = '1' + return self.commit_filelist(self.apiurl, self.prjname, self.name, + local_filelist, msg, **query) + + def commit(self, msg='', verbose=False, skip_local_service_run=False, can_branch=False, force=False): + from ..core import ET_ENCODING + from ..core import branch_pkg + from ..core import dgst + from ..core import getTransActPath + from ..core import http_GET + from ..core import makeurl + from ..core import print_request_list + from ..core import sha256_dgst + from ..core import statfrmt + + # commit only if the upstream revision is the same as the working copy's + upstream_rev = self.latest_rev() + if self.rev != upstream_rev: + raise oscerr.WorkingCopyOutdated((self.absdir, self.rev, upstream_rev)) + + if not skip_local_service_run: + r = self.run_source_services(mode="trylocal", verbose=verbose) + if r != 0: + # FIXME: it is better to raise this in Serviceinfo.execute with more + # information (like which service/command failed) + raise oscerr.ServiceRuntimeError('A service failed with error: %d' % r) + + # check if it is a link, if so, branch the package + if self.is_link_to_different_project(): + if can_branch: + orgprj = self.get_local_origin_project() + print(f"Branching {self.name} from {orgprj} to {self.prjname}") + exists, targetprj, targetpkg, srcprj, srcpkg = branch_pkg( + self.apiurl, orgprj, self.name, target_project=self.prjname) + # update _meta and _files to sychronize the local package + # to the new branched one in OBS + self.update_local_pacmeta() + self.update_local_filesmeta() + else: + print(f"{self.name} Not commited because is link to a different project") + return 1 + + if not self.todo: + self.todo = [i for i in self.to_be_added if i not in self.filenamelist] + self.filenamelist + + pathn = getTransActPath(self.dir) + + todo_send = {} + todo_delete = [] + real_send = [] + sha256sums = {} + for filename in self.filenamelist + [i for i in self.to_be_added if i not in self.filenamelist]: + if filename.startswith('_service:') or filename.startswith('_service_'): + continue + st = self.status(filename) + if st == 'C': + print('Please resolve all conflicts before committing using "osc resolved FILE"!') + return 1 + elif filename in self.todo: + if st in ('A', 'R', 'M'): + todo_send[filename] = dgst(os.path.join(self.absdir, filename)) + sha256sums[filename] = sha256_dgst(os.path.join(self.absdir, filename)) + real_send.append(filename) + print(statfrmt('Sending', os.path.join(pathn, filename))) + elif st in (' ', '!', 'S'): + if st == '!' and filename in self.to_be_added: + print(f'file \'{filename}\' is marked as \'A\' but does not exist') + return 1 + f = self.findfilebyname(filename) + if f is None: + raise oscerr.PackageInternalError(self.prjname, self.name, + 'error: file \'%s\' with state \'%s\' is not known by meta' + % (filename, st)) + todo_send[filename] = f.md5 + elif st == 'D': + todo_delete.append(filename) + print(statfrmt('Deleting', os.path.join(pathn, filename))) + elif st in ('R', 'M', 'D', ' ', '!', 'S'): + # ignore missing new file (it's not part of the current commit) + if st == '!' and filename in self.to_be_added: + continue + f = self.findfilebyname(filename) + if f is None: + raise oscerr.PackageInternalError(self.prjname, self.name, + 'error: file \'%s\' with state \'%s\' is not known by meta' + % (filename, st)) + todo_send[filename] = f.md5 + if ((self.ispulled() or self.islinkrepair() or self.isfrozen()) + and st != 'A' and filename not in sha256sums): + # Ignore files with state 'A': if we should consider it, + # it would have been in pac.todo, which implies that it is + # in sha256sums. + # The storefile is guaranteed to exist (since we have a + # pulled/linkrepair wc, the file cannot have state 'S') + storefile = os.path.join(self.storedir, filename) + sha256sums[filename] = sha256_dgst(storefile) + + if not force and not real_send and not todo_delete and not self.islinkrepair() and not self.ispulled(): + print(f'nothing to do for package {self.name}') + return 1 + + print('Transmitting file data', end=' ') + filelist = self.__generate_commitlist(todo_send) + sfilelist = self.__send_commitlog(msg, filelist, validate=True) + hash_entries = [e for e in sfilelist.findall('entry') if e.get('hash') is not None] + if sfilelist.get('error') and hash_entries: + name2elem = {e.get('name'): e for e in filelist.findall('entry')} + for entry in hash_entries: + filename = entry.get('name') + fileelem = name2elem.get(filename) + if filename not in sha256sums: + msg = 'There is no sha256 sum for file %s.\n' \ + 'This could be due to an outdated working copy.\n' \ + 'Please update your working copy with osc update and\n' \ + 'commit again afterwards.' + print(msg % filename) + return 1 + fileelem.set('hash', f'sha256:{sha256sums[filename]}') + sfilelist = self.__send_commitlog(msg, filelist) + send = self.commit_get_missing(sfilelist) + real_send = [i for i in real_send if i not in send] + # abort after 3 tries + tries = 3 + tdir = None + try: + tdir = os.path.join(self.storedir, '_in_commit') + if os.path.isdir(tdir): + shutil.rmtree(tdir) + os.mkdir(tdir) + while send and tries: + for filename in send[:]: + sys.stdout.write('.') + sys.stdout.flush() + self.put_source_file(filename, tdir) + send.remove(filename) + tries -= 1 + sfilelist = self.__send_commitlog(msg, filelist) + send = self.commit_get_missing(sfilelist) + if send: + raise oscerr.PackageInternalError(self.prjname, self.name, + 'server does not accept filelist:\n%s\nmissing:\n%s\n' + % (ET.tostring(filelist, encoding=ET_ENCODING), ET.tostring(sfilelist, encoding=ET_ENCODING))) + # these files already exist on the server + for filename in real_send: + self.put_source_file(filename, tdir, copy_only=True) + # update store with the committed files + self.__commit_update_store(tdir) + finally: + if tdir is not None and os.path.isdir(tdir): + shutil.rmtree(tdir) + self.rev = sfilelist.get('rev') + print() + print(f'Committed revision {self.rev}.') + + if self.ispulled(): + os.unlink(os.path.join(self.storedir, '_pulled')) + if self.islinkrepair(): + os.unlink(os.path.join(self.storedir, '_linkrepair')) + self.linkrepair = False + # XXX: mark package as invalid? + print('The source link has been repaired. This directory can now be removed.') + + if self.islink() and self.isexpanded(): + li = Linkinfo() + li.read(sfilelist.find('linkinfo')) + if li.xsrcmd5 is None: + raise oscerr.APIError(f'linkinfo has no xsrcmd5 attr:\n{ET.tostring(sfilelist, encoding=ET_ENCODING)}\n') + sfilelist = ET.fromstring(self.get_files_meta(revision=li.xsrcmd5)) + for i in sfilelist.findall('entry'): + if i.get('name') in self.skipped: + i.set('skipped', 'true') + store_write_string(self.absdir, '_files', ET.tostring(sfilelist, encoding=ET_ENCODING) + '\n') + for filename in todo_delete: + self.to_be_deleted.remove(filename) + self.delete_storefile(filename) + self.write_deletelist() + self.write_addlist() + self.update_datastructs() + + print_request_list(self.apiurl, self.prjname, self.name) + + # FIXME: add testcases for this codepath + sinfo = sfilelist.find('serviceinfo') + if sinfo is not None: + print('Waiting for server side source service run') + u = makeurl(self.apiurl, ['source', self.prjname, self.name]) + while sinfo is not None and sinfo.get('code') == 'running': + sys.stdout.write('.') + sys.stdout.flush() + # does it make sense to add some delay? + sfilelist = ET.fromstring(http_GET(u).read()) + # if sinfo is None another commit might have occured in the "meantime" + sinfo = sfilelist.find('serviceinfo') + print('') + rev = self.latest_rev() + self.update(rev=rev) + elif self.get_local_meta() is None: + # if this was a newly added package there is no _meta + # file + self.update_local_pacmeta() + + def __write_storelist(self, name, data): + if len(data) == 0: + try: + os.unlink(os.path.join(self.storedir, name)) + except: + pass + else: + store_write_string(self.absdir, name, '%s\n' % '\n'.join(data)) + + def write_conflictlist(self): + self.__write_storelist('_in_conflict', self.in_conflict) + + def updatefile(self, n, revision, mtime=None): + from ..core import get_source_file + from ..core import utime + + filename = os.path.join(self.dir, n) + storefilename = os.path.join(self.storedir, n) + origfile_tmp = os.path.join(self.storedir, '_in_update', f'{n}.copy') + origfile = os.path.join(self.storedir, '_in_update', n) + if os.path.isfile(filename): + shutil.copyfile(filename, origfile_tmp) + os.rename(origfile_tmp, origfile) + else: + origfile = None + + get_source_file(self.apiurl, self.prjname, self.name, n, targetfilename=storefilename, + revision=revision, progress_obj=self.progress_obj, mtime=mtime, meta=self.meta) + + shutil.copyfile(storefilename, filename) + if mtime: + utime(filename, (-1, mtime)) + if origfile is not None: + os.unlink(origfile) + + def mergefile(self, n, revision, mtime=None): + from ..core import binary_file + from ..core import get_source_file + from ..core import run_external + + filename = os.path.join(self.dir, n) + storefilename = os.path.join(self.storedir, n) + myfilename = os.path.join(self.dir, n + '.mine') + upfilename = os.path.join(self.dir, n + '.new') + origfile_tmp = os.path.join(self.storedir, '_in_update', f'{n}.copy') + origfile = os.path.join(self.storedir, '_in_update', n) + shutil.copyfile(filename, origfile_tmp) + os.rename(origfile_tmp, origfile) + os.rename(filename, myfilename) + + get_source_file(self.apiurl, self.prjname, self.name, n, + revision=revision, targetfilename=upfilename, + progress_obj=self.progress_obj, mtime=mtime, meta=self.meta) + + if binary_file(myfilename) or binary_file(upfilename): + # don't try merging + shutil.copyfile(upfilename, filename) + shutil.copyfile(upfilename, storefilename) + os.unlink(origfile) + self.in_conflict.append(n) + self.write_conflictlist() + return 'C' + else: + # try merging + # diff3 OPTIONS... MINE OLDER YOURS + ret = -1 + with open(filename, 'w') as f: + args = ('-m', '-E', myfilename, storefilename, upfilename) + ret = run_external('diff3', *args, stdout=f) + + # "An exit status of 0 means `diff3' was successful, 1 means some + # conflicts were found, and 2 means trouble." + if ret == 0: + # merge was successful... clean up + shutil.copyfile(upfilename, storefilename) + os.unlink(upfilename) + os.unlink(myfilename) + os.unlink(origfile) + return 'G' + elif ret == 1: + # unsuccessful merge + shutil.copyfile(upfilename, storefilename) + os.unlink(origfile) + self.in_conflict.append(n) + self.write_conflictlist() + return 'C' + else: + merge_cmd = 'diff3 ' + ' '.join(args) + raise oscerr.ExtRuntimeError(f'diff3 failed with exit code: {ret}', merge_cmd) + + def update_local_filesmeta(self, revision=None): + """ + Update the local _files file in the store. + It is replaced with the version pulled from upstream. + """ + meta = self.get_files_meta(revision=revision) + store_write_string(self.absdir, '_files', meta + '\n') + + def get_files_meta(self, revision='latest', skip_service=True): + from ..core import ET_ENCODING + from ..core import show_files_meta + + fm = show_files_meta(self.apiurl, self.prjname, self.name, revision=revision, meta=self.meta) + # look for "too large" files according to size limit and mark them + root = ET.fromstring(fm) + for e in root.findall('entry'): + size = e.get('size') + if size and self.size_limit and int(size) > self.size_limit \ + or skip_service and (e.get('name').startswith('_service:') or e.get('name').startswith('_service_')): + e.set('skipped', 'true') + continue + + if conf.config["exclude_files"]: + exclude = False + for pattern in conf.config["exclude_files"]: + if fnmatch.fnmatch(e.get("name"), pattern): + exclude = True + break + if exclude: + e.set("skipped", "true") + continue + + if conf.config["include_files"]: + include = False + for pattern in conf.config["include_files"]: + if fnmatch.fnmatch(e.get("name"), pattern): + include = True + break + if not include: + e.set("skipped", "true") + continue + + return ET.tostring(root, encoding=ET_ENCODING) + + def get_local_meta(self): + """Get the local _meta file for the package.""" + meta = store_read_file(self.absdir, '_meta') + return meta + + def get_local_origin_project(self): + """Get the originproject from the _meta file.""" + # if the wc was checked out via some old osc version + # there might be no meta file: in this case we assume + # that the origin project is equal to the wc's project + meta = self.get_local_meta() + if meta is None: + return self.prjname + root = ET.fromstring(meta) + return root.get('project') + + def is_link_to_different_project(self): + """Check if the package is a link to a different project.""" + if self.name == "_project": + return False + orgprj = self.get_local_origin_project() + return self.prjname != orgprj + + def update_datastructs(self): + """ + Update the internal data structures if the local _files + file has changed (e.g. update_local_filesmeta() has been + called). + """ + from ..core import DirectoryServiceinfo + + if self.scm_url: + self.filenamelist = [] + self.filelist = [] + self.skipped = [] + self.to_be_added = [] + self.to_be_deleted = [] + self.in_conflict = [] + self.linkrepair = None + self.rev = None + self.srcmd5 = None + self.linkinfo = Linkinfo() + self.serviceinfo = DirectoryServiceinfo() + self.size_limit = None + self.meta = None + self.excluded = [] + self.filenamelist_unvers = [] + return + + files_tree = read_filemeta(self.dir) + files_tree_root = files_tree.getroot() + + self.rev = files_tree_root.get('rev') + self.srcmd5 = files_tree_root.get('srcmd5') + + self.linkinfo = Linkinfo() + self.linkinfo.read(files_tree_root.find('linkinfo')) + self.serviceinfo = DirectoryServiceinfo() + self.serviceinfo.read(files_tree_root.find('serviceinfo')) + self.filenamelist = [] + self.filelist = [] + self.skipped = [] + + for node in files_tree_root.findall('entry'): + try: + f = File(node.get('name'), + node.get('md5'), + int(node.get('size')), + int(node.get('mtime'))) + if node.get('skipped'): + self.skipped.append(f.name) + f.skipped = True + except: + # okay, a very old version of _files, which didn't contain any metadata yet... + f = File(node.get('name'), '', 0, 0) + self.filelist.append(f) + self.filenamelist.append(f.name) + + self.to_be_added = read_tobeadded(self.absdir) + self.to_be_deleted = read_tobedeleted(self.absdir) + self.in_conflict = read_inconflict(self.absdir) + self.linkrepair = os.path.isfile(os.path.join(self.storedir, '_linkrepair')) + self.size_limit = read_sizelimit(self.dir) + self.meta = self.ismetamode() + + # gather unversioned files, but ignore some stuff + self.excluded = [] + for i in os.listdir(self.dir): + for j in conf.config['exclude_glob']: + if fnmatch.fnmatch(i, j): + self.excluded.append(i) + break + self.filenamelist_unvers = [i for i in os.listdir(self.dir) + if i not in self.excluded + if i not in self.filenamelist] + + def islink(self): + """tells us if the package is a link (has 'linkinfo'). + A package with linkinfo is a package which links to another package. + Returns ``True`` if the package is a link, otherwise ``False``.""" + return self.linkinfo.islink() + + def isexpanded(self): + """tells us if the package is a link which is expanded. + Returns ``True`` if the package is expanded, otherwise ``False``.""" + return self.linkinfo.isexpanded() + + def islinkrepair(self): + """tells us if we are repairing a broken source link.""" + return self.linkrepair + + def ispulled(self): + """tells us if we have pulled a link.""" + return os.path.isfile(os.path.join(self.storedir, '_pulled')) + + def isfrozen(self): + """tells us if the link is frozen.""" + return os.path.isfile(os.path.join(self.storedir, '_frozenlink')) + + def ismetamode(self): + """tells us if the package is in meta mode""" + return os.path.isfile(os.path.join(self.storedir, '_meta_mode')) + + def get_pulled_srcmd5(self): + pulledrev = None + for line in open(os.path.join(self.storedir, '_pulled')): + pulledrev = line.strip() + return pulledrev + + def haslinkerror(self): + """ + Returns ``True`` if the link is broken otherwise ``False``. + If the package is not a link it returns ``False``. + """ + return self.linkinfo.haserror() + + def linkerror(self): + """ + Returns an error message if the link is broken otherwise ``None``. + If the package is not a link it returns ``None``. + """ + return self.linkinfo.error + + def hasserviceinfo(self): + """ + Returns ``True``, if this package contains services. + """ + return self.serviceinfo.lsrcmd5 is not None or self.serviceinfo.xsrcmd5 is not None + + def update_local_pacmeta(self): + """ + Update the local _meta file in the store. + It is replaced with the version pulled from upstream. + """ + from ..core import show_package_meta + + meta = show_package_meta(self.apiurl, self.prjname, self.name) + if meta != "": + # is empty for _project for example + meta = b''.join(meta) + store_write_string(self.absdir, '_meta', meta + b'\n') + + def findfilebyname(self, n): + for i in self.filelist: + if i.name == n: + return i + + def get_status(self, excluded=False, *exclude_states): + global store + todo = self.todo + if not todo: + todo = self.filenamelist + self.to_be_added + \ + [i for i in self.filenamelist_unvers if not os.path.isdir(os.path.join(self.absdir, i))] + if excluded: + todo.extend([i for i in self.excluded if i != store]) + todo = set(todo) + res = [] + for fname in sorted(todo): + st = self.status(fname) + if st not in exclude_states: + res.append((st, fname)) + return res + + def status(self, n): + """ + status can be:: + + file storefile file present STATUS + exists exists in _files + x - - 'A' and listed in _to_be_added + x x - 'R' and listed in _to_be_added + x x x ' ' if digest differs: 'M' + and if in conflicts file: 'C' + x - - '?' + - x x 'D' and listed in _to_be_deleted + x x x 'D' and listed in _to_be_deleted (e.g. if deleted file was modified) + x x x 'C' and listed in _in_conflict + x - x 'S' and listed in self.skipped + - - x 'S' and listed in self.skipped + - x x '!' + - - - NOT DEFINED + """ + from ..core import dgst + + known_by_meta = False + exists = False + exists_in_store = False + localfile = os.path.join(self.absdir, n) + if n in self.filenamelist: + known_by_meta = True + if os.path.exists(localfile): + exists = True + if os.path.exists(os.path.join(self.storedir, n)): + exists_in_store = True + + if n in self.to_be_deleted: + state = 'D' + elif n in self.in_conflict: + state = 'C' + elif n in self.skipped: + state = 'S' + elif n in self.to_be_added and exists and exists_in_store: + state = 'R' + elif n in self.to_be_added and exists: + state = 'A' + elif exists and exists_in_store and known_by_meta: + filemeta = self.findfilebyname(n) + state = ' ' + if conf.config['status_mtime_heuristic']: + if os.path.getmtime(localfile) != filemeta.mtime and dgst(localfile) != filemeta.md5: + state = 'M' + elif dgst(localfile) != filemeta.md5: + state = 'M' + elif n in self.to_be_added and not exists: + state = '!' + elif not exists and exists_in_store and known_by_meta and n not in self.to_be_deleted: + state = '!' + elif exists and not exists_in_store and not known_by_meta: + state = '?' + elif not exists_in_store and known_by_meta: + # XXX: this codepath shouldn't be reached (we restore the storefile + # in update_datastructs) + raise oscerr.PackageInternalError(self.prjname, self.name, + 'error: file \'%s\' is known by meta but no storefile exists.\n' + 'This might be caused by an old wc format. Please backup your current\n' + 'wc and checkout the package again. Afterwards copy all files (except the\n' + '.osc/ dir) into the new package wc.' % n) + elif os.path.islink(localfile): + # dangling symlink, whose name is _not_ tracked: treat it + # as unversioned + state = '?' + else: + # this case shouldn't happen (except there was a typo in the filename etc.) + raise oscerr.OscIOError(None, f'osc: \'{n}\' is not under version control') + + return state + + def get_diff(self, revision=None, ignoreUnversioned=False): + from ..core import binary_file + from ..core import get_source_file + from ..core import get_source_file_diff + from ..core import revision_is_empty + + diff_hdr = b'Index: %s\n' + diff_hdr += b'===================================================================\n' + kept = [] + added = [] + deleted = [] + + def diff_add_delete(fname, add, revision): + diff = [] + diff.append(diff_hdr % fname.encode()) + origname = fname + if add: + diff.append(b'--- %s\t(revision 0)\n' % fname.encode()) + rev = 'revision 0' + if not revision_is_empty(revision) and fname not in self.to_be_added: + rev = 'working copy' + diff.append(b'+++ %s\t(%s)\n' % (fname.encode(), rev.encode())) + fname = os.path.join(self.absdir, fname) + if not os.path.isfile(fname): + raise oscerr.OscIOError(None, 'file \'%s\' is marked as \'A\' but does not exist\n' + '(either add the missing file or revert it)' % fname) + else: + if not revision_is_empty(revision): + b_revision = str(revision).encode() + else: + b_revision = self.rev.encode() + diff.append(b'--- %s\t(revision %s)\n' % (fname.encode(), b_revision)) + diff.append(b'+++ %s\t(working copy)\n' % fname.encode()) + fname = os.path.join(self.storedir, fname) + + fd = None + tmpfile = None + try: + if not revision_is_empty(revision) and not add: + (fd, tmpfile) = tempfile.mkstemp(prefix='osc_diff') + get_source_file(self.apiurl, self.prjname, self.name, origname, tmpfile, revision) + fname = tmpfile + if binary_file(fname): + what = b'added' + if not add: + what = b'deleted' + diff = diff[:1] + diff.append(b'Binary file \'%s\' %s.\n' % (origname.encode(), what)) + return diff + tmpl = b'+%s' + ltmpl = b'@@ -0,0 +1,%d @@\n' + if not add: + tmpl = b'-%s' + ltmpl = b'@@ -1,%d +0,0 @@\n' + with open(fname, 'rb') as f: + lines = [tmpl % i for i in f.readlines()] + if len(lines): + diff.append(ltmpl % len(lines)) + if not lines[-1].endswith(b'\n'): + lines.append(b'\n\\ No newline at end of file\n') + diff.extend(lines) + finally: + if fd is not None: + os.close(fd) + if tmpfile is not None and os.path.exists(tmpfile): + os.unlink(tmpfile) + return diff + + if revision is None: + todo = self.todo or [i for i in self.filenamelist if i not in self.to_be_added] + self.to_be_added + for fname in todo: + if fname in self.to_be_added and self.status(fname) == 'A': + added.append(fname) + elif fname in self.to_be_deleted: + deleted.append(fname) + elif fname in self.filenamelist: + kept.append(self.findfilebyname(fname)) + elif fname in self.to_be_added and self.status(fname) == '!': + raise oscerr.OscIOError(None, 'file \'%s\' is marked as \'A\' but does not exist\n' + '(either add the missing file or revert it)' % fname) + elif not ignoreUnversioned: + raise oscerr.OscIOError(None, f'file \'{fname}\' is not under version control') + else: + fm = self.get_files_meta(revision=revision) + root = ET.fromstring(fm) + rfiles = self.__get_files(root) + # swap added and deleted + kept, deleted, added, services = self.__get_rev_changes(rfiles) + added = [f.name for f in added] + added.extend([f for f in self.to_be_added if f not in kept]) + deleted = [f.name for f in deleted] + deleted.extend(self.to_be_deleted) + for f in added[:]: + if f in deleted: + added.remove(f) + deleted.remove(f) +# print kept, added, deleted + for f in kept: + state = self.status(f.name) + if state in ('S', '?', '!'): + continue + elif state == ' ' and revision is None: + continue + elif not revision_is_empty(revision) and self.findfilebyname(f.name).md5 == f.md5 and state != 'M': + continue + yield [diff_hdr % f.name.encode()] + if revision is None: + yield get_source_file_diff(self.absdir, f.name, self.rev) + else: + fd = None + tmpfile = None + diff = [] + try: + (fd, tmpfile) = tempfile.mkstemp(prefix='osc_diff') + get_source_file(self.apiurl, self.prjname, self.name, f.name, tmpfile, revision) + diff = get_source_file_diff(self.absdir, f.name, revision, + os.path.basename(tmpfile), os.path.dirname(tmpfile), f.name) + finally: + if fd is not None: + os.close(fd) + if tmpfile is not None and os.path.exists(tmpfile): + os.unlink(tmpfile) + yield diff + + for f in added: + yield diff_add_delete(f, True, revision) + for f in deleted: + yield diff_add_delete(f, False, revision) + + def merge(self, otherpac): + for todo_entry in otherpac.todo: + if todo_entry not in self.todo: + self.todo.append(todo_entry) + + def __str__(self): + r = """ +name: %s +prjname: %s +workingdir: %s +localfilelist: %s +linkinfo: %s +rev: %s +'todo' files: %s +""" % (self.name, + self.prjname, + self.dir, + '\n '.join(self.filenamelist), + self.linkinfo, + self.rev, + self.todo) + + return r + + def read_meta_from_spec(self, spec=None): + from ..core import read_meta_from_spec + + if spec: + specfile = spec + else: + # scan for spec files + speclist = glob.glob(os.path.join(self.dir, '*.spec')) + if len(speclist) == 1: + specfile = speclist[0] + elif len(speclist) > 1: + print('the following specfiles were found:') + for filename in speclist: + print(filename) + print('please specify one with --specfile') + sys.exit(1) + else: + print('no specfile was found - please specify one ' + 'with --specfile') + sys.exit(1) + + data = read_meta_from_spec(specfile, 'Summary', 'Url', '%description') + self.summary = data.get('Summary', '') + self.url = data.get('Url', '') + self.descr = data.get('%description', '') + + def update_package_meta(self, force=False): + """ + for the updatepacmetafromspec subcommand + argument force supress the confirm question + """ + from .. import obs_api + from ..output import get_user_input + + package_obj = obs_api.Package.from_api(self.apiurl, self.prjname, self.name) + old = package_obj.to_string() + package_obj.title = self.summary.strip() + package_obj.description = "".join(self.descr).strip() + package_obj.url = self.url.strip() + new = package_obj.to_string() + + if not package_obj.has_changed(): + return + + if force: + reply = "y" + else: + while True: + print("\n".join(difflib.unified_diff(old.splitlines(), new.splitlines(), fromfile="old", tofile="new"))) + print() + + reply = get_user_input( + "Write?", + answers={"y": "yes", "n": "no", "e": "edit"}, + ) + if reply == "y": + break + if reply == "n": + break + if reply == "e": + _, _, edited_obj = package_obj.do_edit() + package_obj.do_update(edited_obj) + new = package_obj.to_string() + continue + + if reply == "y": + package_obj.to_api(self.apiurl) + + def mark_frozen(self): + store_write_string(self.absdir, '_frozenlink', '') + print() + print(f"The link in this package (\"{self.name}\") is currently broken. Checking") + print("out the last working version instead; please use 'osc pull'") + print("to merge the conflicts.") + print() + + def unmark_frozen(self): + if os.path.exists(os.path.join(self.storedir, '_frozenlink')): + os.unlink(os.path.join(self.storedir, '_frozenlink')) + + def latest_rev(self, include_service_files=False, expand=False): + from ..core import show_upstream_rev + from ..core import show_upstream_xsrcmd5 + + # if expand is True the xsrcmd5 will be returned (even if the wc is unexpanded) + if self.islinkrepair(): + upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrepair=1, meta=self.meta, include_service_files=include_service_files) + elif self.islink() and (self.isexpanded() or expand): + if self.isfrozen() or self.ispulled(): + upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrev=self.linkinfo.srcmd5, meta=self.meta, include_service_files=include_service_files) + else: + try: + upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, meta=self.meta, include_service_files=include_service_files) + except: + try: + upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrev=self.linkinfo.srcmd5, meta=self.meta, include_service_files=include_service_files) + except: + upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, linkrev="base", meta=self.meta, include_service_files=include_service_files) + self.mark_frozen() + elif not self.islink() and expand: + upstream_rev = show_upstream_xsrcmd5(self.apiurl, self.prjname, self.name, meta=self.meta, include_service_files=include_service_files) + else: + upstream_rev = show_upstream_rev(self.apiurl, self.prjname, self.name, meta=self.meta, include_service_files=include_service_files) + return upstream_rev + + def __get_files(self, fmeta_root): + from ..core import ET_ENCODING + + f = [] + if fmeta_root.get('rev') is None and len(fmeta_root.findall('entry')) > 0: + raise oscerr.APIError(f"missing rev attribute in _files:\n{''.join(ET.tostring(fmeta_root, encoding=ET_ENCODING))}") + for i in fmeta_root.findall('entry'): + error = i.get('error') + if error is not None: + raise oscerr.APIError(f'broken files meta: {error}') + skipped = i.get('skipped') is not None + f.append(File(i.get('name'), i.get('md5'), + int(i.get('size')), int(i.get('mtime')), skipped)) + return f + + def __get_rev_changes(self, revfiles): + kept = [] + added = [] + deleted = [] + services = [] + revfilenames = [] + for f in revfiles: + revfilenames.append(f.name) + # treat skipped like deleted files + if f.skipped: + if f.name.startswith('_service:'): + services.append(f) + else: + deleted.append(f) + continue + # treat skipped like added files + # problem: this overwrites existing files during the update + # (because skipped files aren't in self.filenamelist_unvers) + if f.name in self.filenamelist and f.name not in self.skipped: + kept.append(f) + else: + added.append(f) + for f in self.filelist: + if f.name not in revfilenames: + deleted.append(f) + + return kept, added, deleted, services + + def update_needed(self, sinfo): + # this method might return a false-positive (that is a True is returned, + # even though no update is needed) (for details, see comments below) + if self.islink(): + if self.isexpanded(): + # check if both revs point to the same expanded sources + # Note: if the package contains a _service file, sinfo.srcmd5's lsrcmd5 + # points to the "expanded" services (xservicemd5) => chances + # for a false-positive are high, because osc usually works on the + # "unexpanded" services. + # Once the srcserver supports something like noservice=1, we can get rid of + # this false-positives (patch was already sent to the ml) (but this also + # requires some slight changes in osc) + return sinfo.get('srcmd5') != self.srcmd5 + elif self.hasserviceinfo(): + # check if we have expanded or unexpanded services + if self.serviceinfo.isexpanded(): + return sinfo.get('lsrcmd5') != self.srcmd5 + else: + # again, we might have a false-positive here, because + # a mismatch of the "xservicemd5"s does not neccessarily + # imply a change in the "unexpanded" services. + return sinfo.get('lsrcmd5') != self.serviceinfo.xsrcmd5 + # simple case: unexpanded sources and no services + # self.srcmd5 should also work + return sinfo.get('lsrcmd5') != self.linkinfo.lsrcmd5 + elif self.hasserviceinfo(): + if self.serviceinfo.isexpanded(): + return sinfo.get('srcmd5') != self.srcmd5 + else: + # cannot handle this case, because the sourceinfo does not contain + # information about the lservicemd5. Once the srcserver supports + # a noservice=1 query parameter, we can handle this case. + return True + return sinfo.get('srcmd5') != self.srcmd5 + + def update(self, rev=None, service_files=False, size_limit=None): + from ..core import ET_ENCODING + from ..core import dgst + + rfiles = [] + # size_limit is only temporary for this update + old_size_limit = self.size_limit + if size_limit is not None: + self.size_limit = int(size_limit) + if os.path.isfile(os.path.join(self.storedir, '_in_update', '_files')): + print('resuming broken update...') + root = ET.parse(os.path.join(self.storedir, '_in_update', '_files')).getroot() + rfiles = self.__get_files(root) + kept, added, deleted, services = self.__get_rev_changes(rfiles) + # check if we aborted in the middle of a file update + broken_file = os.listdir(os.path.join(self.storedir, '_in_update')) + broken_file.remove('_files') + if len(broken_file) == 1: + origfile = os.path.join(self.storedir, '_in_update', broken_file[0]) + wcfile = os.path.join(self.absdir, broken_file[0]) + origfile_md5 = dgst(origfile) + origfile_meta = self.findfilebyname(broken_file[0]) + if origfile.endswith('.copy'): + # ok it seems we aborted at some point during the copy process + # (copy process == copy wcfile to the _in_update dir). remove file+continue + os.unlink(origfile) + elif self.findfilebyname(broken_file[0]) is None: + # should we remove this file from _in_update? if we don't + # the user has no chance to continue without removing the file manually + raise oscerr.PackageInternalError(self.prjname, self.name, + '\'%s\' is not known by meta but exists in \'_in_update\' dir') + elif os.path.isfile(wcfile) and dgst(wcfile) != origfile_md5: + (fd, tmpfile) = tempfile.mkstemp(dir=self.absdir, prefix=broken_file[0] + '.') + os.close(fd) + os.rename(wcfile, tmpfile) + os.rename(origfile, wcfile) + print('warning: it seems you modified \'%s\' after the broken ' + 'update. Restored original file and saved modified version ' + 'to \'%s\'.' % (wcfile, tmpfile)) + elif not os.path.isfile(wcfile): + # this is strange... because it existed before the update. restore it + os.rename(origfile, wcfile) + else: + # everything seems to be ok + os.unlink(origfile) + elif len(broken_file) > 1: + raise oscerr.PackageInternalError(self.prjname, self.name, 'too many files in \'_in_update\' dir') + tmp = rfiles[:] + for f in tmp: + if os.path.exists(os.path.join(self.storedir, f.name)): + if dgst(os.path.join(self.storedir, f.name)) == f.md5: + if f in kept: + kept.remove(f) + elif f in added: + added.remove(f) + # this can't happen + elif f in deleted: + deleted.remove(f) + if not service_files: + services = [] + self.__update(kept, added, deleted, services, ET.tostring(root, encoding=ET_ENCODING), root.get('rev')) + os.unlink(os.path.join(self.storedir, '_in_update', '_files')) + os.rmdir(os.path.join(self.storedir, '_in_update')) + # ok everything is ok (hopefully)... + fm = self.get_files_meta(revision=rev) + root = ET.fromstring(fm) + rfiles = self.__get_files(root) + store_write_string(self.absdir, '_files', fm + '\n', subdir='_in_update') + kept, added, deleted, services = self.__get_rev_changes(rfiles) + if not service_files: + services = [] + self.__update(kept, added, deleted, services, fm, root.get('rev')) + os.unlink(os.path.join(self.storedir, '_in_update', '_files')) + if os.path.isdir(os.path.join(self.storedir, '_in_update')): + os.rmdir(os.path.join(self.storedir, '_in_update')) + self.size_limit = old_size_limit + + def __update(self, kept, added, deleted, services, fm, rev): + from ..core import get_source_file + from ..core import getTransActPath + from ..core import statfrmt + + pathn = getTransActPath(self.dir) + # check for conflicts with existing files + for f in added: + if f.name in self.filenamelist_unvers: + raise oscerr.PackageFileConflict(self.prjname, self.name, f.name, + f'failed to add file \'{f.name}\' file/dir with the same name already exists') + # ok, the update can't fail due to existing files + for f in added: + self.updatefile(f.name, rev, f.mtime) + print(statfrmt('A', os.path.join(pathn, f.name))) + for f in deleted: + # if the storefile doesn't exist we're resuming an aborted update: + # the file was already deleted but we cannot know this + # OR we're processing a _service: file (simply keep the file) + if os.path.isfile(os.path.join(self.storedir, f.name)) and self.status(f.name) not in ('M', 'C'): + # if self.status(f.name) != 'M': + self.delete_localfile(f.name) + self.delete_storefile(f.name) + print(statfrmt('D', os.path.join(pathn, f.name))) + if f.name in self.to_be_deleted: + self.to_be_deleted.remove(f.name) + self.write_deletelist() + elif f.name in self.in_conflict: + self.in_conflict.remove(f.name) + self.write_conflictlist() + + for f in kept: + state = self.status(f.name) +# print f.name, state + if state == 'M' and self.findfilebyname(f.name).md5 == f.md5: + # remote file didn't change + pass + elif state == 'M': + # try to merge changes + merge_status = self.mergefile(f.name, rev, f.mtime) + print(statfrmt(merge_status, os.path.join(pathn, f.name))) + elif state == '!': + self.updatefile(f.name, rev, f.mtime) + print(f'Restored \'{os.path.join(pathn, f.name)}\'') + elif state == 'C': + get_source_file(self.apiurl, self.prjname, self.name, f.name, + targetfilename=os.path.join(self.storedir, f.name), revision=rev, + progress_obj=self.progress_obj, mtime=f.mtime, meta=self.meta) + print(f'skipping \'{f.name}\' (this is due to conflicts)') + elif state == 'D' and self.findfilebyname(f.name).md5 != f.md5: + # XXX: in the worst case we might end up with f.name being + # in _to_be_deleted and in _in_conflict... this needs to be checked + if os.path.exists(os.path.join(self.absdir, f.name)): + merge_status = self.mergefile(f.name, rev, f.mtime) + print(statfrmt(merge_status, os.path.join(pathn, f.name))) + if merge_status == 'C': + # state changes from delete to conflict + self.to_be_deleted.remove(f.name) + self.write_deletelist() + else: + # XXX: we cannot recover this case because we've no file + # to backup + self.updatefile(f.name, rev, f.mtime) + print(statfrmt('U', os.path.join(pathn, f.name))) + elif state == ' ' and self.findfilebyname(f.name).md5 != f.md5: + self.updatefile(f.name, rev, f.mtime) + print(statfrmt('U', os.path.join(pathn, f.name))) + + # checkout service files + for f in services: + get_source_file(self.apiurl, self.prjname, self.name, f.name, + targetfilename=os.path.join(self.absdir, f.name), revision=rev, + progress_obj=self.progress_obj, mtime=f.mtime, meta=self.meta) + print(statfrmt('A', os.path.join(pathn, f.name))) + store_write_string(self.absdir, '_files', fm + '\n') + if not self.meta: + self.update_local_pacmeta() + self.update_datastructs() + + print(f'At revision {self.rev}.') + + def run_source_services(self, mode=None, singleservice=None, verbose=None): + if self.name.startswith("_"): + return 0 + curdir = os.getcwd() + os.chdir(self.absdir) # e.g. /usr/lib/obs/service/verify_file fails if not inside the project dir. + si = Serviceinfo() + if os.path.exists('_service'): + try: + service = ET.parse(os.path.join(self.absdir, '_service')).getroot() + except ET.ParseError as v: + line, column = v.position + print(f'XML error in _service file on line {line}, column {column}') + sys.exit(1) + si.read(service) + si.getProjectGlobalServices(self.apiurl, self.prjname, self.name) + r = si.execute(self.absdir, mode, singleservice, verbose) + os.chdir(curdir) + return r + + def revert(self, filename): + if filename not in self.filenamelist and filename not in self.to_be_added: + raise oscerr.OscIOError(None, f'file \'{filename}\' is not under version control') + elif filename in self.skipped: + raise oscerr.OscIOError(None, f'file \'{filename}\' is marked as skipped and cannot be reverted') + if filename in self.filenamelist and not os.path.exists(os.path.join(self.storedir, filename)): + msg = f"file '{filename}' is listed in filenamelist but no storefile exists" + raise oscerr.PackageInternalError(self.prjname, self.name, msg) + state = self.status(filename) + if not (state == 'A' or state == '!' and filename in self.to_be_added): + shutil.copyfile(os.path.join(self.storedir, filename), os.path.join(self.absdir, filename)) + if state == 'D': + self.to_be_deleted.remove(filename) + self.write_deletelist() + elif state == 'C': + self.clear_from_conflictlist(filename) + elif state in ('A', 'R') or state == '!' and filename in self.to_be_added: + self.to_be_added.remove(filename) + self.write_addlist() + + @staticmethod + def init_package(apiurl: str, project, package, dir, size_limit=None, meta=False, progress_obj=None, scm_url=None): + global store + + if not os.path.exists(dir): + os.mkdir(dir) + elif not os.path.isdir(dir): + raise oscerr.OscIOError(None, f'error: \'{dir}\' is no directory') + if os.path.exists(os.path.join(dir, store)): + raise oscerr.OscIOError(None, f'error: \'{dir}\' is already an initialized osc working copy') + else: + os.mkdir(os.path.join(dir, store)) + store_write_project(dir, project) + store_write_string(dir, '_package', package + '\n') + Store(dir).apiurl = apiurl + if meta: + store_write_string(dir, '_meta_mode', '') + if size_limit: + store_write_string(dir, '_size_limit', str(size_limit) + '\n') + if scm_url: + Store(dir).scmurl = scm_url + else: + store_write_string(dir, '_files', '' + '\n') + store_write_string(dir, '_osclib_version', __store_version__ + '\n') + return Package(dir, progress_obj=progress_obj, size_limit=size_limit)