adamm/git-importer
1
0
Fork 1
forked from importers/git-importer
Code Pull Requests Activity

Split Request and Revision

Browse Source
This commit is contained in:
Stephan Kulow 2022-10-17 14:53:18 +02:00
parent dbc1b6ab0b
commit 40a7675e61
5 changed files with 146 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
__pycache__

1
README.md
View File

@ -1,2 +1,3 @@
sudo zypper in python3-psycopg2
sudo su - postgres
# `createdb -O <LOCAL_USER> imported_git`

145
git-importer.py
View File

@ -1,15 +1,12 @@
#!/usr/bin/env python3
#!/usr/bin/python3
import argparse
import asyncio
import datetime
import errno
import fnmatch
import functools
import hashlib
import itertools
import logging
import os
import pathlib
import re
import shutil
@ -24,8 +21,8 @@ import osc.core
import pygit2
import requests
from osclib.cache import Cache
from request import Request
from revision import Revision
# Add a retry wrapper for some of the HTTP actions.
def retry(func):
@ -638,140 +635,6 @@ class ProxySHA256:
return result
class Request:
def parse(self, xml):
self.requestid = int(xml.get("id"))
self.creator = xml.get("creator")
self.type_ = xml.find("action").get("type")
if self.type_ == "delete":
# not much to do
return self
self.source = xml.find("action/source").get("project")
# expanded MD5 or commit revision
self.revisionid = xml.find("action/source").get("rev")
self.target = xml.find("action/target").get("project")
self.state = xml.find("state").get("name")
# TODO: support muti-action requests
# TODO: parse review history
# TODO: add description
return self
def type(self):
return self.type_
def __str__(self):
return f"Req {self.requestid} {self.creator} {self.type_} {self.source}->{self.target} {self.state}"
def __repr__(self):
return f"[{self.__str__()}]"
class Revision:
def __init__(self, obs, history, project, package):
self.obs = obs
self.history = history
self.project = project
self.package = package
self.commit = None
self.ignored = False
def parse(self, xml):
self.rev = int(xml.get("rev"))
# Replaced in check_expanded
self.srcmd5 = xml.find("srcmd5").text
self.version = xml.find("version").text
time = int(xml.find("time").text)
self.time = datetime.datetime.fromtimestamp(time)
userid = xml.find("user")
if userid is not None:
self.userid = userid.text
else:
self.userid = "unknown"
comment = xml.find("comment")
if comment is not None:
self.comment = comment.text or ""
else:
self.comment = ""
# Populated by check_link
self.linkrev = None
self.requestid = None
requestid = xml.find("requestid")
if requestid is not None:
self.requestid = int(requestid.text)
else:
# Sometimes requestid is missing, but can be extracted
# from "comment"
matched = re.match(
r"^Copy from .* based on submit request (\d+) from user .*$",
self.comment,
)
if matched:
self.requestid = int(matched.group(1))
return self
def __str__(self):
return f"Rev {self.project}/{self.rev} Md5 {self.srcmd5} {self.time} {self.userid} {self.requestid}"
def __repr__(self):
return f"[{self.__str__()}]"
def check_link(self):
"""Add 'linkrev' attribute into the revision. Returns False if the link is invalid"""
try:
root = self.obs._xml(
f"source/{self.project}/{self.package}/_link", rev=self.srcmd5
)
except HTTPError as e:
if e.code == 404:
logging.debug("No _link for the revision")
return True
raise e
except ET.ParseError:
logging.error(
f"_link can't be parsed [{self.project}/{self.package} rev={self.srcmd5}]"
)
return False
target_project = root.get("project")
rev = self.history.find_last_rev_after_time(target_project, self.time)
if rev:
logging.debug(f"Linkrev found: {rev}")
self.linkrev = rev.srcmd5
return True
def check_expanded(self):
# Even if it's not a link we still need to check the expanded
# srcmd5 as it's possible used in submit requests
if not self.check_link():
return False
# If there is a "linkrev", "rev" is ignored
params = {"rev": self.srcmd5, "expand": "1"}
if self.linkrev:
params["linkrev"] = self.linkrev
try:
root = self.obs._xml(f"source/{self.project}/{self.package}", **params)
except HTTPError as e:
if e.code == 400:
logging.error(f"Package [{self.project}/{self.package} {params}] can't be expanded: {e}")
return False
raise e
self.srcmd5 = root.get("srcmd5")
return True
class History:
@ -1277,8 +1140,6 @@ def main():
logging.info(f"Removing old repository {args.repodir}")
shutil.rmtree(args.repodir)
Cache.init()
# TODO: use a CLI parameter to describe the projects
importer = Importer(
PROJECTS, args.package, args.repodir, args.search_ancestor, args.rebase_devel

31
request.py Normal file
View File

@ -0,0 +1,31 @@
class Request:
def parse(self, xml):
self.requestid = int(xml.get("id"))
self.creator = xml.get("creator")
self.type_ = xml.find("action").get("type")
if self.type_ == "delete":
# not much to do
return self
self.source = xml.find("action/source").get("project")
# expanded MD5 or commit revision
self.revisionid = xml.find("action/source").get("rev")
self.target = xml.find("action/target").get("project")
self.state = xml.find("state").get("name")
# TODO: support muti-action requests
# TODO: parse review history
# TODO: add description
return self
def type(self):
return self.type_
def __str__(self):
return f"Req {self.requestid} {self.creator} {self.type_} {self.source}->{self.target} {self.state}"
def __repr__(self):
return f"[{self.__str__()}]"

110
revision.py Normal file
View File

@ -0,0 +1,110 @@
import datetime
import logging
import re
from urllib.error import HTTPError
import xml.etree.ElementTree as ET
class Revision:
def __init__(self, obs, history, project, package):
self.obs = obs
self.history = history
self.project = project
self.package = package
self.commit = None
self.ignored = False
def parse(self, xml):
self.rev = int(xml.get("rev"))
# Replaced in check_expanded
self.srcmd5 = xml.find("srcmd5").text
self.version = xml.find("version").text
time = int(xml.find("time").text)
self.time = datetime.datetime.fromtimestamp(time)
userid = xml.find("user")
if userid is not None:
self.userid = userid.text
else:
self.userid = "unknown"
comment = xml.find("comment")
if comment is not None:
self.comment = comment.text or ""
else:
self.comment = ""
# Populated by check_link
self.linkrev = None
self.requestid = None
requestid = xml.find("requestid")
if requestid is not None:
self.requestid = int(requestid.text)
else:
# Sometimes requestid is missing, but can be extracted
# from "comment"
matched = re.match(
r"^Copy from .* based on submit request (\d+) from user .*$",
self.comment,
)
if matched:
self.requestid = int(matched.group(1))
return self
def __str__(self):
return f"Rev {self.project}/{self.rev} Md5 {self.srcmd5} {self.time} {self.userid} {self.requestid}"
def __repr__(self):
return f"[{self.__str__()}]"
def check_link(self):
"""Add 'linkrev' attribute into the revision. Returns False if the link is invalid"""
try:
root = self.obs._xml(
f"source/{self.project}/{self.package}/_link", rev=self.srcmd5
)
except HTTPError as e:
if e.code == 404:
logging.debug("No _link for the revision")
return True
raise e
except ET.ParseError:
logging.error(
f"_link can't be parsed [{self.project}/{self.package} rev={self.srcmd5}]"
)
return False
target_project = root.get("project")
rev = self.history.find_last_rev_after_time(target_project, self.time)
if rev:
logging.debug(f"Linkrev found: {rev}")
self.linkrev = rev.srcmd5
return True
def check_expanded(self):
# Even if it's not a link we still need to check the expanded
# srcmd5 as it's possible used in submit requests
if not self.check_link():
return False
# If there is a "linkrev", "rev" is ignored
params = {"rev": self.srcmd5, "expand": "1"}
if self.linkrev:
params["linkrev"] = self.linkrev
try:
root = self.obs._xml(f"source/{self.project}/{self.package}", **params)
except HTTPError as e:
if e.code == 400:
logging.error(
f"Package [{self.project}/{self.package} {params}] can't be expanded: {e}"
)
return False
raise e
self.srcmd5 = root.get("srcmd5")
return True