forked from importers/git-importer
Download .spec and .changes file only for now
This commit is contained in:
parent
bfdade8ecf
commit
cfab0a522b
@ -1,5 +1,6 @@
|
|||||||
from hashlib import md5
|
from hashlib import md5
|
||||||
|
|
||||||
|
from lib.db import DB
|
||||||
from lib.request import Request
|
from lib.request import Request
|
||||||
|
|
||||||
|
|
||||||
@ -202,6 +203,24 @@ class DBRevision:
|
|||||||
self._files.sort(key=lambda x: x["name"])
|
self._files.sort(key=lambda x: x["name"])
|
||||||
return self._files
|
return self._files
|
||||||
|
|
||||||
|
def calc_delta(self, db: DB, current_rev=None):
|
||||||
|
"""Calculate the list of files to download and to delete"""
|
||||||
|
old_files = dict()
|
||||||
|
to_download = []
|
||||||
|
to_delete = []
|
||||||
|
if current_rev:
|
||||||
|
for entry in current_rev.files_list(db):
|
||||||
|
old_files[entry["name"]] = f"{entry['md5']}-{entry['size']}"
|
||||||
|
for entry in self.files_list(db):
|
||||||
|
if old_files.get(entry["name"], "") != f"{entry['md5']}-{entry['size']}":
|
||||||
|
print("Download", entry["name"])
|
||||||
|
to_download.append(entry["name"])
|
||||||
|
old_files.pop(entry["name"], None)
|
||||||
|
for entry in old_files.keys():
|
||||||
|
print("Delete", entry)
|
||||||
|
to_delete.append(entry)
|
||||||
|
return to_download, to_delete
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def requests_to_fetch(db):
|
def requests_to_fetch(db):
|
||||||
with db.cursor() as cur:
|
with db.cursor() as cur:
|
||||||
|
@ -278,9 +278,11 @@ class Importer:
|
|||||||
return f"{self.branch} c:{self.commit.short_string()}{p1_str}{p2_str}"
|
return f"{self.branch} c:{self.commit.short_string()}{p1_str}{p2_str}"
|
||||||
|
|
||||||
class FlatTreeWalker(AbstractWalker):
|
class FlatTreeWalker(AbstractWalker):
|
||||||
def __init__(self) -> None:
|
def __init__(self, rebase_devel) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.flats = []
|
self.flats = []
|
||||||
|
# the rebase_devel won't work as such as rebasing the branch needs an explicit action
|
||||||
|
self.rebase_devel = rebase_devel
|
||||||
# remember the last merge point so we can know the parent of it for the root of the sources
|
# remember the last merge point so we can know the parent of it for the root of the sources
|
||||||
self.last_merge = None
|
self.last_merge = None
|
||||||
|
|
||||||
@ -288,7 +290,7 @@ class Importer:
|
|||||||
self.flats.append(FlatNode(branch, commit, parent1, parent2))
|
self.flats.append(FlatNode(branch, commit, parent1, parent2))
|
||||||
|
|
||||||
def handle_source_node(self, node) -> None:
|
def handle_source_node(self, node) -> None:
|
||||||
if node.parent and node.parent.merged_into and False:
|
if self.rebase_devel and node.parent and node.parent.merged_into:
|
||||||
self.add("devel", node.revision, node.parent.merged_into.revision)
|
self.add("devel", node.revision, node.parent.merged_into.revision)
|
||||||
return
|
return
|
||||||
if node.parent:
|
if node.parent:
|
||||||
@ -312,18 +314,40 @@ class Importer:
|
|||||||
|
|
||||||
self.last_merge = node
|
self.last_merge = node
|
||||||
|
|
||||||
ftw = FlatTreeWalker()
|
ftw = FlatTreeWalker(self.rebase_devel)
|
||||||
tree.walk(ftw)
|
tree.walk(ftw)
|
||||||
|
branch_state = {"factory": None, "devel": None}
|
||||||
for flat in reversed(ftw.flats):
|
for flat in reversed(ftw.flats):
|
||||||
self.commit_flat(flat)
|
self.commit_flat(db, flat, branch_state)
|
||||||
|
|
||||||
def commit_flat(self, flat):
|
def limit_download(self, file):
|
||||||
|
if file.endswith(".spec") or file.endswith(".changes"):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def commit_flat(self, db, flat, branch_state):
|
||||||
parents = []
|
parents = []
|
||||||
self.git.checkout(flat.branch)
|
self.git.checkout(flat.branch)
|
||||||
if flat.parent1:
|
if flat.parent1:
|
||||||
parents.append(flat.parent1.git_commit)
|
parents.append(flat.parent1.git_commit)
|
||||||
if flat.parent2:
|
if flat.parent2:
|
||||||
parents.append(flat.parent2.git_commit)
|
parents.append(flat.parent2.git_commit)
|
||||||
|
to_download, to_delete = flat.commit.calc_delta(db, branch_state[flat.branch])
|
||||||
|
for file in to_delete:
|
||||||
|
if not self.limit_download(file):
|
||||||
|
continue
|
||||||
|
self.git.remove(file)
|
||||||
|
for file in to_download:
|
||||||
|
if not self.limit_download(file):
|
||||||
|
continue
|
||||||
|
self.obs.download(
|
||||||
|
flat.commit.project,
|
||||||
|
flat.commit.package,
|
||||||
|
file,
|
||||||
|
flat.commit.expanded_srcmd5,
|
||||||
|
self.git.path,
|
||||||
|
)
|
||||||
|
self.git.add(file)
|
||||||
|
|
||||||
commit = self.git.commit(
|
commit = self.git.commit(
|
||||||
f"OBS User {flat.commit.userid}",
|
f"OBS User {flat.commit.userid}",
|
||||||
@ -335,6 +359,7 @@ class Importer:
|
|||||||
parents=parents,
|
parents=parents,
|
||||||
)
|
)
|
||||||
flat.commit.git_commit = commit
|
flat.commit.git_commit = commit
|
||||||
|
branch_state[flat.branch] = flat.commit
|
||||||
|
|
||||||
def import_into_db(self):
|
def import_into_db(self):
|
||||||
db = DB()
|
db = DB()
|
||||||
|
@ -151,7 +151,9 @@ class OBS:
|
|||||||
)
|
)
|
||||||
return osc.core.http_GET(url)
|
return osc.core.http_GET(url)
|
||||||
|
|
||||||
def download(self, project, package, name, revision, dirpath):
|
def download(
|
||||||
|
self, project: str, package: str, name: str, revision: str, dirpath: str
|
||||||
|
) -> None:
|
||||||
with (dirpath / name).open("wb") as f:
|
with (dirpath / name).open("wb") as f:
|
||||||
f.write(self._download(project, package, name, revision).read())
|
f.write(self._download(project, package, name, revision).read())
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user