Download .spec and .changes file only for now

This commit is contained in:
Stephan Kulow 2022-10-31 16:31:10 +01:00
parent bfdade8ecf
commit cfab0a522b
3 changed files with 52 additions and 6 deletions

View File

@ -1,5 +1,6 @@
from hashlib import md5 from hashlib import md5
from lib.db import DB
from lib.request import Request from lib.request import Request
@ -202,6 +203,24 @@ class DBRevision:
self._files.sort(key=lambda x: x["name"]) self._files.sort(key=lambda x: x["name"])
return self._files return self._files
def calc_delta(self, db: DB, current_rev=None):
"""Calculate the list of files to download and to delete"""
old_files = dict()
to_download = []
to_delete = []
if current_rev:
for entry in current_rev.files_list(db):
old_files[entry["name"]] = f"{entry['md5']}-{entry['size']}"
for entry in self.files_list(db):
if old_files.get(entry["name"], "") != f"{entry['md5']}-{entry['size']}":
print("Download", entry["name"])
to_download.append(entry["name"])
old_files.pop(entry["name"], None)
for entry in old_files.keys():
print("Delete", entry)
to_delete.append(entry)
return to_download, to_delete
@staticmethod @staticmethod
def requests_to_fetch(db): def requests_to_fetch(db):
with db.cursor() as cur: with db.cursor() as cur:

View File

@ -278,9 +278,11 @@ class Importer:
return f"{self.branch} c:{self.commit.short_string()}{p1_str}{p2_str}" return f"{self.branch} c:{self.commit.short_string()}{p1_str}{p2_str}"
class FlatTreeWalker(AbstractWalker): class FlatTreeWalker(AbstractWalker):
def __init__(self) -> None: def __init__(self, rebase_devel) -> None:
super().__init__() super().__init__()
self.flats = [] self.flats = []
# the rebase_devel won't work as such as rebasing the branch needs an explicit action
self.rebase_devel = rebase_devel
# remember the last merge point so we can know the parent of it for the root of the sources # remember the last merge point so we can know the parent of it for the root of the sources
self.last_merge = None self.last_merge = None
@ -288,7 +290,7 @@ class Importer:
self.flats.append(FlatNode(branch, commit, parent1, parent2)) self.flats.append(FlatNode(branch, commit, parent1, parent2))
def handle_source_node(self, node) -> None: def handle_source_node(self, node) -> None:
if node.parent and node.parent.merged_into and False: if self.rebase_devel and node.parent and node.parent.merged_into:
self.add("devel", node.revision, node.parent.merged_into.revision) self.add("devel", node.revision, node.parent.merged_into.revision)
return return
if node.parent: if node.parent:
@ -312,18 +314,40 @@ class Importer:
self.last_merge = node self.last_merge = node
ftw = FlatTreeWalker() ftw = FlatTreeWalker(self.rebase_devel)
tree.walk(ftw) tree.walk(ftw)
branch_state = {"factory": None, "devel": None}
for flat in reversed(ftw.flats): for flat in reversed(ftw.flats):
self.commit_flat(flat) self.commit_flat(db, flat, branch_state)
def commit_flat(self, flat): def limit_download(self, file):
if file.endswith(".spec") or file.endswith(".changes"):
return True
return False
def commit_flat(self, db, flat, branch_state):
parents = [] parents = []
self.git.checkout(flat.branch) self.git.checkout(flat.branch)
if flat.parent1: if flat.parent1:
parents.append(flat.parent1.git_commit) parents.append(flat.parent1.git_commit)
if flat.parent2: if flat.parent2:
parents.append(flat.parent2.git_commit) parents.append(flat.parent2.git_commit)
to_download, to_delete = flat.commit.calc_delta(db, branch_state[flat.branch])
for file in to_delete:
if not self.limit_download(file):
continue
self.git.remove(file)
for file in to_download:
if not self.limit_download(file):
continue
self.obs.download(
flat.commit.project,
flat.commit.package,
file,
flat.commit.expanded_srcmd5,
self.git.path,
)
self.git.add(file)
commit = self.git.commit( commit = self.git.commit(
f"OBS User {flat.commit.userid}", f"OBS User {flat.commit.userid}",
@ -335,6 +359,7 @@ class Importer:
parents=parents, parents=parents,
) )
flat.commit.git_commit = commit flat.commit.git_commit = commit
branch_state[flat.branch] = flat.commit
def import_into_db(self): def import_into_db(self):
db = DB() db = DB()

View File

@ -151,7 +151,9 @@ class OBS:
) )
return osc.core.http_GET(url) return osc.core.http_GET(url)
def download(self, project, package, name, revision, dirpath): def download(
self, project: str, package: str, name: str, revision: str, dirpath: str
) -> None:
with (dirpath / name).open("wb") as f: with (dirpath / name).open("wb") as f:
f.write(self._download(project, package, name, revision).read()) f.write(self._download(project, package, name, revision).read())