diff --git a/git-importer.py b/git-importer.py index 09a3ba2..f276bf3 100755 --- a/git-importer.py +++ b/git-importer.py @@ -6,8 +6,8 @@ import pathlib import sys import osc.core -from lib.git_exporter import GitExporter +from lib.git_exporter import GitExporter from lib.importer import Importer from lib.test_exporter import TestExporter diff --git a/lib/git_exporter.py b/lib/git_exporter.py index 0d4d9c9..5d40182 100644 --- a/lib/git_exporter.py +++ b/lib/git_exporter.py @@ -1,13 +1,16 @@ import logging import os -from lib.binary import is_binary_or_large + import yaml + +from lib.binary import is_binary_or_large from lib.db import DB from lib.git import Git from lib.obs import OBS from lib.proxy_sha256 import ProxySHA256, md5 from lib.tree_builder import TreeBuilder + class GitExporter: def __init__(self, api_url, project, package, repodir): self.obs = OBS() @@ -24,7 +27,6 @@ class GitExporter: self.state_file = os.path.join(self.git.path, ".git", "_flat_state.yaml") self.gc_interval = 200 - def download(self, revision): obs_files = self.obs.files(revision.project, revision.package, revision.srcmd5) git_files = { diff --git a/lib/importer.py b/lib/importer.py index 1f56756..283729e 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -5,9 +5,9 @@ from lib.db import DB from lib.db_revision import DBRevision from lib.obs import OBS from lib.obs_revision import OBSRevision - from lib.user import User + class Importer: def __init__(self, api_url, project, package): # Import a Factory package into the database @@ -17,7 +17,8 @@ class Importer: self.obs = OBS() assert project == "openSUSE:Factory" self.obs.change_url(api_url) - + self.refreshed_packages = set() + def update_db_package(self, db, project, package): root = self.obs._history(project, package) if root is None: @@ -149,21 +150,9 @@ class Importer: ) return [DBRevision(row) for row in cur.fetchall()] - - def import_into_db(self): - db = DB() - self.update_db_package(db, self.project, self.package) - self.fetch_all_linked_packages(db, self.project, self.package) - - # all remaining, no filtering here + def fill_file_lists(self, db): self.find_linked_revs(db) - missing_users = User.missing_users(db) - for userid in missing_users: - missing_user = self.obs.user(userid) - if missing_user: - missing_user.import_into_db(db) - self.find_fake_revisions(db) for rev in self.revisions_without_files(db): with db.cursor() as cur: @@ -189,7 +178,35 @@ class Importer: else: rev.set_broken(db) + def refresh_package(self, db, project, package): + key = f"{project}/{package}" + if key in self.refreshed_packages: + # refreshing once is good enough + return + self.refreshed_packages.add(key) + self.update_db_package(db, project, package) + self.fetch_all_linked_packages(db, project, package) + + def import_into_db(self): + db = DB() + + self.refresh_package(db, self.project, self.package) for number in DBRevision.requests_to_fetch(db): self.obs.request(number).import_into_db(db) + with db.cursor() as cur: + cur.execute( + """SELECT DISTINCT source_project,source_package FROM requests + WHERE id IN (SELECT request_id FROM revisions WHERE project=%s and package=%s);""", + (self.project, self.package), + ) + for project, package in cur.fetchall(): + self.refresh_package(db, project, package) + missing_users = User.missing_users(db) + for userid in missing_users: + missing_user = self.obs.user(userid) + if missing_user: + missing_user.import_into_db(db) + + self.fill_file_lists(db) db.conn.commit() diff --git a/tests/tree_test.py b/tests/tree_test.py index a519bd9..42c67ae 100644 --- a/tests/tree_test.py +++ b/tests/tree_test.py @@ -19,7 +19,7 @@ class TestTreeMethods(unittest.TestCase): for rev in data["revisions"]: DBRevision.import_fixture_dict(self.db, rev) - revisions = TreeBuilder(self.db).build(package) + revisions = TreeBuilder(self.db).build("openSUSE:Factory", package) path = os.path.join( os.path.dirname(__file__), f"fixtures/{package}-expected-tree.yaml" )