Fetch source projects of requests

We do not care for current devel project, but for the projects we saw
requests from

git-importer.py

@@ -6,8 +6,8 @@ import pathlib
 import sys
 
 import osc.core
-from lib.git_exporter import GitExporter
 
+from lib.git_exporter import GitExporter
 from lib.importer import Importer
 from lib.test_exporter import TestExporter
 

lib/git_exporter.py

@@ -1,13 +1,16 @@
 import logging
 import os
-from lib.binary import is_binary_or_large
+
 import yaml
+
+from lib.binary import is_binary_or_large
 from lib.db import DB
 from lib.git import Git
 from lib.obs import OBS
 from lib.proxy_sha256 import ProxySHA256, md5
 from lib.tree_builder import TreeBuilder
 
+
 class GitExporter:
     def __init__(self, api_url, project, package, repodir):
         self.obs = OBS()
@@ -24,7 +27,6 @@ class GitExporter:
         self.state_file = os.path.join(self.git.path, ".git", "_flat_state.yaml")
         self.gc_interval = 200
 
-
     def download(self, revision):
         obs_files = self.obs.files(revision.project, revision.package, revision.srcmd5)
         git_files = {

lib/importer.py

@@ -5,9 +5,9 @@ from lib.db import DB
 from lib.db_revision import DBRevision
 from lib.obs import OBS
 from lib.obs_revision import OBSRevision
-
 from lib.user import User
 
+
 class Importer:
     def __init__(self, api_url, project, package):
         # Import a Factory package into the database
@@ -17,6 +17,7 @@ class Importer:
         self.obs = OBS()
         assert project == "openSUSE:Factory"
         self.obs.change_url(api_url)
+        self.refreshed_packages = set()
 
     def update_db_package(self, db, project, package):
         root = self.obs._history(project, package)
@@ -149,21 +150,9 @@ class Importer:
             )
             return [DBRevision(row) for row in cur.fetchall()]
 
-    
-    def import_into_db(self):
-        db = DB()
-        self.update_db_package(db, self.project, self.package)
-        self.fetch_all_linked_packages(db, self.project, self.package)
-        
-        # all remaining, no filtering here
+    def fill_file_lists(self, db):
         self.find_linked_revs(db)
 
-        missing_users = User.missing_users(db)
-        for userid in missing_users:
-            missing_user = self.obs.user(userid)
-            if missing_user:
-                missing_user.import_into_db(db)
-
         self.find_fake_revisions(db)
         for rev in self.revisions_without_files(db):
             with db.cursor() as cur:
@@ -189,7 +178,35 @@ class Importer:
             else:
                 rev.set_broken(db)
 
+    def refresh_package(self, db, project, package):
+        key = f"{project}/{package}"
+        if key in self.refreshed_packages:
+            # refreshing once is good enough
+            return
+        self.refreshed_packages.add(key)
+        self.update_db_package(db, project, package)
+        self.fetch_all_linked_packages(db, project, package)
+
+    def import_into_db(self):
+        db = DB()
+
+        self.refresh_package(db, self.project, self.package)
         for number in DBRevision.requests_to_fetch(db):
             self.obs.request(number).import_into_db(db)
+        with db.cursor() as cur:
+            cur.execute(
+                """SELECT DISTINCT source_project,source_package FROM requests 
+                           WHERE id IN (SELECT request_id FROM revisions WHERE project=%s and package=%s);""",
+                (self.project, self.package),
+            )
+            for project, package in cur.fetchall():
+                self.refresh_package(db, project, package)
 
+        missing_users = User.missing_users(db)
+        for userid in missing_users:
+            missing_user = self.obs.user(userid)
+            if missing_user:
+                missing_user.import_into_db(db)
+
+        self.fill_file_lists(db)
         db.conn.commit()

tests/tree_test.py

@@ -19,7 +19,7 @@ class TestTreeMethods(unittest.TestCase):
         for rev in data["revisions"]:
             DBRevision.import_fixture_dict(self.db, rev)
 
-        revisions = TreeBuilder(self.db).build(package)
+        revisions = TreeBuilder(self.db).build("openSUSE:Factory", package)
         path = os.path.join(
             os.path.dirname(__file__), f"fixtures/{package}-expected-tree.yaml"
         )