Ignore .osc directory

Reviewed-on: importers/git-importer#26
Reviewed-by: Dirk Mueller <dirkmueller@noreply@src.opensuse.org>

README.md

@@ -1,18 +1,5 @@
-Installation
+sudo zypper in python3-psycopg
-------------
+sudo su - postgres
-
+# `createdb -O <LOCAL_USER> imported_git`
-    sudo zypper in python3-psycopg
-    sudo su - postgres
-    createdb -O <LOCAL_USER> imported_git`
 
 To reset the database, drop table scheme
-
-
-Gitea parameters
-----------------
-
-  * `GITEA_HOST` - default: src.opensuse.org
-  * `GITEA_USER` - Used to generate SSH links for push. Default: gitea
-  * `GITEA_ORG`  - target organization to push to
-  * `GITEA_DEFAULT_BRANCH` - default branch
-

git-importer.py

@@ -42,8 +42,8 @@ PROJECTS = [
 ]
 
 
-def export_package(project, package, repodir, cachedir, gc):
+def export_package(package, repodir, cachedir, gc):
-    exporter = GitExporter(URL_OBS, project, package, repodir, cachedir)
+    exporter = GitExporter(URL_OBS, "openSUSE:Factory", package, repodir, cachedir)
     exporter.set_gc_interval(gc)
     exporter.export_as_git()
 
@@ -51,12 +51,6 @@ def export_package(project, package, repodir, cachedir, gc):
 def main():
     parser = argparse.ArgumentParser(description="OBS history importer into git")
     parser.add_argument("packages", help="OBS package names", nargs="*")
-    parser.add_argument(
-        "-p",
-        "--project",
-        default="openSUSE:Factory",
-        help="Project to import/export, default is openSUSE:Factory",
-    )
     parser.add_argument(
         "-r",
         "--repodir",
@@ -116,13 +110,10 @@ def main():
     if not args.cachedir:
         args.cachedir = pathlib.Path("~/.cache/git-import/").expanduser()
 
-    importer = Importer(URL_OBS, args.project, args.packages)
+    importer = Importer(URL_OBS, "openSUSE:Factory", args.packages)
     importer.import_into_db()
     for package in args.packages:
-        if not importer.package_with_scmsync(package):
+        export_package(package, args.repodir, args.cachedir, args.gc)
-            export_package(args.project, package, args.repodir, args.cachedir, args.gc)
-        else:
-            logging.debug(f"{args.project}/{package} has scmsync links - skipping export")
 
 
 if __name__ == "__main__":

lib/git.py

@@ -48,7 +48,6 @@ class Git:
     def open(self):
         if not self.exists():
             self.git_run(["init", "--object-format=sha256", "-b", "factory"])
-        self.git_run(["config", "lfs.allowincompletepush", "true"])
 
     def is_dirty(self):
         """Check if there is something to commit"""
@@ -181,7 +180,7 @@ class Git:
     #                logging.warning(f"Error removing file {path}: {e}")
 
     def add(self, filename):
-        self.git_run(["add", ":(literal)" + str(filename)])
+        self.git_run(["add", filename])
 
     def add_default_gitignore(self):
         if not (self.path / ".gitignore").exists():
@@ -242,7 +241,7 @@ class Git:
 
     def remove(self, file: pathlib.Path):
         self.git_run(
-            ["rm", "-q", "-f", "--ignore-unmatch", ":(literal)" + file.name],
+            ["rm", "-q", "-f", "--ignore-unmatch", file.name],
         )
         patterns = self.get_specific_lfs_gitattributes()
         if file.name in patterns:
@@ -252,27 +251,15 @@ class Git:
     def add_gitea_remote(self, package):
         repo_name = package.replace("+", "_")
         org_name = "rpm"
-        gitea_user = "gitea"
-        gitea_host = "src.opensuse.org"
-        default_branch = "factory"
-
-        if os.getenv("GITEA_HOST"):
-            gitea_host = getenv("GITEA_HOST")
-        if os.getenv("GITEA_USER"):
-            gitea_user = getenv("GITEA_USER")
-        if os.getenv("GITEA_ORG"):
-            org_name = getenv("GITEA_ORG")
-        if os.getenv("GITEA_DEFAULT_BRANCH"):
-            default_branch = getenv("GITEA_DEFAULT_BRANCH")
 
         if not os.getenv("GITEA_TOKEN"):
             logging.warning("Not adding a remote due to missing $GITEA_TOKEN")
             return
 
-        url = f"https://{gitea_host}/api/v1/org/{org_name}/repos"
+        url = f"https://src.opensuse.org/api/v1/org/{org_name}/repos"
         response = requests.post(
             url,
-            data={"name": repo_name, "object_format_name": "sha256", "default_branch": default_branch},
+            data={"name": repo_name},
             headers={"Authorization": f"token {os.getenv('GITEA_TOKEN')}"},
             timeout=10,
         )
@@ -280,7 +267,7 @@ class Git:
         # 201 Created
         if response.status_code not in (201, 409):
             print(response.data)
-        url = f"{gitea_user}@{gitea_host}:{org_name}/{repo_name}.git"
+        url = f"gitea@src.opensuse.org:{org_name}/{repo_name}.git"
         self.git_run(
             ["remote", "add", "origin", url],
         )
@@ -290,11 +277,13 @@ class Git:
             ["remote"],
             stdout=subprocess.PIPE,
         ).stdout.decode("utf-8"):
-            logging.warning("Not pushing to remote because no 'origin' configured")
+            logger.warning("Not pushing to remote because no 'origin' configured")
             return
 
         cmd = ["push"]
         if force:
             cmd.append("-f")
-        cmd += ["origin", "--all"]
+        cmd.append("origin")
+        cmd.append("refs/heads/factory")
+        cmd.append("refs/heads/devel")
         self.git_run(cmd)

lib/git_exporter.py

@@ -29,7 +29,7 @@ class GitExporter:
             self.git.open()
         else:
             self.git.create()
-            # self.git.add_gitea_remote(package)
+            self.git.add_gitea_remote(package)
         self.state_file = os.path.join(self.git.path, ".git", "_flat_state.yaml")
         self.gc_interval = 200
         self.cachedir = cachedir
@@ -86,11 +86,6 @@ class GitExporter:
             logging.debug(f"Committing {flat}")
             self.commit_flat(flat, branch_state)
 
-        # make sure that we create devel branch
-        if not branch_state["devel"]:
-            logging.debug("force creating devel")
-            self.git.set_branch_head("devel", self.git.branch_head("factory"))
-
         self.git.push(force=True)
 
     def run_gc(self):

lib/importer.py

@@ -26,16 +26,13 @@ class Importer:
         # Import multiple Factory packages into the database
         self.packages = packages
         self.project = project
-        self.scmsync_cache = dict()
-        self.packages_with_scmsync = set()
 
         self.db = DB()
         self.obs = OBS(api_url)
-        assert not self.has_scmsync(project)
+        assert project == "openSUSE:Factory"
         self.refreshed_packages = set()
         self.gone_packages_set = None
 
-
     def import_request(self, number):
         self.obs.request(number).import_into_db(self.db)
 
@@ -216,10 +213,6 @@ class Importer:
             return
         logging.debug(f"Refresh {project}/{package}")
         self.refreshed_packages.add(key)
-        if self.has_scmsync(project) or self.has_scmsync(key):
-            self.packages_with_scmsync.add(package)
-            logging.debug(f"{project}/{package} already in Git - skipping")
-            return
         self.update_db_package(project, package)
         self.fetch_all_linked_packages(project, package)
 
@@ -262,18 +255,3 @@ class Importer:
                 for line in f.readlines():
                     self.gone_packages_set.add(line.strip())
         return key in self.gone_packages_set
-
-    def has_scmsync(self, key):
-        if key in self.scmsync_cache:
-            return self.scmsync_cache[key]
-
-        root = self.obs._meta(key)
-        scmsync_exists = False
-        if root is not None:
-            scmsync_exists = root.find('scmsync') is not None
-        self.scmsync_cache[key] = scmsync_exists
-        return scmsync_exists
-
-    def package_with_scmsync(self, package):
-        return package in self.packages_with_scmsync
-

lib/lfs_oid.py

@@ -83,7 +83,6 @@ class LFSOid:
             self.register()
 
     def check(self):
-        return True
         url = f"http://localhost:9999/check/{self.sha256}/{self.size}"
         response = requests.get(
             url,

lib/obs.py

@@ -73,11 +73,11 @@ class OBS:
         logging.debug(f"GET {url}")
         return ET.parse(osc.core.http_GET(url)).getroot()
 
-    def _meta(self, key, **params):
+    def _meta(self, project, package, **params):
         try:
-            root = self._xml(f"source/{key}/_meta", **params)
+            root = self._xml(f"source/{project}/{package}/_meta", **params)
         except HTTPError:
-            logging.error(f"Project/Package [{key} {params}] has no meta")
+            logging.error(f"Package [{project}/{package} {params}] has no meta")
             return None
         return root
 
@@ -118,13 +118,13 @@ class OBS:
         return root
 
     def exists(self, project, package):
-        root = self._meta(f"{project}/{package}")
+        root = self._meta(project, package)
         if root is None:
             return False
         return root.get("project") == project
 
     def devel_project(self, project, package):
-        root = self._meta(f"{project}/{package}")
+        root = self._meta(project, package)
         devel = root.find("devel")
         if devel is None:
             return None

opensuse-monitor.py

@@ -2,58 +2,36 @@
 import json
 from pathlib import Path
 import pika
-import random
+import sys
-import time
 
 MY_TASKS_DIR = Path(__file__).parent / "tasks"
 
+connection = pika.BlockingConnection(pika.URLParameters("amqps://opensuse:opensuse@rabbit.opensuse.org"))
+channel = connection.channel()
 
-def listen_events():
+channel.exchange_declare(exchange='pubsub', exchange_type='topic', passive=True, durable=True)
-    connection = pika.BlockingConnection(
-        pika.URLParameters("amqps://opensuse:opensuse@rabbit.opensuse.org")
-    )
-    channel = connection.channel()
 
-    channel.exchange_declare(
+result = channel.queue_declare("", exclusive=True)
-        exchange="pubsub", exchange_type="topic", passive=True, durable=False
+queue_name = result.method.queue
-    )
 
-    result = channel.queue_declare("", exclusive=True)
+channel.queue_bind(exchange='pubsub',
-    queue_name = result.method.queue
+                   queue=queue_name,routing_key='#')
 
-    channel.queue_bind(
+print(' [*] Waiting for logs. To exit press CTRL+C')
-        exchange="pubsub", queue=queue_name, routing_key="opensuse.obs.package.commit"
-    )
 
-    print(" [*] Waiting for logs. To exit press CTRL+C")
+def callback(ch, method, properties, body):
-
+    if method.routing_key not in ("opensuse.obs.package.commit",):
-    def callback(ch, method, properties, body):
+        return
-        if method.routing_key not in ("opensuse.obs.package.commit",):
+    body = json.loads(body)
+    if 'project' in body and 'package' in body and body['project'] == 'openSUSE:Factory':
+        if '/' in body['package']:
             return
-        body = json.loads(body)
-        if (
-            "project" in body
-            and "package" in body
-            and body["project"] == "openSUSE:Factory"
-        ):
-            if "/" in body["package"]:
-                return
 
-            (MY_TASKS_DIR / body["package"]).touch()
+        (MY_TASKS_DIR / body['package']).touch()
-            print(" [x] %r:%r" % (method.routing_key, body["package"]))
+        print(" [x] %r:%r" % (method.routing_key, body['package']))
 
-    channel.basic_consume(queue_name, callback, auto_ack=True)
+channel.basic_consume(queue_name,
+                      callback,
+                      auto_ack=True)
 
-    channel.start_consuming()
+channel.start_consuming()
-
-
-def main():
-    while True:
-        try:
-            listen_events()
-        except (pika.exceptions.ConnectionClosed, pika.exceptions.AMQPHeartbeatTimeout):
-            time.sleep(random.randint(10, 100))
-
-
-if __name__ == "__main__":
-    main()

update-tasks.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-#
-cd /space/dmueller/git-importer
-
-source credentials.sh
-
-while true; do
-   for i in $PWD/tasks/*; do
-       if test -f "$i"; then
-            echo "$(date): Importing $(basename $i)"
-            if ! python3 ./git-importer.py -c repos/.cache $(basename $i); then
-                mkdir -p $PWD/failed-tasks
-                mv -f $i $PWD/failed-tasks
-            fi
-            rm -f $i
-        fi
-   done
-   inotifywait -q -e create $PWD/tasks
-done