fixed formatting and other suggestions

lib/db.py

@@ -22,7 +22,7 @@ class DB:
 
     def schema_version(self):
         # create a cursor
-        cur = self.conn.cursor()
+        with self.conn.cursor() as cur:
 
             # execute a statement
             try:
@@ -35,7 +35,6 @@ class DB:
 
             db_version = cur.fetchone()
 
-        cur.close()
         return db_version[0]
 
     def close(self):
@@ -170,15 +169,13 @@ class DB:
         if (schema_version + 1) not in schemes:
             return
         try:
-            cur = self.conn.cursor()
+            with self.conn.cursor() as cur:
                 # create table one by one
                 for version, commands in schemes.items():
                     if version <= schema_version:
                         continue
                     for command in commands:
                         cur.execute(command)
-            # close communication with the PostgreSQL database server
-            cur.close()
             # commit the changes
             self.conn.commit()
         except (Exception, psycopg2.DatabaseError) as error:

lib/db_revision.py

@@ -23,14 +23,15 @@ class DBRevision:
         return f"[{self.__str__()}]"
 
     def links_to(self, db, project, package):
-        db.cursor().execute(
+        with db.cursor() as cur:
+            cur.execute(
                 "INSERT INTO links (revision_id, project, package) VALUES (%s,%s,%s)",
                 (self.dbid, project, package),
             )
 
     @classmethod
     def import_obs_rev(cls, db, revision):
-        cur = db.cursor()
+        with db.cursor() as cur:
             cur.execute(
                 """INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, request_number)  
                         VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""",
@@ -45,36 +46,33 @@ class DBRevision:
                     revision.request_number,
                 ),
             )
-        cur.close()
         return cls.fetch_revision(db, revision.project, revision.package, revision.rev)
 
-    @classmethod
+    @staticmethod
-    def fetch_revision(cls, db, project, package, rev):
+    def fetch_revision(db, project, package, rev):
-        cur = db.cursor()
+        with db.cursor() as cur:
             cur.execute(
                 "SELECT * FROM revisions where project=%s and package=%s and rev=%s",
                 (project, package, str(rev)),
             )
             row = cur.fetchone()
-        cur.close()
         return DBRevision(row)
 
-    @classmethod
+    @staticmethod
-    def latest_revision(cls, db, project, package):
+    def latest_revision(db, project, package):
-        cur = db.cursor()
+        with db.cursor() as cur:
             cur.execute(
                 "SELECT MAX(rev) FROM revisions where project=%s and package=%s",
                 (project, package),
             )
             max = cur.fetchone()[0]
-        cur.close()
         if max:
             return DBRevision.fetch_revision(db, project, package, int(max))
         return None
 
-    @classmethod
+    @staticmethod
-    def all_revisions(cls, db, project, package):
+    def all_revisions(db, project, package):
-        cur = db.cursor()
+        with db.cursor() as cur:
             cur.execute(
                 "SELECT * FROM revisions where project=%s and package=%s",
                 (project, package),
@@ -82,17 +80,15 @@ class DBRevision:
             ret = []
             for row in cur.fetchall():
                 ret.append(DBRevision(row))
-        cur.close()
         return ret
 
     def linked_rev(self, db):
-        cur = db.cursor()
+        with db.cursor() as cur:
             cur.execute(
                 "SELECT project,package FROM links where revision_id=%s", (self.dbid,)
             )
             row = cur.fetchone()
             if not row:
-            cur.close()
                 return None
             project, package = row
             cur.execute(
@@ -100,18 +96,17 @@ class DBRevision:
                 (project, package, self.commit_time),
             )
             revisions = [DBRevision(row) for row in cur.fetchall()]
-        cur.close()
         if revisions:
             return revisions[0]
         return None
 
     def set_broken(self, db):
-        cur = db.cursor()
+        with db.cursor() as cur:
             cur.execute("UPDATE revisions SET broken=TRUE where id=%s", (self.dbid,))
-        cur.close()
+
 
     def import_dir_list(self, db, xml):
-        cur = db.cursor()
+        with db.cursor() as cur:
             cur.execute(
                 "UPDATE revisions SET expanded_srcmd5=%s where id=%s",
                 (xml.get("srcmd5"), self.dbid),
@@ -128,11 +123,10 @@ class DBRevision:
                         self.dbid,
                     ),
                 )
-        cur.close()
 
-    @classmethod
+    @staticmethod
-    def requests_to_fetch(self, db, project, package):
+    def requests_to_fetch(db, project, package):
-        cur = db.cursor()
+        with db.cursor() as cur:
             cur.execute(
                 """SELECT request_number FROM revisions revs left join requests
                 reqs on reqs.number=revs.request_number where reqs.id is null and
@@ -140,5 +134,4 @@ class DBRevision:
                 (project, package),
             )
             ret = [row[0] for row in cur.fetchall()]
-        cur.close()
         return ret

lib/importer.py

@@ -11,7 +11,7 @@ from lib.obs_revision import OBSRevision
 from lib.proxy_sha256 import ProxySHA256, md5, sha256
 from lib.request import Request
 import xml.etree.ElementTree as ET
-from lib.users import User
+from lib.user import User
 
 
 def _files_hash(hash_alg, dirpath):
@@ -151,7 +151,7 @@ class Importer:
         for project, _, api_url in self.projects:
             self.obs.change_url(api_url)
             self.update_db_package(db, project, self.package)
-            cur = db.cursor()
+            with db.cursor() as cur:
                 cur.execute(
                     "SELECT DISTINCT l.project, l.package from links l join revisions r on r.id=l.revision_id WHERE r.project=%s AND r.package=%s",
                     (project, self.package),
@@ -160,14 +160,11 @@ class Importer:
                     (lproject, lpackage) = row
                     self.update_db_package(db, lproject, lpackage)
 
-            missing_users = User.get_batch(db)
+                missing_users = User.missing_users(db)
-            print(missing_users)
                 for userid in missing_users:
                     missing_user = self.obs.user(userid)
-                if missing_user is not None:
+                    if missing_user:
                         missing_user.import_into_db(db)
-                else:
-                    logging.info("No missing users")
 
                 for rev in DBRevision.all_revisions(db, project, self.package):
                     # TODO move into SELECT
@@ -184,14 +181,9 @@ class Importer:
                     else:
                         rev.set_broken(db)
 
-                # fake_accounts = ["unknown","buildservice-autocommit", "autobuild", "_service"]
-                # if User.user_lookup(db, rev.userid) is None and rev.userid not in fake_accounts:
-                #     self.obs.users(rev.userid).import_into_db(db)
-
                 for number in DBRevision.requests_to_fetch(db, project, self.package):
                     self.obs.request(number).import_into_db(db)
 
-            cur.close()
         db.conn.commit()
 
     def import_all_revisions(self, gc):

lib/obs.py

@@ -8,7 +8,7 @@ from urllib.error import HTTPError
 import osc.core
 
 from lib.request import Request
-from lib.users import User
+from lib.user import User
 
 
 # Add a retry wrapper for some of the HTTP actions.

lib/request.py

@@ -34,7 +34,7 @@ class Request:
         return f"[{self.__str__()}]"
 
     def import_into_db(self, db):
-        cur = db.cursor()
+        with db.cursor() as cur:
             cur.execute(
                 """INSERT INTO requests (number, creator, type, state, 
                     source_project, source_package, source_rev) 
@@ -54,4 +54,3 @@ class Request:
                 "UPDATE revisions SET request_id=%s WHERE request_number=%s",
                 (rowid, self.number),
             )
-        cur.close()

lib/user.py

@@ -0,0 +1,50 @@
+FAKE_ACCOUNTS = ('unknown', 'buildservice-autocommit', 'autobuild', '_service')
+
+
+class User:
+    def parse(self, xml, userid):
+        self.userid = userid
+        self.realname = xml.find("realname").text
+        self.email = xml.find("email").text
+        if self.email is None:
+            self.email = ""
+
+        return self
+
+    def __str__(self):
+        return f"User {self.userid}: {self.realname} {self.email}"
+
+    def __repr__(self):
+        return f"[{self.__str__()}]"
+
+    def import_into_db(self, db):
+        with db.cursor() as cur:
+            cur.execute(
+                """INSERT INTO users (userid, realname, email) 
+                    VALUES (%s,%s,%s)""",
+                (
+                    self.userid,
+                    self.realname,
+                    self.email,
+                ),
+            )
+
+    @staticmethod
+    def lookup(db, userid):
+        with db.cursor() as cur:
+            cur.execute("SELECT * FROM users where userid=%s", (userid,))
+            row = cur.fetchone()
+        if not row:
+            return None
+        return row
+
+    @staticmethod
+    def missing_users(db):
+        with db.cursor() as cur:
+            cur.execute(
+                """SELECT DISTINCT revisions.userid 
+                    FROM revisions LEFT JOIN users ON revisions.userid = users.userid 
+                    WHERE users.userid IS NULL AND revisions.userid NOT IN {}""".format(FAKE_ACCOUNTS)
+            )
+            missing_users = [row[0] for row in cur.fetchall()]
+        return missing_users

lib/users.py

@@ -1,55 +0,0 @@
-FAKE_ACCOUNTS = ('unknown','buildservice-autocommit', 'autobuild', '_service')
-
-class User:
-    def parse(self, xml, userid):
-        self.userid = userid
-        self.realname = xml.find("realname").text
-        self.email = xml.find("email").text
-        if self.email == None:
-            self.email = ""
-
-        return self
-
-    def __str__(self):
-        return f"User {self.userid}: {self.realname} {self.email}"
-
-    def __repr__(self):
-        return f"[{self.__str__()}]"
-
-    def import_into_db(self, db):
-        cur = db.cursor()
-        cur.execute(
-            """INSERT INTO users (userid, realname, email) 
-                VALUES (%s,%s,%s) RETURNING id""",
-            (
-                self.userid,
-                self.realname,
-                self.email,
-            ),
-        )
-        cur.close()
-
-    def lookup(db, userid):
-        cur= db.cursor()
-        cur.execute("SELECT * FROM users where userid=%s", (userid,))
-        row = cur.fetchone()
-        if not row:
-            cur.close()
-            return None
-        return row
-
-    @classmethod
-    def get_batch(self, db):
-        cur = db.cursor()
-        cur.execute(
-            """SELECT DISTINCT revisions.userid 
-                FROM revisions LEFT JOIN users ON revisions.userid = users.userid 
-                WHERE users.userid IS NULL AND revisions.userid NOT IN 
-                ('unknown','buildservice-autocommit', 'autobuild', '_service')"""
-        )
-        missing_users = [row[0] for row in cur.fetchall()]
-        if not missing_users:
-            cur.close()
-            return []
-        return missing_users
-