From 5ed847de8bf0e377a1e9c21c2407dbc115080409 Mon Sep 17 00:00:00 2001 From: nkrapp Date: Fri, 21 Oct 2022 15:16:34 +0200 Subject: [PATCH 1/6] added user table and import function --- lib/db.py | 12 ++++++++++++ lib/obs.py | 14 ++++++++++++++ lib/users.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 lib/users.py diff --git a/lib/db.py b/lib/db.py index 53d5d3a..80fa486 100644 --- a/lib/db.py +++ b/lib/db.py @@ -154,6 +154,18 @@ class DB: "create index request_number_idx2 on requests(number)", "UPDATE scheme SET version=12", ) + schemes[13] = ( + """ + CREATE TABLE users ( + id SERIAL PRIMARY KEY, + userid VARCHAR(255) NOT NULL, + login VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL, + realname VARCHAR(255) NOT NULL + ) + """, + "UPDATE scheme SET version=13", + ) schema_version = self.schema_version() if (schema_version + 1) not in schemes: diff --git a/lib/obs.py b/lib/obs.py index 77c7c7a..750ac87 100644 --- a/lib/obs.py +++ b/lib/obs.py @@ -8,6 +8,7 @@ from urllib.error import HTTPError import osc.core from lib.request import Request +from lib.users import Users # Add a retry wrapper for some of the HTTP actions. @@ -84,6 +85,14 @@ class OBS: return None return root + def _users(self, userid, **params): + try: + root = self._xml(f"/person/{userid}", **params) + except HTTPError: + logging.error(f"UserID {userid} not found") + return None + return root + def _link(self, project, package, rev): try: root = self._xml(f"source/{project}/{package}/_link", rev=rev) @@ -122,6 +131,11 @@ class OBS: if root is not None: return Request().parse(root) + def users(self, userid): + root = self._users(userid) + if root is not None: + return Users().parse(root, userid) + def files(self, project, package, revision): root = self._xml(f"source/{project}/{package}", rev=revision, expand=1) return [ diff --git a/lib/users.py b/lib/users.py new file mode 100644 index 0000000..3573fdb --- /dev/null +++ b/lib/users.py @@ -0,0 +1,38 @@ +class Users: + def parse(self, xml, userid): + self.userid = userid + self.login = xml.get("login") + self.realname = xml.find("login").get("realname") + self.email = xml.find("login").get("email") + + return self + + def __str__(self): + return f"User {self.userid} {self.login}: {self.realname} {self.email}" + + def __repr__(self): + return f"[{self.__str__()}]" + + def import_into_db(self, db): + cur = db.cursor() + cur.execute( + """INSERT INTO users (userid, login, realname, email) + VALUES (%s,%s,%s,%s) RETURNING id""", + ( + self.userid, + self.login, + self.realname, + self.email, + ), + ) + cur.close() + + def user_lookup(self, db, userid): + cur= db.cursor() + cur.execute("SELECT * FROM users where userid=%s", (userid,)) + row = cur.fetchone() + if not row: + cur.close() + return None + return row + \ No newline at end of file From 10cd7e4e38dee4cc80c2e8e8e10560894749ea2f Mon Sep 17 00:00:00 2001 From: nkrapp Date: Mon, 24 Oct 2022 18:12:44 +0200 Subject: [PATCH 2/6] execute user import --- lib/db.py | 1 - lib/importer.py | 5 +++++ lib/users.py | 14 ++++++-------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/db.py b/lib/db.py index 80fa486..097e675 100644 --- a/lib/db.py +++ b/lib/db.py @@ -159,7 +159,6 @@ class DB: CREATE TABLE users ( id SERIAL PRIMARY KEY, userid VARCHAR(255) NOT NULL, - login VARCHAR(255) NOT NULL, email VARCHAR(255) NOT NULL, realname VARCHAR(255) NOT NULL ) diff --git a/lib/importer.py b/lib/importer.py index e38fa1a..5552d6b 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -10,6 +10,7 @@ from lib.obs import OBS from lib.obs_revision import OBSRevision from lib.proxy_sha256 import ProxySHA256, md5, sha256 from lib.request import Request +from lib.users import Users def _files_hash(hash_alg, dirpath): @@ -173,6 +174,10 @@ class Importer: else: rev.set_broken(db) + fake_accounts = ["unknown","buildservice-autocommit", "autobuild", "_service"] + if Users.user_lookup(db, rev.userid) is None and rev.userid not in fake_accounts: + self.obs.users(rev.userid).import_into_db(db) + for number in DBRevision.requests_to_fetch(db, project, self.package): self.obs.request(number).import_into_db(db) diff --git a/lib/users.py b/lib/users.py index 3573fdb..65abdf8 100644 --- a/lib/users.py +++ b/lib/users.py @@ -1,14 +1,13 @@ class Users: def parse(self, xml, userid): self.userid = userid - self.login = xml.get("login") - self.realname = xml.find("login").get("realname") - self.email = xml.find("login").get("email") + self.realname = xml.find("realname").text + self.email = xml.find("email").text return self def __str__(self): - return f"User {self.userid} {self.login}: {self.realname} {self.email}" + return f"User {self.userid}: {self.realname} {self.email}" def __repr__(self): return f"[{self.__str__()}]" @@ -16,18 +15,17 @@ class Users: def import_into_db(self, db): cur = db.cursor() cur.execute( - """INSERT INTO users (userid, login, realname, email) - VALUES (%s,%s,%s,%s) RETURNING id""", + """INSERT INTO users (userid, realname, email) + VALUES (%s,%s,%s) RETURNING id""", ( self.userid, - self.login, self.realname, self.email, ), ) cur.close() - def user_lookup(self, db, userid): + def user_lookup(db, userid): cur= db.cursor() cur.execute("SELECT * FROM users where userid=%s", (userid,)) row = cur.fetchone() From bab8309d32a880bb7a29678aa484fa31cc14f79a Mon Sep 17 00:00:00 2001 From: nkrapp Date: Tue, 25 Oct 2022 10:21:28 +0200 Subject: [PATCH 3/6] Parse users without email --- lib/users.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/users.py b/lib/users.py index 65abdf8..8862603 100644 --- a/lib/users.py +++ b/lib/users.py @@ -3,6 +3,8 @@ class Users: self.userid = userid self.realname = xml.find("realname").text self.email = xml.find("email").text + if self.email == None: + self.email = "" return self From f14c609db28996c065e49424528d22d2ba61f481 Mon Sep 17 00:00:00 2001 From: nkrapp Date: Wed, 26 Oct 2022 09:30:02 +0200 Subject: [PATCH 4/6] Add batch import --- lib/importer.py | 18 ++++++++++++++---- lib/obs.py | 10 +++++----- lib/users.py | 21 +++++++++++++++++++-- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/lib/importer.py b/lib/importer.py index 5552d6b..7a5a17c 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -10,7 +10,8 @@ from lib.obs import OBS from lib.obs_revision import OBSRevision from lib.proxy_sha256 import ProxySHA256, md5, sha256 from lib.request import Request -from lib.users import Users +import xml.etree.ElementTree as ET +from lib.users import User def _files_hash(hash_alg, dirpath): @@ -159,6 +160,15 @@ class Importer: (lproject, lpackage) = row self.update_db_package(db, lproject, lpackage) + missing_users = User.get_batch(db) + print(missing_users) + for userid in missing_users: + missing_user = self.obs.user(userid) + if missing_user is not None: + missing_user.import_into_db(db) + else: + logging.info("No missing users") + for rev in DBRevision.all_revisions(db, project, self.package): # TODO move into SELECT if rev.broken or rev.expanded_srcmd5: @@ -174,9 +184,9 @@ class Importer: else: rev.set_broken(db) - fake_accounts = ["unknown","buildservice-autocommit", "autobuild", "_service"] - if Users.user_lookup(db, rev.userid) is None and rev.userid not in fake_accounts: - self.obs.users(rev.userid).import_into_db(db) + # fake_accounts = ["unknown","buildservice-autocommit", "autobuild", "_service"] + # if User.user_lookup(db, rev.userid) is None and rev.userid not in fake_accounts: + # self.obs.users(rev.userid).import_into_db(db) for number in DBRevision.requests_to_fetch(db, project, self.package): self.obs.request(number).import_into_db(db) diff --git a/lib/obs.py b/lib/obs.py index 750ac87..661defe 100644 --- a/lib/obs.py +++ b/lib/obs.py @@ -8,7 +8,7 @@ from urllib.error import HTTPError import osc.core from lib.request import Request -from lib.users import Users +from lib.users import User # Add a retry wrapper for some of the HTTP actions. @@ -85,7 +85,7 @@ class OBS: return None return root - def _users(self, userid, **params): + def _user(self, userid, **params): try: root = self._xml(f"/person/{userid}", **params) except HTTPError: @@ -131,10 +131,10 @@ class OBS: if root is not None: return Request().parse(root) - def users(self, userid): - root = self._users(userid) + def user(self, userid): + root = self._user(userid) if root is not None: - return Users().parse(root, userid) + return User().parse(root, userid) def files(self, project, package, revision): root = self._xml(f"source/{project}/{package}", rev=revision, expand=1) diff --git a/lib/users.py b/lib/users.py index 8862603..f26b34f 100644 --- a/lib/users.py +++ b/lib/users.py @@ -1,4 +1,6 @@ -class Users: +FAKE_ACCOUNTS = ('unknown','buildservice-autocommit', 'autobuild', '_service') + +class User: def parse(self, xml, userid): self.userid = userid self.realname = xml.find("realname").text @@ -27,7 +29,7 @@ class Users: ) cur.close() - def user_lookup(db, userid): + def lookup(db, userid): cur= db.cursor() cur.execute("SELECT * FROM users where userid=%s", (userid,)) row = cur.fetchone() @@ -35,4 +37,19 @@ class Users: cur.close() return None return row + + @classmethod + def get_batch(self, db): + cur = db.cursor() + cur.execute( + """SELECT DISTINCT revisions.userid + FROM revisions LEFT JOIN users ON revisions.userid = users.userid + WHERE users.userid IS NULL AND revisions.userid NOT IN + ('unknown','buildservice-autocommit', 'autobuild', '_service')""" + ) + missing_users = [row[0] for row in cur.fetchall()] + if not missing_users: + cur.close() + return [] + return missing_users \ No newline at end of file From 1656d73340a6ce55400a82e79105119cae2804a1 Mon Sep 17 00:00:00 2001 From: nkrapp Date: Wed, 26 Oct 2022 11:58:01 +0200 Subject: [PATCH 5/6] fixed formatting and other suggestions --- lib/db.py | 37 +++++---- lib/db_revision.py | 181 ++++++++++++++++++++++----------------------- lib/importer.py | 68 ++++++++--------- lib/obs.py | 2 +- lib/request.py | 41 +++++----- lib/user.py | 50 +++++++++++++ lib/users.py | 55 -------------- 7 files changed, 205 insertions(+), 229 deletions(-) create mode 100644 lib/user.py delete mode 100644 lib/users.py diff --git a/lib/db.py b/lib/db.py index 097e675..7958c50 100644 --- a/lib/db.py +++ b/lib/db.py @@ -22,20 +22,19 @@ class DB: def schema_version(self): # create a cursor - cur = self.conn.cursor() + with self.conn.cursor() as cur: - # execute a statement - try: - cur.execute("SELECT MAX(version) from scheme") - except psycopg2.errors.UndefinedTable as error: - cur.close() - self.close() - self.connect() - return 0 + # execute a statement + try: + cur.execute("SELECT MAX(version) from scheme") + except psycopg2.errors.UndefinedTable as error: + cur.close() + self.close() + self.connect() + return 0 - db_version = cur.fetchone() + db_version = cur.fetchone() - cur.close() return db_version[0] def close(self): @@ -170,15 +169,13 @@ class DB: if (schema_version + 1) not in schemes: return try: - cur = self.conn.cursor() - # create table one by one - for version, commands in schemes.items(): - if version <= schema_version: - continue - for command in commands: - cur.execute(command) - # close communication with the PostgreSQL database server - cur.close() + with self.conn.cursor() as cur: + # create table one by one + for version, commands in schemes.items(): + if version <= schema_version: + continue + for command in commands: + cur.execute(command) # commit the changes self.conn.commit() except (Exception, psycopg2.DatabaseError) as error: diff --git a/lib/db_revision.py b/lib/db_revision.py index b283652..dbe874d 100644 --- a/lib/db_revision.py +++ b/lib/db_revision.py @@ -23,122 +23,115 @@ class DBRevision: return f"[{self.__str__()}]" def links_to(self, db, project, package): - db.cursor().execute( - "INSERT INTO links (revision_id, project, package) VALUES (%s,%s,%s)", - (self.dbid, project, package), - ) + with db.cursor() as cur: + cur.execute( + "INSERT INTO links (revision_id, project, package) VALUES (%s,%s,%s)", + (self.dbid, project, package), + ) @classmethod def import_obs_rev(cls, db, revision): - cur = db.cursor() - cur.execute( - """INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, request_number) - VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""", - ( - revision.project, - revision.package, - revision.rev, - revision.unexpanded_srcmd5, - revision.time, - revision.userid, - revision.comment, - revision.request_number, - ), - ) - cur.close() + with db.cursor() as cur: + cur.execute( + """INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, request_number) + VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""", + ( + revision.project, + revision.package, + revision.rev, + revision.unexpanded_srcmd5, + revision.time, + revision.userid, + revision.comment, + revision.request_number, + ), + ) return cls.fetch_revision(db, revision.project, revision.package, revision.rev) - @classmethod - def fetch_revision(cls, db, project, package, rev): - cur = db.cursor() - cur.execute( - "SELECT * FROM revisions where project=%s and package=%s and rev=%s", - (project, package, str(rev)), - ) - row = cur.fetchone() - cur.close() + @staticmethod + def fetch_revision(db, project, package, rev): + with db.cursor() as cur: + cur.execute( + "SELECT * FROM revisions where project=%s and package=%s and rev=%s", + (project, package, str(rev)), + ) + row = cur.fetchone() return DBRevision(row) - @classmethod - def latest_revision(cls, db, project, package): - cur = db.cursor() - cur.execute( - "SELECT MAX(rev) FROM revisions where project=%s and package=%s", - (project, package), - ) - max = cur.fetchone()[0] - cur.close() + @staticmethod + def latest_revision(db, project, package): + with db.cursor() as cur: + cur.execute( + "SELECT MAX(rev) FROM revisions where project=%s and package=%s", + (project, package), + ) + max = cur.fetchone()[0] if max: return DBRevision.fetch_revision(db, project, package, int(max)) return None - @classmethod - def all_revisions(cls, db, project, package): - cur = db.cursor() - cur.execute( - "SELECT * FROM revisions where project=%s and package=%s", - (project, package), - ) - ret = [] - for row in cur.fetchall(): - ret.append(DBRevision(row)) - cur.close() + @staticmethod + def all_revisions(db, project, package): + with db.cursor() as cur: + cur.execute( + "SELECT * FROM revisions where project=%s and package=%s", + (project, package), + ) + ret = [] + for row in cur.fetchall(): + ret.append(DBRevision(row)) return ret def linked_rev(self, db): - cur = db.cursor() - cur.execute( - "SELECT project,package FROM links where revision_id=%s", (self.dbid,) - ) - row = cur.fetchone() - if not row: - cur.close() - return None - project, package = row - cur.execute( - "SELECT * FROM revisions where project=%s and package=%s and commit_time <= %s ORDER BY commit_time DESC LIMIT 1", - (project, package, self.commit_time), - ) - revisions = [DBRevision(row) for row in cur.fetchall()] - cur.close() + with db.cursor() as cur: + cur.execute( + "SELECT project,package FROM links where revision_id=%s", (self.dbid,) + ) + row = cur.fetchone() + if not row: + return None + project, package = row + cur.execute( + "SELECT * FROM revisions where project=%s and package=%s and commit_time <= %s ORDER BY commit_time DESC LIMIT 1", + (project, package, self.commit_time), + ) + revisions = [DBRevision(row) for row in cur.fetchall()] if revisions: return revisions[0] return None def set_broken(self, db): - cur = db.cursor() - cur.execute("UPDATE revisions SET broken=TRUE where id=%s", (self.dbid,)) - cur.close() + with db.cursor() as cur: + cur.execute("UPDATE revisions SET broken=TRUE where id=%s", (self.dbid,)) + def import_dir_list(self, db, xml): - cur = db.cursor() - cur.execute( - "UPDATE revisions SET expanded_srcmd5=%s where id=%s", - (xml.get("srcmd5"), self.dbid), - ) - for entry in xml.findall("entry"): + with db.cursor() as cur: cur.execute( - """INSERT INTO files (name, md5, size, mtime, revision_id) - VALUES (%s,%s,%s,%s,%s)""", - ( - entry.get("name"), - entry.get("md5"), - entry.get("size"), - entry.get("mtime"), - self.dbid, - ), + "UPDATE revisions SET expanded_srcmd5=%s where id=%s", + (xml.get("srcmd5"), self.dbid), ) - cur.close() + for entry in xml.findall("entry"): + cur.execute( + """INSERT INTO files (name, md5, size, mtime, revision_id) + VALUES (%s,%s,%s,%s,%s)""", + ( + entry.get("name"), + entry.get("md5"), + entry.get("size"), + entry.get("mtime"), + self.dbid, + ), + ) - @classmethod - def requests_to_fetch(self, db, project, package): - cur = db.cursor() - cur.execute( - """SELECT request_number FROM revisions revs left join requests - reqs on reqs.number=revs.request_number where reqs.id is null and - revs.request_number is not null and project=%s and package=%s;""", - (project, package), - ) - ret = [row[0] for row in cur.fetchall()] - cur.close() + @staticmethod + def requests_to_fetch(db, project, package): + with db.cursor() as cur: + cur.execute( + """SELECT request_number FROM revisions revs left join requests + reqs on reqs.number=revs.request_number where reqs.id is null and + revs.request_number is not null and project=%s and package=%s;""", + (project, package), + ) + ret = [row[0] for row in cur.fetchall()] return ret diff --git a/lib/importer.py b/lib/importer.py index 7a5a17c..67eb34d 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -11,7 +11,7 @@ from lib.obs_revision import OBSRevision from lib.proxy_sha256 import ProxySHA256, md5, sha256 from lib.request import Request import xml.etree.ElementTree as ET -from lib.users import User +from lib.user import User def _files_hash(hash_alg, dirpath): @@ -151,47 +151,39 @@ class Importer: for project, _, api_url in self.projects: self.obs.change_url(api_url) self.update_db_package(db, project, self.package) - cur = db.cursor() - cur.execute( - "SELECT DISTINCT l.project, l.package from links l join revisions r on r.id=l.revision_id WHERE r.project=%s AND r.package=%s", - (project, self.package), - ) - for row in cur.fetchall(): - (lproject, lpackage) = row - self.update_db_package(db, lproject, lpackage) - - missing_users = User.get_batch(db) - print(missing_users) - for userid in missing_users: - missing_user = self.obs.user(userid) - if missing_user is not None: - missing_user.import_into_db(db) - else: - logging.info("No missing users") - - for rev in DBRevision.all_revisions(db, project, self.package): - # TODO move into SELECT - if rev.broken or rev.expanded_srcmd5: - continue - linked_rev = rev.linked_rev(db) - if linked_rev: - linked_rev = linked_rev.unexpanded_srcmd5 - list = self.obs.list( - project, self.package, rev.unexpanded_srcmd5, linked_rev + with db.cursor() as cur: + cur.execute( + "SELECT DISTINCT l.project, l.package from links l join revisions r on r.id=l.revision_id WHERE r.project=%s AND r.package=%s", + (project, self.package), ) - if list: - rev.import_dir_list(db, list) - else: - rev.set_broken(db) + for row in cur.fetchall(): + (lproject, lpackage) = row + self.update_db_package(db, lproject, lpackage) - # fake_accounts = ["unknown","buildservice-autocommit", "autobuild", "_service"] - # if User.user_lookup(db, rev.userid) is None and rev.userid not in fake_accounts: - # self.obs.users(rev.userid).import_into_db(db) + missing_users = User.missing_users(db) + for userid in missing_users: + missing_user = self.obs.user(userid) + if missing_user: + missing_user.import_into_db(db) - for number in DBRevision.requests_to_fetch(db, project, self.package): - self.obs.request(number).import_into_db(db) + for rev in DBRevision.all_revisions(db, project, self.package): + # TODO move into SELECT + if rev.broken or rev.expanded_srcmd5: + continue + linked_rev = rev.linked_rev(db) + if linked_rev: + linked_rev = linked_rev.unexpanded_srcmd5 + list = self.obs.list( + project, self.package, rev.unexpanded_srcmd5, linked_rev + ) + if list: + rev.import_dir_list(db, list) + else: + rev.set_broken(db) + + for number in DBRevision.requests_to_fetch(db, project, self.package): + self.obs.request(number).import_into_db(db) - cur.close() db.conn.commit() def import_all_revisions(self, gc): diff --git a/lib/obs.py b/lib/obs.py index 661defe..51e72e3 100644 --- a/lib/obs.py +++ b/lib/obs.py @@ -8,7 +8,7 @@ from urllib.error import HTTPError import osc.core from lib.request import Request -from lib.users import User +from lib.user import User # Add a retry wrapper for some of the HTTP actions. diff --git a/lib/request.py b/lib/request.py index 33c62c1..f5ee787 100644 --- a/lib/request.py +++ b/lib/request.py @@ -34,24 +34,23 @@ class Request: return f"[{self.__str__()}]" def import_into_db(self, db): - cur = db.cursor() - cur.execute( - """INSERT INTO requests (number, creator, type, state, - source_project, source_package, source_rev) - VALUES (%s,%s,%s,%s,%s,%s,%s) RETURNING id""", - ( - self.number, - self.creator, - self.type_, - self.state, - self.source_project, - self.source_package, - self.source_rev, - ), - ) - rowid = cur.fetchone()[0] - cur.execute( - "UPDATE revisions SET request_id=%s WHERE request_number=%s", - (rowid, self.number), - ) - cur.close() + with db.cursor() as cur: + cur.execute( + """INSERT INTO requests (number, creator, type, state, + source_project, source_package, source_rev) + VALUES (%s,%s,%s,%s,%s,%s,%s) RETURNING id""", + ( + self.number, + self.creator, + self.type_, + self.state, + self.source_project, + self.source_package, + self.source_rev, + ), + ) + rowid = cur.fetchone()[0] + cur.execute( + "UPDATE revisions SET request_id=%s WHERE request_number=%s", + (rowid, self.number), + ) diff --git a/lib/user.py b/lib/user.py new file mode 100644 index 0000000..68fe415 --- /dev/null +++ b/lib/user.py @@ -0,0 +1,50 @@ +FAKE_ACCOUNTS = ('unknown', 'buildservice-autocommit', 'autobuild', '_service') + + +class User: + def parse(self, xml, userid): + self.userid = userid + self.realname = xml.find("realname").text + self.email = xml.find("email").text + if self.email is None: + self.email = "" + + return self + + def __str__(self): + return f"User {self.userid}: {self.realname} {self.email}" + + def __repr__(self): + return f"[{self.__str__()}]" + + def import_into_db(self, db): + with db.cursor() as cur: + cur.execute( + """INSERT INTO users (userid, realname, email) + VALUES (%s,%s,%s)""", + ( + self.userid, + self.realname, + self.email, + ), + ) + + @staticmethod + def lookup(db, userid): + with db.cursor() as cur: + cur.execute("SELECT * FROM users where userid=%s", (userid,)) + row = cur.fetchone() + if not row: + return None + return row + + @staticmethod + def missing_users(db): + with db.cursor() as cur: + cur.execute( + """SELECT DISTINCT revisions.userid + FROM revisions LEFT JOIN users ON revisions.userid = users.userid + WHERE users.userid IS NULL AND revisions.userid NOT IN {}""".format(FAKE_ACCOUNTS) + ) + missing_users = [row[0] for row in cur.fetchall()] + return missing_users diff --git a/lib/users.py b/lib/users.py deleted file mode 100644 index f26b34f..0000000 --- a/lib/users.py +++ /dev/null @@ -1,55 +0,0 @@ -FAKE_ACCOUNTS = ('unknown','buildservice-autocommit', 'autobuild', '_service') - -class User: - def parse(self, xml, userid): - self.userid = userid - self.realname = xml.find("realname").text - self.email = xml.find("email").text - if self.email == None: - self.email = "" - - return self - - def __str__(self): - return f"User {self.userid}: {self.realname} {self.email}" - - def __repr__(self): - return f"[{self.__str__()}]" - - def import_into_db(self, db): - cur = db.cursor() - cur.execute( - """INSERT INTO users (userid, realname, email) - VALUES (%s,%s,%s) RETURNING id""", - ( - self.userid, - self.realname, - self.email, - ), - ) - cur.close() - - def lookup(db, userid): - cur= db.cursor() - cur.execute("SELECT * FROM users where userid=%s", (userid,)) - row = cur.fetchone() - if not row: - cur.close() - return None - return row - - @classmethod - def get_batch(self, db): - cur = db.cursor() - cur.execute( - """SELECT DISTINCT revisions.userid - FROM revisions LEFT JOIN users ON revisions.userid = users.userid - WHERE users.userid IS NULL AND revisions.userid NOT IN - ('unknown','buildservice-autocommit', 'autobuild', '_service')""" - ) - missing_users = [row[0] for row in cur.fetchall()] - if not missing_users: - cur.close() - return [] - return missing_users - \ No newline at end of file From dbeb534108c0142ec2e75e2c087003cb6a65b674 Mon Sep 17 00:00:00 2001 From: nkrapp Date: Wed, 26 Oct 2022 13:46:02 +0200 Subject: [PATCH 6/6] cleanup context managers --- lib/importer.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/importer.py b/lib/importer.py index 67eb34d..34c5f10 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -160,26 +160,26 @@ class Importer: (lproject, lpackage) = row self.update_db_package(db, lproject, lpackage) - missing_users = User.missing_users(db) - for userid in missing_users: - missing_user = self.obs.user(userid) - if missing_user: - missing_user.import_into_db(db) + missing_users = User.missing_users(db) + for userid in missing_users: + missing_user = self.obs.user(userid) + if missing_user: + missing_user.import_into_db(db) - for rev in DBRevision.all_revisions(db, project, self.package): - # TODO move into SELECT - if rev.broken or rev.expanded_srcmd5: - continue - linked_rev = rev.linked_rev(db) - if linked_rev: - linked_rev = linked_rev.unexpanded_srcmd5 - list = self.obs.list( - project, self.package, rev.unexpanded_srcmd5, linked_rev - ) - if list: - rev.import_dir_list(db, list) - else: - rev.set_broken(db) + for rev in DBRevision.all_revisions(db, project, self.package): + # TODO move into SELECT + if rev.broken or rev.expanded_srcmd5: + continue + linked_rev = rev.linked_rev(db) + if linked_rev: + linked_rev = linked_rev.unexpanded_srcmd5 + list = self.obs.list( + project, self.package, rev.unexpanded_srcmd5, linked_rev + ) + if list: + rev.import_dir_list(db, list) + else: + rev.set_broken(db) for number in DBRevision.requests_to_fetch(db, project, self.package): self.obs.request(number).import_into_db(db)