adamm/git-importer
1
0
Fork 1
forked from importers/git-importer
Code Pull Requests Activity

fixed formatting and other suggestions

Browse Source
This commit is contained in:
Nico Krapp 2022-10-26 11:58:01 +02:00
parent f14c609db2
commit 1656d73340
No known key found for this signature in database
GPG Key ID: AC35CFFF55212BC7
7 changed files with 205 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
lib/db.py
View File

@ -22,20 +22,19 @@ class DB:
def schema_version(self): def schema_version(self):
# create a cursor # create a cursor
cur = self.conn.cursor() with self.conn.cursor() as cur:
# execute a statement # execute a statement
try: try:
cur.execute("SELECT MAX(version) from scheme") cur.execute("SELECT MAX(version) from scheme")
except psycopg2.errors.UndefinedTable as error: except psycopg2.errors.UndefinedTable as error:
cur.close() cur.close()
self.close() self.close()
self.connect() self.connect()
return 0 return 0
db_version = cur.fetchone() db_version = cur.fetchone()
cur.close()
return db_version[0] return db_version[0]
def close(self): def close(self):
@ -170,15 +169,13 @@ class DB:
if (schema_version + 1) not in schemes: if (schema_version + 1) not in schemes:
return return
try: try:
cur = self.conn.cursor() with self.conn.cursor() as cur:
# create table one by one # create table one by one
for version, commands in schemes.items(): for version, commands in schemes.items():
if version <= schema_version: if version <= schema_version:
continue continue
for command in commands: for command in commands:
cur.execute(command) cur.execute(command)
# close communication with the PostgreSQL database server
cur.close()
# commit the changes # commit the changes
self.conn.commit() self.conn.commit()
except (Exception, psycopg2.DatabaseError) as error: except (Exception, psycopg2.DatabaseError) as error:

181
lib/db_revision.py
View File

@ -23,122 +23,115 @@ class DBRevision:
return f"[{self.__str__()}]" return f"[{self.__str__()}]"
def links_to(self, db, project, package): def links_to(self, db, project, package):
db.cursor().execute( with db.cursor() as cur:
"INSERT INTO links (revision_id, project, package) VALUES (%s,%s,%s)", cur.execute(
(self.dbid, project, package), "INSERT INTO links (revision_id, project, package) VALUES (%s,%s,%s)",
) (self.dbid, project, package),
)
@classmethod @classmethod
def import_obs_rev(cls, db, revision): def import_obs_rev(cls, db, revision):
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"""INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, request_number) """INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, request_number)
VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""", VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""",
( (
revision.project, revision.project,
revision.package, revision.package,
revision.rev, revision.rev,
revision.unexpanded_srcmd5, revision.unexpanded_srcmd5,
revision.time, revision.time,
revision.userid, revision.userid,
revision.comment, revision.comment,
revision.request_number, revision.request_number,
), ),
) )
cur.close()
return cls.fetch_revision(db, revision.project, revision.package, revision.rev) return cls.fetch_revision(db, revision.project, revision.package, revision.rev)
@classmethod @staticmethod
def fetch_revision(cls, db, project, package, rev): def fetch_revision(db, project, package, rev):
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"SELECT * FROM revisions where project=%s and package=%s and rev=%s", "SELECT * FROM revisions where project=%s and package=%s and rev=%s",
(project, package, str(rev)), (project, package, str(rev)),
) )
row = cur.fetchone() row = cur.fetchone()
cur.close()
return DBRevision(row) return DBRevision(row)
@classmethod @staticmethod
def latest_revision(cls, db, project, package): def latest_revision(db, project, package):
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"SELECT MAX(rev) FROM revisions where project=%s and package=%s", "SELECT MAX(rev) FROM revisions where project=%s and package=%s",
(project, package), (project, package),
) )
max = cur.fetchone()[0] max = cur.fetchone()[0]
cur.close()
if max: if max:
return DBRevision.fetch_revision(db, project, package, int(max)) return DBRevision.fetch_revision(db, project, package, int(max))
return None return None
@classmethod @staticmethod
def all_revisions(cls, db, project, package): def all_revisions(db, project, package):
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"SELECT * FROM revisions where project=%s and package=%s", "SELECT * FROM revisions where project=%s and package=%s",
(project, package), (project, package),
) )
ret = [] ret = []
for row in cur.fetchall(): for row in cur.fetchall():
ret.append(DBRevision(row)) ret.append(DBRevision(row))
cur.close()
return ret return ret
def linked_rev(self, db): def linked_rev(self, db):
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"SELECT project,package FROM links where revision_id=%s", (self.dbid,) "SELECT project,package FROM links where revision_id=%s", (self.dbid,)
) )
row = cur.fetchone() row = cur.fetchone()
if not row: if not row:
cur.close() return None
return None project, package = row
project, package = row cur.execute(
cur.execute( "SELECT * FROM revisions where project=%s and package=%s and commit_time <= %s ORDER BY commit_time DESC LIMIT 1",
"SELECT * FROM revisions where project=%s and package=%s and commit_time <= %s ORDER BY commit_time DESC LIMIT 1", (project, package, self.commit_time),
(project, package, self.commit_time), )
) revisions = [DBRevision(row) for row in cur.fetchall()]
revisions = [DBRevision(row) for row in cur.fetchall()]
cur.close()
if revisions: if revisions:
return revisions[0] return revisions[0]
return None return None
def set_broken(self, db): def set_broken(self, db):
cur = db.cursor() with db.cursor() as cur:
cur.execute("UPDATE revisions SET broken=TRUE where id=%s", (self.dbid,)) cur.execute("UPDATE revisions SET broken=TRUE where id=%s", (self.dbid,))
cur.close()
def import_dir_list(self, db, xml): def import_dir_list(self, db, xml):
cur = db.cursor() with db.cursor() as cur:
cur.execute(
"UPDATE revisions SET expanded_srcmd5=%s where id=%s",
(xml.get("srcmd5"), self.dbid),
)
for entry in xml.findall("entry"):
cur.execute( cur.execute(
"""INSERT INTO files (name, md5, size, mtime, revision_id) "UPDATE revisions SET expanded_srcmd5=%s where id=%s",
VALUES (%s,%s,%s,%s,%s)""", (xml.get("srcmd5"), self.dbid),
(
entry.get("name"),
entry.get("md5"),
entry.get("size"),
entry.get("mtime"),
self.dbid,
),
) )
cur.close() for entry in xml.findall("entry"):
cur.execute(
"""INSERT INTO files (name, md5, size, mtime, revision_id)
VALUES (%s,%s,%s,%s,%s)""",
(
entry.get("name"),
entry.get("md5"),
entry.get("size"),
entry.get("mtime"),
self.dbid,
),
)
@classmethod @staticmethod
def requests_to_fetch(self, db, project, package): def requests_to_fetch(db, project, package):
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"""SELECT request_number FROM revisions revs left join requests """SELECT request_number FROM revisions revs left join requests
reqs on reqs.number=revs.request_number where reqs.id is null and reqs on reqs.number=revs.request_number where reqs.id is null and
revs.request_number is not null and project=%s and package=%s;""", revs.request_number is not null and project=%s and package=%s;""",
(project, package), (project, package),
) )
ret = [row[0] for row in cur.fetchall()] ret = [row[0] for row in cur.fetchall()]
cur.close()
return ret return ret

68
lib/importer.py
View File

@ -11,7 +11,7 @@ from lib.obs_revision import OBSRevision
from lib.proxy_sha256 import ProxySHA256, md5, sha256 from lib.proxy_sha256 import ProxySHA256, md5, sha256
from lib.request import Request from lib.request import Request
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from lib.users import User from lib.user import User
def _files_hash(hash_alg, dirpath): def _files_hash(hash_alg, dirpath):
@ -151,47 +151,39 @@ class Importer:
for project, _, api_url in self.projects: for project, _, api_url in self.projects:
self.obs.change_url(api_url) self.obs.change_url(api_url)
self.update_db_package(db, project, self.package) self.update_db_package(db, project, self.package)
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"SELECT DISTINCT l.project, l.package from links l join revisions r on r.id=l.revision_id WHERE r.project=%s AND r.package=%s", "SELECT DISTINCT l.project, l.package from links l join revisions r on r.id=l.revision_id WHERE r.project=%s AND r.package=%s",
(project, self.package), (project, self.package),
)
for row in cur.fetchall():
(lproject, lpackage) = row
self.update_db_package(db, lproject, lpackage)
missing_users = User.get_batch(db)
print(missing_users)
for userid in missing_users:
missing_user = self.obs.user(userid)
if missing_user is not None:
missing_user.import_into_db(db)
else:
logging.info("No missing users")
for rev in DBRevision.all_revisions(db, project, self.package):
# TODO move into SELECT
if rev.broken or rev.expanded_srcmd5:
continue
linked_rev = rev.linked_rev(db)
if linked_rev:
linked_rev = linked_rev.unexpanded_srcmd5
list = self.obs.list(
project, self.package, rev.unexpanded_srcmd5, linked_rev
) )
if list: for row in cur.fetchall():
rev.import_dir_list(db, list) (lproject, lpackage) = row
else: self.update_db_package(db, lproject, lpackage)
rev.set_broken(db)
# fake_accounts = ["unknown","buildservice-autocommit", "autobuild", "_service"] missing_users = User.missing_users(db)
# if User.user_lookup(db, rev.userid) is None and rev.userid not in fake_accounts: for userid in missing_users:
# self.obs.users(rev.userid).import_into_db(db) missing_user = self.obs.user(userid)
if missing_user:
missing_user.import_into_db(db)
for number in DBRevision.requests_to_fetch(db, project, self.package): for rev in DBRevision.all_revisions(db, project, self.package):
self.obs.request(number).import_into_db(db) # TODO move into SELECT
if rev.broken or rev.expanded_srcmd5:
continue
linked_rev = rev.linked_rev(db)
if linked_rev:
linked_rev = linked_rev.unexpanded_srcmd5
list = self.obs.list(
project, self.package, rev.unexpanded_srcmd5, linked_rev
)
if list:
rev.import_dir_list(db, list)
else:
rev.set_broken(db)
for number in DBRevision.requests_to_fetch(db, project, self.package):
self.obs.request(number).import_into_db(db)
cur.close()
db.conn.commit() db.conn.commit()
def import_all_revisions(self, gc): def import_all_revisions(self, gc):

2
lib/obs.py
View File

@ -8,7 +8,7 @@ from urllib.error import HTTPError
import osc.core import osc.core
from lib.request import Request from lib.request import Request
from lib.users import User from lib.user import User
# Add a retry wrapper for some of the HTTP actions. # Add a retry wrapper for some of the HTTP actions.

41
lib/request.py
View File

@ -34,24 +34,23 @@ class Request:
return f"[{self.__str__()}]" return f"[{self.__str__()}]"
def import_into_db(self, db): def import_into_db(self, db):
cur = db.cursor() with db.cursor() as cur:
cur.execute( cur.execute(
"""INSERT INTO requests (number, creator, type, state, """INSERT INTO requests (number, creator, type, state,
source_project, source_package, source_rev) source_project, source_package, source_rev)
VALUES (%s,%s,%s,%s,%s,%s,%s) RETURNING id""", VALUES (%s,%s,%s,%s,%s,%s,%s) RETURNING id""",
( (
self.number, self.number,
self.creator, self.creator,
self.type_, self.type_,
self.state, self.state,
self.source_project, self.source_project,
self.source_package, self.source_package,
self.source_rev, self.source_rev,
), ),
) )
rowid = cur.fetchone()[0] rowid = cur.fetchone()[0]
cur.execute( cur.execute(
"UPDATE revisions SET request_id=%s WHERE request_number=%s", "UPDATE revisions SET request_id=%s WHERE request_number=%s",
(rowid, self.number), (rowid, self.number),
) )
cur.close()

50
lib/user.py Normal file
View File

@ -0,0 +1,50 @@
FAKE_ACCOUNTS = ('unknown', 'buildservice-autocommit', 'autobuild', '_service')
class User:
def parse(self, xml, userid):
self.userid = userid
self.realname = xml.find("realname").text
self.email = xml.find("email").text
if self.email is None:
self.email = ""
return self
def __str__(self):
return f"User {self.userid}: {self.realname} {self.email}"
def __repr__(self):
return f"[{self.__str__()}]"
def import_into_db(self, db):
with db.cursor() as cur:
cur.execute(
"""INSERT INTO users (userid, realname, email)
VALUES (%s,%s,%s)""",
(
self.userid,
self.realname,
self.email,
),
)
@staticmethod
def lookup(db, userid):
with db.cursor() as cur:
cur.execute("SELECT * FROM users where userid=%s", (userid,))
row = cur.fetchone()
if not row:
return None
return row
@staticmethod
def missing_users(db):
with db.cursor() as cur:
cur.execute(
"""SELECT DISTINCT revisions.userid
FROM revisions LEFT JOIN users ON revisions.userid = users.userid
WHERE users.userid IS NULL AND revisions.userid NOT IN {}""".format(FAKE_ACCOUNTS)
)
missing_users = [row[0] for row in cur.fetchall()]
return missing_users

55
lib/users.py
View File

@ -1,55 +0,0 @@
FAKE_ACCOUNTS = ('unknown','buildservice-autocommit', 'autobuild', '_service')
class User:
def parse(self, xml, userid):
self.userid = userid
self.realname = xml.find("realname").text
self.email = xml.find("email").text
if self.email == None:
self.email = ""
return self
def __str__(self):
return f"User {self.userid}: {self.realname} {self.email}"
def __repr__(self):
return f"[{self.__str__()}]"
def import_into_db(self, db):
cur = db.cursor()
cur.execute(
"""INSERT INTO users (userid, realname, email)
VALUES (%s,%s,%s) RETURNING id""",
(
self.userid,
self.realname,
self.email,
),
)
cur.close()
def lookup(db, userid):
cur= db.cursor()
cur.execute("SELECT * FROM users where userid=%s", (userid,))
row = cur.fetchone()
if not row:
cur.close()
return None
return row
@classmethod
def get_batch(self, db):
cur = db.cursor()
cur.execute(
"""SELECT DISTINCT revisions.userid
FROM revisions LEFT JOIN users ON revisions.userid = users.userid
WHERE users.userid IS NULL AND revisions.userid NOT IN
('unknown','buildservice-autocommit', 'autobuild', '_service')"""
)
missing_users = [row[0] for row in cur.fetchall()]
if not missing_users:
cur.close()
return []
return missing_users