aplanas/git-importer
1
0
Fork 0
forked from importers/git-importer
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed formatting and other suggestions

Browse Source
This commit is contained in:
Nico Krapp 2022-10-26 11:58:01 +02:00
parent f14c609db2
commit 1656d73340
No known key found for this signature in database
GPG Key ID: AC35CFFF55212BC7
7 changed files with 205 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lib/db.py
View File

@ -22,7 +22,7 @@ class DB:
def schema_version(self):
# create a cursor
cur = self.conn.cursor()
with self.conn.cursor() as cur:
# execute a statement
try:
@ -35,7 +35,6 @@ class DB:
db_version = cur.fetchone()
cur.close()
return db_version[0]
def close(self):
@ -170,15 +169,13 @@ class DB:
if (schema_version + 1) not in schemes:
return
try:
cur = self.conn.cursor()
with self.conn.cursor() as cur:
# create table one by one
for version, commands in schemes.items():
if version <= schema_version:
continue
for command in commands:
cur.execute(command)
# close communication with the PostgreSQL database server
cur.close()
# commit the changes
self.conn.commit()
except (Exception, psycopg2.DatabaseError) as error:

45
lib/db_revision.py
View File

@ -23,14 +23,15 @@ class DBRevision:
return f"[{self.__str__()}]"
def links_to(self, db, project, package):
db.cursor().execute(
with db.cursor() as cur:
cur.execute(
"INSERT INTO links (revision_id, project, package) VALUES (%s,%s,%s)",
(self.dbid, project, package),
)
@classmethod
def import_obs_rev(cls, db, revision):
cur = db.cursor()
with db.cursor() as cur:
cur.execute(
"""INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, request_number)
VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""",
@ -45,36 +46,33 @@ class DBRevision:
revision.request_number,
),
)
cur.close()
return cls.fetch_revision(db, revision.project, revision.package, revision.rev)
@classmethod
def fetch_revision(cls, db, project, package, rev):
cur = db.cursor()
@staticmethod
def fetch_revision(db, project, package, rev):
with db.cursor() as cur:
cur.execute(
"SELECT * FROM revisions where project=%s and package=%s and rev=%s",
(project, package, str(rev)),
)
row = cur.fetchone()
cur.close()
return DBRevision(row)
@classmethod
def latest_revision(cls, db, project, package):
cur = db.cursor()
@staticmethod
def latest_revision(db, project, package):
with db.cursor() as cur:
cur.execute(
"SELECT MAX(rev) FROM revisions where project=%s and package=%s",
(project, package),
)
max = cur.fetchone()[0]
cur.close()
if max:
return DBRevision.fetch_revision(db, project, package, int(max))
return None
@classmethod
def all_revisions(cls, db, project, package):
cur = db.cursor()
@staticmethod
def all_revisions(db, project, package):
with db.cursor() as cur:
cur.execute(
"SELECT * FROM revisions where project=%s and package=%s",
(project, package),
@ -82,17 +80,15 @@ class DBRevision:
ret = []
for row in cur.fetchall():
ret.append(DBRevision(row))
cur.close()
return ret
def linked_rev(self, db):
cur = db.cursor()
with db.cursor() as cur:
cur.execute(
"SELECT project,package FROM links where revision_id=%s", (self.dbid,)
)
row = cur.fetchone()
if not row:
cur.close()
return None
project, package = row
cur.execute(
@ -100,18 +96,17 @@ class DBRevision:
(project, package, self.commit_time),
)
revisions = [DBRevision(row) for row in cur.fetchall()]
cur.close()
if revisions:
return revisions[0]
return None
def set_broken(self, db):
cur = db.cursor()
with db.cursor() as cur:
cur.execute("UPDATE revisions SET broken=TRUE where id=%s", (self.dbid,))
cur.close()
def import_dir_list(self, db, xml):
cur = db.cursor()
with db.cursor() as cur:
cur.execute(
"UPDATE revisions SET expanded_srcmd5=%s where id=%s",
(xml.get("srcmd5"), self.dbid),
@ -128,11 +123,10 @@ class DBRevision:
self.dbid,
),
)
cur.close()
@classmethod
def requests_to_fetch(self, db, project, package):
cur = db.cursor()
@staticmethod
def requests_to_fetch(db, project, package):
with db.cursor() as cur:
cur.execute(
"""SELECT request_number FROM revisions revs left join requests
reqs on reqs.number=revs.request_number where reqs.id is null and
@ -140,5 +134,4 @@ class DBRevision:
(project, package),
)
ret = [row[0] for row in cur.fetchall()]
cur.close()
return ret

16
lib/importer.py
View File

@ -11,7 +11,7 @@ from lib.obs_revision import OBSRevision
from lib.proxy_sha256 import ProxySHA256, md5, sha256
from lib.request import Request
import xml.etree.ElementTree as ET
from lib.users import User
from lib.user import User
def _files_hash(hash_alg, dirpath):
@ -151,7 +151,7 @@ class Importer:
for project, _, api_url in self.projects:
self.obs.change_url(api_url)
self.update_db_package(db, project, self.package)
cur = db.cursor()
with db.cursor() as cur:
cur.execute(
"SELECT DISTINCT l.project, l.package from links l join revisions r on r.id=l.revision_id WHERE r.project=%s AND r.package=%s",
(project, self.package),
@ -160,14 +160,11 @@ class Importer:
(lproject, lpackage) = row
self.update_db_package(db, lproject, lpackage)
missing_users = User.get_batch(db)
print(missing_users)
missing_users = User.missing_users(db)
for userid in missing_users:
missing_user = self.obs.user(userid)
if missing_user is not None:
if missing_user:
missing_user.import_into_db(db)
else:
logging.info("No missing users")
for rev in DBRevision.all_revisions(db, project, self.package):
# TODO move into SELECT
@ -184,14 +181,9 @@ class Importer:
else:
rev.set_broken(db)
# fake_accounts = ["unknown","buildservice-autocommit", "autobuild", "_service"]
# if User.user_lookup(db, rev.userid) is None and rev.userid not in fake_accounts:
# self.obs.users(rev.userid).import_into_db(db)
for number in DBRevision.requests_to_fetch(db, project, self.package):
self.obs.request(number).import_into_db(db)
cur.close()
db.conn.commit()
def import_all_revisions(self, gc):

2
lib/obs.py
View File

@ -8,7 +8,7 @@ from urllib.error import HTTPError
import osc.core
from lib.request import Request
from lib.users import User
from lib.user import User
# Add a retry wrapper for some of the HTTP actions.

3
lib/request.py
View File

@ -34,7 +34,7 @@ class Request:
return f"[{self.__str__()}]"
def import_into_db(self, db):
cur = db.cursor()
with db.cursor() as cur:
cur.execute(
"""INSERT INTO requests (number, creator, type, state,
source_project, source_package, source_rev)
@ -54,4 +54,3 @@ class Request:
"UPDATE revisions SET request_id=%s WHERE request_number=%s",
(rowid, self.number),
)
cur.close()

50
lib/user.py Normal file
View File

@ -0,0 +1,50 @@
FAKE_ACCOUNTS = ('unknown', 'buildservice-autocommit', 'autobuild', '_service')
class User:
def parse(self, xml, userid):
self.userid = userid
self.realname = xml.find("realname").text
self.email = xml.find("email").text
if self.email is None:
self.email = ""
return self
def __str__(self):
return f"User {self.userid}: {self.realname} {self.email}"
def __repr__(self):
return f"[{self.__str__()}]"
def import_into_db(self, db):
with db.cursor() as cur:
cur.execute(
"""INSERT INTO users (userid, realname, email)
VALUES (%s,%s,%s)""",
(
self.userid,
self.realname,
self.email,
),
)
@staticmethod
def lookup(db, userid):
with db.cursor() as cur:
cur.execute("SELECT * FROM users where userid=%s", (userid,))
row = cur.fetchone()
if not row:
return None
return row
@staticmethod
def missing_users(db):
with db.cursor() as cur:
cur.execute(
"""SELECT DISTINCT revisions.userid
FROM revisions LEFT JOIN users ON revisions.userid = users.userid
WHERE users.userid IS NULL AND revisions.userid NOT IN {}""".format(FAKE_ACCOUNTS)
)
missing_users = [row[0] for row in cur.fetchall()]
return missing_users

55
lib/users.py
View File

@ -1,55 +0,0 @@
FAKE_ACCOUNTS = ('unknown','buildservice-autocommit', 'autobuild', '_service')
class User:
def parse(self, xml, userid):
self.userid = userid
self.realname = xml.find("realname").text
self.email = xml.find("email").text
if self.email == None:
self.email = ""
return self
def __str__(self):
return f"User {self.userid}: {self.realname} {self.email}"
def __repr__(self):
return f"[{self.__str__()}]"
def import_into_db(self, db):
cur = db.cursor()
cur.execute(
"""INSERT INTO users (userid, realname, email)
VALUES (%s,%s,%s) RETURNING id""",
(
self.userid,
self.realname,
self.email,
),
)
cur.close()
def lookup(db, userid):
cur= db.cursor()
cur.execute("SELECT * FROM users where userid=%s", (userid,))
row = cur.fetchone()
if not row:
cur.close()
return None
return row
@classmethod
def get_batch(self, db):
cur = db.cursor()
cur.execute(
"""SELECT DISTINCT revisions.userid
FROM revisions LEFT JOIN users ON revisions.userid = users.userid
WHERE users.userid IS NULL AND revisions.userid NOT IN
('unknown','buildservice-autocommit', 'autobuild', '_service')"""
)
missing_users = [row[0] for row in cur.fetchall()]
if not missing_users:
cur.close()
return []
return missing_users