diff --git a/lib/db.py b/lib/db.py index 4150b7b..dfb4f9a 100644 --- a/lib/db.py +++ b/lib/db.py @@ -51,21 +51,25 @@ class DB: CREATE TABLE scheme ( id SERIAL PRIMARY KEY, version SMALLINT NOT NULL - ) - """, + ) + """, "INSERT INTO scheme (version) VALUES(1)", + """ DROP TABLE IF EXISTS revisions""", """ CREATE TABLE revisions ( id SERIAL PRIMARY KEY, project VARCHAR(255) NOT NULL, package VARCHAR(255) NOT NULL, - rev VARCHAR(255) NOT NULL, + rev INTEGER NOT NULL, unexpanded_srcmd5 VARCHAR(255) NOT NULL, commit_time timestamp NOT NULL, userid VARCHAR(255) NOT NULL, comment TEXT, requestid INTEGER ) - """, + """, + """ + CREATE UNIQUE INDEX ppr ON revisions (project, package, rev); + """ ) if self.schema_version() > 0: @@ -84,9 +88,11 @@ class DB: self.close() raise error + def cursor(self): + return self.conn.cursor() + def import_rev(self, revision): cur = self.conn.cursor() - print(revision) cur.execute( """INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, requestid) VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""", diff --git a/lib/db_revision.py b/lib/db_revision.py new file mode 100644 index 0000000..62f9cb0 --- /dev/null +++ b/lib/db_revision.py @@ -0,0 +1,22 @@ +class DBRevision: + + def __init__(self, row): + (self.dbid, self.project, self.package, self.rev, self.unexpanded_srcmd5, self.commit_time, self.userid, self.comment, self.requestid) = row + + @classmethod + def fetch_revision(cls, db, project, package, rev): + cur = db.cursor() + cur.execute("SELECT * FROM revisions where project=%s and package=%s and rev=%s", (project, package, rev)) + row = cur.fetchone() + cur.close() + return DBRevision(row) + + @classmethod + def latest_revision(cls, db, project, package): + cur = db.cursor() + cur.execute("SELECT MAX(rev) FROM revisions where project=%s and package=%s", (project, package)) + max = cur.fetchone()[0] + cur.close() + if max: + return DBRevision.fetch_revision(db, project, package, int(max)) + return None diff --git a/lib/history.py b/lib/history.py index 8984353..94ea7b3 100644 --- a/lib/history.py +++ b/lib/history.py @@ -2,7 +2,7 @@ import itertools import logging import re -from lib.revision import Revision +from lib.obs_revision import OBSRevision class History: @@ -33,7 +33,7 @@ class History: root = self.obs._history(project, self.package, **params) if root is not None: return [ - Revision(self.obs, self, project, self.package).parse(r) + OBSRevision(self.obs, self, project, self.package).parse(r) for r in root.findall("revision") ] diff --git a/lib/importer.py b/lib/importer.py index 4b90767..ecc008d 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -6,6 +6,8 @@ from lib.db import DB from lib.git import Git from lib.history import History from lib.obs import OBS +from lib.obs_revision import OBSRevision +from lib.db_revision import DBRevision from lib.proxy_sha256 import ProxySHA256, md5, sha256 @@ -121,13 +123,22 @@ class Importer: print(f"Remove {name}") self.git.remove(name) + def update_db_package(self, db, project, package): + root = self.obs._history(project, package) + if root is None: + return + latest = DBRevision.latest_revision(db, project, self.package) + for r in root.findall("revision"): + rev = OBSRevision(self.obs, self, project, self.package).parse(r) + if not latest or rev.rev > latest.rev: + db.import_rev(rev) + db.conn.commit() + def import_into_db(self): db = DB() - self.history.fetch_all_revisions(self.projects) - for project, _, _ in self.projects: - for rev in self.history[project]: - db.import_rev(rev) - db.conn.commit() + for project, _, api_url in self.projects: + self.obs.change_url(api_url) + self.update_db_package(db, project, self.package) def import_all_revisions(self, gc): # Fetch all the requests and sort them. Ideally we should diff --git a/lib/revision.py b/lib/obs_revision.py similarity index 99% rename from lib/revision.py rename to lib/obs_revision.py index fa67ce0..1c0875e 100644 --- a/lib/revision.py +++ b/lib/obs_revision.py @@ -5,7 +5,7 @@ import xml.etree.ElementTree as ET from urllib.error import HTTPError -class Revision: +class OBSRevision: def __init__(self, obs, history, project, package): self.obs = obs self.history = history diff --git a/tests/db_import_test.py b/tests/db_import_test.py index b52d11e..aa9ca2a 100644 --- a/tests/db_import_test.py +++ b/tests/db_import_test.py @@ -4,7 +4,7 @@ import xml.etree.ElementTree as ET from lib.db import DB from lib.history import History from lib.obs import OBS -from lib.revision import Revision +from lib.obs_revision import OBSRevision class TestDBMethods(unittest.TestCase): @@ -14,7 +14,7 @@ class TestDBMethods(unittest.TestCase): self.history = History(self.obs, "xz") def test_import(self): - test_rev = Revision(self.obs, self.history, "openSUSE:Factory", "xz") + test_rev = OBSRevision(self.obs, self.history, "openSUSE:Factory", "xz") test_rev.parse( ET.fromstring( """