From fa9ec8618ea6d753c767ac5f5ef0258b111bf5c1 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Wed, 19 Oct 2022 14:00:07 +0200 Subject: [PATCH] Load requests into the DB - not sure yet what for :) --- lib/db.py | 38 +++++++++++++++++++++++++++++++++++++- lib/db_revision.py | 22 ++++++++++++++++++---- lib/importer.py | 6 +++++- lib/obs_revision.py | 12 ++++++------ lib/request.py | 36 ++++++++++++++++++++++++++++++------ 5 files changed, 96 insertions(+), 18 deletions(-) diff --git a/lib/db.py b/lib/db.py index 0a93bfb..5c61ad1 100644 --- a/lib/db.py +++ b/lib/db.py @@ -109,7 +109,43 @@ class DB: """, "UPDATE scheme SET version=5", ) - + schemes[6] = ( + """ + CREATE TABLE requests ( + id SERIAL PRIMARY KEY, + number INTEGER NOT NULL, + revision_id INTEGER NOT NULL, + creator VARCHAR(255) NOT NULL, + type VARCHAR(10) NOT NULL, + state VARCHAR(10) NOT NULL, + source_package VARCHAR(255), + source_project VARCHAR(255) + ) + """, + "UPDATE scheme SET version=6", + ) + schemes[7] = ( + "ALTER TABLE requests DROP COLUMN revision_id", + "UPDATE scheme SET version=7", + ) + schemes[8] = ( + "ALTER TABLE requests ADD COLUMN source_rev VARCHAR(40)", + "UPDATE scheme SET version=8", + ) + schemes[9] = ( + "ALTER TABLE revisions ADD COLUMN request_number INTEGER", + "UPDATE revisions SET request_number=requestid", + "ALTER TABLE revisions DROP COLUMN requestid", + "UPDATE scheme SET version=9", + ) + schemes[10] = ( + "ALTER TABLE revisions ADD COLUMN request_id INTEGER", + """ALTER TABLE revisions + ADD CONSTRAINT request_id_foreign_key + FOREIGN KEY (request_id) + REFERENCES requests (id)""", + "UPDATE scheme SET version=10", + ) schema_version = self.schema_version() if (schema_version + 1) not in schemes: return diff --git a/lib/db_revision.py b/lib/db_revision.py index d804669..b283652 100644 --- a/lib/db_revision.py +++ b/lib/db_revision.py @@ -10,13 +10,14 @@ class DBRevision: self.commit_time, self.userid, self.comment, - self.requestid, self.broken, self.expanded_srcmd5, + self.request_number, + self.request_id, ) = row def __str__(self): - return f"Rev {self.project}/{self.rev} Md5 {self.unexpanded_srcmd5} {self.commit_time} {self.userid} {self.requestid}" + return f"Rev {self.project}/{self.rev} Md5 {self.unexpanded_srcmd5} {self.commit_time} {self.userid} {self.request_number}" def __repr__(self): return f"[{self.__str__()}]" @@ -31,7 +32,7 @@ class DBRevision: def import_obs_rev(cls, db, revision): cur = db.cursor() cur.execute( - """INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, requestid) + """INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, request_number) VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""", ( revision.project, @@ -41,7 +42,7 @@ class DBRevision: revision.time, revision.userid, revision.comment, - revision.requestid, + revision.request_number, ), ) cur.close() @@ -128,3 +129,16 @@ class DBRevision: ), ) cur.close() + + @classmethod + def requests_to_fetch(self, db, project, package): + cur = db.cursor() + cur.execute( + """SELECT request_number FROM revisions revs left join requests + reqs on reqs.number=revs.request_number where reqs.id is null and + revs.request_number is not null and project=%s and package=%s;""", + (project, package), + ) + ret = [row[0] for row in cur.fetchall()] + cur.close() + return ret diff --git a/lib/importer.py b/lib/importer.py index 562ec00..e38fa1a 100644 --- a/lib/importer.py +++ b/lib/importer.py @@ -1,6 +1,5 @@ import functools import logging -import xml.etree.ElementTree as ET from lib.binary import is_binary_or_large from lib.db import DB @@ -10,6 +9,7 @@ from lib.history import History from lib.obs import OBS from lib.obs_revision import OBSRevision from lib.proxy_sha256 import ProxySHA256, md5, sha256 +from lib.request import Request def _files_hash(hash_alg, dirpath): @@ -159,6 +159,7 @@ class Importer: self.update_db_package(db, lproject, lpackage) for rev in DBRevision.all_revisions(db, project, self.package): + # TODO move into SELECT if rev.broken or rev.expanded_srcmd5: continue linked_rev = rev.linked_rev(db) @@ -172,6 +173,9 @@ class Importer: else: rev.set_broken(db) + for number in DBRevision.requests_to_fetch(db, project, self.package): + self.obs.request(number).import_into_db(db) + cur.close() db.conn.commit() diff --git a/lib/obs_revision.py b/lib/obs_revision.py index 0143c2c..f25d5a2 100644 --- a/lib/obs_revision.py +++ b/lib/obs_revision.py @@ -39,10 +39,10 @@ class OBSRevision: # Populated by check_link self.linkrev = None - self.requestid = None - requestid = xml.find("requestid") - if requestid is not None: - self.requestid = int(requestid.text) + self.request_number = None + request_number = xml.find("requestid") + if request_number is not None: + self.request_number = int(request_number.text) else: # Sometimes requestid is missing, but can be extracted # from "comment" @@ -51,12 +51,12 @@ class OBSRevision: self.comment, ) if matched: - self.requestid = int(matched.group(1)) + self.request_number = int(matched.group(1)) return self def __str__(self): - return f"Rev {self.project}/{self.rev} Md5 {self.srcmd5} {self.time} {self.userid} {self.requestid}" + return f"Rev {self.project}/{self.rev} Md5 {self.srcmd5} {self.time} {self.userid} {self.request_number}" def __repr__(self): return f"[{self.__str__()}]" diff --git a/lib/request.py b/lib/request.py index 2b7baec..afc24bf 100644 --- a/lib/request.py +++ b/lib/request.py @@ -1,21 +1,22 @@ class Request: def parse(self, xml): - self.requestid = int(xml.get("id")) + self.number = int(xml.get("id")) self.creator = xml.get("creator") + self.state = xml.find("state").get("name") self.type_ = xml.find("action").get("type") if self.type_ == "delete": # not much to do return self - self.source = xml.find("action/source").get("project") + self.source_project = xml.find("action/source").get("project") + self.source_package = xml.find("action/source").get("package") + # expanded MD5 or commit revision - self.revisionid = xml.find("action/source").get("rev") + self.source_rev = xml.find("action/source").get("rev") self.target = xml.find("action/target").get("project") - self.state = xml.find("state").get("name") - # TODO: support muti-action requests # TODO: parse review history # TODO: add description @@ -25,7 +26,30 @@ class Request: return self.type_ def __str__(self): - return f"Req {self.requestid} {self.creator} {self.type_} {self.source}->{self.target} {self.state}" + return f"Req {self.number} {self.creator} {self.type_} {self.source_project}->{self.target} {self.state}" def __repr__(self): return f"[{self.__str__()}]" + + def import_into_db(self, db): + cur = db.cursor() + cur.execute( + """INSERT INTO requests (number, creator, type, state, + source_project, source_package, source_rev) + VALUES (%s,%s,%s,%s,%s,%s,%s) RETURNING id""", + ( + self.number, + self.creator, + self.type_, + self.state, + self.source_project, + self.source_package, + self.source_rev, + ), + ) + rowid = cur.fetchone()[0] + cur.execute( + "UPDATE revisions SET request_id=%s WHERE request_number=%s", + (rowid, self.number), + ) + cur.close()