importers/git-importer
3
0
Fork 2
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Load requests into the DB - not sure yet what for :)

Browse Source
This commit is contained in:
Stephan Kulow 2022-10-19 14:00:07 +02:00
parent 771595ba85
commit fa9ec8618e
5 changed files with 96 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
lib/db.py
View File

@ -109,7 +109,43 @@ class DB:
""", """,
"UPDATE scheme SET version=5", "UPDATE scheme SET version=5",
) )
schemes[6] = (
"""
CREATE TABLE requests (
id SERIAL PRIMARY KEY,
number INTEGER NOT NULL,
revision_id INTEGER NOT NULL,
creator VARCHAR(255) NOT NULL,
type VARCHAR(10) NOT NULL,
state VARCHAR(10) NOT NULL,
source_package VARCHAR(255),
source_project VARCHAR(255)
)
""",
"UPDATE scheme SET version=6",
)
schemes[7] = (
"ALTER TABLE requests DROP COLUMN revision_id",
"UPDATE scheme SET version=7",
)
schemes[8] = (
"ALTER TABLE requests ADD COLUMN source_rev VARCHAR(40)",
"UPDATE scheme SET version=8",
)
schemes[9] = (
"ALTER TABLE revisions ADD COLUMN request_number INTEGER",
"UPDATE revisions SET request_number=requestid",
"ALTER TABLE revisions DROP COLUMN requestid",
"UPDATE scheme SET version=9",
)
schemes[10] = (
"ALTER TABLE revisions ADD COLUMN request_id INTEGER",
"""ALTER TABLE revisions
ADD CONSTRAINT request_id_foreign_key
FOREIGN KEY (request_id)
REFERENCES requests (id)""",
"UPDATE scheme SET version=10",
)
schema_version = self.schema_version() schema_version = self.schema_version()
if (schema_version + 1) not in schemes: if (schema_version + 1) not in schemes:
return return

22
lib/db_revision.py
View File

@ -10,13 +10,14 @@ class DBRevision:
self.commit_time, self.commit_time,
self.userid, self.userid,
self.comment, self.comment,
self.requestid,
self.broken, self.broken,
self.expanded_srcmd5, self.expanded_srcmd5,
self.request_number,
self.request_id,
) = row ) = row
def __str__(self): def __str__(self):
return f"Rev {self.project}/{self.rev} Md5 {self.unexpanded_srcmd5} {self.commit_time} {self.userid} {self.requestid}" return f"Rev {self.project}/{self.rev} Md5 {self.unexpanded_srcmd5} {self.commit_time} {self.userid} {self.request_number}"
def __repr__(self): def __repr__(self):
return f"[{self.__str__()}]" return f"[{self.__str__()}]"
@ -31,7 +32,7 @@ class DBRevision:
def import_obs_rev(cls, db, revision): def import_obs_rev(cls, db, revision):
cur = db.cursor() cur = db.cursor()
cur.execute( cur.execute(
"""INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, requestid) """INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, request_number)
VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""", VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""",
( (
revision.project, revision.project,
@ -41,7 +42,7 @@ class DBRevision:
revision.time, revision.time,
revision.userid, revision.userid,
revision.comment, revision.comment,
revision.requestid, revision.request_number,
), ),
) )
cur.close() cur.close()
@ -128,3 +129,16 @@ class DBRevision:
), ),
) )
cur.close() cur.close()
@classmethod
def requests_to_fetch(self, db, project, package):
cur = db.cursor()
cur.execute(
"""SELECT request_number FROM revisions revs left join requests
reqs on reqs.number=revs.request_number where reqs.id is null and
revs.request_number is not null and project=%s and package=%s;""",
(project, package),
)
ret = [row[0] for row in cur.fetchall()]
cur.close()
return ret

6
lib/importer.py
View File

@ -1,6 +1,5 @@
import functools import functools
import logging import logging
import xml.etree.ElementTree as ET
from lib.binary import is_binary_or_large from lib.binary import is_binary_or_large
from lib.db import DB from lib.db import DB
@ -10,6 +9,7 @@ from lib.history import History
from lib.obs import OBS from lib.obs import OBS
from lib.obs_revision import OBSRevision from lib.obs_revision import OBSRevision
from lib.proxy_sha256 import ProxySHA256, md5, sha256 from lib.proxy_sha256 import ProxySHA256, md5, sha256
from lib.request import Request
def _files_hash(hash_alg, dirpath): def _files_hash(hash_alg, dirpath):
@ -159,6 +159,7 @@ class Importer:
self.update_db_package(db, lproject, lpackage) self.update_db_package(db, lproject, lpackage)
for rev in DBRevision.all_revisions(db, project, self.package): for rev in DBRevision.all_revisions(db, project, self.package):
# TODO move into SELECT
if rev.broken or rev.expanded_srcmd5: if rev.broken or rev.expanded_srcmd5:
continue continue
linked_rev = rev.linked_rev(db) linked_rev = rev.linked_rev(db)
@ -172,6 +173,9 @@ class Importer:
else: else:
rev.set_broken(db) rev.set_broken(db)
for number in DBRevision.requests_to_fetch(db, project, self.package):
self.obs.request(number).import_into_db(db)
cur.close() cur.close()
db.conn.commit() db.conn.commit()

12
lib/obs_revision.py
View File

@ -39,10 +39,10 @@ class OBSRevision:
# Populated by check_link # Populated by check_link
self.linkrev = None self.linkrev = None
self.requestid = None self.request_number = None
requestid = xml.find("requestid") request_number = xml.find("requestid")
if requestid is not None: if request_number is not None:
self.requestid = int(requestid.text) self.request_number = int(request_number.text)
else: else:
# Sometimes requestid is missing, but can be extracted # Sometimes requestid is missing, but can be extracted
# from "comment" # from "comment"
@ -51,12 +51,12 @@ class OBSRevision:
self.comment, self.comment,
) )
if matched: if matched:
self.requestid = int(matched.group(1)) self.request_number = int(matched.group(1))
return self return self
def __str__(self): def __str__(self):
return f"Rev {self.project}/{self.rev} Md5 {self.srcmd5} {self.time} {self.userid} {self.requestid}" return f"Rev {self.project}/{self.rev} Md5 {self.srcmd5} {self.time} {self.userid} {self.request_number}"
def __repr__(self): def __repr__(self):
return f"[{self.__str__()}]" return f"[{self.__str__()}]"

36
lib/request.py
View File

@ -1,21 +1,22 @@
class Request: class Request:
def parse(self, xml): def parse(self, xml):
self.requestid = int(xml.get("id")) self.number = int(xml.get("id"))
self.creator = xml.get("creator") self.creator = xml.get("creator")
self.state = xml.find("state").get("name")
self.type_ = xml.find("action").get("type") self.type_ = xml.find("action").get("type")
if self.type_ == "delete": if self.type_ == "delete":
# not much to do # not much to do
return self return self
self.source = xml.find("action/source").get("project") self.source_project = xml.find("action/source").get("project")
self.source_package = xml.find("action/source").get("package")
# expanded MD5 or commit revision # expanded MD5 or commit revision
self.revisionid = xml.find("action/source").get("rev") self.source_rev = xml.find("action/source").get("rev")
self.target = xml.find("action/target").get("project") self.target = xml.find("action/target").get("project")
self.state = xml.find("state").get("name")
# TODO: support muti-action requests # TODO: support muti-action requests
# TODO: parse review history # TODO: parse review history
# TODO: add description # TODO: add description
@ -25,7 +26,30 @@ class Request:
return self.type_ return self.type_
def __str__(self): def __str__(self):
return f"Req {self.requestid} {self.creator} {self.type_} {self.source}->{self.target} {self.state}" return f"Req {self.number} {self.creator} {self.type_} {self.source_project}->{self.target} {self.state}"
def __repr__(self): def __repr__(self):
return f"[{self.__str__()}]" return f"[{self.__str__()}]"
def import_into_db(self, db):
cur = db.cursor()
cur.execute(
"""INSERT INTO requests (number, creator, type, state,
source_project, source_package, source_rev)
VALUES (%s,%s,%s,%s,%s,%s,%s) RETURNING id""",
(
self.number,
self.creator,
self.type_,
self.state,
self.source_project,
self.source_package,
self.source_rev,
),
)
rowid = cur.fetchone()[0]
cur.execute(
"UPDATE revisions SET request_id=%s WHERE request_number=%s",
(rowid, self.number),
)
cur.close()