Add experimental import into DB

This commit is contained in:
Stephan Kulow 2022-10-17 20:39:03 +02:00
parent a82562b794
commit 8563334b07
4 changed files with 42 additions and 3 deletions

View File

@ -83,6 +83,11 @@ def main():
default="INFO",
help="logging level",
)
parser.add_argument(
"--db",
action="store_true",
help="Import revisions into database only",
)
args = parser.parse_args()
@ -112,6 +117,9 @@ def main():
importer = Importer(
PROJECTS, args.package, args.repodir, args.search_ancestor, args.rebase_devel
)
if args.db:
importer.import_into_db()
return
importer.import_all_revisions(args.gc)

View File

@ -1,10 +1,12 @@
import psycopg2
from config import config
from lib.config import config
class DB:
def __init__(self):
self.connect()
self.create_tables()
def connect(self):
try:
@ -59,7 +61,8 @@ class DB:
unexpanded_srcmd5 VARCHAR(255) NOT NULL,
commit_time timestamp NOT NULL,
userid VARCHAR(255) NOT NULL,
comment VARCHAR(255)
comment TEXT,
requestid INTEGER
)
""",
)
@ -80,6 +83,24 @@ class DB:
self.close()
raise error
def import_rev(self, revision):
cur = self.conn.cursor()
print(revision)
cur.execute(
"""INSERT INTO revisions (project, package, rev, unexpanded_srcmd5, commit_time, userid, comment, requestid)
VALUES(%s, %s, %s, %s, %s, %s, %s, %s)""",
(
revision.project,
revision.package,
revision.rev,
revision.unexpanded_srcmd5,
revision.time,
revision.userid,
revision.comment,
revision.requestid,
),
)
if __name__ == "__main__":
db = DB()

View File

@ -2,6 +2,7 @@ import functools
import logging
from lib.binary import is_binary_or_large
from lib.db import DB
from lib.git import Git
from lib.history import History
from lib.obs import OBS
@ -120,6 +121,14 @@ class Importer:
print(f"Remove {name}")
self.git.remove(name)
def import_into_db(self):
db = DB()
self.history.fetch_all_revisions(self.projects)
for project, _, _ in self.projects:
for rev in self.history[project]:
db.import_rev(rev)
db.conn.commit()
def import_all_revisions(self, gc):
# Fetch all the requests and sort them. Ideally we should
# build the graph here, to avoid new commits before the merge.

View File

@ -17,8 +17,9 @@ class Revision:
def parse(self, xml):
self.rev = int(xml.get("rev"))
self.unexpanded_srcmd5 = xml.find("srcmd5").text
# Replaced in check_expanded
self.srcmd5 = xml.find("srcmd5").text
self.srcmd5 = self.unexpanded_srcmd5
time = int(xml.find("time").text)
self.time = datetime.datetime.fromtimestamp(time)