- Initial checkin. Version 0.1
OBS-URL: https://build.opensuse.org/package/show/home:jones_tony/kernel-source-component?expand=0&rev=1
This commit is contained in:
Git OBS Bridge
619
kernel-source-component.py
Normal file
619
kernel-source-component.py
Normal file
@@ -0,0 +1,619 @@
|
||||
#!/usr/bin/python3
|
||||
#
|
||||
# Extract a tarball and changelog corresponding to an independently buildable
|
||||
# userspace component from kernel source git
|
||||
#
|
||||
# (C) 2023 by Tony Jones <tonyj@suse.de>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License
|
||||
# as published by the Free Software Foundation; either version 2
|
||||
# of the License, or (at your option) any later version.
|
||||
# See http://www.gnu.org/licenses/gpl-2.0.html for full license text.
|
||||
|
||||
import argparse
|
||||
import giturlparse
|
||||
import glob
|
||||
import os
|
||||
import pygit2
|
||||
import re
|
||||
import sys
|
||||
import subprocess
|
||||
import tempfile
|
||||
import whatthepatch
|
||||
import time
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from wcmatch import glob
|
||||
|
||||
scratch_env = "SCRATCH_AREA"
|
||||
scratch_size_gb = 5
|
||||
scratch_size_bytes = scratch_size_gb * (2**30)
|
||||
component_files = ""
|
||||
patterns = []
|
||||
|
||||
|
||||
def vprint(*pargs, **kwargs):
|
||||
global args
|
||||
|
||||
if args.verbose:
|
||||
print(*pargs, **kwargs)
|
||||
|
||||
|
||||
def warning(*pargs, **kwargs):
|
||||
global args
|
||||
|
||||
if not args.quiet:
|
||||
print("Warning: ", file=sys.stderr, end="")
|
||||
print(*pargs, file=sys.stderr, **kwargs)
|
||||
|
||||
|
||||
def error(*pargs, **kwargs):
|
||||
print("Error: ", file=sys.stderr, end="")
|
||||
print(*pargs, file=sys.stderr, **kwargs)
|
||||
|
||||
|
||||
def cl_write(*pargs, **kwargs):
|
||||
print(*pargs, file=clfile, **kwargs)
|
||||
|
||||
|
||||
def shortsha(commit):
|
||||
return str(commit)[0:10]
|
||||
|
||||
|
||||
def read_obsinfo():
|
||||
names = glob.glob("*.obsinfo")
|
||||
|
||||
if not names:
|
||||
error("Unable to determine component name")
|
||||
return None
|
||||
elif len(names) > 1:
|
||||
error("Too many obsinfo files")
|
||||
return None
|
||||
|
||||
obsinfof = names.pop()
|
||||
|
||||
try:
|
||||
obsinfo = {}
|
||||
|
||||
with open(obsinfof) as file:
|
||||
for line in file:
|
||||
name, value = line.partition(":")[::2]
|
||||
obsinfo[name.strip()] = value.strip()
|
||||
except:
|
||||
error(f"Error parsing obsinfo file '{obsinfof}'")
|
||||
return None
|
||||
|
||||
pathbase = re.sub(".obsinfo$", "", obsinfof)
|
||||
name = obsinfo["name"]
|
||||
|
||||
if pathbase != name:
|
||||
error("Filename/name: mismatch for obsinfo '{obsinfo}' 'name:{name}'")
|
||||
return None
|
||||
|
||||
for required in ["name", "commit"]:
|
||||
if not required in obsinfo:
|
||||
error(f"Error name '{required}' missing from obsinfo '{obsinfof}'")
|
||||
return None
|
||||
|
||||
for suffix in ["spec", "changes"]:
|
||||
if not os.path.isfile(name + "." + suffix):
|
||||
error(f"Unable to verify {suffix} file for component {name}")
|
||||
return None
|
||||
|
||||
return obsinfo
|
||||
|
||||
|
||||
def write_obsinfo(outdir, obsinfo):
|
||||
for key in ["name", "version", "mtime", "commit"]:
|
||||
if not key in obsinfo:
|
||||
error("Error name '{key}' missing from obsinfo")
|
||||
return False
|
||||
|
||||
name = obsinfo["name"]
|
||||
file = f"{outdir}/{name}.obsinfo"
|
||||
|
||||
newp = open(file, "w")
|
||||
if not newp:
|
||||
error(f"Unable to open new obsinfo file '{file}' for writing")
|
||||
return False
|
||||
|
||||
for key in ["name", "version", "mtime", "commit"]:
|
||||
print(f"{key}: {obsinfo[key]}", file=newp)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def update_changelog(outdir, obsinfo, newclp):
|
||||
name = obsinfo["name"]
|
||||
file = f"{name}.changes"
|
||||
newfile = f"{outdir}/{file}"
|
||||
|
||||
oldp = open(file, "r")
|
||||
if not oldp:
|
||||
error(f"Unable to open old changes file '{oldfile}' for reading")
|
||||
return False
|
||||
|
||||
newp = open(newfile, "w")
|
||||
if not newp:
|
||||
error(f"Unable to open new changes file '{file}' for writing")
|
||||
oldp.close()
|
||||
return False
|
||||
|
||||
newclp.seek(0)
|
||||
for fp in [newclp, oldp]:
|
||||
for line in fp.readlines():
|
||||
newp.write(line)
|
||||
|
||||
oldp.close()
|
||||
newp.close()
|
||||
return True
|
||||
|
||||
|
||||
def generate_interdiff(path, filea, fileb):
|
||||
global interdiff_fail
|
||||
|
||||
cmd = f"cd {path} && interdiff <(git cat-file -p {filea.id}) <(git cat-file -p {fileb.id}) 2> /dev/null"
|
||||
|
||||
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
|
||||
file = os.fdopen(proc.stdout.fileno(), "r", encoding="utf8")
|
||||
blob = file.read()
|
||||
|
||||
returncode = proc.wait()
|
||||
|
||||
if returncode != 0:
|
||||
interdiff_fail += 1
|
||||
return None
|
||||
|
||||
paths = []
|
||||
for diff in whatthepatch.parse_patch(blob):
|
||||
path = re.sub("^[ab]/", "", diff.header.new_path)
|
||||
paths.append(path)
|
||||
|
||||
return paths
|
||||
|
||||
|
||||
def git_checkout(gitrepo, gitbranch):
|
||||
scratch = os.environ.get(scratch_env)
|
||||
|
||||
scratch_area = None
|
||||
|
||||
for dir in [scratch, "/abuild", "/tmp"]:
|
||||
if dir and os.path.isdir(dir) and os.access(dir, os.W_OK):
|
||||
scratch_area = dir
|
||||
break
|
||||
|
||||
if not scratch_area:
|
||||
error(
|
||||
f"Unable to locate suitable scratch area. Set environment variable {scratch_env} to a directory with at least {scratch_size_gb}GB available"
|
||||
)
|
||||
return None
|
||||
|
||||
statvfs = os.statvfs(scratch_area)
|
||||
|
||||
if not statvfs or statvfs.f_frsize * statvfs.f_bavail < scratch_size_bytes:
|
||||
error(
|
||||
f"Scratch area '{scratch_area}' requires >= {scratch_size_gb}GB available space"
|
||||
)
|
||||
return None
|
||||
|
||||
tmpdirctx = tempfile.TemporaryDirectory(None, None, scratch_area)
|
||||
|
||||
cmd = ["git", "clone"]
|
||||
msg = f"{gitrepo}"
|
||||
if gitbranch:
|
||||
cmd.append(f"-b{gitbranch}")
|
||||
msg += f":{gitbranch}"
|
||||
cmd.append(gitrepo)
|
||||
cmd.append(tmpdirctx.name)
|
||||
|
||||
vprint(f"Cloning git tree from {msg} to {tmpdirctx.name}")
|
||||
proc = subprocess.run(cmd)
|
||||
|
||||
if proc.returncode != 0:
|
||||
error("git clone failed")
|
||||
return None
|
||||
|
||||
vprint("Git clone completed")
|
||||
|
||||
return tmpdirctx
|
||||
|
||||
|
||||
def get_kernel_version(path):
|
||||
makefile = path + "/Makefile"
|
||||
|
||||
try:
|
||||
with open(makefile) as file:
|
||||
version = None
|
||||
patchlevel = None
|
||||
sublevel = None
|
||||
extraversion = None
|
||||
|
||||
for line in file:
|
||||
_name, _value = line.partition("=")[::2]
|
||||
name = _name.strip()
|
||||
value = _value.strip()
|
||||
|
||||
if name and value:
|
||||
match name:
|
||||
case "VERSION":
|
||||
version = value
|
||||
case "PATCHLEVEL":
|
||||
patchlevel = value
|
||||
case "SUBLEVEL":
|
||||
sublevel = value
|
||||
case "EXTRAVERSION":
|
||||
extraversion = value
|
||||
|
||||
# value should always be present even if ""
|
||||
if name == "EXTRAVERSION":
|
||||
break
|
||||
|
||||
if version and patchlevel:
|
||||
full = f"{version}.{patchlevel}"
|
||||
|
||||
if sublevel:
|
||||
full += f".{sublevel}"
|
||||
if extraversion:
|
||||
full += f".{extraversion}"
|
||||
|
||||
return f"{version}.{patchlevel}.{sublevel}"
|
||||
|
||||
except:
|
||||
error(f"Unable to parse config {config}")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_patch_changelog(data):
|
||||
line = 1
|
||||
subject = None
|
||||
references = None
|
||||
for line in data.splitlines():
|
||||
if line == 10:
|
||||
return None
|
||||
|
||||
res = re.search(r"^([A-Z][a-z]+):(.*)$", line)
|
||||
|
||||
if not res:
|
||||
continue
|
||||
|
||||
tag, _value = res.groups()
|
||||
value = _value.lstrip()
|
||||
|
||||
match tag:
|
||||
case "Subject":
|
||||
subject = value.replace("[PATCH] ", "")
|
||||
case "References":
|
||||
references = value
|
||||
|
||||
if subject and references:
|
||||
return subject, references
|
||||
|
||||
|
||||
def read_config():
|
||||
componentfilesf = "_component_files"
|
||||
patternsf = "_patterns"
|
||||
global patterns, component_files
|
||||
|
||||
try:
|
||||
patterns = [line.rstrip() for line in open(patternsf, "r")]
|
||||
except:
|
||||
error(f"Unable to open pattern file {patternsf}")
|
||||
return False
|
||||
|
||||
try:
|
||||
lines = [line.rstrip() for line in open(componentfilesf)]
|
||||
for line in lines:
|
||||
for item in line.split(" "):
|
||||
component_files += " " + item
|
||||
except:
|
||||
error(f"Unable to open component file {componentfilesf}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def sequence_patches(path):
|
||||
cmd = f"cd {path} && scripts/sequence-patch.sh --rapid"
|
||||
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
|
||||
path = None
|
||||
|
||||
while True:
|
||||
line = proc.stdout.readline()
|
||||
if not line:
|
||||
break
|
||||
match line.split():
|
||||
case [b"Creating", b"tree", b"in", _path]:
|
||||
path = _path.decode()
|
||||
|
||||
returncode = proc.wait()
|
||||
|
||||
if not path or returncode != 0:
|
||||
error("sequence_patch failed")
|
||||
return None
|
||||
else:
|
||||
print(f"Expanded kernel tree to: {path}")
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def generate_tarfile(expanded_tree, outdir, obsinfo):
|
||||
tarversion = obsinfo["version"]
|
||||
name = obsinfo["name"]
|
||||
tarname = f"{name}-{tarversion}"
|
||||
tarfile = f"{outdir}/{tarname}.tar.gz"
|
||||
|
||||
cmd = f"cd {expanded_tree} && tar --transform 's/^/.\/{tarname}\//' -czf {tarfile} {component_files}"
|
||||
proc = subprocess.run(["/bin/bash", "-c", cmd])
|
||||
|
||||
return proc.returncode == 0
|
||||
|
||||
|
||||
def format_commit_message(message, lines=None):
|
||||
existing_fmt = False
|
||||
|
||||
for lineno, line in enumerate(message.splitlines()):
|
||||
if lines and lineno == lines:
|
||||
break
|
||||
|
||||
if lineno == 0 and line[0] == "-":
|
||||
existing_fmt = True
|
||||
|
||||
if existing_fmt:
|
||||
cl_write(line)
|
||||
else:
|
||||
cl_write(f"- {line}" if lineno == 0 else f" {line}")
|
||||
|
||||
|
||||
#
|
||||
#
|
||||
# PROGRAM START
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Extract tarball and changelog from kernel source git"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--outdir",
|
||||
help="osc service parameter for internal use only "
|
||||
"(determines where generated files go before "
|
||||
"collection",
|
||||
dest="outdir",
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose", "-v", help="Enable verbose output", dest="verbose", action="store_true"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quiet", "-q", help="Enable quiet mode", dest="quiet", action="store_true"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo",
|
||||
"-r",
|
||||
help="kernel source repository",
|
||||
dest="gitrepo",
|
||||
default="kerncvs.suse.de:/home/git/kernel-source.git",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--branch", "-b", dest="gitbranch", help="kernel source branch name"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force", "-f",
|
||||
action="store_true",
|
||||
dest="force",
|
||||
default="False",
|
||||
help="Force generation of changelog/tarball even when no changes are detected",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isdir(args.outdir) or not os.access(args.outdir, os.W_OK):
|
||||
error(f"Specified directory '{args.outdir}' does not exist or is not writable")
|
||||
exit(1)
|
||||
|
||||
outdir=os.path.abspath(args.outdir)
|
||||
|
||||
if outdir == os.getcwd():
|
||||
error(f"Specified directory '{args.outdir}' is pwd")
|
||||
exit(1)
|
||||
|
||||
obsinfo = read_obsinfo()
|
||||
|
||||
if not obsinfo:
|
||||
error("Unable to read obsinfo file")
|
||||
exit(1)
|
||||
|
||||
if "version" in obsinfo:
|
||||
match = re.match("^.*\.git([0-9]+)\.[0-9a-z]*$", obsinfo["version"])
|
||||
if match:
|
||||
offset = int(match.group(1))
|
||||
else:
|
||||
warning("commit: but no prior version: found in obsinfo [setting gitoffset = 0]")
|
||||
offset = 0
|
||||
|
||||
if not read_config():
|
||||
error("Unable to read config")
|
||||
exit(1)
|
||||
|
||||
p = giturlparse.parse(args.gitrepo)
|
||||
if not p:
|
||||
error("Unable to parse repository {args.gitrepo}")
|
||||
exit(1)
|
||||
|
||||
if os.path.isdir(p.href):
|
||||
if args.gitbranch:
|
||||
error(
|
||||
"Branch option invalid if specified repository is directory. Please manually checkout required branch and rerun"
|
||||
)
|
||||
exit(1)
|
||||
|
||||
unexpanded_git_tree = args.gitrepo
|
||||
else:
|
||||
gitdir = git_checkout(args.gitrepo, args.gitbranch)
|
||||
if not gitdir:
|
||||
error("Unable to checkout kernel source tree from git")
|
||||
exit(1)
|
||||
|
||||
unexpanded_git_tree = gitdir.name
|
||||
|
||||
clfile = tempfile.TemporaryFile(mode="w+")
|
||||
|
||||
repo = pygit2.Repository(unexpanded_git_tree)
|
||||
|
||||
total_changes = 0
|
||||
head = repo.head.target
|
||||
|
||||
newobsinfo = {}
|
||||
newobsinfo["name"] = obsinfo["name"]
|
||||
newobsinfo["commit"] = head
|
||||
|
||||
interdiff_fail = 0
|
||||
whatthepatch_fail = 0
|
||||
|
||||
for commit in repo.walk(head, pygit2.GIT_SORT_NONE):
|
||||
|
||||
if str(commit.id) == obsinfo["commit"]:
|
||||
break
|
||||
|
||||
offset += 1
|
||||
|
||||
if len(commit.parents) == 1:
|
||||
changelog = []
|
||||
|
||||
diff = commit.parents[0].tree.diff_to_tree(commit.tree)
|
||||
|
||||
num_deltas = len(diff)
|
||||
for delta in diff.deltas:
|
||||
status = delta.status
|
||||
|
||||
if status == pygit2.GIT_DELTA_ADDED:
|
||||
file1 = delta.new_file
|
||||
elif status == pygit2.GIT_DELTA_DELETED:
|
||||
file1 = delta.old_file
|
||||
elif status == pygit2.GIT_DELTA_MODIFIED:
|
||||
file1 = delta.new_file
|
||||
file2 = delta.old_file
|
||||
else:
|
||||
continue
|
||||
|
||||
if status == pygit2.GIT_DELTA_ADDED or status == pygit2.GIT_DELTA_DELETED:
|
||||
if glob.globmatch(file1.path, "patches.*/**", flags=glob.GLOBSTAR):
|
||||
filematched = False
|
||||
blob = repo.revparse_single(str(file1.id)).data.decode("utf-8")
|
||||
|
||||
for diff in whatthepatch.parse_patch(blob):
|
||||
if not diff or not diff.header:
|
||||
whatthepatch_fail += 1
|
||||
continue
|
||||
|
||||
if glob.globmatch(
|
||||
diff.header.new_path, patterns, flags=glob.GLOBSTAR
|
||||
):
|
||||
filematched = True
|
||||
break
|
||||
|
||||
if filematched:
|
||||
res = get_patch_changelog(blob)
|
||||
if res:
|
||||
subject, references = res
|
||||
changelog.append((status, subject, references, file1.path))
|
||||
|
||||
elif status == pygit2.GIT_DELTA_MODIFIED:
|
||||
new = glob.globmatch(file1.path, "patches.*/**", flags=glob.GLOBSTAR)
|
||||
old = glob.globmatch(file2.path, "patches.*/**", flags=glob.GLOBSTAR)
|
||||
|
||||
filematched = False
|
||||
if new or old:
|
||||
paths = generate_interdiff(unexpanded_git_tree, file1, file2)
|
||||
if paths:
|
||||
for path in paths:
|
||||
if glob.globmatch(path, patterns, flags=glob.GLOBSTAR):
|
||||
filematched = True
|
||||
break
|
||||
|
||||
if filematched:
|
||||
blob = repo.revparse_single(str(file1.id)).data.decode("utf-8")
|
||||
res = get_patch_changelog(blob)
|
||||
if res:
|
||||
subject, references = res
|
||||
changelog.append((status, subject, references, file1.path))
|
||||
|
||||
num_changes = len(changelog)
|
||||
|
||||
total_changes += num_changes
|
||||
|
||||
if num_changes > 0:
|
||||
cl_write(
|
||||
"-------------------------------------------------------------------"
|
||||
)
|
||||
stime = datetime.fromtimestamp(commit.commit_time, timezone.utc).strftime(
|
||||
"%a %b %d %H:%M:%S %Z %Y"
|
||||
)
|
||||
cl_write(f"{stime} - {commit.author}")
|
||||
|
||||
if num_changes == 1:
|
||||
if num_changes == 1:
|
||||
# most typical case, one change (patch addition, deletion) or
|
||||
# refresh of patch
|
||||
status, subject, references, file = changelog.pop()
|
||||
|
||||
# ignore above, just use original git commit message
|
||||
format_commit_message(commit.message)
|
||||
|
||||
elif num_changes > 1:
|
||||
format_commit_message(commit.message, 6)
|
||||
|
||||
cl_write("")
|
||||
cl_write("Relevant changes:")
|
||||
for status, subject, references, file in changelog:
|
||||
if subject:
|
||||
for index, line in enumerate(subject.splitlines()):
|
||||
mode = ""
|
||||
match status:
|
||||
case pygit2.GIT_DELTA_ADDED:
|
||||
mode = "Add"
|
||||
case pygit2.GIT_DELTA_DELETED:
|
||||
mode = "Delete"
|
||||
case pygit2.GIT_DELTA_MODIFIED:
|
||||
mode = "Refresh"
|
||||
|
||||
format_commit_message(f"{mode} patch: {file}\n{subject}")
|
||||
|
||||
if references:
|
||||
cl_write(f" ({references})")
|
||||
|
||||
cl_write(f"\n [kernel-source commit {shortsha(commit.id)}]\n")
|
||||
|
||||
if interdiff_fail > 0:
|
||||
warning(f"{interdiff_fail} patches could not be processed by interdiff")
|
||||
|
||||
if whatthepatch_fail > 0:
|
||||
warning(f"{whatthepatch_fail} patches could not be processed by whatthepatch")
|
||||
|
||||
if total_changes > 0 or args.force == True:
|
||||
print(f"New changes: {total_changes}")
|
||||
|
||||
expanded_git_tree = sequence_patches(unexpanded_git_tree)
|
||||
if not expanded_git_tree:
|
||||
error("Unable to expand git tree")
|
||||
exit(1)
|
||||
|
||||
kernversion = get_kernel_version(expanded_git_tree)
|
||||
|
||||
if not kernversion:
|
||||
error("Unable to determine kernel version")
|
||||
exit(1)
|
||||
|
||||
newobsinfo["version"] = f"{kernversion}.git{offset}.{shortsha(head)}"
|
||||
newobsinfo["mtime"] = int(time.time())
|
||||
|
||||
if not generate_tarfile(expanded_git_tree, outdir, newobsinfo):
|
||||
error("Unable to generate tarfile")
|
||||
exit(1)
|
||||
|
||||
update_changelog(outdir, obsinfo, clfile)
|
||||
write_obsinfo(outdir, newobsinfo)
|
||||
print(f"Updated revision to {str(head)}")
|
||||
|
||||
else:
|
||||
print("No new changes")
|
||||
Reference in New Issue
Block a user