c350ef1226
Initial checkin. Version 0.1 OBS-URL: https://build.opensuse.org/package/show/Kernel:tools/kernel-source-component?expand=0&rev=2
620 lines
17 KiB
Python
620 lines
17 KiB
Python
#!/usr/bin/python3
|
|
#
|
|
# Extract a tarball and changelog corresponding to an independently buildable
|
|
# userspace component from kernel source git
|
|
#
|
|
# (C) 2023 by Tony Jones <tonyj@suse.de>
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License
|
|
# as published by the Free Software Foundation; either version 2
|
|
# of the License, or (at your option) any later version.
|
|
# See http://www.gnu.org/licenses/gpl-2.0.html for full license text.
|
|
|
|
import argparse
|
|
import giturlparse
|
|
import glob
|
|
import os
|
|
import pygit2
|
|
import re
|
|
import sys
|
|
import subprocess
|
|
import tempfile
|
|
import whatthepatch
|
|
import time
|
|
|
|
from datetime import datetime, timezone
|
|
from wcmatch import glob
|
|
|
|
scratch_env = "SCRATCH_AREA"
|
|
scratch_size_gb = 5
|
|
scratch_size_bytes = scratch_size_gb * (2**30)
|
|
component_files = ""
|
|
patterns = []
|
|
|
|
|
|
def vprint(*pargs, **kwargs):
|
|
global args
|
|
|
|
if args.verbose:
|
|
print(*pargs, **kwargs)
|
|
|
|
|
|
def warning(*pargs, **kwargs):
|
|
global args
|
|
|
|
if not args.quiet:
|
|
print("Warning: ", file=sys.stderr, end="")
|
|
print(*pargs, file=sys.stderr, **kwargs)
|
|
|
|
|
|
def error(*pargs, **kwargs):
|
|
print("Error: ", file=sys.stderr, end="")
|
|
print(*pargs, file=sys.stderr, **kwargs)
|
|
|
|
|
|
def cl_write(*pargs, **kwargs):
|
|
print(*pargs, file=clfile, **kwargs)
|
|
|
|
|
|
def shortsha(commit):
|
|
return str(commit)[0:10]
|
|
|
|
|
|
def read_obsinfo():
|
|
names = glob.glob("*.obsinfo")
|
|
|
|
if not names:
|
|
error("Unable to determine component name")
|
|
return None
|
|
elif len(names) > 1:
|
|
error("Too many obsinfo files")
|
|
return None
|
|
|
|
obsinfof = names.pop()
|
|
|
|
try:
|
|
obsinfo = {}
|
|
|
|
with open(obsinfof) as file:
|
|
for line in file:
|
|
name, value = line.partition(":")[::2]
|
|
obsinfo[name.strip()] = value.strip()
|
|
except:
|
|
error(f"Error parsing obsinfo file '{obsinfof}'")
|
|
return None
|
|
|
|
pathbase = re.sub(".obsinfo$", "", obsinfof)
|
|
name = obsinfo["name"]
|
|
|
|
if pathbase != name:
|
|
error("Filename/name: mismatch for obsinfo '{obsinfo}' 'name:{name}'")
|
|
return None
|
|
|
|
for required in ["name", "commit"]:
|
|
if not required in obsinfo:
|
|
error(f"Error name '{required}' missing from obsinfo '{obsinfof}'")
|
|
return None
|
|
|
|
for suffix in ["spec", "changes"]:
|
|
if not os.path.isfile(name + "." + suffix):
|
|
error(f"Unable to verify {suffix} file for component {name}")
|
|
return None
|
|
|
|
return obsinfo
|
|
|
|
|
|
def write_obsinfo(outdir, obsinfo):
|
|
for key in ["name", "version", "mtime", "commit"]:
|
|
if not key in obsinfo:
|
|
error("Error name '{key}' missing from obsinfo")
|
|
return False
|
|
|
|
name = obsinfo["name"]
|
|
file = f"{outdir}/{name}.obsinfo"
|
|
|
|
newp = open(file, "w")
|
|
if not newp:
|
|
error(f"Unable to open new obsinfo file '{file}' for writing")
|
|
return False
|
|
|
|
for key in ["name", "version", "mtime", "commit"]:
|
|
print(f"{key}: {obsinfo[key]}", file=newp)
|
|
|
|
return True
|
|
|
|
|
|
def update_changelog(outdir, obsinfo, newclp):
|
|
name = obsinfo["name"]
|
|
file = f"{name}.changes"
|
|
newfile = f"{outdir}/{file}"
|
|
|
|
oldp = open(file, "r")
|
|
if not oldp:
|
|
error(f"Unable to open old changes file '{oldfile}' for reading")
|
|
return False
|
|
|
|
newp = open(newfile, "w")
|
|
if not newp:
|
|
error(f"Unable to open new changes file '{file}' for writing")
|
|
oldp.close()
|
|
return False
|
|
|
|
newclp.seek(0)
|
|
for fp in [newclp, oldp]:
|
|
for line in fp.readlines():
|
|
newp.write(line)
|
|
|
|
oldp.close()
|
|
newp.close()
|
|
return True
|
|
|
|
|
|
def generate_interdiff(path, filea, fileb):
|
|
global interdiff_fail
|
|
|
|
cmd = f"cd {path} && interdiff <(git cat-file -p {filea.id}) <(git cat-file -p {fileb.id}) 2> /dev/null"
|
|
|
|
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
|
|
file = os.fdopen(proc.stdout.fileno(), "r", encoding="utf8")
|
|
blob = file.read()
|
|
|
|
returncode = proc.wait()
|
|
|
|
if returncode != 0:
|
|
interdiff_fail += 1
|
|
return None
|
|
|
|
paths = []
|
|
for diff in whatthepatch.parse_patch(blob):
|
|
path = re.sub("^[ab]/", "", diff.header.new_path)
|
|
paths.append(path)
|
|
|
|
return paths
|
|
|
|
|
|
def git_checkout(gitrepo, gitbranch):
|
|
scratch = os.environ.get(scratch_env)
|
|
|
|
scratch_area = None
|
|
|
|
for dir in [scratch, "/abuild", "/tmp"]:
|
|
if dir and os.path.isdir(dir) and os.access(dir, os.W_OK):
|
|
scratch_area = dir
|
|
break
|
|
|
|
if not scratch_area:
|
|
error(
|
|
f"Unable to locate suitable scratch area. Set environment variable {scratch_env} to a directory with at least {scratch_size_gb}GB available"
|
|
)
|
|
return None
|
|
|
|
statvfs = os.statvfs(scratch_area)
|
|
|
|
if not statvfs or statvfs.f_frsize * statvfs.f_bavail < scratch_size_bytes:
|
|
error(
|
|
f"Scratch area '{scratch_area}' requires >= {scratch_size_gb}GB available space"
|
|
)
|
|
return None
|
|
|
|
tmpdirctx = tempfile.TemporaryDirectory(None, None, scratch_area)
|
|
|
|
cmd = ["git", "clone"]
|
|
msg = f"{gitrepo}"
|
|
if gitbranch:
|
|
cmd.append(f"-b{gitbranch}")
|
|
msg += f":{gitbranch}"
|
|
cmd.append(gitrepo)
|
|
cmd.append(tmpdirctx.name)
|
|
|
|
vprint(f"Cloning git tree from {msg} to {tmpdirctx.name}")
|
|
proc = subprocess.run(cmd)
|
|
|
|
if proc.returncode != 0:
|
|
error("git clone failed")
|
|
return None
|
|
|
|
vprint("Git clone completed")
|
|
|
|
return tmpdirctx
|
|
|
|
|
|
def get_kernel_version(path):
|
|
makefile = path + "/Makefile"
|
|
|
|
try:
|
|
with open(makefile) as file:
|
|
version = None
|
|
patchlevel = None
|
|
sublevel = None
|
|
extraversion = None
|
|
|
|
for line in file:
|
|
_name, _value = line.partition("=")[::2]
|
|
name = _name.strip()
|
|
value = _value.strip()
|
|
|
|
if name and value:
|
|
match name:
|
|
case "VERSION":
|
|
version = value
|
|
case "PATCHLEVEL":
|
|
patchlevel = value
|
|
case "SUBLEVEL":
|
|
sublevel = value
|
|
case "EXTRAVERSION":
|
|
extraversion = value
|
|
|
|
# value should always be present even if ""
|
|
if name == "EXTRAVERSION":
|
|
break
|
|
|
|
if version and patchlevel:
|
|
full = f"{version}.{patchlevel}"
|
|
|
|
if sublevel:
|
|
full += f".{sublevel}"
|
|
if extraversion:
|
|
full += f".{extraversion}"
|
|
|
|
return f"{version}.{patchlevel}.{sublevel}"
|
|
|
|
except:
|
|
error(f"Unable to parse config {config}")
|
|
|
|
return None
|
|
|
|
|
|
def get_patch_changelog(data):
|
|
line = 1
|
|
subject = None
|
|
references = None
|
|
for line in data.splitlines():
|
|
if line == 10:
|
|
return None
|
|
|
|
res = re.search(r"^([A-Z][a-z]+):(.*)$", line)
|
|
|
|
if not res:
|
|
continue
|
|
|
|
tag, _value = res.groups()
|
|
value = _value.lstrip()
|
|
|
|
match tag:
|
|
case "Subject":
|
|
subject = value.replace("[PATCH] ", "")
|
|
case "References":
|
|
references = value
|
|
|
|
if subject and references:
|
|
return subject, references
|
|
|
|
|
|
def read_config():
|
|
componentfilesf = "_component_files"
|
|
patternsf = "_patterns"
|
|
global patterns, component_files
|
|
|
|
try:
|
|
patterns = [line.rstrip() for line in open(patternsf, "r")]
|
|
except:
|
|
error(f"Unable to open pattern file {patternsf}")
|
|
return False
|
|
|
|
try:
|
|
lines = [line.rstrip() for line in open(componentfilesf)]
|
|
for line in lines:
|
|
for item in line.split(" "):
|
|
component_files += " " + item
|
|
except:
|
|
error(f"Unable to open component file {componentfilesf}")
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def sequence_patches(path):
|
|
cmd = f"cd {path} && scripts/sequence-patch.sh --rapid"
|
|
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
|
|
path = None
|
|
|
|
while True:
|
|
line = proc.stdout.readline()
|
|
if not line:
|
|
break
|
|
match line.split():
|
|
case [b"Creating", b"tree", b"in", _path]:
|
|
path = _path.decode()
|
|
|
|
returncode = proc.wait()
|
|
|
|
if not path or returncode != 0:
|
|
error("sequence_patch failed")
|
|
return None
|
|
else:
|
|
print(f"Expanded kernel tree to: {path}")
|
|
|
|
return path
|
|
|
|
|
|
def generate_tarfile(expanded_tree, outdir, obsinfo):
|
|
tarversion = obsinfo["version"]
|
|
name = obsinfo["name"]
|
|
tarname = f"{name}-{tarversion}"
|
|
tarfile = f"{outdir}/{tarname}.tar.gz"
|
|
|
|
cmd = f"cd {expanded_tree} && tar --transform 's/^/.\/{tarname}\//' -czf {tarfile} {component_files}"
|
|
proc = subprocess.run(["/bin/bash", "-c", cmd])
|
|
|
|
return proc.returncode == 0
|
|
|
|
|
|
def format_commit_message(message, lines=None):
|
|
existing_fmt = False
|
|
|
|
for lineno, line in enumerate(message.splitlines()):
|
|
if lines and lineno == lines:
|
|
break
|
|
|
|
if lineno == 0 and line[0] == "-":
|
|
existing_fmt = True
|
|
|
|
if existing_fmt:
|
|
cl_write(line)
|
|
else:
|
|
cl_write(f"- {line}" if lineno == 0 else f" {line}")
|
|
|
|
|
|
#
|
|
#
|
|
# PROGRAM START
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Extract tarball and changelog from kernel source git"
|
|
)
|
|
parser.add_argument(
|
|
"--outdir",
|
|
help="osc service parameter for internal use only "
|
|
"(determines where generated files go before "
|
|
"collection",
|
|
dest="outdir",
|
|
required=True,
|
|
)
|
|
parser.add_argument(
|
|
"--verbose", "-v", help="Enable verbose output", dest="verbose", action="store_true"
|
|
)
|
|
parser.add_argument(
|
|
"--quiet", "-q", help="Enable quiet mode", dest="quiet", action="store_true"
|
|
)
|
|
parser.add_argument(
|
|
"--repo",
|
|
"-r",
|
|
help="kernel source repository",
|
|
dest="gitrepo",
|
|
default="kerncvs.suse.de:/home/git/kernel-source.git",
|
|
)
|
|
parser.add_argument(
|
|
"--branch", "-b", dest="gitbranch", help="kernel source branch name"
|
|
)
|
|
parser.add_argument(
|
|
"--force", "-f",
|
|
action="store_true",
|
|
dest="force",
|
|
default="False",
|
|
help="Force generation of changelog/tarball even when no changes are detected",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
if not os.path.isdir(args.outdir) or not os.access(args.outdir, os.W_OK):
|
|
error(f"Specified directory '{args.outdir}' does not exist or is not writable")
|
|
exit(1)
|
|
|
|
outdir=os.path.abspath(args.outdir)
|
|
|
|
if outdir == os.getcwd():
|
|
error(f"Specified directory '{args.outdir}' is pwd")
|
|
exit(1)
|
|
|
|
obsinfo = read_obsinfo()
|
|
|
|
if not obsinfo:
|
|
error("Unable to read obsinfo file")
|
|
exit(1)
|
|
|
|
if "version" in obsinfo:
|
|
match = re.match("^.*\.git([0-9]+)\.[0-9a-z]*$", obsinfo["version"])
|
|
if match:
|
|
offset = int(match.group(1))
|
|
else:
|
|
warning("commit: but no prior version: found in obsinfo [setting gitoffset = 0]")
|
|
offset = 0
|
|
|
|
if not read_config():
|
|
error("Unable to read config")
|
|
exit(1)
|
|
|
|
p = giturlparse.parse(args.gitrepo)
|
|
if not p:
|
|
error("Unable to parse repository {args.gitrepo}")
|
|
exit(1)
|
|
|
|
if os.path.isdir(p.href):
|
|
if args.gitbranch:
|
|
error(
|
|
"Branch option invalid if specified repository is directory. Please manually checkout required branch and rerun"
|
|
)
|
|
exit(1)
|
|
|
|
unexpanded_git_tree = args.gitrepo
|
|
else:
|
|
gitdir = git_checkout(args.gitrepo, args.gitbranch)
|
|
if not gitdir:
|
|
error("Unable to checkout kernel source tree from git")
|
|
exit(1)
|
|
|
|
unexpanded_git_tree = gitdir.name
|
|
|
|
clfile = tempfile.TemporaryFile(mode="w+")
|
|
|
|
repo = pygit2.Repository(unexpanded_git_tree)
|
|
|
|
total_changes = 0
|
|
head = repo.head.target
|
|
|
|
newobsinfo = {}
|
|
newobsinfo["name"] = obsinfo["name"]
|
|
newobsinfo["commit"] = head
|
|
|
|
interdiff_fail = 0
|
|
whatthepatch_fail = 0
|
|
|
|
for commit in repo.walk(head, pygit2.GIT_SORT_NONE):
|
|
|
|
if str(commit.id) == obsinfo["commit"]:
|
|
break
|
|
|
|
offset += 1
|
|
|
|
if len(commit.parents) == 1:
|
|
changelog = []
|
|
|
|
diff = commit.parents[0].tree.diff_to_tree(commit.tree)
|
|
|
|
num_deltas = len(diff)
|
|
for delta in diff.deltas:
|
|
status = delta.status
|
|
|
|
if status == pygit2.GIT_DELTA_ADDED:
|
|
file1 = delta.new_file
|
|
elif status == pygit2.GIT_DELTA_DELETED:
|
|
file1 = delta.old_file
|
|
elif status == pygit2.GIT_DELTA_MODIFIED:
|
|
file1 = delta.new_file
|
|
file2 = delta.old_file
|
|
else:
|
|
continue
|
|
|
|
if status == pygit2.GIT_DELTA_ADDED or status == pygit2.GIT_DELTA_DELETED:
|
|
if glob.globmatch(file1.path, "patches.*/**", flags=glob.GLOBSTAR):
|
|
filematched = False
|
|
blob = repo.revparse_single(str(file1.id)).data.decode("utf-8")
|
|
|
|
for diff in whatthepatch.parse_patch(blob):
|
|
if not diff or not diff.header:
|
|
whatthepatch_fail += 1
|
|
continue
|
|
|
|
if glob.globmatch(
|
|
diff.header.new_path, patterns, flags=glob.GLOBSTAR
|
|
):
|
|
filematched = True
|
|
break
|
|
|
|
if filematched:
|
|
res = get_patch_changelog(blob)
|
|
if res:
|
|
subject, references = res
|
|
changelog.append((status, subject, references, file1.path))
|
|
|
|
elif status == pygit2.GIT_DELTA_MODIFIED:
|
|
new = glob.globmatch(file1.path, "patches.*/**", flags=glob.GLOBSTAR)
|
|
old = glob.globmatch(file2.path, "patches.*/**", flags=glob.GLOBSTAR)
|
|
|
|
filematched = False
|
|
if new or old:
|
|
paths = generate_interdiff(unexpanded_git_tree, file1, file2)
|
|
if paths:
|
|
for path in paths:
|
|
if glob.globmatch(path, patterns, flags=glob.GLOBSTAR):
|
|
filematched = True
|
|
break
|
|
|
|
if filematched:
|
|
blob = repo.revparse_single(str(file1.id)).data.decode("utf-8")
|
|
res = get_patch_changelog(blob)
|
|
if res:
|
|
subject, references = res
|
|
changelog.append((status, subject, references, file1.path))
|
|
|
|
num_changes = len(changelog)
|
|
|
|
total_changes += num_changes
|
|
|
|
if num_changes > 0:
|
|
cl_write(
|
|
"-------------------------------------------------------------------"
|
|
)
|
|
stime = datetime.fromtimestamp(commit.commit_time, timezone.utc).strftime(
|
|
"%a %b %d %H:%M:%S %Z %Y"
|
|
)
|
|
cl_write(f"{stime} - {commit.author}\n")
|
|
|
|
if num_changes == 1:
|
|
if num_changes == 1:
|
|
# most typical case, one change (patch addition, deletion) or
|
|
# refresh of patch
|
|
status, subject, references, file = changelog.pop()
|
|
|
|
# ignore above, just use original git commit message
|
|
format_commit_message(commit.message)
|
|
|
|
elif num_changes > 1:
|
|
format_commit_message(commit.message, 6)
|
|
|
|
cl_write("")
|
|
cl_write("Relevant changes:")
|
|
for status, subject, references, file in changelog:
|
|
if subject:
|
|
for index, line in enumerate(subject.splitlines()):
|
|
mode = ""
|
|
match status:
|
|
case pygit2.GIT_DELTA_ADDED:
|
|
mode = "Add"
|
|
case pygit2.GIT_DELTA_DELETED:
|
|
mode = "Delete"
|
|
case pygit2.GIT_DELTA_MODIFIED:
|
|
mode = "Refresh"
|
|
|
|
format_commit_message(f"{mode} patch: {file}\n{subject}")
|
|
|
|
if references:
|
|
cl_write(f" ({references})")
|
|
|
|
cl_write(f"\n [kernel-source commit {shortsha(commit.id)}]\n")
|
|
|
|
if interdiff_fail > 0:
|
|
warning(f"{interdiff_fail} patches could not be processed by interdiff")
|
|
|
|
if whatthepatch_fail > 0:
|
|
warning(f"{whatthepatch_fail} patches could not be processed by whatthepatch")
|
|
|
|
if total_changes > 0 or args.force == True:
|
|
print(f"New changes: {total_changes}")
|
|
|
|
expanded_git_tree = sequence_patches(unexpanded_git_tree)
|
|
if not expanded_git_tree:
|
|
error("Unable to expand git tree")
|
|
exit(1)
|
|
|
|
kernversion = get_kernel_version(expanded_git_tree)
|
|
|
|
if not kernversion:
|
|
error("Unable to determine kernel version")
|
|
exit(1)
|
|
|
|
newobsinfo["version"] = f"{kernversion}.git{offset}.{shortsha(head)}"
|
|
newobsinfo["mtime"] = int(time.time())
|
|
|
|
if not generate_tarfile(expanded_git_tree, outdir, newobsinfo):
|
|
error("Unable to generate tarfile")
|
|
exit(1)
|
|
|
|
update_changelog(outdir, obsinfo, clfile)
|
|
write_obsinfo(outdir, newobsinfo)
|
|
print(f"Updated revision to {str(head)}")
|
|
|
|
else:
|
|
print("No new changes")
|