kernel-source-component/kernel-source-component.py

#!/usr/bin/python3
#
# Extract a tarball and changelog corresponding to an independently buildable
# userspace component from kernel source git
#
# (C) 2023 by Tony Jones <tonyj@suse.de>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# See http://www.gnu.org/licenses/gpl-2.0.html for full license text.

import argparse
import giturlparse
import glob
import os
import pygit2
import re
import sys
import subprocess
import tempfile
import whatthepatch
import time

from datetime import datetime, timezone
from wcmatch import glob

scratch_env = "SCRATCH_AREA"
scratch_size_gb = 5
scratch_size_bytes = scratch_size_gb * (2**30)
component_files = ""
patterns = []


def vprint(*pargs, **kwargs):
    global args

    if args.verbose:
        print(*pargs, **kwargs)


def warning(*pargs, **kwargs):
    global args

    if not args.quiet:
        print("Warning: ", file=sys.stderr, end="")
        print(*pargs, file=sys.stderr, **kwargs)


def error(*pargs, **kwargs):
    print("Error: ", file=sys.stderr, end="")
    print(*pargs, file=sys.stderr, **kwargs)


def cl_write(*pargs, **kwargs):
    print(*pargs, file=clfile, **kwargs)


def shortsha(commit):
    return str(commit)[0:10]


def read_obsinfo():
    names = glob.glob("*.obsinfo")

    if not names:
        error("Unable to determine component name")
        return None
    elif len(names) > 1:
        error("Too many obsinfo files")
        return None

    obsinfof = names.pop()

    try:
        obsinfo = {}

        with open(obsinfof) as file:
            for line in file:
                name, value = line.partition(":")[::2]
                obsinfo[name.strip()] = value.strip()
    except:
        error(f"Error parsing obsinfo file '{obsinfof}'")
        return None

    pathbase = re.sub(".obsinfo$", "", obsinfof)
    name = obsinfo["name"]

    if pathbase != name:
        error("Filename/name: mismatch for obsinfo '{obsinfo}' 'name:{name}'")
        return None

    for required in ["name", "commit"]:
        if not required in obsinfo:
            error(f"Error name '{required}' missing from obsinfo '{obsinfof}'")
            return None

    for suffix in ["spec", "changes"]:
        if not os.path.isfile(name + "." + suffix):
            error(f"Unable to verify {suffix} file for component {name}")
            return None

    return obsinfo


def write_obsinfo(outdir, obsinfo):
    for key in ["name", "version", "mtime", "commit"]:
        if not key in obsinfo:
            error("Error name '{key}' missing from obsinfo")
            return False

    name = obsinfo["name"]
    file = f"{outdir}/{name}.obsinfo"

    newp = open(file, "w")
    if not newp:
        error(f"Unable to open new obsinfo file '{file}' for writing")
        return False

    for key in ["name", "version", "mtime", "commit"]:
        print(f"{key}: {obsinfo[key]}", file=newp)

    return True


def update_changelog(outdir, obsinfo, newclp):
    name = obsinfo["name"]
    file = f"{name}.changes"
    newfile = f"{outdir}/{file}"

    oldp = open(file, "r")
    if not oldp:
        error(f"Unable to open old changes file '{oldfile}' for reading")
        return False

    newp = open(newfile, "w")
    if not newp:
        error(f"Unable to open new changes file '{file}' for writing")
        oldp.close()
        return False

    newclp.seek(0)
    for fp in [newclp, oldp]:
        for line in fp.readlines():
            newp.write(line)

    oldp.close()
    newp.close()
    return True


def generate_interdiff(path, filea, fileb):
    global interdiff_fail

    cmd = f"cd {path} && interdiff <(git cat-file -p {filea.id}) <(git cat-file -p {fileb.id}) 2> /dev/null"

    proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
    file = os.fdopen(proc.stdout.fileno(), "r", encoding="utf8")
    blob = file.read()

    returncode = proc.wait()

    if returncode != 0:
        interdiff_fail += 1
        return None

    paths = []
    for diff in whatthepatch.parse_patch(blob):
        path = re.sub("^[ab]/", "", diff.header.new_path)
        paths.append(path)

    return paths


def git_checkout(gitrepo, gitbranch):
    scratch = os.environ.get(scratch_env)

    scratch_area = None

    for dir in [scratch, "/abuild", "/tmp"]:
        if dir and os.path.isdir(dir) and os.access(dir, os.W_OK):
            scratch_area = dir
            break

    if not scratch_area:
        error(
            f"Unable to locate suitable scratch area.  Set environment variable {scratch_env} to a directory with at least {scratch_size_gb}GB available"
        )
        return None

    statvfs = os.statvfs(scratch_area)

    if not statvfs or statvfs.f_frsize * statvfs.f_bavail < scratch_size_bytes:
        error(
            f"Scratch area '{scratch_area}' requires >= {scratch_size_gb}GB available space"
        )
        return None

    tmpdirctx = tempfile.TemporaryDirectory(None, None, scratch_area)

    cmd = ["git", "clone"]
    msg = f"{gitrepo}"
    if gitbranch:
        cmd.append(f"-b{gitbranch}")
        msg += f":{gitbranch}"
    cmd.append(gitrepo)
    cmd.append(tmpdirctx.name)

    vprint(f"Cloning git tree from {msg} to {tmpdirctx.name}")
    proc = subprocess.run(cmd)

    if proc.returncode != 0:
        error("git clone failed")
        return None

    vprint("Git clone completed")

    return tmpdirctx


def get_kernel_version(path):
    makefile = path + "/Makefile"

    try:
        with open(makefile) as file:
            version = None
            patchlevel = None
            sublevel = None
            extraversion = None

            for line in file:
                _name, _value = line.partition("=")[::2]
                name = _name.strip()
                value = _value.strip()

                if name and value:
                    match name:
                        case "VERSION":
                            version = value
                        case "PATCHLEVEL":
                            patchlevel = value
                        case "SUBLEVEL":
                            sublevel = value
                        case "EXTRAVERSION":
                            extraversion = value

                # value should always be present even if ""
                if name == "EXTRAVERSION":
                    break

            if version and patchlevel:
                full = f"{version}.{patchlevel}"

                if sublevel:
                    full += f".{sublevel}"
                    if extraversion:
                        full += f".{extraversion}"

                return f"{version}.{patchlevel}.{sublevel}"

    except:
        error(f"Unable to parse config {config}")

    return None


def get_patch_changelog(data):
    line = 1
    subject = None
    references = None
    for line in data.splitlines():
        if line == 10:
            return None

        res = re.search(r"^([A-Z][a-z]+):(.*)$", line)

        if not res:
            continue

        tag, _value = res.groups()
        value = _value.lstrip()

        match tag:
            case "Subject":
                subject = value.replace("[PATCH] ", "")
            case "References":
                references = value

        if subject and references:
            return subject, references


def read_config():
    componentfilesf = "_component_files"
    patternsf = "_patterns"
    global patterns, component_files

    try:
        patterns = [line.rstrip() for line in open(patternsf, "r")]
    except:
        error(f"Unable to open pattern file {patternsf}")
        return False

    try:
        lines = [line.rstrip() for line in open(componentfilesf)]
        for line in lines:
            for item in line.split(" "):
                component_files += " " + item
    except:
        error(f"Unable to open component file {componentfilesf}")
        return False

    return True


def sequence_patches(path):
    cmd = f"cd {path} && scripts/sequence-patch.sh --rapid"
    proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
    path = None

    while True:
        line = proc.stdout.readline()
        if not line:
            break
        match line.split():
            case [b"Creating", b"tree", b"in", _path]:
                path = _path.decode()

    returncode = proc.wait()

    if not path or returncode != 0:
        error("sequence_patch failed")
        return None
    else:
        print(f"Expanded kernel tree to: {path}")

    return path


def generate_tarfile(expanded_tree, outdir, obsinfo):
    tarversion = obsinfo["version"]
    name = obsinfo["name"]
    tarname = f"{name}-{tarversion}"
    tarfile = f"{outdir}/{tarname}.tar.gz"

    cmd = f"cd {expanded_tree} && tar --transform 's/^/.\/{tarname}\//' -czf {tarfile} {component_files}"
    proc = subprocess.run(["/bin/bash", "-c", cmd])

    return proc.returncode == 0


def format_commit_message(message, lines=None):
    existing_fmt = False

    for lineno, line in enumerate(message.splitlines()):
        if lines and lineno == lines:
            break

        if lineno == 0 and line[0] == "-":
            existing_fmt = True

        if existing_fmt:
            cl_write(line)
        else:
            cl_write(f"- {line}" if lineno == 0 else f"  {line}")


#
#
# PROGRAM START

parser = argparse.ArgumentParser(
    description="Extract tarball and changelog from kernel source git"
)
parser.add_argument(
    "--outdir",
    help="osc service parameter for internal use only "
    "(determines where generated files go before "
    "collection",
    dest="outdir",
    required=True,
)
parser.add_argument(
    "--verbose", "-v", help="Enable verbose output", dest="verbose", action="store_true"
)
parser.add_argument(
    "--quiet", "-q", help="Enable quiet mode", dest="quiet", action="store_true"
)
parser.add_argument(
    "--repo",
    "-r",
    help="kernel source repository",
    dest="gitrepo",
    default="kerncvs.suse.de:/home/git/kernel-source.git",
)
parser.add_argument(
    "--branch", "-b", dest="gitbranch", help="kernel source branch name"
)
parser.add_argument(
    "--force", "-f", 
    action="store_true",
    dest="force",
    default="False",
    help="Force generation of changelog/tarball even when no changes are detected",
)

args = parser.parse_args()

if not os.path.isdir(args.outdir) or not os.access(args.outdir, os.W_OK):
    error(f"Specified directory '{args.outdir}' does not exist or is not writable")
    exit(1)

outdir=os.path.abspath(args.outdir)

if outdir == os.getcwd():
    error(f"Specified directory '{args.outdir}' is pwd")
    exit(1)

obsinfo = read_obsinfo()

if not obsinfo:
    error("Unable to read obsinfo file")
    exit(1)

if "version" in obsinfo:
    match = re.match("^.*\.git([0-9]+)\.[0-9a-z]*$", obsinfo["version"])
    if match:
        offset = int(match.group(1))
else:
    warning("commit: but no prior version: found in obsinfo [setting gitoffset = 0]")
    offset = 0

if not read_config():
    error("Unable to read config")
    exit(1)

p = giturlparse.parse(args.gitrepo)
if not p:
    error("Unable to parse repository {args.gitrepo}")
    exit(1)

if os.path.isdir(p.href):
    if args.gitbranch:
        error(
            "Branch option invalid if specified repository is directory.  Please manually checkout required branch and rerun"
        )
        exit(1)

    unexpanded_git_tree = args.gitrepo
else:
    gitdir = git_checkout(args.gitrepo, args.gitbranch)
    if not gitdir:
        error("Unable to checkout kernel source tree from git")
        exit(1)

    unexpanded_git_tree = gitdir.name

clfile = tempfile.TemporaryFile(mode="w+")

repo = pygit2.Repository(unexpanded_git_tree)

total_changes = 0
head = repo.head.target

newobsinfo = {}
newobsinfo["name"] = obsinfo["name"]
newobsinfo["commit"] = head

interdiff_fail = 0
whatthepatch_fail = 0

for commit in repo.walk(head, pygit2.GIT_SORT_NONE):

    if str(commit.id) == obsinfo["commit"]:
        break

    offset += 1

    if len(commit.parents) == 1:
        changelog = []

        diff = commit.parents[0].tree.diff_to_tree(commit.tree)

        num_deltas = len(diff)
        for delta in diff.deltas:
            status = delta.status

            if status == pygit2.GIT_DELTA_ADDED:
                file1 = delta.new_file
            elif status == pygit2.GIT_DELTA_DELETED:
                file1 = delta.old_file
            elif status == pygit2.GIT_DELTA_MODIFIED:
                file1 = delta.new_file
                file2 = delta.old_file
            else:
                continue

            if status == pygit2.GIT_DELTA_ADDED or status == pygit2.GIT_DELTA_DELETED:
                if glob.globmatch(file1.path, "patches.*/**", flags=glob.GLOBSTAR):
                    filematched = False
                    blob = repo.revparse_single(str(file1.id)).data.decode("utf-8")

                    for diff in whatthepatch.parse_patch(blob):
                        if not diff or not diff.header:
                            whatthepatch_fail += 1
                            continue

                        if glob.globmatch(
                            diff.header.new_path, patterns, flags=glob.GLOBSTAR
                        ):
                            filematched = True
                            break

                    if filematched:
                        res = get_patch_changelog(blob)
                        if res:
                            subject, references = res
                            changelog.append((status, subject, references, file1.path))

            elif status == pygit2.GIT_DELTA_MODIFIED:
                new = glob.globmatch(file1.path, "patches.*/**", flags=glob.GLOBSTAR)
                old = glob.globmatch(file2.path, "patches.*/**", flags=glob.GLOBSTAR)

                filematched = False
                if new or old:
                    paths = generate_interdiff(unexpanded_git_tree, file1, file2)
                    if paths:
                        for path in paths:
                            if glob.globmatch(path, patterns, flags=glob.GLOBSTAR):
                                filematched = True
                                break

                if filematched:
                    blob = repo.revparse_single(str(file1.id)).data.decode("utf-8")
                    res = get_patch_changelog(blob)
                    if res:
                        subject, references = res
                        changelog.append((status, subject, references, file1.path))

        num_changes = len(changelog)

        total_changes += num_changes

        if num_changes > 0:
            cl_write(
                "-------------------------------------------------------------------"
            )
            stime = datetime.fromtimestamp(commit.commit_time, timezone.utc).strftime(
                "%a %b %d %H:%M:%S %Z %Y"
            )
            cl_write(f"{stime} - {commit.author}\n")

            if num_changes == 1:
                if num_changes == 1:
                    # most typical case, one change (patch addition, deletion) or
                    # refresh of patch
                    status, subject, references, file = changelog.pop()

                    # ignore above, just use original git commit message
                    format_commit_message(commit.message)

            elif num_changes > 1:
                format_commit_message(commit.message, 6)

                cl_write("")
                cl_write("Relevant changes:")
                for status, subject, references, file in changelog:
                    if subject:
                        for index, line in enumerate(subject.splitlines()):
                            mode = ""
                            match status:
                                case pygit2.GIT_DELTA_ADDED:
                                    mode = "Add"
                                case pygit2.GIT_DELTA_DELETED:
                                    mode = "Delete"
                                case pygit2.GIT_DELTA_MODIFIED:
                                    mode = "Refresh"

                            format_commit_message(f"{mode} patch: {file}\n{subject}")

                    if references:
                        cl_write(f"  ({references})")

            cl_write(f"\n  [kernel-source commit {shortsha(commit.id)}]\n")

if interdiff_fail > 0:
    warning(f"{interdiff_fail} patches could not be processed by interdiff")

if whatthepatch_fail > 0:
    warning(f"{whatthepatch_fail} patches could not be processed by whatthepatch")

if total_changes > 0 or args.force == True:
    print(f"New changes: {total_changes}")

    expanded_git_tree = sequence_patches(unexpanded_git_tree)
    if not expanded_git_tree:
        error("Unable to expand git tree")
        exit(1)

    kernversion = get_kernel_version(expanded_git_tree)

    if not kernversion:
        error("Unable to determine kernel version")
        exit(1)

    newobsinfo["version"] = f"{kernversion}.git{offset}.{shortsha(head)}"
    newobsinfo["mtime"] = int(time.time())

    if not generate_tarfile(expanded_git_tree, outdir, newobsinfo):
        error("Unable to generate tarfile")
        exit(1)

    update_changelog(outdir, obsinfo, clfile)
    write_obsinfo(outdir, newobsinfo)
    print(f"Updated revision to {str(head)}")

else:
    print("No new changes")