commit 199ba0de24db750b9d10d292bed4e9042143cb6e Author: Egbert Eich Date: Mon Sep 1 12:49:11 2025 +0200 Initial import Signed-off-by: Egbert Eich diff --git a/README.md b/README.md new file mode 100644 index 0000000..4545488 --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +# ROCm Packaging on SUSE + +## Motivations +To achieve an more or less identical 'look and feel' between Tumbleweed and +Fedora, we strive to the same spec files on both. +A set of ROCm packages has been available on Fedora for a while already, +so it is best to continue to develop packages there. The burden on packagers +at SUSE is slightly higher when submitting updates, but the hope is that +this will help to reduce the overall burden considerably. + +## Caveats +Since Fedora has been using Git as their SCM for packages for a while already +and SUSE is currently switching to Git, this appears to be straight forward. +Unfortunately it is not quite like this: SUSE uses sha256 to identify commits, +while Fedora still uses sha1. Also, the Fedora packaging format is different +from SUSE's standard - most notibly, SUSE maintains the changelog in a +separate file. +So there are some conversions required to get from a Fedora package to +a package that will be accetible in openSUSE Factory. Luckily, this +entire process can be scripted. + +## Theory of Operation +The process involves: +Fetch updates from Fedora -> Test build with Factory -> Convert to SUSE +standard -> Add to devel project. +For this, we need to maintain two git repositories for each package locally: +1. a clone of the 'upstream' source in Fedora at + https://src.fedoraproject.org/rpms/ +2. a clone of the SUSE ROCm devel project repository: + gitea@src.opensuse.org:ROCm/.git +A fill list of packages can be found in the file 'packagelist' + +### High Level Script +1. `convert_package_to_suse.sh` Converts 'upstream' packages sha256 pushes + them to the correspondig devel package and converts the spec file into + a SUSE compliant format[^note_on_tools]. + +### Plumbing +These are used by the tools above, normally you don't have to call them +directly. They are listed here as a reference. +1. `git_to_256` Takes the latest current state of the 'origin/rawhide' branch + from the ingress package converts the object format from sha1 to sha256 + and imports it into the devel project. +1. `susefy-package.sh` converts the spec file format, generates a changelog, + downloads the sources and creates a commit that can subsequently be + submitted to openSUSE Factory[^note_on_tools]. + +## Workflow +For the following, we assume that the devel project clone will be +in the directory `$BASEDIR/ROCm` while the ingress project will be +in the directory`$BASEDIR/ROCmUpstream`. +While the directory containing this readme is located at `$SCRIPTDIR`. +For this to work, you will need recent versions of `osc`, +`obs-service-download_files`, `obs-service-format_spec_file` +and `obs-git-init` from +[openSUSE:Tools](https://download.opensuse.org/repositories/openSUSE:/Tools) +installed. + +### Getting Started - 1st Time Setup +This typically needs to be done only once to populate the directories +`ROCm` and `ROCmUpstream` with package Git repositories. +1. Copy the file `packagelist` from this repo to either directory. + ``` + cp $SCRIPTDIR/packagelist $BASEDIR/ROCmUpstream + ``` +2. Enter the directory `ROCmUpstream` and clone the upstream packages: + ``` + cd $BASEDIR/ROCmUpstream + $SCRIPTDIR/initialize_rocm.sh + ``` +3. Enter the directory `ROCm` and clone the devel project: + ``` + cd $BASEDIR/ROCm + $SCRIPTDIR/initialize_rocm_suse.sh + cd .. + ``` +Now you should be all set. + + +## Pull latest updates into the local copy from the upstream project: +To pull the latest updates you may run a command like this: + ``` + cd $BASEDIR/ROCmUpstream + for i in `cat packagelist`; do + cd $i + git fetch + cd - &>/dev/null + done + ``` +## Migration to Devel Project +Once the packages have been updated, you can migrate them to the +devel project: + ``` + cd $BASEDIR/ROCmUpstream + $SCRIPTDIR/convert_package_to_suse.sh -u rawhide -t main $BASEDIR/ROCm + ``` +[^note_on_tools]: Currently, you will need patches to the packages +`build` and `obs-service_download_spec_files` that have not been released, +yet. diff --git a/convert_package_to_suse.sh b/convert_package_to_suse.sh new file mode 100755 index 0000000..49edb28 --- /dev/null +++ b/convert_package_to_suse.sh @@ -0,0 +1,59 @@ +#! /usr/bin/bash +set -x + +UPSTREAM_BRANCH=rawhide +TARGET_BRANCH=main + +help() { + echo -e "$0 [[-u ][-t ]] |[-h]\n" \ + "Convert package from ingress directory to sha256 git in\n"\ + "directory and reformat it for SUSE.\n" \ + "The ingress can either be a single package directory or\n" \ + "a top level directory containing the file 'packagelist.\n" \ + "\t-u : upstream branch. Default:\n" \ + "\t-t : target branch. Default: \n" \ + "\t-h: help" +} + +update_one_package() { + local script_dir=$1 + local target_dir=$2 + [ -n "$target_dir" ] || target_dir=../../ROCm + local dir=$(pwd) + package_name=$(basename $dir) + [ -e ${package_name}.spec ] || { echo "No package directory?"; exit 1; } + $script_dir/git_to_256 push $target_dir/$package_name origin/$UPSTREAM_BRANCH || exit 1 + cd $target_dir/$package_name +## Cave: force push? +#git push origin refs/remotes/upstream/rawhide:refs/remotes/upstream/rawhide + $script_dir/susefy-package.sh -r upstream/$UPSTREAM_BRANCH -t $TARGET_BRANCH + $no_push || git push origin $TARGET_BRANCH +} + +no_push=false + +while [ -n "$1" ]; do + case $1 in + -u) shift; UPSTREAM=$i; shift ;; + -t) shift; TARGET=$i; shift ;; + -n) shift; no_push=true ;; + -h) help; exit 0 ;; + -*) echo "unknown option $1" &>2; exit 1 ;; + *) target_dir=$1; shift ;; + esac +done + +script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +if [ -e ./packagelist ]; then + dir=$(pwd) + for i in $(cat ./packagelist); do + cd $dir/$i + update_one_package $script_dir $target_dir + done +elif [ -d .git -a *.spec != \*.spec ]; then + update_one_package $script_dir $target_dir +else + echo "No package file nor spec or no git directory?" >&2 + exit 1 +fi + diff --git a/git_to_256 b/git_to_256 new file mode 100755 index 0000000..01a3959 --- /dev/null +++ b/git_to_256 @@ -0,0 +1,58 @@ +#! /usr/bin/bash +#set -x +#DEBUG=echo + +upstream_branch=upstream +our_name=suse + +die() { + local retval=$1 + rm -rf $tmpdir + exit $retval +} + +usage() { + echo -e "$0 [ -r ] ...\n" \ + " Convert repo to sha256 and run git command.\n" \ + " ie. \`push {|${our_name}...\`\n" \ + " -r : provide remote repository.\n" \ + " Alternatively set environment variable SUSE_GIT.\n" \ + " Local or remote branches from 'origin' are set up to be\n" \ + " pushed to remotes/upstream/*\n" + exit 0 +} + +[ "$1" = "-h" ] && usage + +git_url=$(git config remote.origin.url) +repo=$(basename $git_url) + +target_git=$our_name +[ "$1" = "-r" ] && { shift; target_git=$1; shift; } +cmdargs="${1+$@}" +[ "$target_git" = "$our_name" ] && [[ $cmdargs =~ push\ ([^\ ]+)\\* ]] \ + && target_git=${BASH_REMATCH[1]} +[ "$target_git" = "$our_name" ] && target_git=$SUSE_GIT +[ -n "$target_git" -a "$target_git" != "$our_name" ] || \ + { echo -e "Target repo cannot be determined\n"\ + " set environment variable SUSE_GIT\n"\ + " to target repository or use the -r option"; exit 1; } +[[ $target_git =~ .*/${repo} ]] || [[ $target_git =~ .*/${repo%.git} ]] || target_git+="/${repo}" +cmdargs=${cmdargs//${target_git}/${our_name}} + +tmpdir=$(mktemp -d /tmp/tmp-XXXXXXXXX) + +mkdir -p $tmpdir/$repo +git init --object-format=sha256 $tmpdir/$repo + +git fast-export --all | env GIT_DIR=$tmpdir/$repo/.git git fast-import +export GIT_DIR=$tmpdir/$repo/.git +git remote add origin ${git_url} +git remote add ${our_name} ${target_git} +git config set remote.${our_name}.push refs/heads/\*:refs/remotes/${upstream_branch}/\* +git config set --append remote.${our_name}.push refs/remotes/origin/\*:refs/remotes/${upstream_branch}/\* +#git remote rename origin ${upstream_branch} +[ -n "$DEBUG" ] && $DEBUG $tmpdir +${DEBUG} git ${cmdargs} || die $? + +[ -n "$DEBUG" ] && echo $tmpdir || rm -rf $tmpdir diff --git a/initialize_rocm.sh b/initialize_rocm.sh new file mode 120000 index 0000000..94cca26 --- /dev/null +++ b/initialize_rocm.sh @@ -0,0 +1 @@ +initialize_rocm_suse.sh \ No newline at end of file diff --git a/initialize_rocm_suse.sh b/initialize_rocm_suse.sh new file mode 100755 index 0000000..9210d44 --- /dev/null +++ b/initialize_rocm_suse.sh @@ -0,0 +1,27 @@ +#! /usr/bin/bash +#set -x +if [ $(basename $0) == initialize_rocm_suse.sh ]; then + URL=gitea@src.opensuse.org:ROCm + repo=suse +else + URL=https://src.fedoraproject.org/rpms + repu=upstream +fi + +[ -e packagelist ] || { echo "File 'packagelist' does not exist in the current directory. Wrong directory?" >&2; exit 1; } + +for i in `cat packagelist`; do + [ -e $i ] && + { echo -e "File or directory $i exists in current directory.\n" \ + "unable to proceed"; exit 1; } +done + +for i in `cat packagelist`; do + git clone $URL/$i.git + if [ $repo = suse ]; then + # Also fetch upstream/rawhide! + cd $i + git fetch origin remotes/upstream/rawhide:remotes/upstream/rawhide + cd - &>/dev/null + fi +done diff --git a/susefy-package.sh b/susefy-package.sh new file mode 100755 index 0000000..492bccc --- /dev/null +++ b/susefy-package.sh @@ -0,0 +1,140 @@ +#! /usr/bin/bash + +commit=upstream/rawhide +target_branch=main +# intermediate_branch=work + +#set -x +shopt -s nullglob + +init_git() { + /usr/bin/obs-git-init +# clean_files sources README.md _buildconfig\* _buildinfo\* debian .osc +} + +download_packages() { + # Download spec files + osc service run download_files + for i in _service:download_files:*; do + j=${i#_service:download_files:} + [ -e $j ] && { rm $i; continue; } + mv $i $j + done +} + +clean_files() { + while [ -n "$1" ] ; do + echo "$1" >> .gitignore + [ -d "$1" ] && rm -rf "$1" || rm -f "$1" + shift + done +} + +susefy() { + local specfile=$1 + local tmpfile=$(mktemp specfile-XXXXX) + # Generate changes file + rm -f .gitignore + clean_files sources README.md _buildconfig\* _buildinfo\* debian .osc changelog + cat ${specfile} | /usr/lib/build/spec2changelog | \ + awk "BEGIN{p=1}/^\s*$/{ if (p) print; p=0; next; }/.*/{ p=1; print; }" |\ + awk '/^[^-<>]* [[:digit:]]{4} - [^<>]* <[^<>]*> .*$/{gsub("[^<>]*$","");print;}{print;next;}' > ${specfile%.spec}.changes + # Strip changes + cat ${specfile} | \ + awk "/^%changelog/{ print; nextfile; }{ print; }" > $tmpfile + mv $tmpfile ${specfile} + # Fix format SUSE style + osc service run format_spec_file + mv _service:format_spec_file:${specfile} ${specfile} +} + +setup_merge_branch() { + local target_branch=$1 + if ! $(git branch | grep -q $target_branch); then + if $(git branch -a | grep -q origin/$target_branch); then + git branch $target_branch origin/$target_branch + else + git branch $target_branch + fi + fi +} + +git_add_commit() { + local msg="$1" + OFS=$IFS + IFS=" +" + git status --porcelain | while read line; do + file=${line##* }; + [[ $line =~ ^\?\?\ .\\* ]] && { git add $file; continue; } + [[ $line =~ ^.[DM]\ .\\* ]] && { git add $file; continue; } + [[ $line =~ ^.[DU]\ .\\* ]] && { git rm $file; continue; } + done + IFS=$OFS + git commit -s -m "$msg" +} + +merge_branches() { + local line file + local source=$1 + local target_branch=$2 + + git merge -s ours -m "Merge updates from ${source} into ${target_branch} branch" ${target_branch} + git branch -D ${target_branch} + git checkout -b ${target_branch} + git branch -D tmp +} + +[ -x /usr/bin/obs-git-init -a -x /usr/bin/osc -a -x /usr/lib/build/spec2changelog ] \ + || { echo "make sure packages \'obs-git-init\', \'osc\' " \ + "and \'obs-service_format_spec_file\' are installed"; exit 1; } + +while [ -n "$1" ]; do + case $1 in + -r) shift; commit=$1; shift ;; + -t) shift; target_branch=$1; shift ;; + -i) shift; intermediate_branch=$1; shift ;; + -*) echo "$unknown option $1"; exit 1 ;; + *) specfile=$1; shift ;; + esac +done + +id1=$(git show-ref $commit | cut -f 1 -d' ') +id2=$(git merge-base $commit $target_branch) +if [ "$id1" = "$id2" ]; then + echo "Nothing to do!" + exit 0 +fi + +git checkout -b tmp $commit + +if [ -z "$specfile" ]; then + s=(*.spec) + [ ${#s[*]} -eq 1 ] || \ + { echo "Cannot uniquely determine specfile. Specify as argument" >&2; + exit 1; } + specfile=${s[0]} +fi +[ -n "$specfile" ] || \ + { echo "Specify specfile" >&2; exit 1; } +[[ $specfile =~ .\\*\.spec ]] || specfile=${specfile}.spec +[ -e ${specfile} ] || \ + { echo "$specfile: incorrect spec file name" >&2; exit 1; } + +# Initialize git +init_git +download_packages +git_add_commit "Add/update .git* files, add sources & remove unneeded files" +if [ -n "${intermediate_branch}" ]; then + setup_merge_branch ${intermediate_branch} + merge_branches $commit ${intermediate_branch} + git checkout -b tmp ${intermediate_branch} +fi +susefy $specfile +git_add_commit "Convert to SUSE Format" +setup_merge_branch $target_branch +if [ -n "${intermediate_branch}" ]; then + merge_branches ${intermediate_branch} $target_branch +else + merge_branches $commit $target_branch +fi