mold/scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split and update tools

Browse Source
This commit is contained in:
Ludwig Nussel 2023-02-20 14:27:24 +01:00
parent abff01a7fe
commit 8d9042228c
4 changed files with 590 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
README.md Normal file
View File

@ -0,0 +1,53 @@
Scripts to sync git submodules of a project via pull requests
The git repo in the current directory is expected to have at least
one remote pointing to the target project. If that repo is not
writable, another one can be used for pushing.
create a config file `.settings` in the top level directory of the git submodule project:
```
# name of the remote to push to as repored by `git remote`
PUSH_REMOTE="joesix"
# username on gitea
PR_SRC_USER="joesix"
# target project for pull requests
PR_PROJECT="mold"
# target repo on gitea
PR_REPO="core"
# token
TOKEN="deadbeef"
# the url for cloning the project. Packages may use an url relative
to that.
PACKAGE_BASE_URL="https://gitea.opensuse.org/mold/core.git"
# relative url for new packages
PACKAGE_RELATIVE_URL="../../rpm"
# remote to to send pull requests to
REMOTE="origin"
# base branch of the target remote
BASE_BRANCH="main"
# optional: fixed date or a filename prefixed with @ to get a stable date for testing
DATE="@token"
# optional: an OBS project with a list of packages
OBSPKGLIST="https://api.opensuse.org/public/source/openSUSE:Factory:Rings:0-Bootstrap"
```
Scripts:
All scripts use getopt, --help may not always be up to date though
:-)
- updatemodules: checks all submodules for updates. For packages
that need to be updated creates `refs/pq/$packagename`. The
updated refence there updates the submodule commit reference for
updates. It also adds or removes entries from/to `.gitmodules` if
needed. There's also a `--single` option which produces a single
reference for all updates. The `--status` option show the current
state.
- pusher: compares the references created by the `updatemodules`
script with the specified remote. Pushed pending refs to an
`update_$packagename` ref. Gitea doesn't seem to support refs in
subdirs other than `heads`, that's why. Unless the `--dry` option
is specified, the `pusher` script also creates pull requests for
pending refs.
- gitea-pulls: dialog based script to list, view, merge or close
pull requests from the command line.

155
gitea-pulls Executable file
View File

@ -0,0 +1,155 @@
#!/bin/bash
# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: Copyright 2023 SUSE LLC
set -e
# config options
PR_SRC_USER=
PR_PROJECT=
PR_REPO=
TOKEN=
# command line only
verbose=0
cfg_file=
###################
requestfile=$(mktemp gitea-pulls.XXXXXX)
tmpfile=$(mktemp gitea-pulls.XXXXXX)
cleanup()
{
rm -f "$requestfile" "$tmpfile"
}
trap cleanup EXIT
helpandquit()
{
cat <<-EOF
Usage: $0 [OPTIONS]
OPTIONS:
--verbsoe verbose
-h help screen
EOF
exit 0
}
log_info()
{
[ "$verbose" -gt 0 ] || return 0
echo "$@"
}
d(){
local retval=0
# Bash makes it a bit annoying to read the output of a different FD into a variable, it
# only supports reading stdout by itself. So redirect 3 to stdout and 1 to the real stdout.
exec {stdoutfd}>&1
result="$(dialog --backtitle "Gita Pull requests" --output-fd 3 "$@" 3>&1 1>&${stdoutfd})" || retval=$?
# Word splitting makes it necessary to use eval here.
eval "exec ${stdoutfd}>&-"
return "$retval"
}
# Given the number of total item pairs, outputs the number of items to display at once
menuheight() {
local height=$(($1 / 2))
[ "$height" -le "$dh_menu" ] || height="$dh_menu"
echo "$height"
}
stty_size() {
set -- $(stty size)
LINES="$1"
COLUMNS="$2"
# stty size can return zero when not ready or
# its a serial console
if [ "$COLUMNS" = "0" ] || [ "$LINES" = "0" ]; then
LINES=24
COLUMNS=80
fi
dh_menu=$((LINES-15))
dh_text=$((LINES-5))
}
stty_size
getopttmp=$(getopt -o hc:v --long help,config:,verbose -n "${0##*/}" -- "$@")
eval set -- "$getopttmp"
while true ; do
case "$1" in
-h|--help) helpandquit; shift ;;
-v|--verbose) verbose=$((++verbose)); shift ;;
-c|--config) cfg_file="$2"; shift 2 ;;
--) shift ; break ;;
*) echo "Internal error!" ; exit 1 ;;
esac
done
# shellcheck disable=SC1090
. "${cfg_file:-.settings}"
needed=(PR_SRC_USER PR_PROJECT PR_REPO TOKEN)
for i in "${needed[@]}"; do
eval test -n "\"\$$i\"" || { echo "The following settings are mandatory: ${needed[*]}"; exit 1; }
done
request()
{
local urlpart="$1"
shift
curl -s -f "https://gitea.opensuse.org/api/v1/repos/$PR_PROJECT/$PR_REPO$urlpart" \
-H "accept: application/json" \
-H "Authorization: token $TOKEN" \
-H "Content-Type: application/json" "$@"
}
fetch_requests()
{
request "/pulls?state=open" > "$requestfile"
}
fetch_requests
while true; do
list=()
while read -r number _branch _cid owner title; do
list+=("$number" "$owner: $title")
done < <(jq '.[]|[((.number|tostring)), .head.ref, .head.sha, .head.repo.owner.login, .title]|join(" ")' -r < "$requestfile")
if [ "${#list}" = 0 ]; then
d --msgbox "No pull requests in $PR_PROJECT/$PR_REPO" 0 0
exit 0
fi
d --no-hot-list --menu "$PR_PROJECT/$PR_REPO pull requests" 0 0 "$(menuheight ${#list[@]})" "${list[@]}" || exit 1
prno="$result"
while true; do
list=(raw raw diff diff merge merge close close)
d --no-tags --menu "$PR_PROJECT/$PR_REPO #$prno" 0 0 "$(menuheight ${#list[@]})" "${list[@]}" || break
action="$result"
case "$action" in
raw)
jq ".[]|select(.number==$prno)" < "$requestfile" > "$tmpfile"
d --textbox "$tmpfile" 0 0
;;
diff)
request "/pulls/$prno.diff" > "$tmpfile"
d --textbox "$tmpfile" 0 0
;;
merge)
request "/pulls/$prno/merge" -d '{ "do": "merge", "delete_branch_after_merge": true}' | jq > "$tmpfile"
d --textbox "$tmpfile" 0 0
fetch_requests
break
;;
close)
request "/pulls/$prno" -d '{ "state": "closed"}' -X PATCH | jq > "$tmpfile"
d --textbox "$tmpfile" 0 0
fetch_requests
break
;;
esac
done
done

176
pusher Executable file
View File

@ -0,0 +1,176 @@
#!/bin/bash
set -e
# config options
# shellcheck disable=SC2034
declare -A keep
PR_SRC_USER=
BASE_BRANCH="main"
PR_PROJECT=
PR_REPO=
TOKEN=
PUSH_REMOTE="origin"
# command line only
verbose=0
cfg_file=
dryrun=
show_status=
###################
push_url=
did_push=
tmpfile=$(mktemp pusher.XXXXXX)
cleanup()
{
rm -f "$tmpfile"
}
trap cleanup EXIT
helpandquit()
{
cat <<-EOF
Usage: $0 [OPTIONS] [<module> ...]
OPTIONS:
-h help screen
EOF
exit 0
}
log_info()
{
[ "$verbose" -gt 0 ] || return 0
echo "$@"
}
request_pulls()
{
curl -s -f "$pr_target?state=open" \
-H "accept: application/json" \
-H "Authorization: token $TOKEN" \
-H "Content-Type: application/json"
}
getopttmp=$(getopt -o hc:v --long help,single,branch:,config:,date:,remote:,status,verbose,dry -n "${0##*/}" -- "$@")
eval set -- "$getopttmp"
while true ; do
case "$1" in
-h|--help) helpandquit; shift ;;
-v|--verbose) verbose=$((++verbose)); shift ;;
-c|--config) cfg_file="$2"; shift 2 ;;
--status) show_status=1; shift ;;
--dry) dryrun=1; shift ;;
--) shift ; break ;;
*) echo "Internal error!" ; exit 1 ;;
esac
done
declare -A modules
for m in "$@"; do
modules["$m"]=1
done
# shellcheck disable=SC1090
. "${cfg_file:-.settings}"
needed=(PR_SRC_USER PR_PROJECT PR_REPO TOKEN)
for i in "${needed[@]}"; do
eval test -n "\"\$$i\"" || { echo "The following settings are mandatory: ${needed[*]}"; exit 1; }
done
push_url="$(git config --get remote."$PUSH_REMOTE".url)"
pr_target="https://gitea.opensuse.org/api/v1/repos/$PR_PROJECT/$PR_REPO/pulls"
# push queue
declare -A pq
while read -r cid _c ref; do
m="${ref#refs/pq/}"
pq["$m"]="$cid"
done < <(git for-each-ref 'refs/pq/*')
# request queue
declare -A rq
while read -r cid _c ref; do
m="${ref#refs/remotes/"$PUSH_REMOTE"/update_}"
[ "$m" != "$ref" ] || continue
rq["$m"]="$cid"
done < <(git for-each-ref "refs/remotes/$PUSH_REMOTE/*")
declare -A pr
while read -r _id cid ref; do
m="${ref#update_}"
pr["$m"]="$cid"
log_info "pr $m: $cid"
done < <(request_pulls | jq ".[]|select(.head.repo.owner.login == \"$PR_SRC_USER\")|[((.number|tostring)), .head.sha, .head.ref]|join(\" \")" -r)
log_info "to push: ${!pq[*]}"
log_info "remote: ${!rq[*]}"
if [ -n "$show_status" ]; then
echo "# local remote pr package"
if [ "${#modules[@]}" -eq 0 ]; then
for m in "${!pq[@]}" "${!rq[@]}" "${!pr[@]}"; do
modules["$m"]=1
done
fi
for m in "${!modules[@]}"; do
a=${pq[$m]:0:7}
b=${rq[$m]:0:7}
c=${pr[$m]:0:7}
echo "${a:- } ${b:- } ${c:- } $m"
done
exit 0
fi
ret=0
for m in "${!pq[@]}"; do
[ "${#modules[@]}" -eq 0 -o -n "${modules[$m]}" ] || continue
cid="${pq[$m]}"
[ -n "$cid" ]
if [ -n "${rq[$m]}" ] && [ "$cid" = "${rq[$m]}" ]; then
log_info "$m already pushed"
else
log_info "pushing $m: $cid"
if ! git send-pack --force "$push_url" "${cid}:refs/heads/update_$m"; then
echo "failed to push $m update" >&2
ret=1
continue
fi
did_push=1
fi
if [ -n "${pr[$m]}" ]; then
if [ "$cid" = "${pr[$m]}" ]; then
log_info "pr for $m exists, no update"
else
echo "Warning: exiting pr for $m with different content"
fi
continue
fi
title=$(git log -1 "--format=%s" "$cid")
head="update_$m"
[ "$PR_PROJECT" = "$PR_SRC_USER" ] || head="$PR_SRC_USER:$head"
log_info "Filing pr '$title': $cid at $pr_target"
[ -z "$dryrun" ] || continue
#https://github.com/go-gitea/gitea/issues/18842
if curl -s -f "$pr_target" \
-H "accept: application/json" \
-H "Authorization: token $TOKEN" \
-H "Content-Type: application/json" \
-d "{ \"base\": \"$BASE_BRANCH\", \"head\": \"$head\", \"title\": \"$title\"}" > "$tmpfile"; then
prid=$(jq -r .number < "$tmpfile")
echo "filed pr #$prid for $m"
else
echo "failed to file pr for $m"
jq -r .message < "$tmpfile"
fi
done
if [ -n "$did_push" ]; then
log_info "Updating remote $PUSH_REMOTE"
git remote update -p "$PUSH_REMOTE" > /dev/null
fi
exit "$ret"

262
updatemodules
View File

@ -1,100 +1,250 @@
#!/bin/bash
allatonce=
pushurl="gitea@gitea.opensuse.org:lnussel/core.git"
prsrcusr="lnussel"
prtarget="https://gitea.opensuse.org/api/v1/repos/mold/core/pulls"
set -e
if [ "$1" = '--allatonce' ]; then
allatonce=1
# config options
declare -A keep
REMOTE="origin"
BASE_BRANCH="main"
PACKAGE_BASE_URL=
PACKAGE_RELATIVE_URL="../../rpm"
# https://api.opensuse.org/public/source/openSUSE:Factory:Rings:0-Bootstrap
OBSPKGLIST=
DATE="$(date "+%s %z")"
keep["dummy-release"]=1
# command line only
single=
verbose=0
cfg_file=
###################
# constant needed to aid quoting
nl=$'\n'
declare -A obs
declare -A new
declare -A drop
declare -A revs
gitmodules_rev=
helpandquit()
{
cat <<-EOF
Usage: $0 [OPTIONS] [<module> ...]
OPTIONS:
--single create single commit for all changes
-h help screen
EOF
exit 0
}
show_status()
{
git for-each-ref 'refs/pq/*'
exit 0
}
clear_queue()
{
while read -r ref; do
git update-ref -d "$ref"
done < <(git for-each-ref 'refs/pq/*' --format '%(refname)')
exit 0
}
isnew()
{
local p="${1:?}"
[ -n "${new[$p]}" ]
}
todrop()
{
local p="${1:?}"
[ -n "${drop[$p]}" ]
}
log_info()
{
[ "$verbose" -gt 0 ] || return 0
echo "$@"
}
makedict()
{
local dict="$1"
shift
mapfile -t a < <("$@")
for k in "${a[@]}"; do
eval "$dict"["$k"]=1
done
}
getopttmp=$(getopt -o hc:v --long help,single,branch:,config:,date:,remote:,status,verbose,clear -n "${0##*/}" -- "$@")
eval set -- "$getopttmp"
while true ; do
case "$1" in
-h|--help) helpandquit; shift ;;
-v|--verbose) verbose=$((++verbose)); shift ;;
-c|--config) cfg_file="$2"; shift 2 ;;
--single) single=1; shift ;;
--remote) REMOTE="$2"; shift 2 ;;
--branch) BASE_BRANCH="$2"; shift 2 ;;
--status) show_status; exit 0 ;;
--clear) clear_queue; exit 0 ;;
--date) DATE="$2"; shift 2 ;;
--) shift ; break ;;
*) echo "Internal error!" ; exit 1 ;;
esac
done
modules=("$@")
PACKAGE_BASE_URL="$(git config --get remote."$REMOTE".url)"
# shellcheck disable=SC1090
. "${cfg_file:-.settings}"
if [ "${DATE:0:1}" = '@' ]; then
DATE="$(stat -c %Y "${DATE:1}") +0100"
fi
modules=($@)
origin="$(git config --get remote.origin.url)"
now="$(date "+%s %z")"
# FIXME: hardcoded to avoid changing commits for now
now="$(stat -c %Y token) +0100"
export GIT_AUTHOR_NAME="Auto"
export GIT_AUTHOR_EMAIL="auto@suse.de"
export GIT_AUTHOR_DATE="$now"
export GIT_AUTHOR_DATE="$DATE"
export GIT_COMMITTER_NAME="Auto"
export GIT_COMMITTER_EMAIL="auto@suse.de"
export GIT_COMMITTER_DATE="$now"
export GIT_COMMITTER_DATE="$DATE"
tmpfile=$(mktemp updatemodules.XXXXXX)
tmpfile2=$(mktemp updatemodules.XXXXXX)
cleanup()
{
rm -f "$tmpfile"
rm -f "$tmpfile" "$tmpfile2"
}
trap cleanup EXIT
read -r token < token || exit 1
# read all submodules
declare -A revs
while read -r m t cid p; do
[ "$t" = "commit" ] || continue
if [ "$t" = blob ] && [ "$p" = ".gitmodules" ]; then
gitmodules_rev="$cid"
fi
[ "$t" = "commit" ] || continue
revs["$p"]="$cid"
done < <(git cat-file -p "HEAD^{tree}")
done < <(git cat-file -p "$REMOTE/$BASE_BRANCH^{tree}")
if [ -z "$modules" ]; then
modules=("${!revs[@]}")
if [ -n "$OBSPKGLIST" ]; then
while read -r p; do
obs["$p"]=1
[ -n "${revs[$p]}" ] || new["$p"]=1
done < <(curl -s -f "$OBSPKGLIST"|sed -ne 's/.*entry name="\([^":]*\).*\/>/\1/p'|sort -u | grep -v AGGR)
for m in "${!revs[@]}"; do
[ -n "${obs[$m]}" -o -n "${keep[$m]}" ] || drop["$m"]=1
done
fi
[ -z "${new[*]}" ] || log_info "new: ${!new[*]}"
[ -z "${drop[*]}" ] || log_info "drop: ${!drop[*]}"
if [ "${#modules[@]}" = 0 ]; then
modules=("${!revs[@]}" "${!new[@]}")
fi
# push queue
declare -A pq
makedict pq git for-each-ref 'refs/pq/*' --format '%(refname)'
# check remotes for updates
declare -A commits
treetext=$(git cat-file -p "HEAD^{tree}")
treetext=$(git cat-file -p "$REMOTE/$BASE_BRANCH^{tree}")
git cat-file -p "$REMOTE/$BASE_BRANCH":.gitmodules > "$tmpfile"
cat "$tmpfile" > "$tmpfile2"
for m in "${modules[@]}"; do
path="$(git config -f .gitmodules --get "submodule.$m.path")"
url="$(git config -f .gitmodules --get "submodule.$m.url")"
branch="$(git config -f .gitmodules --get "submodule.$m.branch")"
if isnew "$m"; then
path="$m"
url="$PACKAGE_RELATIVE_URL/$m"
smbranch=
git config -f "$tmpfile" --add "submodule.$m.path" "$path"
git config -f "$tmpfile" --add "submodule.$m.url" "$url"
else
path="$(git config -f "$tmpfile" --get "submodule.$m.path")"
url="$(git config -f "$tmpfile" --get "submodule.$m.url")"
smbranch="$(git config -f "$tmpfile" --get "submodule.$m.branch" || :)"
fi
if [ -z "$path" ] || [ -z "$url" ]; then
echo "$m unknown" >&2
continue
fi
if [ "${url:0:3}" = '../' ]; then
url="$origin/$url"
url="$PACKAGE_BASE_URL/$url"
fi
read -r cid _d < <(git ls-remote "$url" "${branch:-HEAD}")
if [ -z "${revs[$path]}" ]; then
echo "$path unknown" >&2
continue
cid=
if ! todrop "$m"; then
read -r cid _d < <(git ls-remote "$url" "${smbranch:-HEAD}")
if [ -z "$cid" ]; then
echo "$path not in pool" >&2
continue
fi
fi
# create a new commit for this package
if [ "${revs[$path]}" != "$cid" ]; then
echo "Needs update: $path@${revs[$path]} -> $cid"
if [ "$allatonce" = 1 ]; then
treetext="${treetext/${revs[$path]} $path/$cid $path}"
log_info "Needs update: $path@${revs[$path]:-NEW} -> ${cid:-DROP}"
if isnew "$m"; then
newtreetext="$treetext${nl}160000 commit $cid $m"
elif todrop "$m"; then
newtreetext="${treetext/160000 commit ${revs[$path]} $path$nl/}"
git config -f "$tmpfile" --remove-section "submodule.$m"
else
newtree="$(git cat-file -p "HEAD^{tree}" | sed -e "s/${revs[$path]}\t$path/$cid\t$path/" | git mktree)"
newcid="$(git commit-tree -p HEAD -m "Update $m" "$newtree")"
commits["$m"]="$newcid"
newtreetext="${treetext/${revs[$path]} $path/$cid $path}"
fi
if isnew "$m" || todrop "$m"; then
nh=$(git hash-object -w --path .gitmodules "$tmpfile")
newtreetext="${newtreetext/$gitmodules_rev .gitmodules/$nh .gitmodules}"
fi
if [ "$single" = 1 ]; then
if isnew "$m" || todrop "$m"; then
gitmodules_rev="$nh"
fi
treetext="$newtreetext"
# fall through
else
newtree=$(echo "$newtreetext" | git mktree)
msg="Update $m"
isnew "$m" && msg="Add $m"
todrop "$m" && msg="Remove $m"
newcid="$(git commit-tree -p "$REMOTE/$BASE_BRANCH" -m "$msg" "$newtree")"
commits["$m"]="$newcid"
cat "$tmpfile2" > "$tmpfile"
continue
fi
# fall through
fi
if [ -n "${pq[refs/pq/$m]}" ]; then
log_info "remove pq for $m"
git update-ref -d "refs/pq/$m"
fi
done
if [ "$allatonce" = 1 ]; then
newtree="$(echo "$treetext" | git mktree)"
newcid="$(git commit-tree -p HEAD -m "Update all" "$newtree")"
commits["all"]="$newcid"
if [ "$single" = 1 ]; then
newtree="$(echo "$treetext" | git mktree)"
newcid="$(git commit-tree -p "$REMOTE/$BASE_BRANCH" -m "Update all" "$newtree")"
commits["all"]="$newcid"
elif [ -n "${pq[refs/pq/all]}" ]; then
log_info "remove pq for single commit"
git update-ref -d "refs/pq/all"
fi
for m in "${!commits[@]}"; do
if git send-pack --force "$pushurl" "${commits[$m]}:refs/heads/update_$m"; then
#https://github.com/go-gitea/gitea/issues/18842
if curl -s -f "$prtarget" \
-H "accept: application/json" \
-H "Authorization: token $token" \
-H "Content-Type: application/json" \
-d "{ \"base\": \"main\", \"head\": \"$prsrcusr:update_$m\", \"title\": \"Update $m\"}" > "$tmpfile"; then
prid=$(jq .id < "$tmpfile")
echo "filed pr #$prid for $m"
else
echo "failed to file pr for $m"
jq .message < "$tmpfile"
fi
else
echo "failed to push $m update" >&2
jq .message < "$tmpfile"
ref="refs/pq/$m"
if [ -n "${pq[$ref]}" ]; then
cid="$(git rev-parse "$ref")"
[ "$cid" = "${commits[$m]}" ] || echo "Warning: previous commit $cid for $m" >&2
fi
git update-ref "$ref" "${commits[$m]}"
done