diff --git a/README.md b/README.md new file mode 100644 index 0000000..b64d1ca --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +Scripts to sync git submodules of a project via pull requests + +The git repo in the current directory is expected to have at least +one remote pointing to the target project. If that repo is not +writable, another one can be used for pushing. + +create a config file `.settings` in the top level directory of the git submodule project: +``` +# name of the remote to push to as repored by `git remote` +PUSH_REMOTE="joesix" +# username on gitea +PR_SRC_USER="joesix" +# target project for pull requests +PR_PROJECT="mold" +# target repo on gitea +PR_REPO="core" +# token +TOKEN="deadbeef" +# the url for cloning the project. Packages may use an url relative +to that. +PACKAGE_BASE_URL="https://gitea.opensuse.org/mold/core.git" +# relative url for new packages +PACKAGE_RELATIVE_URL="../../rpm" +# remote to to send pull requests to +REMOTE="origin" +# base branch of the target remote +BASE_BRANCH="main" +# optional: fixed date or a filename prefixed with @ to get a stable date for testing +DATE="@token" +# optional: an OBS project with a list of packages +OBSPKGLIST="https://api.opensuse.org/public/source/openSUSE:Factory:Rings:0-Bootstrap" +``` + +Scripts: + +All scripts use getopt, --help may not always be up to date though +:-) + +- updatemodules: checks all submodules for updates. For packages + that need to be updated creates `refs/pq/$packagename`. The + updated refence there updates the submodule commit reference for + updates. It also adds or removes entries from/to `.gitmodules` if + needed. There's also a `--single` option which produces a single + reference for all updates. The `--status` option show the current + state. +- pusher: compares the references created by the `updatemodules` + script with the specified remote. Pushed pending refs to an + `update_$packagename` ref. Gitea doesn't seem to support refs in + subdirs other than `heads`, that's why. Unless the `--dry` option + is specified, the `pusher` script also creates pull requests for + pending refs. +- gitea-pulls: dialog based script to list, view, merge or close + pull requests from the command line. diff --git a/gitea-pulls b/gitea-pulls new file mode 100755 index 0000000..091a316 --- /dev/null +++ b/gitea-pulls @@ -0,0 +1,155 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: Copyright 2023 SUSE LLC +set -e + +# config options +PR_SRC_USER= +PR_PROJECT= +PR_REPO= +TOKEN= + +# command line only +verbose=0 +cfg_file= + +################### + +requestfile=$(mktemp gitea-pulls.XXXXXX) +tmpfile=$(mktemp gitea-pulls.XXXXXX) +cleanup() +{ + rm -f "$requestfile" "$tmpfile" +} +trap cleanup EXIT + +helpandquit() +{ + cat <<-EOF + Usage: $0 [OPTIONS] + OPTIONS: + --verbsoe verbose + -h help screen + EOF + exit 0 +} + +log_info() +{ + [ "$verbose" -gt 0 ] || return 0 + echo "$@" +} + +d(){ + local retval=0 + # Bash makes it a bit annoying to read the output of a different FD into a variable, it + # only supports reading stdout by itself. So redirect 3 to stdout and 1 to the real stdout. + exec {stdoutfd}>&1 + result="$(dialog --backtitle "Gita Pull requests" --output-fd 3 "$@" 3>&1 1>&${stdoutfd})" || retval=$? + # Word splitting makes it necessary to use eval here. + eval "exec ${stdoutfd}>&-" + return "$retval" +} + +# Given the number of total item pairs, outputs the number of items to display at once +menuheight() { + local height=$(($1 / 2)) + [ "$height" -le "$dh_menu" ] || height="$dh_menu" + echo "$height" +} + +stty_size() { + set -- $(stty size) + LINES="$1" + COLUMNS="$2" + # stty size can return zero when not ready or + # its a serial console + if [ "$COLUMNS" = "0" ] || [ "$LINES" = "0" ]; then + LINES=24 + COLUMNS=80 + fi + + dh_menu=$((LINES-15)) + dh_text=$((LINES-5)) +} +stty_size + +getopttmp=$(getopt -o hc:v --long help,config:,verbose -n "${0##*/}" -- "$@") +eval set -- "$getopttmp" + +while true ; do + case "$1" in + -h|--help) helpandquit; shift ;; + -v|--verbose) verbose=$((++verbose)); shift ;; + -c|--config) cfg_file="$2"; shift 2 ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; + esac +done + +# shellcheck disable=SC1090 +. "${cfg_file:-.settings}" + +needed=(PR_SRC_USER PR_PROJECT PR_REPO TOKEN) +for i in "${needed[@]}"; do + eval test -n "\"\$$i\"" || { echo "The following settings are mandatory: ${needed[*]}"; exit 1; } +done + +request() +{ + local urlpart="$1" + shift + curl -s -f "https://gitea.opensuse.org/api/v1/repos/$PR_PROJECT/$PR_REPO$urlpart" \ + -H "accept: application/json" \ + -H "Authorization: token $TOKEN" \ + -H "Content-Type: application/json" "$@" +} + +fetch_requests() +{ + request "/pulls?state=open" > "$requestfile" +} + +fetch_requests + +while true; do + list=() + while read -r number _branch _cid owner title; do + list+=("$number" "$owner: $title") + done < <(jq '.[]|[((.number|tostring)), .head.ref, .head.sha, .head.repo.owner.login, .title]|join(" ")' -r < "$requestfile") + if [ "${#list}" = 0 ]; then + d --msgbox "No pull requests in $PR_PROJECT/$PR_REPO" 0 0 + exit 0 + fi + d --no-hot-list --menu "$PR_PROJECT/$PR_REPO pull requests" 0 0 "$(menuheight ${#list[@]})" "${list[@]}" || exit 1 + prno="$result" + + while true; do + list=(raw raw diff diff merge merge close close) + d --no-tags --menu "$PR_PROJECT/$PR_REPO #$prno" 0 0 "$(menuheight ${#list[@]})" "${list[@]}" || break + action="$result" + + case "$action" in + raw) + jq ".[]|select(.number==$prno)" < "$requestfile" > "$tmpfile" + d --textbox "$tmpfile" 0 0 + ;; + diff) + request "/pulls/$prno.diff" > "$tmpfile" + d --textbox "$tmpfile" 0 0 + ;; + merge) + request "/pulls/$prno/merge" -d '{ "do": "merge", "delete_branch_after_merge": true}' | jq > "$tmpfile" + d --textbox "$tmpfile" 0 0 + fetch_requests + break + ;; + close) + request "/pulls/$prno" -d '{ "state": "closed"}' -X PATCH | jq > "$tmpfile" + d --textbox "$tmpfile" 0 0 + fetch_requests + break + ;; + esac + done +done diff --git a/pusher b/pusher new file mode 100755 index 0000000..c8c9325 --- /dev/null +++ b/pusher @@ -0,0 +1,176 @@ +#!/bin/bash +set -e + +# config options +# shellcheck disable=SC2034 +declare -A keep +PR_SRC_USER= +BASE_BRANCH="main" +PR_PROJECT= +PR_REPO= +TOKEN= +PUSH_REMOTE="origin" + +# command line only +verbose=0 +cfg_file= +dryrun= +show_status= + +################### +push_url= +did_push= + +tmpfile=$(mktemp pusher.XXXXXX) +cleanup() +{ + rm -f "$tmpfile" +} +trap cleanup EXIT + +helpandquit() +{ + cat <<-EOF + Usage: $0 [OPTIONS] [ ...] + OPTIONS: + -h help screen + EOF + exit 0 +} + +log_info() +{ + [ "$verbose" -gt 0 ] || return 0 + echo "$@" +} + +request_pulls() +{ + curl -s -f "$pr_target?state=open" \ + -H "accept: application/json" \ + -H "Authorization: token $TOKEN" \ + -H "Content-Type: application/json" +} + +getopttmp=$(getopt -o hc:v --long help,single,branch:,config:,date:,remote:,status,verbose,dry -n "${0##*/}" -- "$@") +eval set -- "$getopttmp" + +while true ; do + case "$1" in + -h|--help) helpandquit; shift ;; + -v|--verbose) verbose=$((++verbose)); shift ;; + -c|--config) cfg_file="$2"; shift 2 ;; + --status) show_status=1; shift ;; + --dry) dryrun=1; shift ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; + esac +done + +declare -A modules +for m in "$@"; do + modules["$m"]=1 +done + +# shellcheck disable=SC1090 +. "${cfg_file:-.settings}" + +needed=(PR_SRC_USER PR_PROJECT PR_REPO TOKEN) +for i in "${needed[@]}"; do + eval test -n "\"\$$i\"" || { echo "The following settings are mandatory: ${needed[*]}"; exit 1; } +done + +push_url="$(git config --get remote."$PUSH_REMOTE".url)" +pr_target="https://gitea.opensuse.org/api/v1/repos/$PR_PROJECT/$PR_REPO/pulls" + +# push queue +declare -A pq +while read -r cid _c ref; do + m="${ref#refs/pq/}" + pq["$m"]="$cid" +done < <(git for-each-ref 'refs/pq/*') +# request queue +declare -A rq +while read -r cid _c ref; do + m="${ref#refs/remotes/"$PUSH_REMOTE"/update_}" + [ "$m" != "$ref" ] || continue + rq["$m"]="$cid" +done < <(git for-each-ref "refs/remotes/$PUSH_REMOTE/*") + +declare -A pr +while read -r _id cid ref; do + m="${ref#update_}" + pr["$m"]="$cid" + log_info "pr $m: $cid" +done < <(request_pulls | jq ".[]|select(.head.repo.owner.login == \"$PR_SRC_USER\")|[((.number|tostring)), .head.sha, .head.ref]|join(\" \")" -r) + +log_info "to push: ${!pq[*]}" +log_info "remote: ${!rq[*]}" + +if [ -n "$show_status" ]; then + echo "# local remote pr package" + if [ "${#modules[@]}" -eq 0 ]; then + for m in "${!pq[@]}" "${!rq[@]}" "${!pr[@]}"; do + modules["$m"]=1 + done + fi + for m in "${!modules[@]}"; do + a=${pq[$m]:0:7} + b=${rq[$m]:0:7} + c=${pr[$m]:0:7} + echo "${a:- } ${b:- } ${c:- } $m" + done + exit 0 +fi + +ret=0 +for m in "${!pq[@]}"; do + [ "${#modules[@]}" -eq 0 -o -n "${modules[$m]}" ] || continue + cid="${pq[$m]}" + [ -n "$cid" ] + if [ -n "${rq[$m]}" ] && [ "$cid" = "${rq[$m]}" ]; then + log_info "$m already pushed" + else + log_info "pushing $m: $cid" + if ! git send-pack --force "$push_url" "${cid}:refs/heads/update_$m"; then + echo "failed to push $m update" >&2 + ret=1 + continue + fi + did_push=1 + fi + + if [ -n "${pr[$m]}" ]; then + if [ "$cid" = "${pr[$m]}" ]; then + log_info "pr for $m exists, no update" + else + echo "Warning: exiting pr for $m with different content" + fi + continue + fi + + title=$(git log -1 "--format=%s" "$cid") + head="update_$m" + [ "$PR_PROJECT" = "$PR_SRC_USER" ] || head="$PR_SRC_USER:$head" + log_info "Filing pr '$title': $cid at $pr_target" + [ -z "$dryrun" ] || continue + #https://github.com/go-gitea/gitea/issues/18842 + if curl -s -f "$pr_target" \ + -H "accept: application/json" \ + -H "Authorization: token $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{ \"base\": \"$BASE_BRANCH\", \"head\": \"$head\", \"title\": \"$title\"}" > "$tmpfile"; then + prid=$(jq -r .number < "$tmpfile") + echo "filed pr #$prid for $m" + else + echo "failed to file pr for $m" + jq -r .message < "$tmpfile" + fi +done + +if [ -n "$did_push" ]; then + log_info "Updating remote $PUSH_REMOTE" + git remote update -p "$PUSH_REMOTE" > /dev/null +fi + +exit "$ret" diff --git a/updatemodules b/updatemodules index 7e1fe0e..70eca6d 100755 --- a/updatemodules +++ b/updatemodules @@ -1,100 +1,250 @@ #!/bin/bash -allatonce= -pushurl="gitea@gitea.opensuse.org:lnussel/core.git" -prsrcusr="lnussel" -prtarget="https://gitea.opensuse.org/api/v1/repos/mold/core/pulls" +set -e -if [ "$1" = '--allatonce' ]; then - allatonce=1 +# config options +declare -A keep +REMOTE="origin" +BASE_BRANCH="main" +PACKAGE_BASE_URL= +PACKAGE_RELATIVE_URL="../../rpm" +# https://api.opensuse.org/public/source/openSUSE:Factory:Rings:0-Bootstrap +OBSPKGLIST= +DATE="$(date "+%s %z")" +keep["dummy-release"]=1 + +# command line only +single= +verbose=0 +cfg_file= + +################### + +# constant needed to aid quoting +nl=$'\n' + +declare -A obs +declare -A new +declare -A drop +declare -A revs +gitmodules_rev= + +helpandquit() +{ + cat <<-EOF + Usage: $0 [OPTIONS] [ ...] + OPTIONS: + --single create single commit for all changes + -h help screen + EOF + exit 0 +} + +show_status() +{ + git for-each-ref 'refs/pq/*' + exit 0 +} + +clear_queue() +{ + while read -r ref; do + git update-ref -d "$ref" + done < <(git for-each-ref 'refs/pq/*' --format '%(refname)') + exit 0 +} + +isnew() +{ + local p="${1:?}" + [ -n "${new[$p]}" ] +} + +todrop() +{ + local p="${1:?}" + [ -n "${drop[$p]}" ] +} + +log_info() +{ + [ "$verbose" -gt 0 ] || return 0 + echo "$@" +} + +makedict() +{ + local dict="$1" shift + mapfile -t a < <("$@") + for k in "${a[@]}"; do + eval "$dict"["$k"]=1 + done +} + +getopttmp=$(getopt -o hc:v --long help,single,branch:,config:,date:,remote:,status,verbose,clear -n "${0##*/}" -- "$@") +eval set -- "$getopttmp" + +while true ; do + case "$1" in + -h|--help) helpandquit; shift ;; + -v|--verbose) verbose=$((++verbose)); shift ;; + -c|--config) cfg_file="$2"; shift 2 ;; + --single) single=1; shift ;; + --remote) REMOTE="$2"; shift 2 ;; + --branch) BASE_BRANCH="$2"; shift 2 ;; + --status) show_status; exit 0 ;; + --clear) clear_queue; exit 0 ;; + --date) DATE="$2"; shift 2 ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; + esac +done + +modules=("$@") + +PACKAGE_BASE_URL="$(git config --get remote."$REMOTE".url)" + +# shellcheck disable=SC1090 +. "${cfg_file:-.settings}" + +if [ "${DATE:0:1}" = '@' ]; then + DATE="$(stat -c %Y "${DATE:1}") +0100" fi -modules=($@) -origin="$(git config --get remote.origin.url)" -now="$(date "+%s %z")" -# FIXME: hardcoded to avoid changing commits for now -now="$(stat -c %Y token) +0100" export GIT_AUTHOR_NAME="Auto" export GIT_AUTHOR_EMAIL="auto@suse.de" -export GIT_AUTHOR_DATE="$now" +export GIT_AUTHOR_DATE="$DATE" export GIT_COMMITTER_NAME="Auto" export GIT_COMMITTER_EMAIL="auto@suse.de" -export GIT_COMMITTER_DATE="$now" +export GIT_COMMITTER_DATE="$DATE" tmpfile=$(mktemp updatemodules.XXXXXX) +tmpfile2=$(mktemp updatemodules.XXXXXX) cleanup() { - rm -f "$tmpfile" + rm -f "$tmpfile" "$tmpfile2" } trap cleanup EXIT -read -r token < token || exit 1 - # read all submodules -declare -A revs while read -r m t cid p; do - [ "$t" = "commit" ] || continue + if [ "$t" = blob ] && [ "$p" = ".gitmodules" ]; then + gitmodules_rev="$cid" + fi + [ "$t" = "commit" ] || continue revs["$p"]="$cid" -done < <(git cat-file -p "HEAD^{tree}") +done < <(git cat-file -p "$REMOTE/$BASE_BRANCH^{tree}") -if [ -z "$modules" ]; then - modules=("${!revs[@]}") +if [ -n "$OBSPKGLIST" ]; then + while read -r p; do + obs["$p"]=1 + [ -n "${revs[$p]}" ] || new["$p"]=1 + done < <(curl -s -f "$OBSPKGLIST"|sed -ne 's/.*entry name="\([^":]*\).*\/>/\1/p'|sort -u | grep -v AGGR) + + for m in "${!revs[@]}"; do + [ -n "${obs[$m]}" -o -n "${keep[$m]}" ] || drop["$m"]=1 + done fi +[ -z "${new[*]}" ] || log_info "new: ${!new[*]}" +[ -z "${drop[*]}" ] || log_info "drop: ${!drop[*]}" + +if [ "${#modules[@]}" = 0 ]; then + modules=("${!revs[@]}" "${!new[@]}") +fi + +# push queue +declare -A pq +makedict pq git for-each-ref 'refs/pq/*' --format '%(refname)' + # check remotes for updates declare -A commits -treetext=$(git cat-file -p "HEAD^{tree}") +treetext=$(git cat-file -p "$REMOTE/$BASE_BRANCH^{tree}") +git cat-file -p "$REMOTE/$BASE_BRANCH":.gitmodules > "$tmpfile" +cat "$tmpfile" > "$tmpfile2" for m in "${modules[@]}"; do - path="$(git config -f .gitmodules --get "submodule.$m.path")" - url="$(git config -f .gitmodules --get "submodule.$m.url")" - branch="$(git config -f .gitmodules --get "submodule.$m.branch")" + if isnew "$m"; then + path="$m" + url="$PACKAGE_RELATIVE_URL/$m" + smbranch= + git config -f "$tmpfile" --add "submodule.$m.path" "$path" + git config -f "$tmpfile" --add "submodule.$m.url" "$url" + else + path="$(git config -f "$tmpfile" --get "submodule.$m.path")" + url="$(git config -f "$tmpfile" --get "submodule.$m.url")" + smbranch="$(git config -f "$tmpfile" --get "submodule.$m.branch" || :)" + fi if [ -z "$path" ] || [ -z "$url" ]; then echo "$m unknown" >&2 continue fi if [ "${url:0:3}" = '../' ]; then - url="$origin/$url" + url="$PACKAGE_BASE_URL/$url" fi - read -r cid _d < <(git ls-remote "$url" "${branch:-HEAD}") - if [ -z "${revs[$path]}" ]; then - echo "$path unknown" >&2 - continue + + cid= + if ! todrop "$m"; then + read -r cid _d < <(git ls-remote "$url" "${smbranch:-HEAD}") + if [ -z "$cid" ]; then + echo "$path not in pool" >&2 + continue + fi fi + # create a new commit for this package if [ "${revs[$path]}" != "$cid" ]; then - echo "Needs update: $path@${revs[$path]} -> $cid" - if [ "$allatonce" = 1 ]; then - treetext="${treetext/${revs[$path]} $path/$cid $path}" + log_info "Needs update: $path@${revs[$path]:-NEW} -> ${cid:-DROP}" + if isnew "$m"; then + newtreetext="$treetext${nl}160000 commit $cid $m" + elif todrop "$m"; then + newtreetext="${treetext/160000 commit ${revs[$path]} $path$nl/}" + git config -f "$tmpfile" --remove-section "submodule.$m" else - newtree="$(git cat-file -p "HEAD^{tree}" | sed -e "s/${revs[$path]}\t$path/$cid\t$path/" | git mktree)" - newcid="$(git commit-tree -p HEAD -m "Update $m" "$newtree")" - commits["$m"]="$newcid" + newtreetext="${treetext/${revs[$path]} $path/$cid $path}" fi + if isnew "$m" || todrop "$m"; then + nh=$(git hash-object -w --path .gitmodules "$tmpfile") + newtreetext="${newtreetext/$gitmodules_rev .gitmodules/$nh .gitmodules}" + fi + + if [ "$single" = 1 ]; then + if isnew "$m" || todrop "$m"; then + gitmodules_rev="$nh" + fi + treetext="$newtreetext" + # fall through + else + newtree=$(echo "$newtreetext" | git mktree) + msg="Update $m" + isnew "$m" && msg="Add $m" + todrop "$m" && msg="Remove $m" + newcid="$(git commit-tree -p "$REMOTE/$BASE_BRANCH" -m "$msg" "$newtree")" + commits["$m"]="$newcid" + cat "$tmpfile2" > "$tmpfile" + continue + fi + # fall through + fi + if [ -n "${pq[refs/pq/$m]}" ]; then + log_info "remove pq for $m" + git update-ref -d "refs/pq/$m" fi done -if [ "$allatonce" = 1 ]; then - newtree="$(echo "$treetext" | git mktree)" - newcid="$(git commit-tree -p HEAD -m "Update all" "$newtree")" - commits["all"]="$newcid" +if [ "$single" = 1 ]; then + newtree="$(echo "$treetext" | git mktree)" + newcid="$(git commit-tree -p "$REMOTE/$BASE_BRANCH" -m "Update all" "$newtree")" + commits["all"]="$newcid" +elif [ -n "${pq[refs/pq/all]}" ]; then + log_info "remove pq for single commit" + git update-ref -d "refs/pq/all" fi for m in "${!commits[@]}"; do - if git send-pack --force "$pushurl" "${commits[$m]}:refs/heads/update_$m"; then - #https://github.com/go-gitea/gitea/issues/18842 - if curl -s -f "$prtarget" \ - -H "accept: application/json" \ - -H "Authorization: token $token" \ - -H "Content-Type: application/json" \ - -d "{ \"base\": \"main\", \"head\": \"$prsrcusr:update_$m\", \"title\": \"Update $m\"}" > "$tmpfile"; then - prid=$(jq .id < "$tmpfile") - echo "filed pr #$prid for $m" - else - echo "failed to file pr for $m" - jq .message < "$tmpfile" - fi - else - echo "failed to push $m update" >&2 - jq .message < "$tmpfile" + ref="refs/pq/$m" + if [ -n "${pq[$ref]}" ]; then + cid="$(git rev-parse "$ref")" + [ "$cid" = "${commits[$m]}" ] || echo "Warning: previous commit $cid for $m" >&2 fi + git update-ref "$ref" "${commits[$m]}" done