openSUSE-release-tools/publish_distro

#!/bin/bash

#
# The script is executed periodically by cronjob
# as part of ~mirror/bin/publish_factory_leap wrapper on pontifex
#
# Tool publises data from stage (/src/ftp-stage) to public (/srv/ftp - download.opensuse.org) 
#
# Configuration lives in openSUSE-release-tools.git/publish_distro_conf
# Syntax: ./publish_distro config_file
# 

# no --delete built in

# number of hours to delay publishing when new isos arrive
PUBLISH_DELAY=2

# force publish even if isos appear to be synced already
force=

# used for cleanup
date='now'

# set to echo for dry run
dryrun=

# cleanup instead of sync
do_cleanup=

# url with snapshot changes app
diff_url_base="https://openqa.opensuse.org/snapshot-changes/opensuse"
changes_dir_base="/srv/www-local/snapshot-changes"
repos=("oss" "non-oss")
extra_repos=("source" "debug")
isodir="iso"

get_version() {
	exit 1
}
get_iso() {
	exit 1
}
get_iso_link() {
	exit 1
}
get_diff_url() {
	exit 1
}
get_mark_published_url() {
	exit 1
}
get_changes_filename() {
	exit 1
}

# --dry and --cleanup should not be passed to rsync through $@
# config file and possibly other opts including --force need to be
for arg do
  shift
  if [ "$arg" == '--force' ]; then
        force=1
        newargs+=("$arg")
        shift
        continue
  elif [ "$arg" == '--dry' ]; then
        dryrun=echo
        shift
        continue
   elif [ "$arg" == '--cleanup' ]; then
        do_cleanup=1
        date="90 days ago"
        continue
  elif [ "$arg" == '--now' ]; then
	PUBLISH_DELAY=0
        shift
        continue
  else
        newargs+=("$arg")
  fi
done

# set newargs as $@
set -- "${newargs[@]}"

. "$1" || { echo "need to specify config file" >&2; exit 1; }
shift

stage="/srv/ftp-stage/pub/opensuse/$path"
dest="/srv/ftp/pub/opensuse/$path"

if [ ! -e "$stage" -o ! -e "$dest" ]; then
	echo "stage ($stage) and/or dest ($dest) doesn't exist" >&2
	exit 1
fi

if [ -n "$do_cleanup" ]; then
	# find newest delete log of that day
	deletelog=$(ls -1tr ${deletelog%/*}/*-deletes.log | tail -1)
	test -n "$deletelog" || exit 1
	(
	df -h /srv
	for i in $(awk '/\*deleting   .*/ { print $2}' $deletelog ); do
		if [ -e "$stage/$i" ]; then
			echo "WARNING: $i still exist in stage, not deleting"
		else
			$dryrun rm -rvf "$dest/$i"
		fi
	done
	df -h /srv
	) > $deletelog-deleted
	exit 0
fi

mkdir -p "${synclog%/*}" "${deletelog%/*}"
if test -f $synclog; then
    old $synclog
fi
if test -f $deletelog; then
    old $deletelog
fi

version=
do_sync_isos=1
for flavor in "${flavors[@]}"; do
	get_version

	get_iso

	get_diff_url
	get_changes_filename
	if [ -z "$force" -a -e "$dest/$isodir/$iso" ]; then
		if [ -t 1 ]; then # only log to tty
			echo "$iso already published, skipping isos"
		fi
		do_sync_isos=0
		break
	fi
        if [ -n "$changes" ]; then
                if [ -d "$stage/$changes" ]; then
                       # new way, use the obs generated changelogs
                       $dryrun rsync -avvhiH $stage/$changes \
                         --link-dest=$stage \
                                     $dest/$changes $dry_delete "$@" \
                           | LC_ALL=C grep -v '^\.[fdL]      ' \
                           | LC_ALL=C grep -v '^\(sending\|delta\)' \
                           | tee -a $log
                else
                       # old way (already broken?)
                       if [ ! -s "$changes" ]; then
                        echo "generating $changes" | tee -a $synclog
                        [ -z "$dryrun" ] || changes=/dev/stdout
                        $dryrun curl -sf "$url" > $changes
                       fi
                       if [ ! -e $stage/$isodir/$changes ]; then
                        $dryrun cp -v $changes $stage/$isodir | tee -a $synclog
                        [ -e $stage/$isodir/Changes ] || $dryrun ln -sf . $stage/$isodir/Changes 2>&1 | tee -a $synclog
                       fi
                fi
        fi
	if [ ! -e "$stage/$isodir/$iso" ]; then
		echo "$flavor with $version doesn't exist, skipping isos" | tee -a $synclog
		do_sync_isos=0
		break
	else
		get_iso_link
		if [ "`readlink $link`" != "$iso" ]; then
			$dryrun ln -sf "$iso" "$link"
			$dryrun ln -sf "$iso.sha256" "$link.sha256"
		fi
		if [ -z "$force" ]; then
			if test `date -d "$PUBLISH_DELAY hours ago" +%s` -lt `stat -c "%Z" "$stage/$isodir/$iso"`; then
				echo "$iso was created less than $PUBLISH_DELAY hours ago, delay publishing" | tee -a $synclog
				do_sync_isos=0
			fi
		fi
	fi
done

# produce directories stamped with build number
for r in "${repos[@]/#//repo/}"; do
	for i in /suse/setup /boot; do
		d="$stage$r$i"
		[ -d "$d" ] || continue
		if [ -e "$stage$r/media.1/build" ]; then
			read build < "$stage$r/media.1/build"
			stamp="${build#*Build}"
			if [ "$build" = "$stamp" ]; then
				echo "ERROR: build id not parsable: $build. not syncing"
				do_sync_isos=0
				break 2
			fi
		elif [ -e "$stage$r/media.1/media" ]; then
			{ read dummy; read build; } < "$stage$r/media.1/media"
			stamp="${build#*Build}"
			if [ "$build" = "$stamp" ]; then
				echo "ERROR: build id not parsable: $build. not syncing"
				do_sync_isos=0
				break 2
			fi
		else
			echo "ERROR: repo $r is missing build id, not syncing"
			do_sync_isos=0
			break 2
		fi
		stamped="$stage$r${i%/*}/$stamp-${i##*/}"
		if [ ! -L "$d" ]; then
			$dryrun mv "$d" "$stamped"
			$dryrun ln -s "${stamped##*/}" "$d"
			echo "current $stamp" | $dryrun tee $stage/$r/.current.txt
		fi
	done
done

if [ "$do_sync_isos" = 0 ]; then
	if [ -t 1 ]; then
		echo "nothing to do"
	fi
	exit 0
fi

# scan mirrors before making the files visible
#mb scan -q -a -d /tumbleweed

TODO=()
#if [ "$do_sync_isos" = 1 ]; then
	TODO+=(iso)
	echo "current $version" | $dryrun tee $stage/$isodir/.current.txt
#fi

if [ -n "$repos" ]; then
	TODO+=(repo)
	TODO+=(DELETE_repo)
fi

# a single grep -v regexp performs awfully bad (needs at least 30x so long)
# although LC_ALL=C might fix this, too
for i in "${TODO[@]}"; do

    case $i in 
    DELETE_*)
        i=${i#DELETE_}
        log=$deletelog
        dry_delete="--delete -n"
        ;;
    iso) # ISOs we sync with delete
       log=$synclog
       dry_delete="--delete-after"
	i="$isodir"
       ;;
    *)
        log=$synclog
        dry_delete=""
        ;;
    esac

    echo ========== $i $dry_delete ===========
    $dryrun rsync -avvhiH $stage/$i \
      --link-dest=$stage \
                  $dest $dry_delete "$@" \
        | LC_ALL=C grep -v '^\.[fdL]      ' \
        | LC_ALL=C grep -v '^\(sending\|delta\)' \
        | tee -a $log
done

# also sync source and debug, not bind mounted for Leap
for r in "${extra_repos[@]}"; do
	stage="/srv/ftp-stage/pub/opensuse/$r/$path"
	dest="/srv/ftp/pub/opensuse/$r/$path"
	$dryrun rsync -avhiH $stage/ --link-dest=$stage \
                  $dest --delete-after "$@" | tee -a log
done

# mark published
get_mark_published_url
if [ -n "$url" ]; then
	$dryrun curl -X POST -F "version=$version" "$url"
fi

#echo creating hashes
#metalink-hasher update -t /srv/metalink-hashes/srv/ftp/pub/opensuse/tumbleweed \
#                       -b /srv/ftp/pub/opensuse/tumbleweed \
#                          /srv/ftp/pub/opensuse/tumbleweed

# mb makehaskes is not needed by mirrorcache (replaced mirrorbrain)
#$dryrun /usr/bin/mb makehashes -b /srv/ftp/pub/opensuse \
#                       -t /srv/metalink-hashes/srv/ftp/pub/opensuse \
#                       $dest